{
  "slug": "hf-endpoints-deploy-and-update-support-managed-engine-images-and-parallelism-set-1911ad7ea871",
  "title": "hf endpoints deploy and update support managed engine images and parallelism settings",
  "tools": [
    "hf"
  ],
  "tags": [
    "agent"
  ],
  "introduced_at": "2026-08-18T12:03:22.000Z",
  "announced_at": "2026-08-18T12:03:22.000Z",
  "effective_at": null,
  "date_basis": "source-publication",
  "status": "unknown",
  "how": {
    "incantation": "When deploying a vLLM or SGLang endpoint on a multi-accelerator instance, set parallelism to use the allocated accelerators. Use `hf endpoints deploy` to configure its engine image, or `hf endpoints update` to retune a running endpoint.",
    "examples": [
      "hf endpoints deploy gpt-oss-120b-vllm --repo openai/gpt-oss-120b --framework custom --accelerator gpu --instance-size x8 --instance-type nvidia-h200 --region us-east-1 --vendor aws --engine vllm --custom-image vllm/vllm-openai:v0.23.0 --tensor-parallel-size 8",
      "hf endpoints update gpt-oss-120b-vllm --tensor-parallel-size 4 --data-parallel-size 2",
      "custom_image={\"vLLM\": {\"url\": \"vllm/vllm-openai:v0.23.0\", \"healthRoute\": \"/health\", \"port\": 8000, \"tensorParallelSize\": 1}}"
    ],
    "gotchas": [
      "Version: v1.28.0",
      "--engine: Managed engine image to run with `--custom-image`; documented values include vLLM and SGLang engines. Defaults to an arbitrary container.",
      "--custom-image: Docker image URL for the container; requires `--framework custom`. On update, replacing the image requires passing settings to keep.",
      "--tensor-parallel-size: Number of accelerators to shard one model copy across; for vLLM and SGLang.",
      "--data-parallel-size: Number of model copies to run, one per accelerator; for vLLM only.",
      "--health-route: Health check route exposed by the container; requires `--custom-image`.",
      "--port: Port the container listens on; requires `--custom-image`.",
      "custom_image.<engine>: Use an engine-name key such as `vLLM` or `sGLang`; its value is an object forwarded to the API when it has no top-level `url`.",
      "custom_image.<engine>.url: Container image URL.",
      "custom_image.<engine>.port: Container port.",
      "custom_image.<engine>.healthRoute: Container health check route.",
      "custom_image.<engine>.tensorParallelSize: Engine tuning option for tensor parallelism.",
      "Prerequisite: Deploy requires `--repo`, `--framework`, `--accelerator`, `--instance-size`, and `--instance-type`; valid hardware combinations also require `--vendor` and `--region`.",
      "Prerequisite: `--custom-image` requires `--framework custom`.",
      "A custom_image dictionary without a top-level `url` is forwarded to the API untouched.",
      "vLLM and SGLang default to a single accelerator while an endpoint is allocated every accelerator of its instance; the API rejects that misconfiguration."
    ]
  },
  "why_it_matters": "Deploy or retune an Inference Endpoint using a custom engine image or multi-accelerator parallelism.",
  "before": "Configure `custom_image` using the prior flat format and use other available endpoint settings; vLLM and SGLang defaulted to one accelerator despite an instance allocating every accelerator.",
  "sources": [
    "https://github.com/huggingface/huggingface_hub/releases/tag/v1.28.0",
    "https://huggingface.co/docs/huggingface_hub/en/guides/inference_endpoints",
    "https://huggingface.co/docs/huggingface_hub/en/package_reference/cli"
  ],
  "rank": 0.98062,
  "judge": {
    "publish": true,
    "judge_model": "codex-cli/gpt-6-luna+typesafe/jev-1.13.0",
    "prompt_version": "extract-v7+enrich-v2+judge-v2x3+jev-publish-v1"
  },
  "review_state": "approved",
  "provenance": {
    "run_id": "wave2b-20261017",
    "verdict_id": "wave2b-20261017:wave2b-20261017-9caf816dceeb1ab1-1:gate",
    "evidence_hash": "993462f9fdde4b163b463844dd6de5545f7218432e2911419ef1fe81f6818c65"
  },
  "review": {
    "draft_hash": "cbd19cf2636b3957bca7eecf687d3312f005f994a1ade29173c1bb888bdf9d48",
    "reviewer": "publication-gate",
    "reviewed_at": "2026-10-07T00:00:00.000Z",
    "decision": "wave2b-20261017:gate",
    "reason": "3 of 3 judge runs approved with p_yes >= 0.9 (mean 0.98 >= 0.9; 2 needed) and Jev gave 0.72 (>= 0.6 needed)."
  }
}