diff --git a/docs/docs/sglang-diffusion/api/cli.mdx b/docs/docs/sglang-diffusion/api/cli.mdx index 20eaebab2..a8be23b96 100644 --- a/docs/docs/sglang-diffusion/api/cli.mdx +++ b/docs/docs/sglang-diffusion/api/cli.mdx @@ -75,6 +75,7 @@ Use `sglang generate --help` and `sglang serve --help` for the full argument lis ### Model and runtime - `--model-path {MODEL}`: model path or Hugging Face model ID +- `--served-model-name {NAME}`: stable model name exposed by serving APIs. Defaults to `--model-id` when set, otherwise `--model-path`. - `--model-variant {NAME}`: semantic checkpoint variant to load when one model repository contains multiple weight partitions. The pipeline maps this stable name to the repository layout before loading; for example, MiniMax-H3 accepts `fl2va` and `ref2va`. This is a server/load-time choice, unlike a request's `task`. - `--model-subfolder {PATH}`: advanced direct override for a component subfolder inside the model repository. Prefer `--model-variant` when the pipeline exposes semantic routing. If both are supplied, they must resolve to the same weight partition. - `--lora-path {PATH}` and `--lora-nickname {NAME}`: load a LoRA adapter diff --git a/docs/docs/sglang-diffusion/api/openai_api.mdx b/docs/docs/sglang-diffusion/api/openai_api.mdx index a68aa9e82..069b4bfbd 100644 --- a/docs/docs/sglang-diffusion/api/openai_api.mdx +++ b/docs/docs/sglang-diffusion/api/openai_api.mdx @@ -18,6 +18,7 @@ Launch the server using the `sglang serve` command. ```bash SERVER_ARGS=( --model-path Wan-AI/Wan2.1-T2V-1.3B-Diffusers + --served-model-name wan-t2v --text-encoder-cpu-offload --pin-cpu-memory --num-gpus 4 @@ -30,34 +31,67 @@ sglang serve "${SERVER_ARGS[@]}" ``` - **--model-path**: Path to the model or model ID. +- **--served-model-name**: Stable model name exposed by the serving APIs. It defaults to `--model-id` when set, otherwise `--model-path`. - **--port**: HTTP port to listen on (default: `30000`). -**Get Model Information** +### Served model name -**Endpoint:** `GET /models` +`--served-model-name` separates the public API identity from the checkpoint location. This is useful when replicas use different local mount paths or when a gateway needs one stable model name: -Returns information about the model served by this server, including model path, task type, pipeline configuration, and precision settings. +```bash +sglang serve \ + --model-path /models/Wan2.1-T2V-1.3B-Diffusers \ + --served-model-name wan-t2v \ + --port 30010 +``` + +`--model-id` is not a free-form deployment alias: it selects the registered model configuration for checkpoints whose local path cannot be identified. `--served-model-name` only controls the name exposed by serving APIs. When both are set, the served name takes precedence for API responses. + +### Discover the served model + +**Endpoint:** `GET /v1/models` + +Returns the public model name together with diffusion-specific runtime information. **Curl Example:** ```bash curl -curl -sS -X GET "http://localhost:30010/models" +curl -sS "http://localhost:30010/v1/models" ``` **Response Example:** ```json { - "model_path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", - "task_type": "T2V", - "pipeline_name": "wan_pipeline", - "pipeline_class": "WanPipeline", - "num_gpus": 4, - "dit_precision": "bf16", - "vae_precision": "fp16" + "object": "list", + "data": [ + { + "id": "wan-t2v", + "object": "model", + "created": 1786348800, + "owned_by": "sglang", + "root": "wan-t2v", + "parent": null, + "max_model_len": null, + "num_gpus": 4, + "task_type": "T2V", + "dit_precision": "bf16", + "vae_precision": "fp16", + "pipeline_name": "WanPipeline", + "pipeline_class": "WanPipeline" + } + ] } ``` +Retrieve the same model by its served name: + +```bash curl +curl -sS "http://localhost:30010/v1/models/wan-t2v" +``` + +`GET /server_info` also reports `served_model_name` for gateway discovery. Video and action responses use this name when the request does not provide a model explicitly. + --- ## Endpoints diff --git a/docs/docs/sglang-diffusion/deployment_cookbook.mdx b/docs/docs/sglang-diffusion/deployment_cookbook.mdx index bc0d333a2..38f410705 100644 --- a/docs/docs/sglang-diffusion/deployment_cookbook.mdx +++ b/docs/docs/sglang-diffusion/deployment_cookbook.mdx @@ -77,6 +77,22 @@ livenessProbe: See [Health endpoints](/docs/sglang-diffusion/api/cli#health-endpoints) for the status-code contract and warmup-mode behavior. +## Stable Model Identity + +Use `--served-model-name` when the public model name must remain stable across replicas, hosts, or checkpoint mount paths: + +```bash +sglang serve \ + --model-path /mnt/checkpoints/Qwen-Image \ + --model-id Qwen-Image \ + --served-model-name image-production \ + --port 30010 +``` + +The resolved public name follows `--served-model-name`, then `--model-id`, then `--model-path`. `--model-id` remains an internal model registry and configuration-resolution hint; it is not a replacement for a deployment alias. The resolved name is exposed through `/server_info` and `/v1/models` and is used by video and action responses when a request does not supply its own model. + +See [OpenAI API: Served model name](/docs/sglang-diffusion/api/openai_api#served-model-name) for discovery and retrieval examples. + ## Performance Modes `--performance-mode` applies safe presets without overriding explicit offload, FSDP, or parallelism flags. `auto` is the default. Use `manual` when you need to keep performance-related server args under explicit user control. `--mode` is a short alias. diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/action/protocol.py b/python/sglang/multimodal_gen/runtime/entrypoints/action/protocol.py index 2e4ced790..d4f65b2be 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/action/protocol.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/action/protocol.py @@ -151,7 +151,7 @@ def action_metadata(server_args: ServerArgs) -> dict[str, Any]: defaults = Cosmos3SamplingParams() return { "object": "action.metadata", - "model": server_args.model_id or server_args.model_path, + "model": server_args.served_model_name, "model_path": server_args.model_path, "policy_family": "cosmos3", "input": { @@ -201,7 +201,7 @@ def action_metadata(server_args: ServerArgs) -> dict[str, Any]: ) return { "object": "action.metadata", - "model": server_args.model_id or server_args.model_path, + "model": server_args.served_model_name, "model_path": server_args.model_path, "policy_family": policy_family, "input": { @@ -622,7 +622,7 @@ def action_generation_response( "id": output.get("request_id") or f"act_{uuid.uuid4().hex}", "object": "action.generation", "created": int(time.time()), - "model": server_args.model_id or server_args.model_path, + "model": server_args.served_model_name, "data": [ { "index": 0, diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py b/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py index 2b16736d4..1b5098acb 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py @@ -205,7 +205,7 @@ async def server_info_endpoint(request: Request): return { "model_path": server_args.model_path, - "served_model_name": server_args.model_id or server_args.model_path, + "served_model_name": server_args.served_model_name, "tp_size": server_args.tp_size, "dp_size": server_args.dp_size, "version": __version__, diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/openai/common_api.py b/python/sglang/multimodal_gen/runtime/entrypoints/openai/common_api.py index c1f50c2b9..a4a68545a 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/openai/common_api.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/openai/common_api.py @@ -45,15 +45,17 @@ class DiffusionModelCard(ModelCard): pipeline_class: Optional[str] = None -def _build_model_card(server_args: ServerArgs, model_id: str) -> DiffusionModelCard: +def _build_model_card( + server_args: ServerArgs, served_model_name: str +) -> DiffusionModelCard: model_info = get_model_info( server_args.model_path, backend=server_args.backend, model_id=server_args.model_id, ) card_kwargs: dict[str, Any] = { - "id": model_id, - "root": model_id, + "id": served_model_name, + "root": served_model_name, "num_gpus": server_args.num_gpus, "task_type": server_args.pipeline_config.task_type.name, "dit_precision": server_args.pipeline_config.dit_precision, @@ -203,7 +205,7 @@ async def available_models(): if not server_args: raise HTTPException(status_code=500, detail="Server args not initialized") - model_card = _build_model_card(server_args, server_args.model_path) + model_card = _build_model_card(server_args, server_args.served_model_name) # Return dict directly to preserve extended fields (ModelList strips them) return {"object": "list", "data": [model_card.model_dump()]} @@ -216,7 +218,7 @@ async def retrieve_model(model: str): if not server_args: raise HTTPException(status_code=500, detail="Server args not initialized") - if model != server_args.model_path: + if model != server_args.served_model_name: return orjson_response( { "error": { @@ -230,4 +232,4 @@ async def retrieve_model(model: str): ) # Return dict to preserve extended fields - return _build_model_card(server_args, model).model_dump() + return _build_model_card(server_args, server_args.served_model_name).model_dump() diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py b/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py index 676b3ad1a..b28244c11 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py @@ -425,14 +425,17 @@ def _build_video_sampling_params(request_id: str, request: VideoGenerationsReque # extract metadata which http_server needs to know def _video_job_from_sampling( - request_id: str, req: VideoGenerationsRequest, sampling: SamplingParams + request_id: str, + req: VideoGenerationsRequest, + sampling: SamplingParams, + served_model_name: str, ) -> Dict[str, Any]: size_str = f"{sampling.width}x{sampling.height}" seconds = int(round((sampling.num_frames or 0) / float(sampling.fps or 24))) return { "id": request_id, "object": "video", - "model": req.model or "sora-2", + "model": req.model or served_model_name, "status": "queued", "progress": 0, "created_at": int(time.time()), @@ -841,7 +844,12 @@ async def create_video( scheduler_batches = sampling_params.expand_video_request_outputs_for_queue( batch ) - job = _video_job_from_sampling(request_id, req, sampling_params) + job = _video_job_from_sampling( + request_id, + req, + sampling_params, + server_args.served_model_name, + ) job.update(sampling_params.project_video_queued_job_fields(batch)) await VIDEO_STORE.upsert(request_id, job) except Exception as e: diff --git a/python/sglang/multimodal_gen/runtime/launch_server.py b/python/sglang/multimodal_gen/runtime/launch_server.py index 2820c3df0..c89e63e3a 100644 --- a/python/sglang/multimodal_gen/runtime/launch_server.py +++ b/python/sglang/multimodal_gen/runtime/launch_server.py @@ -309,7 +309,7 @@ def launch_server(server_args: ServerArgs, launch_http_server: bool = True): logger.info( "Action generation endpoint ready: model=%s; per-request details are " "debug-only (use --log-level debug).", - server_args.model_id or server_args.model_path, + server_args.served_model_name, ) logger.info("Starting FastAPI server.") if server_args.webui: diff --git a/python/sglang/multimodal_gen/runtime/server_args/server_args.py b/python/sglang/multimodal_gen/runtime/server_args/server_args.py index 646fcdb4c..d1091c99a 100644 --- a/python/sglang/multimodal_gen/runtime/server_args/server_args.py +++ b/python/sglang/multimodal_gen/runtime/server_args/server_args.py @@ -199,6 +199,9 @@ class ServerArgs(DisaggServerArgsMixin): # explicit model ID override (e.g. "Qwen-Image") model_id: str | None = None + # served model name exposed via /v1/models and generation responses + served_model_name: str | None = None + # Model backend (sglang native or diffusers) backend: Backend = Backend.AUTO @@ -492,6 +495,8 @@ class ServerArgs(DisaggServerArgsMixin): auto_tuner.maybe_adjust_auto_fsdp_with_offload_enabled() auto_tuner.maybe_replace_cpu_offloaded_components_with_layerwise() self._adjust_path() + if self.served_model_name is None: + self.served_model_name = self.model_id or self.model_path self._adjust_quant_config() self._adjust_breakable_cuda_graph_support() self._adjust_warmup() @@ -1428,6 +1433,15 @@ class ServerArgs(DisaggServerArgsMixin): "(e.g. 'Qwen-Image' for 'Qwen/Qwen-Image')." ), ) + parser.add_argument( + "--served-model-name", + type=str, + default=ServerArgs.served_model_name, + help=( + "Override the model name exposed by /v1/models and used in generation " + "responses. Defaults to --model-id if set, otherwise --model-path." + ), + ) parser.add_argument( "--pipeline", "--pipeline-class-name", diff --git a/python/sglang/multimodal_gen/test/unit/test_cosmos3.py b/python/sglang/multimodal_gen/test/unit/test_cosmos3.py index 68cd4239e..29cf01ca5 100644 --- a/python/sglang/multimodal_gen/test/unit/test_cosmos3.py +++ b/python/sglang/multimodal_gen/test/unit/test_cosmos3.py @@ -66,6 +66,7 @@ def _cosmos3_server_args(config=None): return types.SimpleNamespace( model_id=None, model_path="nvidia/Cosmos3-Nano", + served_model_name="cosmos3-production", backend=None, pipeline_class_name=None, output_path=None, @@ -453,6 +454,7 @@ class TestCosmos3ActionEndpoint(unittest.TestCase): def test_metadata_describes_cosmos_action_contract(self): metadata = action_metadata(_cosmos3_server_args()) + self.assertEqual(metadata["model"], "cosmos3-production") self.assertEqual(metadata["policy_family"], "cosmos3") self.assertEqual(metadata["input"]["modalities"], ["image", "video"]) self.assertEqual(metadata["output"]["action_horizon"], 16) diff --git a/python/sglang/multimodal_gen/test/unit/test_pi05_action_api.py b/python/sglang/multimodal_gen/test/unit/test_pi05_action_api.py index 0c26b0e47..edd5d54b7 100644 --- a/python/sglang/multimodal_gen/test/unit/test_pi05_action_api.py +++ b/python/sglang/multimodal_gen/test/unit/test_pi05_action_api.py @@ -26,6 +26,7 @@ def _server_args(config: Pi05PipelineConfig | None = None) -> SimpleNamespace: return SimpleNamespace( model_id=None, model_path="lerobot/pi05_base", + served_model_name="pi05-production", output_path=None, comfyui_mode=False, num_gpus=1, @@ -165,6 +166,7 @@ def test_action_metadata_reports_policy_shape_and_capabilities(): metadata = action_metadata(_server_args(config)) assert metadata["object"] == "action.metadata" + assert metadata["model"] == "pi05-production" assert metadata["policy_family"] == "pi05" assert metadata["input"]["image_keys"] == ["front", "wrist"] assert metadata["input"]["image_size"] == [256, 256] @@ -197,6 +199,7 @@ def test_action_generation_response_uses_actual_output_parameters(): assert response["id"] == "action-response-1" assert response["object"] == "action.generation" + assert response["model"] == "pi05-production" assert response["data"][0]["action"]["shape"] == [2, 2] assert response["data"][0]["action"]["values"] == output["actions"] assert response["usage"]["action_horizon"] == 2 diff --git a/python/sglang/multimodal_gen/test/unit/test_served_model_name.py b/python/sglang/multimodal_gen/test/unit/test_served_model_name.py new file mode 100644 index 000000000..999af3bf3 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_served_model_name.py @@ -0,0 +1,58 @@ +import asyncio +from types import SimpleNamespace +from unittest.mock import patch + +from sglang.multimodal_gen.runtime.entrypoints.openai import common_api +from sglang.multimodal_gen.runtime.entrypoints.openai.common_api import ( + DiffusionModelCard, +) +from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import ( + VideoGenerationsRequest, +) +from sglang.multimodal_gen.runtime.entrypoints.openai.video_api import ( + _video_job_from_sampling, +) + + +def test_model_list_and_retrieve_use_served_model_name(): + server_args = SimpleNamespace( + model_path="/private/checkpoint", + served_model_name="public-model", + ) + + def build_model_card(_server_args, model_name): + return DiffusionModelCard(id=model_name, root=model_name) + + with ( + patch.object(common_api, "get_global_server_args", return_value=server_args), + patch.object(common_api, "_build_model_card", side_effect=build_model_card), + ): + models = asyncio.run(common_api.available_models()) + model = asyncio.run(common_api.retrieve_model("public-model")) + missing = asyncio.run(common_api.retrieve_model("/private/checkpoint")) + + assert models["data"][0]["id"] == "public-model" + assert model["id"] == "public-model" + assert missing.status_code == 404 + + +def test_video_job_uses_served_model_name_unless_requested(): + sampling = SimpleNamespace( + width=512, + height=512, + num_frames=49, + fps=24, + output_file_path=lambda: "/tmp/output.mp4", + ) + request = VideoGenerationsRequest(prompt="test") + + job = _video_job_from_sampling("request-id", request, sampling, "public-model") + explicit_job = _video_job_from_sampling( + "request-id", + request.model_copy(update={"model": "requested-model"}), + sampling, + "public-model", + ) + + assert job["model"] == "public-model" + assert explicit_job["model"] == "requested-model" diff --git a/python/sglang/multimodal_gen/test/unit/test_server_args.py b/python/sglang/multimodal_gen/test/unit/test_server_args.py index 50f27e786..da595e168 100644 --- a/python/sglang/multimodal_gen/test/unit/test_server_args.py +++ b/python/sglang/multimodal_gen/test/unit/test_server_args.py @@ -304,6 +304,41 @@ class TestServerArgsPathExpansion(unittest.TestCase): ["text_encoder", "image_encoder", "vae"], ) + def test_served_model_name_cli_arg(self): + parser = FlexibleArgumentParser() + ServerArgs.add_cli_args(parser) + cases = [ + ( + [ + "--model-path", + "/fake", + "--model-id", + "Qwen-Image", + "--served-model-name", + "my-served-name", + ], + "my-served-name", + ), + ( + ["--model-path", "/fake", "--model-id", "Qwen-Image"], + "Qwen-Image", + ), + (["--model-path", "/fake"], "/fake"), + ] + + for argv, expected in cases: + with self.subTest(argv=argv): + with patch.object(sys, "argv", ["sglang"] + argv): + args, unknown_args = parser.parse_known_args(argv) + with patch.object( + PipelineConfig, + "from_kwargs", + return_value=QwenImagePipelineConfig(), + ): + server_args = ServerArgs.from_cli_args(args, unknown_args) + + self.assertEqual(server_args.served_model_name, expected) + def test_dit_layerwise_offload_cli_arg(self): parser = FlexibleArgumentParser() ServerArgs.add_cli_args(parser)