[diffusion] chore: retire released warmup and decoder flags (#33850)

This commit is contained in:
Mick
2026-08-06 23:02:32 +08:00
committed by GitHub
parent 591cfb0881
commit c212a6938c
19 changed files with 145 additions and 306 deletions
+1 -1
View File
@@ -207,7 +207,7 @@ Notes on launch behavior:
- **CPU offload flags are optional.** `--dit-cpu-offload`, `--text-encoder-cpu-offload`, and `--image-encoder-cpu-offload` are available; defaults are auto-adjusted from GPU memory (GPUs under 30 GB get more aggressive offloading). - **CPU offload flags are optional.** `--dit-cpu-offload`, `--text-encoder-cpu-offload`, and `--image-encoder-cpu-offload` are available; defaults are auto-adjusted from GPU memory (GPUs under 30 GB get more aggressive offloading).
- **Multi-GPU realtime.** Prefer explicit sequence parallelism (`--sp-degree` equal to the number of GPUs for a single session). Do not enable CFG parallel for the realtime profile: the default realtime request uses `guidance_scale=1.0`, while CFG parallel requires active cond/uncond branches. - **Multi-GPU realtime.** Prefer explicit sequence parallelism (`--sp-degree` equal to the number of GPUs for a single session). Do not enable CFG parallel for the realtime profile: the default realtime request uses `guidance_scale=1.0`, while CFG parallel requires active cond/uncond branches.
- **FSDP.** Use `--use-fsdp-inference` only when you specifically need weight sharding for memory. For the low-latency realtime profile, prefer keeping components resident and using SP first. - **FSDP.** Use `--use-fsdp-inference` only when you specifically need weight sharding for memory. For the low-latency realtime profile, prefer keeping components resident and using SP first.
- **Warmup.** Server warmup is **automatically skipped** for the realtime pipeline — a synthetic warmup request has no WebSocket session, so the server detects the registered realtime adapter and skips it. No `--warmup` flag is needed. - **Warmup.** Server warmup is **automatically skipped** for the realtime pipeline — a synthetic warmup request has no WebSocket session, so the server detects the registered realtime adapter and skips it. No explicit `--warmup-mode` setting is needed.
Once up, the realtime WebSocket endpoint lives at `ws://127.0.0.1:30000/v1/realtime_video/generate` (use the Python client in §7 to connect — plain `curl` does not speak the `ws://` upgrade). Once up, the realtime WebSocket endpoint lives at `ws://127.0.0.1:30000/v1/realtime_video/generate` (use the Python client in §7 to connect — plain `curl` does not speak the `ws://` upgrade).
+1 -1
View File
@@ -254,7 +254,7 @@ Then, apply the quantization config from yaml. Please also enable torch.compile
sglang generate \ sglang generate \
--backend diffusers \ --backend diffusers \
--model-path Qwen/Qwen-Image \ --model-path Qwen/Qwen-Image \
--warmup \ --warmup-mode request \
--cache-dit-config quantize.yaml \ --cache-dit-config quantize.yaml \
--enable-torch-compile \ --enable-torch-compile \
--dit-cpu-offload false \ --dit-cpu-offload false \
@@ -273,10 +273,6 @@ Set `--disagg-p2p-hostname` to the actual IP on each machine. For multi-machine,
<td><code>--decoder-sp</code></td> <td><code>--decoder-sp</code></td>
<td>Decoder sequence parallelism</td> <td>Decoder sequence parallelism</td>
</tr> </tr>
<tr>
<td><code>--decoder-tp</code></td>
<td>Deprecated alias for <code>--decoder-sp</code></td>
</tr>
</tbody> </tbody>
</table> </table>
@@ -122,7 +122,7 @@ SGLANG_CACHE_DIT_FN=2 SGLANG_CACHE_DIT_BN=1 SGLANG_CACHE_DIT_WARMUP=4 SGLANG_CAC
SGLANG_CACHE_DIT_MC=4 SGLANG_CACHE_DIT_TAYLORSEER=true SGLANG_CACHE_DIT_TS_ORDER=2 \ SGLANG_CACHE_DIT_MC=4 SGLANG_CACHE_DIT_TAYLORSEER=true SGLANG_CACHE_DIT_TS_ORDER=2 \
SGLANG_CACHE_DIT_ENABLED=true sglang generate --model-path /path/to/zai-org/GLM-Image/ \ SGLANG_CACHE_DIT_ENABLED=true sglang generate --model-path /path/to/zai-org/GLM-Image/ \
--prompt "A curious raccoon" --height 1920 --width 1088 --num-inference-steps 50 --num-gpus 4 \ --prompt "A curious raccoon" --height 1920 --width 1088 --num-inference-steps 50 --num-gpus 4 \
--sp-degree 4 --srt-encoder-url "http://${HOST}:${PORT}" --warmup --sp-degree 4 --srt-encoder-url "http://${HOST}:${PORT}" --warmup-mode request
``` ```
Result: Result:
```bash ```bash
@@ -292,7 +292,7 @@ sglang generate \
--prompt="At night, while their owner sleeps in a bedroom, three cats march in loudly playing tiny brass instruments, then abruptly file out." \ --prompt="At night, while their owner sleeps in a bedroom, three cats march in loudly playing tiny brass instruments, then abruptly file out." \
--seed=1101 --num-gpus=4 --tp-size=2 --ulysses-degree=2 \ --seed=1101 --num-gpus=4 --tp-size=2 --ulysses-degree=2 \
--performance-mode=speed --enable-torch-compile=false \ --performance-mode=speed --enable-torch-compile=false \
--save-output --warmup \ --save-output --warmup-mode request \
--perf-dump-path="${BENCH_DIR}/minimax-h3-t2va-baseline.json" --perf-dump-path="${BENCH_DIR}/minimax-h3-t2va-baseline.json"
``` ```
@@ -344,7 +344,7 @@ sglang generate \
--width=768 --height=512 \ --width=768 --height=512 \
--num-frames=121 \ --num-frames=121 \
--seed=42 --num-gpus=2 --enable-cfg-parallel \ --seed=42 --num-gpus=2 --enable-cfg-parallel \
--save-output --enable-torch-compile --warmup --save-output --enable-torch-compile --warmup-mode request
``` ```
`LTX2TwoStagePipeline` is a native path. The spatial upsampler and distilled `LTX2TwoStagePipeline` is a native path. The spatial upsampler and distilled
@@ -361,7 +361,7 @@ sglang generate \
--width=768 --height=512 \ --width=768 --height=512 \
--num-frames=121 \ --num-frames=121 \
--seed=42 --num-gpus=2 --cfg-parallel-size=2 \ --seed=42 --num-gpus=2 --cfg-parallel-size=2 \
--save-output --enable-torch-compile --warmup --save-output --enable-torch-compile --warmup-mode request
``` ```
This matches the nightly comparison case `ltx2.3_twostage_ti2v_2gpus`. This matches the nightly comparison case `ltx2.3_twostage_ti2v_2gpus`.
@@ -377,7 +377,7 @@ sglang generate \
--num-frames=121 --fps=24 \ --num-frames=121 --fps=24 \
--num-inference-steps=30 --guidance-scale=3.0 \ --num-inference-steps=30 --guidance-scale=3.0 \
--seed=1234 --num-gpus=2 \ --seed=1234 --num-gpus=2 \
--save-output --enable-torch-compile --warmup --save-output --enable-torch-compile --warmup-mode request
``` ```
Use this when you want the native `LTX2Pipeline` baseline for `LTX-2.3` at the Use this when you want the native `LTX2Pipeline` baseline for `LTX-2.3` at the
@@ -395,7 +395,7 @@ sglang generate \
--num-frames=121 --fps=24 \ --num-frames=121 --fps=24 \
--num-inference-steps=30 --guidance-scale=3.0 \ --num-inference-steps=30 --guidance-scale=3.0 \
--seed=1234 --num-gpus=2 \ --seed=1234 --num-gpus=2 \
--save-output --enable-torch-compile --warmup --save-output --enable-torch-compile --warmup-mode request
``` ```
This matches the skill-only `ltx23-two-stage` preset. Use it as a This matches the skill-only `ltx23-two-stage` preset. Use it as a
@@ -413,7 +413,7 @@ sglang generate \
--num-inference-steps=40 --guidance-scale=4.0 \ --num-inference-steps=40 --guidance-scale=4.0 \
--num-gpus=2 --enable-cfg-parallel --ulysses-degree=1 \ --num-gpus=2 --enable-cfg-parallel --ulysses-degree=1 \
--dit-layerwise-offload false --dit-cpu-offload false \ --dit-layerwise-offload false --dit-cpu-offload false \
--save-output --enable-torch-compile --warmup --save-output --enable-torch-compile --warmup-mode request
``` ```
### Manual command example: FireRed Image Edit ### Manual command example: FireRed Image Edit
@@ -428,7 +428,7 @@ sglang generate \
--num-inference-steps=40 --guidance-scale=4.0 \ --num-inference-steps=40 --guidance-scale=4.0 \
--num-gpus=2 --enable-cfg-parallel --ulysses-degree=1 \ --num-gpus=2 --enable-cfg-parallel --ulysses-degree=1 \
--dit-layerwise-offload false --dit-cpu-offload false \ --dit-layerwise-offload false --dit-cpu-offload false \
--save-output --enable-torch-compile --warmup --save-output --enable-torch-compile --warmup-mode request
``` ```
Use `FireRedTeam/FireRed-Image-Edit-1.0` in the same command when comparing the Use `FireRedTeam/FireRed-Image-Edit-1.0` in the same command when comparing the
@@ -453,7 +453,7 @@ sglang generate \
--config="${CONFIG_DIR}/hunyuan3d-shape.json" \ --config="${CONFIG_DIR}/hunyuan3d-shape.json" \
--num-inference-steps=50 --guidance-scale=5.0 \ --num-inference-steps=50 --guidance-scale=5.0 \
--dit-layerwise-offload false --dit-cpu-offload false \ --dit-layerwise-offload false --dit-cpu-offload false \
--save-output --enable-torch-compile --warmup --save-output --enable-torch-compile --warmup-mode request
``` ```
For Hunyuan3D, compare the denoise stage separately from mesh export and paint For Hunyuan3D, compare the denoise stage separately from mesh export and paint
@@ -473,7 +473,7 @@ sglang generate \
--seed=42 --save-output \ --seed=42 --save-output \
--num-gpus=4 --enable-cfg-parallel --ulysses-degree=2 \ --num-gpus=4 --enable-cfg-parallel --ulysses-degree=2 \
--text-encoder-cpu-offload --pin-cpu-memory \ --text-encoder-cpu-offload --pin-cpu-memory \
--warmup --enable-torch-compile --warmup-mode request --enable-torch-compile
``` ```
`Wan2.2-I2V-A14B` uses the 720p max-area config by default, and explicit `Wan2.2-I2V-A14B` uses the 720p max-area config by default, and explicit
@@ -485,7 +485,7 @@ reference-image aspect ratio.
For every benchmark run, write a perf dump JSON: For every benchmark run, write a perf dump JSON:
```bash ```bash
sglang generate ... --warmup --perf-dump-path "${BENCH_DIR}/<result>.json" sglang generate ... --warmup-mode request --perf-dump-path "${BENCH_DIR}/<result>.json"
``` ```
Before/after comparison: Before/after comparison:
@@ -542,7 +542,7 @@ sglang generate \
--model-path=black-forest-labs/FLUX.1-dev \ --model-path=black-forest-labs/FLUX.1-dev \
--prompt="A futuristic cyberpunk city at night" \ --prompt="A futuristic cyberpunk city at night" \
--width=1024 --height=1024 --num-inference-steps=50 \ --width=1024 --height=1024 --num-inference-steps=50 \
--seed=42 --enable-torch-compile --warmup \ --seed=42 --enable-torch-compile --warmup-mode request \
--profile --profile
``` ```
@@ -665,7 +665,7 @@ def _expected_nightly_cli_args(case: dict) -> dict[str, str]:
serve_args = shlex.split(case["frameworks"]["sglang"].get("serve_args", "")) serve_args = shlex.split(case["frameworks"]["sglang"].get("serve_args", ""))
parsed_serve_args = _parse_cli_args(serve_args) parsed_serve_args = _parse_cli_args(serve_args)
for flag, value in parsed_serve_args.items(): for flag, value in parsed_serve_args.items():
if flag in {"enable-torch-compile", "warmup"}: if flag in {"enable-torch-compile", "warmup-mode"}:
continue continue
expected[flag] = _normalize_cli_value(value) expected[flag] = _normalize_cli_value(value)
@@ -805,7 +805,7 @@ def build_sglang_cmd(
if save_output: if save_output:
cmd.append("--save-output") cmd.append("--save-output")
if warmup: if warmup:
cmd.append("--warmup") cmd.extend(["--warmup-mode", "request"])
if torch_compile and not cfg.get("force_eager", False): if torch_compile and not cfg.get("force_eager", False):
cmd.append("--enable-torch-compile") cmd.append("--enable-torch-compile")
if perf_dump_path: if perf_dump_path:
@@ -32,8 +32,8 @@ These options are intended to preserve output quality. In practice, some paths (
| Option | CLI Flag / Env Var | What It Does | Speedup | Limitations / Notes | | Option | CLI Flag / Env Var | What It Does | Speedup | Limitations / Notes |
|---|---|---|---|---| |---|---|---|---|---|
| **torch.compile** | `--enable-torch-compile` | Applies `torch.compile` to the DiT forward pass, fusing ops and reducing kernel launch overhead. | ~1.2–1.5x on denoising | First request is slow (compilation). May cause minor precision drifts due to [PyTorch issue #145213](https://github.com/pytorch/pytorch/issues/145213). Pair with `--warmup` for best results. | | **torch.compile** | `--enable-torch-compile` | Applies `torch.compile` to the DiT forward pass, fusing ops and reducing kernel launch overhead. | ~1.2–1.5x on denoising | First request is slow (compilation). May cause minor precision drifts due to [PyTorch issue #145213](https://github.com/pytorch/pytorch/issues/145213). Pair with `--warmup-mode request` for best results. |
| **Warmup** | `--warmup` | Runs dummy forward passes to warm up CUDA caches, JIT, and `torch.compile`. Eliminates cold-start penalty. | Removes first-request latency spike | Adds startup time. Without `--warmup-resolutions`, warmup happens on first request. | | **Warmup** | `--warmup-mode request` | Runs dummy forward passes to warm up CUDA caches, JIT, and `torch.compile`. Eliminates cold-start penalty. | Removes first-request latency spike | Adds startup time. Without `--warmup-resolutions`, warmup happens on first request. |
| **Warmup Resolutions** | `--warmup-resolutions 256x256 720x720` | Pre-compiles and warms up specific resolutions at server startup (instead of lazily on first request). | Faster first request per resolution | Each resolution adds to startup time. Serving mode only; useful when you know your target resolutions in advance. | | **Warmup Resolutions** | `--warmup-resolutions 256x256 720x720` | Pre-compiles and warms up specific resolutions at server startup (instead of lazily on first request). | Faster first request per resolution | Each resolution adds to startup time. Serving mode only; useful when you know your target resolutions in advance. |
| **Multi-GPU (SP)** | `--num-gpus N --ulysses-degree N` | Sequence parallelism across GPUs. Shards sequence tokens (not frames) to minimize padding. | Near-linear scaling with N GPUs | Requires NCCL; inter-GPU bandwidth matters. `ulysses_degree * ring_degree = sp_degree`. For Wan2.2 video, start by benchmarking pure Ulysses before assuming a mixed Ulysses/Ring layout is fastest. | | **Multi-GPU (SP)** | `--num-gpus N --ulysses-degree N` | Sequence parallelism across GPUs. Shards sequence tokens (not frames) to minimize padding. | Near-linear scaling with N GPUs | Requires NCCL; inter-GPU bandwidth matters. `ulysses_degree * ring_degree = sp_degree`. For Wan2.2 video, start by benchmarking pure Ulysses before assuming a mixed Ulysses/Ring layout is fastest. |
| **CFG Parallel** | `--enable-cfg-parallel` | Runs conditional and unconditional CFG branches in parallel across GPUs. For CFG models on multi-GPU, benchmark this against pure Ulysses on your topology instead of assuming one always wins. | Often faster than pure SP for CFG models | Requires `num_gpus >= 2`. Halves the Ulysses group size (e.g. 8 GPU → two 4-GPU groups). Only for models that use CFG. Nightly coverage configs may intentionally use smaller Ulysses groups to keep ring behavior exercised; that does not automatically make them the lowest-latency choice. | | **CFG Parallel** | `--enable-cfg-parallel` | Runs conditional and unconditional CFG branches in parallel across GPUs. For CFG models on multi-GPU, benchmark this against pure Ulysses on your topology instead of assuming one always wins. | Often faster than pure SP for CFG models | Requires `num_gpus >= 2`. Halves the Ulysses group size (e.g. 8 GPU → two 4-GPU groups). Only for models that use CFG. Nightly coverage configs may intentionally use smaller Ulysses groups to keep ring behavior exercised; that does not automatically make them the lowest-latency choice. |
@@ -127,7 +127,7 @@ Current H3 restrictions:
```bash ```bash
sglang generate --model-path Wan-AI/Wan2.2-T2V-A14B-Diffusers \ sglang generate --model-path Wan-AI/Wan2.2-T2V-A14B-Diffusers \
--num-gpus 8 --enable-cfg-parallel --ulysses-degree 4 \ --num-gpus 8 --enable-cfg-parallel --ulysses-degree 4 \
--enable-torch-compile --warmup \ --enable-torch-compile --warmup-mode request \
--text-encoder-cpu-offload true \ --text-encoder-cpu-offload true \
--prompt "..." --save-output --prompt "..." --save-output
``` ```
@@ -149,7 +149,7 @@ sglang generate --model-path Lightricks/LTX-2 \
--width 768 --height 512 \ --width 768 --height 512 \
--num-frames 121 \ --num-frames 121 \
--seed 42 --num-gpus 2 --enable-cfg-parallel \ --seed 42 --num-gpus 2 --enable-cfg-parallel \
--enable-torch-compile --warmup --save-output --enable-torch-compile --warmup-mode request --save-output
``` ```
Note: LTX-2 is a current-source benchmark preset rather than a nightly Note: LTX-2 is a current-source benchmark preset rather than a nightly
@@ -168,7 +168,7 @@ sglang generate --model-path Lightricks/LTX-2.3 \
--width 768 --height 512 \ --width 768 --height 512 \
--num-frames 121 \ --num-frames 121 \
--seed 42 --num-gpus 2 --cfg-parallel-size 2 \ --seed 42 --num-gpus 2 --cfg-parallel-size 2 \
--enable-torch-compile --warmup --save-output --enable-torch-compile --warmup-mode request --save-output
``` ```
Note: this matches the nightly comparison case `ltx2.3_twostage_ti2v_2gpus`. The nightly config omits explicit steps and guidance, so this command omits them too and uses runtime defaults. Download `${ASSET_DIR}/cat.png` with the benchmark/profile skill before running it. Note: this matches the nightly comparison case `ltx2.3_twostage_ti2v_2gpus`. The nightly config omits explicit steps and guidance, so this command omits them too and uses runtime defaults. Download `${ASSET_DIR}/cat.png` with the benchmark/profile skill before running it.
@@ -183,7 +183,7 @@ sglang generate --model-path Lightricks/LTX-2.3 \
--num-frames 121 --fps 24 \ --num-frames 121 --fps 24 \
--num-inference-steps 30 --guidance-scale 3.0 \ --num-inference-steps 30 --guidance-scale 3.0 \
--seed 1234 --num-gpus 2 \ --seed 1234 --num-gpus 2 \
--enable-torch-compile --warmup --save-output --enable-torch-compile --warmup-mode request --save-output
``` ```
Note: use this as the native `LTX2Pipeline` baseline for `LTX-2.3`. It keeps the validated one-stage resolution and explicit `LTX-2.3` sampling defaults, and matches the `ltx23-one-stage` benchmark preset in `sglang-diffusion-benchmark-profile`. Note: use this as the native `LTX2Pipeline` baseline for `LTX-2.3`. It keeps the validated one-stage resolution and explicit `LTX-2.3` sampling defaults, and matches the `ltx23-one-stage` benchmark preset in `sglang-diffusion-benchmark-profile`.
@@ -199,7 +199,7 @@ sglang generate --model-path Lightricks/LTX-2.3 \
--num-frames 121 --fps 24 \ --num-frames 121 --fps 24 \
--num-inference-steps 30 --guidance-scale 3.0 \ --num-inference-steps 30 --guidance-scale 3.0 \
--seed 1234 --num-gpus 2 \ --seed 1234 --num-gpus 2 \
--enable-torch-compile --warmup --save-output --enable-torch-compile --warmup-mode request --save-output
``` ```
Note: this is a high-resolution stress target for the native `LTX-2.3` two-stage path. It matches the skill-only `ltx23-two-stage` benchmark preset, not a nightly comparison case. Note: this is a high-resolution stress target for the native `LTX-2.3` two-stage path. It matches the skill-only `ltx23-two-stage` benchmark preset, not a nightly comparison case.
@@ -208,7 +208,7 @@ Note: this is a high-resolution stress target for the native `LTX-2.3` two-stage
```bash ```bash
sglang generate --model-path <IMAGE_MODEL> \ sglang generate --model-path <IMAGE_MODEL> \
--enable-torch-compile --warmup \ --enable-torch-compile --warmup-mode request \
--dit-layerwise-offload false \ --dit-layerwise-offload false \
--dit-cpu-offload false \ --dit-cpu-offload false \
--prompt "..." --save-output --prompt "..." --save-output
@@ -227,7 +227,7 @@ sglang generate --backend=sglang \
--num-inference-steps 40 --guidance-scale 4.0 \ --num-inference-steps 40 --guidance-scale 4.0 \
--num-gpus 2 --enable-cfg-parallel --ulysses-degree 1 \ --num-gpus 2 --enable-cfg-parallel --ulysses-degree 1 \
--dit-layerwise-offload false --dit-cpu-offload false \ --dit-layerwise-offload false --dit-cpu-offload false \
--enable-torch-compile --warmup --save-output --enable-torch-compile --warmup-mode request --save-output
``` ```
```bash ```bash
@@ -239,7 +239,7 @@ sglang generate --backend=sglang \
--num-inference-steps 40 --guidance-scale 4.0 \ --num-inference-steps 40 --guidance-scale 4.0 \
--num-gpus 2 --enable-cfg-parallel --ulysses-degree 1 \ --num-gpus 2 --enable-cfg-parallel --ulysses-degree 1 \
--dit-layerwise-offload false --dit-cpu-offload false \ --dit-layerwise-offload false --dit-cpu-offload false \
--enable-torch-compile --warmup --save-output --enable-torch-compile --warmup-mode request --save-output
``` ```
Use `FireRedTeam/FireRed-Image-Edit-1.0` in the same command when comparing Use `FireRedTeam/FireRed-Image-Edit-1.0` in the same command when comparing
@@ -264,7 +264,7 @@ sglang generate --backend=sglang \
--config "${CONFIG_DIR}/hunyuan3d-shape.json" \ --config "${CONFIG_DIR}/hunyuan3d-shape.json" \
--num-inference-steps 50 --guidance-scale 5.0 \ --num-inference-steps 50 --guidance-scale 5.0 \
--dit-layerwise-offload false --dit-cpu-offload false \ --dit-layerwise-offload false --dit-cpu-offload false \
--enable-torch-compile --warmup --save-output --enable-torch-compile --warmup-mode request --save-output
``` ```
For Hunyuan3D, treat `Hunyuan3DShapeDenoisingStage` as the primary latency For Hunyuan3D, treat `Hunyuan3DShapeDenoisingStage` as the primary latency
@@ -275,7 +275,7 @@ drive DiT optimization decisions.
```bash ```bash
sglang generate --model-path <MODEL> \ sglang generate --model-path <MODEL> \
--enable-torch-compile --warmup \ --enable-torch-compile --warmup-mode request \
--dit-layerwise-offload --dit-offload-prefetch-size 0.1 \ --dit-layerwise-offload --dit-offload-prefetch-size 0.1 \
--text-encoder-cpu-offload true --vae-cpu-offload true \ --text-encoder-cpu-offload true --vae-cpu-offload true \
--prompt "..." --save-output --prompt "..." --save-output
@@ -287,7 +287,7 @@ sglang generate --model-path <MODEL> \
SGLANG_CACHE_DIT_ENABLED=true sglang generate --model-path <MODEL> \ SGLANG_CACHE_DIT_ENABLED=true sglang generate --model-path <MODEL> \
--attention-backend sage_attn \ --attention-backend sage_attn \
--dit-layerwise-offload false \ --dit-layerwise-offload false \
--enable-torch-compile --warmup \ --enable-torch-compile --warmup-mode request \
--prompt "..." --save-output --prompt "..." --save-output
``` ```
@@ -314,22 +314,22 @@ Use these as first commands to benchmark, not as universal winners.
| Model family | First performance shape | Starting flags | Notes | | Model family | First performance shape | Starting flags | Notes |
|---|---|---|---| |---|---|---|---|
| MiniMax-H3 | 1344x768 resolved canvas, 5 seconds / 124 frames at 24 fps, 50 joint video/audio steps | H200: `--num-gpus 4 --ulysses-degree 4 --performance-mode speed --enable-torch-compile false`; H100: TP2 + Ulysses2 | Root ID plus `--model-variant fl2va` for T2VA/FL2VA or `ref2va` for Ref2VA. Ulysses only; no Ring/CFG/SageAttention. Preserve tiled video-VAE decode. Profile joint denoise, video VAE, audio VAE/vocoder, encoder, and collectives separately. | | MiniMax-H3 | 1344x768 resolved canvas, 5 seconds / 124 frames at 24 fps, 50 joint video/audio steps | H200: `--num-gpus 4 --ulysses-degree 4 --performance-mode speed --enable-torch-compile false`; H100: TP2 + Ulysses2 | Root ID plus `--model-variant fl2va` for T2VA/FL2VA or `ref2va` for Ref2VA. Ulysses only; no Ring/CFG/SageAttention. Preserve tiled video-VAE decode. Profile joint denoise, video VAE, audio VAE/vocoder, encoder, and collectives separately. |
| FLUX.1 / FLUX.2 image | 1024x1024, runtime-default steps/guidance, 1 GPU | `--enable-torch-compile --warmup --dit-layerwise-offload false` | `black-forest-labs/FLUX.*` repos are gated; for FP8/NVFP4 use validated `--transformer-path` or `--transformer-weights-path` flows from the quant skill. | | FLUX.1 / FLUX.2 image | 1024x1024, runtime-default steps/guidance, 1 GPU | `--enable-torch-compile --warmup-mode request --dit-layerwise-offload false` | `black-forest-labs/FLUX.*` repos are gated; for FP8/NVFP4 use validated `--transformer-path` or `--transformer-weights-path` flows from the quant skill. |
| FLUX.2 Klein / Klein Base | 1024x1024, runtime-default steps/guidance, 1 GPU | `--enable-torch-compile --warmup --dit-layerwise-offload false` | Current registry has `black-forest-labs/FLUX.2-klein-4B`, `FLUX.2-klein-9B`, and base variants. Klein is step-distilled; Klein Base is not. | | FLUX.2 Klein / Klein Base | 1024x1024, runtime-default steps/guidance, 1 GPU | `--enable-torch-compile --warmup-mode request --dit-layerwise-offload false` | Current registry has `black-forest-labs/FLUX.2-klein-4B`, `FLUX.2-klein-9B`, and base variants. Klein is step-distilled; Klein Base is not. |
| Qwen-Image / Qwen-Image-Edit | 1024x1024, runtime-default steps/guidance, 1 GPU | `--enable-torch-compile --warmup`; optionally native `SGLANG_CACHE_DIT_ENABLED=true` | Cache-DiT is lossy. For edit tasks, keep reference image, seed, and output size fixed. | | Qwen-Image / Qwen-Image-Edit | 1024x1024, runtime-default steps/guidance, 1 GPU | `--enable-torch-compile --warmup-mode request`; optionally native `SGLANG_CACHE_DIT_ENABLED=true` | Cache-DiT is lossy. For edit tasks, keep reference image, seed, and output size fixed. |
| Z-Image / Z-Image-Turbo | 1024x1024, runtime-default steps/guidance, 1 GPU | `--enable-torch-compile --warmup` | Keep base Z-Image separate from Turbo: base uses 50-step CFG defaults, Turbo uses 9-step zero-CFG defaults. Mainline has bf16-native Triton RMSNorm scale and tanh-residual fusions. | | Z-Image / Z-Image-Turbo | 1024x1024, runtime-default steps/guidance, 1 GPU | `--enable-torch-compile --warmup-mode request` | Keep base Z-Image separate from Turbo: base uses 50-step CFG defaults, Turbo uses 9-step zero-CFG defaults. Mainline has bf16-native Triton RMSNorm scale and tanh-residual fusions. |
| Wan2.2 A14B T2V/I2V | 1280x720, 81 frames | Nightly: `--num-gpus 4 --enable-cfg-parallel --ulysses-degree 2 --text-encoder-cpu-offload --pin-cpu-memory` | For lowest latency, also benchmark pure Ulysses on the same GPUs. | | Wan2.2 A14B T2V/I2V | 1280x720, 81 frames | Nightly: `--num-gpus 4 --enable-cfg-parallel --ulysses-degree 2 --text-encoder-cpu-offload --pin-cpu-memory` | For lowest latency, also benchmark pure Ulysses on the same GPUs. |
| Wan2.2 TI2V 5B | 1280x720, 81 frames, 1 GPU | `--enable-torch-compile --warmup` | Keep the input image and motion prompt fixed when comparing sparse attention or Cache-DiT. | | Wan2.2 TI2V 5B | 1280x720, 81 frames, 1 GPU | `--enable-torch-compile --warmup-mode request` | Keep the input image and motion prompt fixed when comparing sparse attention or Cache-DiT. |
| Wan2.1 / FastWan / TurboWan variants | 480p or 720p video, family defaults | `--enable-torch-compile --warmup`; add `--ulysses-degree` / CFG parallel only after measuring | Current registry includes Wan2.1, FastWan2.1, FastWan2.2 TI2V, TurboWan2.1, TurboWan2.2 I2V, and Wan2.1-Fun InP. Use the compatibility matrix and benchmark presets before choosing topology. | | Wan2.1 / FastWan / TurboWan variants | 480p or 720p video, family defaults | `--enable-torch-compile --warmup-mode request`; add `--ulysses-degree` / CFG parallel only after measuring | Current registry includes Wan2.1, FastWan2.1, FastWan2.2 TI2V, TurboWan2.1, TurboWan2.2 I2V, and Wan2.1-Fun InP. Use the compatibility matrix and benchmark presets before choosing topology. |
| Cosmos3 Nano / Super | T2I: 1024x1024 with `--num-frames 1`; T2V/I2V: 480p/720p video | `SGLANG_DISABLE_COSMOS3_GUARDRAILS=1` for benchmark isolation; `--enable-torch-compile --warmup` | One checkpoint serves T2I/T2V/I2V. Mode is request-driven: `num_frames == 1` means T2I, `--image-path` means I2V. | | Cosmos3 Nano / Super | T2I: 1024x1024 with `--num-frames 1`; T2V/I2V: 480p/720p video | `SGLANG_DISABLE_COSMOS3_GUARDRAILS=1` for benchmark isolation; `--enable-torch-compile --warmup-mode request` | One checkpoint serves T2I/T2V/I2V. Mode is request-driven: `num_frames == 1` means T2I, `--image-path` means I2V. |
| Ideogram 4 FP8/NVFP4 | 1024x1024, native preset defaults | `--enable-torch-compile --warmup` | Do not set `--num-inference-steps` or `--guidance-scale` directly unless you also update the Ideogram preset; sampling params derive them from `preset`. | | Ideogram 4 FP8/NVFP4 | 1024x1024, native preset defaults | `--enable-torch-compile --warmup-mode request` | Do not set `--num-inference-steps` or `--guidance-scale` directly unless you also update the Ideogram preset; sampling params derive them from `preset`. |
| ERNIE-Image / GLM-Image / SANA / SD3 | 1024-class image, family defaults | `--enable-torch-compile --warmup`; disable offload only after checking VRAM | Treat these as current native image families. Start with benchmark/profile presets for ERNIE, GLM, and SANA; use registry/config defaults for SD3 unless you add a new preset. | | ERNIE-Image / GLM-Image / SANA / SD3 | 1024-class image, family defaults | `--enable-torch-compile --warmup-mode request`; disable offload only after checking VRAM | Treat these as current native image families. Start with benchmark/profile presets for ERNIE, GLM, and SANA; use registry/config defaults for SD3 unless you add a new preset. |
| LTX-2 / LTX-2.3 | 768x512 or HQ 1920x1088, 121 frames | `--pipeline-class-name LTX2TwoStagePipeline --enable-torch-compile --warmup`; HQ uses `LTX2TwoStageHQPipeline` | Use benchmark/profile presets for nightly alignment, one-stage, high-resolution stress, and HQ. Device mode choices are `original` and `resident`; `resident` is fastest but uses more VRAM. `snapshot` is a deprecated alias for `original`, so do not use it in new commands. | | LTX-2 / LTX-2.3 | 768x512 or HQ 1920x1088, 121 frames | `--pipeline-class-name LTX2TwoStagePipeline --enable-torch-compile --warmup-mode request`; HQ uses `LTX2TwoStageHQPipeline` | Use benchmark/profile presets for nightly alignment, one-stage, high-resolution stress, and HQ. Device mode choices are `original` and `resident`; `resident` is fastest but uses more VRAM. `snapshot` is a deprecated alias for `original`, so do not use it in new commands. |
| HunyuanVideo | 848x480 or 720p class video | `--text-encoder-cpu-offload --pin-cpu-memory --enable-torch-compile --warmup` | Check VAE decode separately. GroupNorm+SiLU is default-eligible in mainline when wrapper guards pass; use `bench_group_norm_silu.py` when VAE residual blocks are hot. | | HunyuanVideo | 848x480 or 720p class video | `--text-encoder-cpu-offload --pin-cpu-memory --enable-torch-compile --warmup-mode request` | Check VAE decode separately. GroupNorm+SiLU is default-eligible in mainline when wrapper guards pass; use `bench_group_norm_silu.py` when VAE residual blocks are hot. |
| JoyAI-Image-Edit | 1024-class TI2I, 40 steps, guidance 4.0 | `--backend=sglang --num-gpus 2 --enable-cfg-parallel --ulysses-degree 1 --enable-torch-compile --warmup --dit-layerwise-offload false --dit-cpu-offload false` | Newly supported image-edit path. Keep the input image, prompt, seed, and output size fixed; 2-GPU CFG parallel is the validated H100 starting point. | | JoyAI-Image-Edit | 1024-class TI2I, 40 steps, guidance 4.0 | `--backend=sglang --num-gpus 2 --enable-cfg-parallel --ulysses-degree 1 --enable-torch-compile --warmup-mode request --dit-layerwise-offload false --dit-cpu-offload false` | Newly supported image-edit path. Keep the input image, prompt, seed, and output size fixed; 2-GPU CFG parallel is the validated H100 starting point. |
| FireRed-Image-Edit 1.0 / 1.1 | 1024x1024 image edit, 40 steps, guidance 4.0 | `--backend=sglang --num-gpus 2 --enable-cfg-parallel --ulysses-degree 1 --enable-torch-compile --warmup --dit-layerwise-offload false --dit-cpu-offload false` | Uses the native `QwenImageEditPlusPipeline` path. 2-GPU CFG parallel is the validated H100 starting point; benchmark 1.0 and 1.1 separately because checkpoint differences can change denoise latency. | | FireRed-Image-Edit 1.0 / 1.1 | 1024x1024 image edit, 40 steps, guidance 4.0 | `--backend=sglang --num-gpus 2 --enable-cfg-parallel --ulysses-degree 1 --enable-torch-compile --warmup-mode request --dit-layerwise-offload false --dit-cpu-offload false` | Uses the native `QwenImageEditPlusPipeline` path. 2-GPU CFG parallel is the validated H100 starting point; benchmark 1.0 and 1.1 separately because checkpoint differences can change denoise latency. |
| Hunyuan3D-2 shape | Shape generation, 50 steps, guidance 5.0 | `--backend=sglang --enable-torch-compile --warmup --dit-layerwise-offload false --dit-cpu-offload false` | Focus on `Hunyuan3DShapeDenoisingStage`; keep mesh export/paint timings separate from denoise. | | Hunyuan3D-2 shape | Shape generation, 50 steps, guidance 5.0 | `--backend=sglang --enable-torch-compile --warmup-mode request --dit-layerwise-offload false --dit-cpu-offload false` | Focus on `Hunyuan3DShapeDenoisingStage`; keep mesh export/paint timings separate from denoise. |
| MOVA / Helios / LingBot World | Use the benchmark/profile presets or server test cases first | `--enable-torch-compile --warmup`; pin offload and topology flags explicitly | These video/realtime families have model-specific stages and condition handling. Keep prompt/image/action inputs fixed and prefer perf dumps over wall time alone. | | MOVA / Helios / LingBot World | Use the benchmark/profile presets or server test cases first | `--enable-torch-compile --warmup-mode request`; pin offload and topology flags explicitly | These video/realtime families have model-specific stages and condition handling. Keep prompt/image/action inputs fixed and prefer perf dumps over wall time alone. |
## Historical PR Watchlist ## Historical PR Watchlist
@@ -343,7 +343,7 @@ about whether the work has merged:
## Tips ## Tips
- **Benchmarking**: always use `--warmup` and look for the line ending with `(with warmup excluded)` for accurate timing. - **Benchmarking**: always use `--warmup-mode request` and look for the line ending with `(with warmup excluded)` for accurate timing.
- **Perf dump**: use `--perf-dump-path result.json` to save structured metrics, then compare with `python python/sglang/multimodal_gen/benchmarks/compare_perf.py baseline.json result.json`. - **Perf dump**: use `--perf-dump-path result.json` to save structured metrics, then compare with `python python/sglang/multimodal_gen/benchmarks/compare_perf.py baseline.json result.json`.
- **Offload tuning**: after the first request, the runtime logs peak GPU memory and which components could stay resident. Use this to decide which `--*-cpu-offload` flags to disable. - **Offload tuning**: after the first request, the runtime logs peak GPU memory and which components could stay resident. Use this to decide which `--*-cpu-offload` flags to disable.
- **Backend selection**: `--backend sglang` (default, auto-detected) enables native optimizations (fused kernels, SP, native Cache-DiT env knobs, etc.). `--backend diffusers` falls back to Diffusers pipelines and is the path that accepts `--cache-dit-config` plus diffusers attention backend names. - **Backend selection**: `--backend sglang` (default, auto-detected) enables native optimizations (fused kernels, SP, native Cache-DiT env knobs, etc.). `--backend diffusers` falls back to Diffusers pipelines and is the path that accepts `--cache-dit-config` plus diffusers attention backend names.
@@ -472,7 +472,7 @@ class DiffGenerator:
def _log_summary(self, results: list[GenerationResult]) -> None: def _log_summary(self, results: list[GenerationResult]) -> None:
if not results: if not results:
return return
if self.server_args.warmup: if self.server_args.warmup_mode != "off":
total_duration_ms = results[0].metrics.get("total_duration_ms", 0) total_duration_ms = results[0].metrics.get("total_duration_ms", 0)
logger.info( logger.info(
f"Warmed-up request processed in {GREEN}%.2f{RESET} seconds (with warmup excluded)", f"Warmed-up request processed in {GREEN}%.2f{RESET} seconds (with warmup excluded)",
@@ -118,7 +118,7 @@ async def lifespan(app: FastAPI):
# 2. Start the ZMQ Broker in the background to handle offline requests # 2. Start the ZMQ Broker in the background to handle offline requests
broker_task = asyncio.create_task(run_zeromq_broker(server_args)) broker_task = asyncio.create_task(run_zeromq_broker(server_args))
warmup_task = None warmup_task = None
if server_args.server_warmup: if server_args.warmup_mode == "server":
warmup_task = asyncio.create_task( warmup_task = asyncio.create_task(
_run_server_warmup_after_http_ready(server_args, warmup_done) _run_server_warmup_after_http_ready(server_args, warmup_done)
) )
@@ -411,8 +411,7 @@ def launch_pool_disagg_server(
"pool_work_endpoint": work_eps[inst_idx], "pool_work_endpoint": work_eps[inst_idx],
"pool_result_endpoint": result_ep, "pool_result_endpoint": result_ep,
"num_gpus": num_role_gpus, "num_gpus": num_role_gpus,
"warmup": role_type == RoleType.ENCODER, "warmup_mode": "request" if role_type == RoleType.ENCODER else "off",
"server_warmup": False,
"scheduler_port": find_port(port_cursor), "scheduler_port": find_port(port_cursor),
"master_port": find_port(port_cursor + 100), "master_port": find_port(port_cursor + 100),
# Per-role parallelism (None = auto-derive from num_gpus) # Per-role parallelism (None = auto-derive from num_gpus)
@@ -692,8 +691,7 @@ def launch_disagg_role(server_args: ServerArgs):
"disagg_mode": True, "disagg_mode": True,
"pool_work_endpoint": work_endpoint, "pool_work_endpoint": work_endpoint,
"pool_result_endpoint": result_endpoint, "pool_result_endpoint": result_endpoint,
"warmup": role_type == RoleType.ENCODER, "warmup_mode": "request" if role_type == RoleType.ENCODER else "off",
"server_warmup": False,
"scheduler_port": internal_scheduler_port, "scheduler_port": internal_scheduler_port,
# Per-role parallelism (None = auto-derive from num_gpus) # Per-role parallelism (None = auto-derive from num_gpus)
"tp_size": role_par["tp_size"], "tp_size": role_par["tp_size"],
@@ -358,7 +358,7 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin):
if ( if (
not args.enable_torch_compile not args.enable_torch_compile
or not args.offload_during_compile or not args.offload_during_compile
or not args.warmup or args.warmup_mode == "off"
or not self._owns_compile_warmup_lifecycle() or not self._owns_compile_warmup_lifecycle()
or args.use_fsdp_inference or args.use_fsdp_inference
or self._cache_dit_requested() or self._cache_dit_requested()
@@ -234,9 +234,3 @@ class DisaggServerArgsMixin:
default=cls.decoder_sp, default=cls.decoder_sp,
help="Sequence parallelism for decoder role.", help="Sequence parallelism for decoder role.",
) )
parser.add_argument(
"--decoder-tp",
type=int,
default=cls.decoder_tp,
help="Deprecated alias for --decoder-sp.",
)
@@ -315,21 +315,15 @@ class ServerArgs(DisaggServerArgsMixin):
# NVTX profiling # NVTX profiling
enable_layerwise_nvtx_marker: bool = False enable_layerwise_nvtx_marker: bool = False
# warmup # Warmup is controlled by the canonical `warmup_mode` knob: one of WARMUP_MODES.
# `warmup_mode` is the canonical knob: one of WARMUP_MODES
# - "off": no warmup. # - "off": no warmup.
# - "server": server-based warmup — a synthetic request right after the # - "server": server-based warmup — a synthetic request right after the
# server is ready, before real traffic # server is ready, before real traffic
# - "request": request-based warmup — warm on the first real request(s). # - "request": request-based warmup — warm on the first real request(s).
# This is a BENCHMARK aid # This is a BENCHMARK aid.
# existing consumers keep working) and as deprecated CLI aliases. None means # None is resolved by _adjust_warmup from the selected runtime features.
# "derive the mode from the legacy booleans"; _adjust_warmup resolves it.
warmup_mode: str | None = None warmup_mode: str | None = None
# deprecated: warmup and server_warmup
warmup: bool = False
server_warmup: bool = False
warmup_resolutions: list[str] = None warmup_resolutions: list[str] = None
warmup_steps: int = 1 warmup_steps: int = 1
@@ -411,7 +405,6 @@ class ServerArgs(DisaggServerArgsMixin):
denoiser_ulysses: int | None = None denoiser_ulysses: int | None = None
denoiser_ring: int | None = None denoiser_ring: int | None = None
decoder_sp: int | None = None decoder_sp: int | None = None
decoder_tp: int | None = None
pool_work_endpoint: str | None = None pool_work_endpoint: str | None = None
pool_result_endpoint: str | None = None pool_result_endpoint: str | None = None
pool_control_endpoint: str | None = None pool_control_endpoint: str | None = None
@@ -457,7 +450,6 @@ class ServerArgs(DisaggServerArgsMixin):
"""set defaults and normalize values.""" """set defaults and normalize values."""
auto_tuner = ServerArgsAutoTuner(self) auto_tuner = ServerArgsAutoTuner(self)
auto_tuner.adjust_based_on_performance_mode() auto_tuner.adjust_based_on_performance_mode()
self._adjust_disagg_parallelism_aliases()
if auto_tuner.could_override_server_args(): if auto_tuner.could_override_server_args():
self._adjust_offload() self._adjust_offload()
auto_tuner.maybe_adjust_auto_default_layerwise_offload() auto_tuner.maybe_adjust_auto_default_layerwise_offload()
@@ -480,21 +472,6 @@ class ServerArgs(DisaggServerArgsMixin):
auto_tuner.finalize_auto_flags() auto_tuner.finalize_auto_flags()
self.adjust_pipeline_config() self.adjust_pipeline_config()
def _adjust_disagg_parallelism_aliases(self):
if self.decoder_tp is None:
return
if self.decoder_sp is not None and self.decoder_sp != self.decoder_tp:
raise ValueError(
"decoder_tp is deprecated in favor of decoder_sp; "
"please set only one of them or keep the same value."
)
if self.decoder_sp is None:
logger.warning(
"decoder_tp is deprecated and is treated as decoder_sp for "
"decoder/VAE parallel decode. Please use decoder_sp instead."
)
self.decoder_sp = self.decoder_tp
def _validate_parameters(self): def _validate_parameters(self):
"""check consistency and raise errors for invalid configs""" """check consistency and raise errors for invalid configs"""
self._validate_pipeline() self._validate_pipeline()
@@ -889,64 +866,37 @@ class ServerArgs(DisaggServerArgsMixin):
return None, None return None, None
def _adjust_warmup(self): def _adjust_warmup(self):
# --warmup-mode > --warmup/--server-warmup if self.warmup_mode is not None and self.warmup_mode not in WARMUP_MODES:
mode_explicit = self.is_arg_explicitly_set("warmup_mode")
legacy_explicit = self.is_arg_explicitly_set(
"warmup"
) or self.is_arg_explicitly_set("server_warmup")
if self.warmup_mode is not None:
if self.warmup_mode not in WARMUP_MODES:
raise ValueError( raise ValueError(
f"Invalid --warmup-mode {self.warmup_mode!r}; " f"Invalid --warmup-mode {self.warmup_mode!r}; "
f"expected one of {WARMUP_MODES}." f"expected one of {WARMUP_MODES}."
) )
if mode_explicit and legacy_explicit:
logger.warning(
"Both --warmup-mode and the deprecated --warmup/--server-warmup "
"were set; --warmup-mode=%s takes precedence.",
self.warmup_mode,
)
if mode_explicit or not legacy_explicit:
self.warmup = self.warmup_mode != "off"
self.server_warmup = self.warmup_mode == "server"
elif self.warmup:
self.server_warmup = self.server_warmup or self.warmup_mode == "server"
# Explicit resolutions imply warmup is on (request-based). if self.enable_torch_compile and self.warmup_mode is None:
if self.warmup_resolutions is not None: self.warmup_mode = "server"
self.warmup = True
if (
self.enable_torch_compile
and self.warmup_mode is None
and not mode_explicit
and not legacy_explicit
):
self.warmup = True
self.server_warmup = True
logger.info( logger.info(
"Automatically enabled server warmup for torch.compile so first " "Automatically enabled server warmup for torch.compile so first "
"real requests do not pay compile latency. Set --warmup-mode off " "real requests do not pay compile latency. Set --warmup-mode off "
"to disable this behavior." "to disable this behavior."
) )
# Explicit resolutions need a request path unless an existing server
# default already supplies the synthetic startup request.
if self.warmup_resolutions is not None and self.warmup_mode in (None, "off"):
self.warmup_mode = "request"
# BCG captures every graph during a synthetic warmup forward at startup # BCG captures every graph during a synthetic warmup forward at startup
# so that serving never records a fresh graph. That requires # so serving never records a fresh graph.
# server-based warmup (a real warmup request issued at startup), not
# request-based warmup which runs no forward until the first request.
if self.enable_breakable_cuda_graph and self.disagg_role == RoleType.MONOLITHIC: if self.enable_breakable_cuda_graph and self.disagg_role == RoleType.MONOLITHIC:
self.warmup = True self.warmup_mode = "server"
self.server_warmup = True
if self.disagg_role != RoleType.MONOLITHIC: # Disaggregated roles do not host the HTTP startup request. Preserve
self.server_warmup = False # warmup intent, but schedule it on the first request instead.
if self.disagg_role != RoleType.MONOLITHIC and self.warmup_mode == "server":
self.warmup_mode = "request"
if not self.warmup: if self.warmup_mode is None:
self.server_warmup = False self.warmup_mode = "off"
self.warmup_mode = (
"off" if not self.warmup else "server" if self.server_warmup else "request"
)
@staticmethod @staticmethod
def _require_port(port: int, name: str) -> None: def _require_port(port: int, name: str) -> None:
@@ -1622,28 +1572,16 @@ class ServerArgs(DisaggServerArgsMixin):
choices=list(WARMUP_MODES), choices=list(WARMUP_MODES),
default=ServerArgs.warmup_mode, default=ServerArgs.warmup_mode,
help=( help=(
"Warmup mode (canonical knob). One of: " "Warmup mode. One of: `off` (no warmup); `request` "
"`off` (no warmup); `request` (request-based: warm on real " "(request-based: warm on real incoming requests); `server` "
"incoming requests); `server` (server-based: a synthetic warmup " "(server-based: a synthetic warmup request right after the server "
"request right after the server is ready, before traffic). " "is ready, before traffic). `sglang serve` defaults to `server`; "
"Takes precedence over the deprecated --warmup/--server-warmup. " "other entrypoints default "
"`sglang serve` defaults to `server`; other entrypoints default "
"to request-based when warmup is enabled. When enabled, look for " "to request-based when warmup is enabled. When enabled, look for "
"the line ending with `(with warmup excluded)` for actual " "the line ending with `(with warmup excluded)` for actual "
"processing time." "processing time."
), ),
) )
parser.add_argument(
"--warmup",
action=StoreBoolean,
default=ServerArgs.warmup,
help=(
"[DEPRECATED: use --warmup-mode] Perform warmup before normal "
"traffic. Maps to --warmup-mode request (or server, combined "
"with --server-warmup). Recommended when benchmarking for fair "
"comparison and best performance."
),
)
parser.add_argument( parser.add_argument(
"--warmup-resolutions", "--warmup-resolutions",
type=str, type=str,
@@ -1657,16 +1595,6 @@ class ServerArgs(DisaggServerArgsMixin):
default=ServerArgs.warmup_steps, default=ServerArgs.warmup_steps,
help="The number of warmup steps to perform for each resolution.", help="The number of warmup steps to perform for each resolution.",
) )
parser.add_argument(
"--server-warmup",
action=StoreBoolean,
default=ServerArgs.server_warmup,
help=(
"[DEPRECATED: use --warmup-mode server] Send a synthetic warmup "
"request after the server is ready (server-based warmup)."
),
)
# layerwise offload # layerwise offload
parser.add_argument( parser.add_argument(
"--dit-cpu-offload", "--dit-cpu-offload",
@@ -2238,6 +2166,7 @@ class ServerArgs(DisaggServerArgsMixin):
@classmethod @classmethod
def from_dict(cls, kwargs: dict[str, Any]) -> "ServerArgs": def from_dict(cls, kwargs: dict[str, Any]) -> "ServerArgs":
"""Create a ServerArgs object from a dictionary.""" """Create a ServerArgs object from a dictionary."""
cls._reject_retired_args(kwargs)
attrs = [attr.name for attr in dataclasses.fields(cls) if attr.init] attrs = [attr.name for attr in dataclasses.fields(cls) if attr.init]
server_args_kwargs: dict[str, Any] = {} server_args_kwargs: dict[str, Any] = {}
explicit_arg_names = kwargs.get("_explicit_arg_names") explicit_arg_names = kwargs.get("_explicit_arg_names")
@@ -2264,6 +2193,20 @@ class ServerArgs(DisaggServerArgsMixin):
return cls(**server_args_kwargs) return cls(**server_args_kwargs)
@staticmethod
def _reject_retired_args(kwargs: dict[str, Any]) -> None:
retired_args = {
"decoder_tp": "decoder_sp for decoder/VAE parallel decode",
"warmup": "warmup_mode=request or warmup_mode=off",
"server_warmup": "warmup_mode=server or warmup_mode=off",
}
removed = [name for name in retired_args if name in kwargs]
if removed:
replacements = "; ".join(
f"{name} -> {retired_args[name]}" for name in removed
)
raise ValueError(f"Removed server argument(s): {replacements}")
@staticmethod @staticmethod
def load_config_file(config_file: str) -> dict[str, Any]: def load_config_file(config_file: str) -> dict[str, Any]:
"""Load a config file.""" """Load a config file."""
@@ -2285,6 +2228,7 @@ class ServerArgs(DisaggServerArgsMixin):
@classmethod @classmethod
def from_kwargs(cls, **kwargs: Any) -> "ServerArgs": def from_kwargs(cls, **kwargs: Any) -> "ServerArgs":
cls._reject_retired_args(kwargs)
explicit_arg_names = set(kwargs) explicit_arg_names = set(kwargs)
# Convert backend string to enum if necessary # Convert backend string to enum if necessary
@@ -69,7 +69,7 @@ def should_return_warmup_result(req_or_group: Any) -> bool:
def should_run_server_warmup(server_args: ServerArgs) -> bool: def should_run_server_warmup(server_args: ServerArgs) -> bool:
return server_args.warmup and server_args.server_warmup return server_args.warmup_mode == "server"
def is_realtime_serving(server_args: ServerArgs) -> bool: def is_realtime_serving(server_args: ServerArgs) -> bool:
@@ -95,7 +95,7 @@ def should_run_synthetic_server_warmup(server_args: ServerArgs) -> bool:
def should_run_explicit_client_warmup(server_args: ServerArgs) -> bool: def should_run_explicit_client_warmup(server_args: ServerArgs) -> bool:
return ( return (
server_args.warmup server_args.warmup_mode != "off"
and server_args.warmup_resolutions is not None and server_args.warmup_resolutions is not None
and supports_synthetic_warmup(server_args) and supports_synthetic_warmup(server_args)
) )
@@ -298,10 +298,9 @@ class SchedulerWarmupMixin:
) -> list[tuple[bytes, Any]]: ) -> list[tuple[bytes, Any]]:
if ( if (
self.req_based_warmup_scheduled self.req_based_warmup_scheduled
or not self.server_args.warmup or self.server_args.warmup_mode != "request"
or not recv_reqs or not recv_reqs
or self.server_args.warmup_resolutions is not None or self.server_args.warmup_resolutions is not None
or self.server_args.server_warmup
): ):
return recv_reqs return recv_reqs
@@ -68,8 +68,8 @@ def _build_server_extra_args(case: DiffusionTestCase) -> str:
if server_args.lora_path: if server_args.lora_path:
a += f" --lora-path {server_args.lora_path}" a += f" --lora-path {server_args.lora_path}"
# default warmup # request-based warmup keeps the first measured generation out of the baseline
a += " --warmup" a += " --warmup-mode request"
for extra_arg in server_args.extras: for extra_arg in server_args.extras:
a += f" {extra_arg}" a += f" {extra_arg}"
@@ -457,7 +457,7 @@ ONE_GPU_CASES: list[DiffusionTestCase] = [
modality="video", modality="video",
num_gpus=1, num_gpus=1,
extras=[ extras=[
"--pipeline-class-name LingBotWorldCausalDMDPipeline --warmup false" "--pipeline-class-name LingBotWorldCausalDMDPipeline --warmup-mode off"
], ],
text_encoder_cpu_offload=True, text_encoder_cpu_offload=True,
), ),
@@ -77,7 +77,7 @@ def pi05_generator():
kwargs = { kwargs = {
"model_path": _MODEL_PATH, "model_path": _MODEL_PATH,
"num_gpus": num_gpus, "num_gpus": num_gpus,
"warmup": False, "warmup_mode": "off",
"trust_remote_code": False, "trust_remote_code": False,
} }
if num_gpus > 1: if num_gpus > 1:
@@ -64,12 +64,11 @@ def _make_bare_scheduler(enable_cfg_parallel: bool) -> Scheduler:
scheduler = object.__new__(Scheduler) scheduler = object.__new__(Scheduler)
server_args = MagicMock() server_args = MagicMock()
server_args.warmup = True server_args.warmup_mode = "request"
server_args.warmup_steps = 1 server_args.warmup_steps = 1
server_args.warmup_resolutions = ["512x512"] server_args.warmup_resolutions = ["512x512"]
server_args.enable_cfg_parallel = enable_cfg_parallel server_args.enable_cfg_parallel = enable_cfg_parallel
server_args.enable_torch_compile = False server_args.enable_torch_compile = False
server_args.server_warmup = False
server_args.is_arg_explicitly_set.return_value = False server_args.is_arg_explicitly_set.return_value = False
task_type = MagicMock() task_type = MagicMock()
@@ -235,7 +234,7 @@ class TestWarmupReqCfgParallel(unittest.TestCase):
def test_req_based_warmup_remains_explicit_legacy_entry(self): def test_req_based_warmup_remains_explicit_legacy_entry(self):
scheduler = _make_bare_scheduler(enable_cfg_parallel=False) scheduler = _make_bare_scheduler(enable_cfg_parallel=False)
scheduler.server_args.warmup_resolutions = None scheduler.server_args.warmup_resolutions = None
scheduler.server_args.server_warmup = False scheduler.server_args.warmup_mode = "request"
req = _make_generation_req() req = _make_generation_req()
recv_reqs = [(b"0", req)] recv_reqs = [(b"0", req)]
@@ -253,7 +252,7 @@ class TestWarmupReqCfgParallel(unittest.TestCase):
def test_req_based_warmup_skips_default_server_warmup_path(self): def test_req_based_warmup_skips_default_server_warmup_path(self):
scheduler = _make_bare_scheduler(enable_cfg_parallel=False) scheduler = _make_bare_scheduler(enable_cfg_parallel=False)
scheduler.server_args.warmup_resolutions = None scheduler.server_args.warmup_resolutions = None
scheduler.server_args.server_warmup = True scheduler.server_args.warmup_mode = "server"
recv_reqs = [(b"0", _make_generation_req())] recv_reqs = [(b"0", _make_generation_req())]
processed = scheduler.process_received_reqs_with_req_based_warmup(recv_reqs) processed = scheduler.process_received_reqs_with_req_based_warmup(recv_reqs)
@@ -265,7 +264,7 @@ class TestWarmupReqCfgParallel(unittest.TestCase):
def test_diff_generator_runs_explicit_warmup_through_scheduler_client(self): def test_diff_generator_runs_explicit_warmup_through_scheduler_client(self):
generator = object.__new__(DiffGenerator) generator = object.__new__(DiffGenerator)
server_args = MagicMock() server_args = MagicMock()
server_args.warmup = True server_args.warmup_mode = "request"
server_args.warmup_resolutions = ["832x480"] server_args.warmup_resolutions = ["832x480"]
server_args.warmup_steps = 1 server_args.warmup_steps = 1
server_args.enable_cfg_parallel = False server_args.enable_cfg_parallel = False
@@ -693,8 +692,7 @@ class TestWarmupReqCfgParallel(unittest.TestCase):
def test_action_pipeline_disables_synthetic_warmup(self): def test_action_pipeline_disables_synthetic_warmup(self):
server_args = MagicMock() server_args = MagicMock()
server_args.warmup = True server_args.warmup_mode = "server"
server_args.server_warmup = True
server_args.warmup_resolutions = ["512x512"] server_args.warmup_resolutions = ["512x512"]
server_args.pipeline_config.task_type = ModelTaskType.VLA_ACTION server_args.pipeline_config.task_type = ModelTaskType.VLA_ACTION
@@ -704,8 +702,7 @@ class TestWarmupReqCfgParallel(unittest.TestCase):
def test_mesh_pipeline_builds_image_conditioned_warmup(self): def test_mesh_pipeline_builds_image_conditioned_warmup(self):
server_args = MagicMock() server_args = MagicMock()
server_args.warmup = True server_args.warmup_mode = "server"
server_args.server_warmup = True
server_args.warmup_steps = 1 server_args.warmup_steps = 1
server_args.warmup_resolutions = None server_args.warmup_resolutions = None
server_args.enable_cfg_parallel = False server_args.enable_cfg_parallel = False
@@ -421,12 +421,10 @@ class TestServerArgsPathExpansion(unittest.TestCase):
execute_serve_cmd(args, unknown_args) execute_serve_cmd(args, unknown_args)
server_args = dispatch_launch.call_args.args[0] server_args = dispatch_launch.call_args.args[0]
self.assertTrue(server_args.warmup) self.assertEqual(server_args.warmup_mode, "server")
self.assertTrue(server_args.server_warmup) self.assertFalse(server_args.is_arg_explicitly_set("warmup_mode"))
self.assertFalse(server_args.is_arg_explicitly_set("warmup"))
self.assertFalse(server_args.is_arg_explicitly_set("server_warmup"))
def test_serve_cli_preserves_explicit_warmup_false(self): def test_serve_cli_preserves_explicit_warmup_mode_off(self):
from sglang.multimodal_gen.runtime.entrypoints.cli.serve import ( from sglang.multimodal_gen.runtime.entrypoints.cli.serve import (
add_multimodal_gen_serve_args, add_multimodal_gen_serve_args,
execute_serve_cmd, execute_serve_cmd,
@@ -437,8 +435,8 @@ class TestServerArgsPathExpansion(unittest.TestCase):
argv = [ argv = [
"--model-path", "--model-path",
"/fake", "/fake",
"--warmup", "--warmup-mode",
"false", "off",
] ]
with ( with (
@@ -454,18 +452,17 @@ class TestServerArgsPathExpansion(unittest.TestCase):
execute_serve_cmd(args, unknown_args) execute_serve_cmd(args, unknown_args)
server_args = dispatch_launch.call_args.args[0] server_args = dispatch_launch.call_args.args[0]
self.assertFalse(server_args.warmup) self.assertEqual(server_args.warmup_mode, "off")
self.assertFalse(server_args.server_warmup) self.assertTrue(server_args.is_arg_explicitly_set("warmup_mode"))
self.assertTrue(server_args.is_arg_explicitly_set("warmup"))
def test_serve_cli_preserves_config_warmup_false(self): def test_serve_cli_preserves_config_warmup_mode_off(self):
from sglang.multimodal_gen.runtime.entrypoints.cli.serve import ( from sglang.multimodal_gen.runtime.entrypoints.cli.serve import (
add_multimodal_gen_serve_args, add_multimodal_gen_serve_args,
execute_serve_cmd, execute_serve_cmd,
) )
with tempfile.NamedTemporaryFile("w", suffix=".json") as config_file: with tempfile.NamedTemporaryFile("w", suffix=".json") as config_file:
json.dump({"model_path": "/fake", "warmup": False}, config_file) json.dump({"model_path": "/fake", "warmup_mode": "off"}, config_file)
config_file.flush() config_file.flush()
parser = FlexibleArgumentParser() parser = FlexibleArgumentParser()
@@ -490,9 +487,18 @@ class TestServerArgsPathExpansion(unittest.TestCase):
execute_serve_cmd(args, unknown_args) execute_serve_cmd(args, unknown_args)
server_args = dispatch_launch.call_args.args[0] server_args = dispatch_launch.call_args.args[0]
self.assertFalse(server_args.warmup) self.assertEqual(server_args.warmup_mode, "off")
self.assertFalse(server_args.server_warmup) self.assertTrue(server_args.is_arg_explicitly_set("warmup_mode"))
self.assertTrue(server_args.is_arg_explicitly_set("warmup"))
def test_retired_warmup_config_is_rejected(self):
with self.assertRaisesRegex(ValueError, "warmup.*warmup_mode"):
_from_dict_without_model_resolution(
{"model_path": "/fake", "warmup": False}
)
def test_retired_warmup_kwargs_are_rejected(self):
with self.assertRaisesRegex(ValueError, "warmup.*warmup_mode"):
ServerArgs.from_kwargs(model_path="/fake", warmup=False)
def test_disagg_role_disables_server_warmup(self): def test_disagg_role_disables_server_warmup(self):
with patch.object( with patch.object(
@@ -501,156 +507,82 @@ class TestServerArgsPathExpansion(unittest.TestCase):
server_args = ServerArgs.from_dict( server_args = ServerArgs.from_dict(
{ {
"model_path": "/fake", "model_path": "/fake",
"warmup": True, "warmup_mode": "server",
"server_warmup": True,
"disagg_role": "server", "disagg_role": "server",
} }
) )
self.assertTrue(server_args.warmup) self.assertEqual(server_args.warmup_mode, "request")
self.assertFalse(server_args.server_warmup)
class TestWarmupModeNormalization(unittest.TestCase): class TestWarmupModeNormalization(unittest.TestCase):
"""`_adjust_warmup` resolves the canonical warmup_mode and its derived booleans.""" """`_adjust_warmup` resolves the canonical warmup mode."""
def _resolve( def _resolve(
self, self,
*, *,
warmup_mode=None, warmup_mode=None,
warmup=False,
server_warmup=False,
warmup_resolutions=None, warmup_resolutions=None,
enable_torch_compile=False, enable_torch_compile=False,
enable_breakable_cuda_graph=False,
disagg_role=None, disagg_role=None,
explicit=(),
): ):
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
sa = ServerArgs.__new__(ServerArgs) sa = ServerArgs.__new__(ServerArgs)
sa.warmup_mode = warmup_mode sa.warmup_mode = warmup_mode
sa.warmup = warmup
sa.server_warmup = server_warmup
sa.warmup_resolutions = warmup_resolutions sa.warmup_resolutions = warmup_resolutions
sa.enable_torch_compile = enable_torch_compile sa.enable_torch_compile = enable_torch_compile
sa.enable_breakable_cuda_graph = enable_breakable_cuda_graph
sa.disagg_role = RoleType.MONOLITHIC if disagg_role is None else disagg_role sa.disagg_role = RoleType.MONOLITHIC if disagg_role is None else disagg_role
sa._explicit_arg_names = set(explicit)
sa._adjust_warmup() sa._adjust_warmup()
return sa return sa
def test_explicit_mode_off_disables_all(self): def test_explicit_mode_off_disables_all(self):
sa = self._resolve(warmup_mode="off", explicit=("warmup_mode",)) sa = self._resolve(warmup_mode="off")
self.assertEqual(sa.warmup_mode, "off") self.assertEqual(sa.warmup_mode, "off")
self.assertFalse(sa.warmup)
self.assertFalse(sa.server_warmup)
def test_explicit_mode_request(self): def test_explicit_mode_request(self):
sa = self._resolve(warmup_mode="request", explicit=("warmup_mode",)) sa = self._resolve(warmup_mode="request")
self.assertEqual(sa.warmup_mode, "request") self.assertEqual(sa.warmup_mode, "request")
self.assertTrue(sa.warmup)
self.assertFalse(sa.server_warmup)
def test_explicit_mode_server(self): def test_explicit_mode_server(self):
sa = self._resolve(warmup_mode="server", explicit=("warmup_mode",))
self.assertEqual(sa.warmup_mode, "server")
self.assertTrue(sa.warmup)
self.assertTrue(sa.server_warmup)
def test_explicit_mode_overrides_explicit_legacy(self):
sa = self._resolve(
warmup_mode="request",
warmup=True,
server_warmup=True,
explicit=("warmup_mode", "warmup", "server_warmup"),
)
self.assertEqual(sa.warmup_mode, "request")
self.assertTrue(sa.warmup)
self.assertFalse(sa.server_warmup)
def test_explicit_legacy_false_beats_defaulted_mode(self):
# serve defaults warmup_mode="server" (not explicit); `--warmup false` wins.
sa = self._resolve(
warmup_mode="server",
warmup=False,
server_warmup=False,
explicit=("warmup",),
)
self.assertEqual(sa.warmup_mode, "off")
self.assertFalse(sa.warmup)
self.assertFalse(sa.server_warmup)
def test_defaulted_mode_applies_without_legacy_flags(self):
# bare `sglang serve`: warmup_mode="server" defaulted, no legacy override.
sa = self._resolve(warmup_mode="server") sa = self._resolve(warmup_mode="server")
self.assertEqual(sa.warmup_mode, "server") self.assertEqual(sa.warmup_mode, "server")
self.assertTrue(sa.warmup)
self.assertTrue(sa.server_warmup)
def test_legacy_only_maps_to_request(self): def test_defaulted_mode_applies_without_legacy_flags(self):
sa = self._resolve(warmup_mode=None, warmup=True, explicit=("warmup",)) # Bare `sglang serve` defaults to server-based warmup.
self.assertEqual(sa.warmup_mode, "request") sa = self._resolve(warmup_mode="server")
self.assertTrue(sa.warmup) self.assertEqual(sa.warmup_mode, "server")
self.assertFalse(sa.server_warmup)
def test_resolutions_force_warmup_on(self): def test_resolutions_force_warmup_on(self):
sa = self._resolve( sa = self._resolve(
warmup_mode="off", warmup_mode="off",
warmup_resolutions=["512x512"], warmup_resolutions=["512x512"],
explicit=("warmup_mode",),
) )
self.assertTrue(sa.warmup)
self.assertFalse(sa.server_warmup)
self.assertEqual(sa.warmup_mode, "request") self.assertEqual(sa.warmup_mode, "request")
def test_torch_compile_defaults_to_server_warmup(self): def test_torch_compile_defaults_to_server_warmup(self):
sa = self._resolve(enable_torch_compile=True) sa = self._resolve(enable_torch_compile=True)
self.assertEqual(sa.warmup_mode, "server") self.assertEqual(sa.warmup_mode, "server")
self.assertTrue(sa.warmup)
self.assertTrue(sa.server_warmup)
def test_legacy_warmup_on_uses_defaulted_server_mode(self):
# `serve --warmup` (legacy ON, mode defaulted to "server" but not
# explicit) must resolve to server-based warmup, not silently downgrade
# to request mode.
sa = self._resolve(warmup_mode="server", warmup=True, explicit=("warmup",))
self.assertEqual(sa.warmup_mode, "server")
self.assertTrue(sa.warmup)
self.assertTrue(sa.server_warmup)
def test_torch_compile_respects_explicit_warmup_off(self): def test_torch_compile_respects_explicit_warmup_off(self):
sa = self._resolve( sa = self._resolve(
warmup_mode="off", warmup_mode="off",
enable_torch_compile=True, enable_torch_compile=True,
explicit=("warmup_mode",),
) )
self.assertEqual(sa.warmup_mode, "off") self.assertEqual(sa.warmup_mode, "off")
self.assertFalse(sa.warmup)
self.assertFalse(sa.server_warmup)
def test_torch_compile_uses_server_warmup_for_explicit_resolutions(self): def test_torch_compile_uses_server_warmup_for_explicit_resolutions(self):
sa = self._resolve( sa = self._resolve(
warmup_resolutions=["1024x1024"], warmup_resolutions=["1024x1024"],
enable_torch_compile=True, enable_torch_compile=True,
explicit=("warmup_resolutions",),
) )
self.assertEqual(sa.warmup_mode, "server") self.assertEqual(sa.warmup_mode, "server")
self.assertTrue(sa.warmup)
self.assertTrue(sa.server_warmup)
def test_legacy_warmup_with_resolutions_runs_server_warmup(self): def test_breakable_cuda_graph_forces_server_warmup(self):
# Dead-zone regression: `serve --warmup --warmup-resolutions X` must run sa = self._resolve(enable_breakable_cuda_graph=True)
# server-based (synthetic) warmup, not end up with no warmup at all
# (request-based warmup bails out when warmup_resolutions is set).
sa = self._resolve(
warmup_mode="server",
warmup=True,
warmup_resolutions=["1024x1024"],
explicit=("warmup",),
)
self.assertTrue(sa.warmup)
self.assertTrue(sa.server_warmup)
self.assertEqual(sa.warmup_mode, "server") self.assertEqual(sa.warmup_mode, "server")
def test_disagg_role_disables_server_warmup(self): def test_disagg_role_disables_server_warmup(self):
@@ -659,10 +591,7 @@ class TestWarmupModeNormalization(unittest.TestCase):
sa = self._resolve( sa = self._resolve(
warmup_mode="server", warmup_mode="server",
disagg_role=RoleType.DENOISER, disagg_role=RoleType.DENOISER,
explicit=("warmup_mode",),
) )
self.assertTrue(sa.warmup)
self.assertFalse(sa.server_warmup)
self.assertEqual(sa.warmup_mode, "request") self.assertEqual(sa.warmup_mode, "request")
def test_torch_compile_server_warmup_disabled_for_disagg_role(self): def test_torch_compile_server_warmup_disabled_for_disagg_role(self):
@@ -670,12 +599,10 @@ class TestWarmupModeNormalization(unittest.TestCase):
sa = self._resolve(enable_torch_compile=True, disagg_role=RoleType.DENOISER) sa = self._resolve(enable_torch_compile=True, disagg_role=RoleType.DENOISER)
self.assertEqual(sa.warmup_mode, "request") self.assertEqual(sa.warmup_mode, "request")
self.assertTrue(sa.warmup)
self.assertFalse(sa.server_warmup)
def test_invalid_mode_raises(self): def test_invalid_mode_raises(self):
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
self._resolve(warmup_mode="bogus", explicit=("warmup_mode",)) self._resolve(warmup_mode="bogus")
class TestWarmupImageIsModelValid(unittest.TestCase): class TestWarmupImageIsModelValid(unittest.TestCase):
@@ -1962,24 +1889,9 @@ class TestPerRoleParallelism(unittest.TestCase):
self.assertIsNone(par["ulysses_degree"]) self.assertIsNone(par["ulysses_degree"])
self.assertIsNone(par["ring_degree"]) self.assertIsNone(par["ring_degree"])
def test_decoder_tp_is_alias_of_decoder_sp(self): def test_removed_decoder_tp_is_rejected(self):
args = self._from_dict({"model_path": "/fake", "decoder_tp": 2}) with self.assertRaisesRegex(ValueError, "decoder_tp.*decoder_sp"):
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType self._from_dict({"model_path": "/fake", "decoder_tp": 2})
self.assertEqual(args.decoder_sp, 2)
par = args.get_role_parallelism(RoleType.DECODER)
self.assertIsNone(par["tp_size"])
self.assertEqual(par["sp_degree"], 2)
def test_conflicting_decoder_tp_and_decoder_sp_raise(self):
with self.assertRaisesRegex(ValueError, "decoder_tp is deprecated"):
self._from_dict(
{
"model_path": "/fake",
"decoder_tp": 2,
"decoder_sp": 4,
}
)
def test_monolithic_returns_all_none(self): def test_monolithic_returns_all_none(self):
args = self._from_dict({"model_path": "/fake", "encoder_tp": 2}) args = self._from_dict({"model_path": "/fake", "encoder_tp": 2})
@@ -2090,7 +2002,6 @@ class TestPerRoleParallelism(unittest.TestCase):
self.assertEqual(args.denoiser_ring, 2) self.assertEqual(args.denoiser_ring, 2)
self.assertEqual(args.encoder_tp, 1) self.assertEqual(args.encoder_tp, 1)
self.assertEqual(args.decoder_sp, 8) self.assertEqual(args.decoder_sp, 8)
self.assertIsNone(args.decoder_tp)
class TestPipelineResolutionCliOverride(unittest.TestCase): class TestPipelineResolutionCliOverride(unittest.TestCase):