From db272201a2dbd72e5699e443240a851f1313ad45 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:42:12 -0700 Subject: [PATCH] [Config] Retire get_global_server_args, and clear the deprecated flags that have a replacement (#38375) --- .../references/authoring-reference.md | 2 +- .../skills/cookbook-migrate-model/SKILL.md | 4 +- .../references/dimension-mapping.md | 2 +- .../skills/sglang-runtime-context/SKILL.md | 12 +- .../bench_int8_checkpoint_reuse.py | 4 +- .../autoregressive/DeepSeek/DeepSeek-V3.mdx | 2 +- .../autoregressive/Meituan/LongCat-2.0.mdx | 2 +- .../autoregressive/Qwen/Qwen3.8-27B.mdx | 2 +- .../cuda_graph_for_multi_modal_encoder.mdx | 11 +- .../advanced_features/pd_disaggregation.mdx | 7 +- .../piecewise_cuda_graph.mdx | 38 ++- .../advanced_features/server_arguments.mdx | 84 ------- .../speculative_decoding.mdx | 4 +- .../best-practices/deepseek_r1.mdx | 20 +- .../best-practices/deepseek_v4_flash.mdx | 6 +- .../best-practices/glm_5_1.mdx | 21 +- .../best-practices/glm_5_2.mdx | 3 +- .../best-practices/kimi_k2_6.mdx | 10 +- .../best-practices/mimo_v2_flash.mdx | 16 +- .../best-practices/minimax_m2_5.mdx | 10 +- .../best-practices/qwen3_235b_a22b.mdx | 4 +- .../best-practices/qwen3_30b_a3b.mdx | 8 +- .../best-practices/qwen3_32b.mdx | 6 +- .../best-practices/qwen3_5_397b.mdx | 18 +- .../best-practices/qwen3_6_27b.mdx | 18 +- .../best-practices/qwen3_6_35b_a3b.mdx | 22 +- .../best-practices/qwen3_8b.mdx | 6 +- .../qwen3_next_80b_a3b_instruct.mdx | 6 +- .../tutorials/deepseek_r1.mdx | 2 +- .../tutorials/deepseek_v3_2.mdx | 2 +- .../tutorials/deepseek_v4_flash.mdx | 2 +- .../model-deployment/tutorials/glm_5_1.mdx | 2 +- .../model-deployment/tutorials/glm_5_2.mdx | 10 +- .../model-deployment/tutorials/hy3.mdx | 6 +- .../model-deployment/tutorials/kimi_k2_6.mdx | 2 +- .../tutorials/mimo_v2_flash.mdx | 2 +- .../tutorials/minimax_m2_5.mdx | 2 +- .../tutorials/qwen3_235b_a22b.mdx | 2 +- .../tutorials/qwen3_30b_a3b.mdx | 2 +- .../model-deployment/tutorials/qwen3_32b.mdx | 2 +- .../tutorials/qwen3_5_397b.mdx | 2 +- .../tutorials/qwen3_6_27b.mdx | 2 +- .../tutorials/qwen3_6_35b_a3b.mdx | 2 +- .../tutorials/qwen3_8_max.mdx | 4 +- .../model-deployment/tutorials/qwen3_8b.mdx | 2 +- .../tutorials/qwen3_next_80b_a3b_instruct.mdx | 2 +- .../optimization/parameter_tuning.mdx | 4 +- .../reference/support_features.mdx | 24 +- docs/docs/sglang-diffusion/disaggregation.mdx | 2 +- docs/docs/sglang-diffusion/models_with_ar.mdx | 2 +- .../src/snippets/configs/Qwen/qwen3.8-27b.jsx | 12 +- docs/src/snippets/configs/Qwen/qwen3.8.jsx | 4 +- .../configs/inclusionAI/ling-3.0-flash.jsx | 10 +- .../configs/meituan-longcat/longcat-2.0.jsx | 8 +- .../snippets/configs/moonshotai/kimi-k3.jsx | 10 +- .../openbmb/minicpm5-2b-benchmarks.jsx | 2 +- .../snippets/configs/openbmb/minicpm5-2b.jsx | 4 +- docs/src/snippets/configs/zai-org/glm-5.2.jsx | 30 +-- .../configs/zai-org/glm-5.3-flash.jsx | 8 +- docs/src/snippets/configs/zai-org/glm-5.3.jsx | 24 +- .../aot/tests/test_fused_qk_norm_rope.py | 1 - .../ascend/test_glm_image_distributed.py | 2 +- .../test/single_test_file/test_ar_models.py | 2 +- .../test/unit/test_disagg_trace.py | 5 +- python/sglang/srt/arg_groups/arg_utils.py | 45 ++-- .../sglang/srt/arg_groups/argparse_actions.py | 53 ++++- .../sglang/srt/arg_groups/attention_hook.py | 5 + .../sglang/srt/arg_groups/cuda_graph_hook.py | 8 +- .../srt/arg_groups/model_overrides/inkling.py | 2 +- python/sglang/srt/arg_groups/moe_hook.py | 2 +- .../single_forward_manager/manager.py | 2 +- python/sglang/srt/layers/communicator.py | 2 +- python/sglang/srt/layers/moe/mega_moe.py | 4 +- python/sglang/srt/layers/radix_attention.py | 2 +- .../srt/mem_cache/storage/nixl/README.md | 4 +- .../sglang/srt/model_executor/model_runner.py | 1 - python/sglang/srt/runtime_context.py | 4 +- python/sglang/srt/server_args.py | 218 +++--------------- python/sglang/test/ascend/test_npu_logging.py | 2 +- .../sglang/test/chunked_prefill_test_utils.py | 2 +- python/sglang/test/kv_canary/utils.py | 2 +- python/sglang/test/mock_model/utils.py | 2 +- .../dsv4flash/1k1k/2p1d-ep16-mtp.yaml | 2 +- .../mi355x-fp4/dsv4flash/1k1k/2p1d-ep16.yaml | 2 +- .../dsv4pro/1k1k/2p1d-ep16-mtp.yaml | 2 +- .../mi355x-fp4/dsv4pro/1k1k/2p1d-ep16.yaml | 2 +- .../mi355x-fp4/glm52/1k1k/2p1d-ep16-mtp.yaml | 2 +- .../mi355x-fp4/glm52/1k1k/2p1d-ep16.yaml | 2 +- .../dsv4flash/1k1k/2p1d-ep16-mtp.yaml | 2 +- .../mi355x-fp8/dsv4flash/1k1k/2p1d-ep16.yaml | 2 +- .../dsv4pro/1k1k/2p1d-ep16-mtp.yaml | 2 +- .../mi355x-fp8/dsv4pro/1k1k/2p1d-ep16.yaml | 2 +- .../kimik26/1k1k/2p1d-ep16-mtp-mxfp4.yaml | 2 +- .../kimik26/1k1k/2p1d-ep16-mxfp4.yaml | 2 +- scripts/playground/bench_speculative.py | 2 +- scripts/playground/disaggregation/cli-so.py | 2 +- .../4-gpu-models/test_qwen35_fp4_triton.py | 2 +- .../test_qwen35_models_archived.py | 4 +- .../4-gpu-models/test_qwen3_next_models.py | 4 +- .../test_qwen3_next_models_mtp_archived.py | 2 +- .../chunked_prefill/test_e2e_hybrid_swa.py | 2 +- test/manual/ep/test_eplb.py | 1 - test/manual/lora/run_spec_lora_matrix.py | 6 +- test/manual/models/test_granite_moe_hybrid.py | 2 +- .../models/test_nvidia_nemotron_nano_v2.py | 4 +- .../nightly/test_vlms_piecewise_cuda_graph.py | 6 +- .../nightly/test_vlms_vit_cuda_graph.py | 6 +- ...ewise_cuda_graph_support_1_gpu_archived.py | 2 +- ...est_disaggregation_piecewise_cuda_graph.py | 2 +- .../manual/test_dsa_alias_cli_registry_env.py | 63 ----- test/manual/test_w4a8_deepseek_v3.py | 6 +- .../test_deepseek_v4_flash_eval_mi45x.py | 2 +- .../amd/test_deepseek_r1_mxfp4_8gpu.py | 6 +- .../amd/test_deepseek_v4_flash_fp8_tbo.py | 2 +- .../amd/test_deepseek_v4_pro_fp4_dspark.py | 2 +- .../amd/test_deepseek_v4_pro_fp4_tbo.py | 2 +- .../amd/test_deepseek_v4_pro_fp4_tbo_mtp.py | 2 +- .../attention/test_qwen35_deterministic.py | 2 +- .../core/test_basic_sanity_dflash.py | 4 +- .../core/test_basic_sanity_dspark.py | 2 +- .../core/test_basic_sanity_eagle3.py | 2 +- test/registered/cp/test_dsa_prefill_cp.py | 2 +- test/registered/cp/test_gqa_prefill_cp.py | 6 +- test/registered/cpu/test_cpu_graph.py | 4 +- .../breakable/test_breakable_cuda_graph.py | 4 +- ...est_pcg_with_speculative_decoding_extra.py | 2 +- test/registered/dcp/test_dsv31_dcp8_gsm8k.py | 2 +- .../test_nightly_precision_regression.py | 2 +- .../test_disaggregation_hybrid_attention.py | 8 +- .../dllm/test_dllm_batching_fdfo.py | 2 +- .../dllm/test_dllm_fdfo_joint_threshold.py | 2 +- .../models/test_deepseek_v4_flash_fp4_b200.py | 2 +- .../e2e/models/test_qwen35_fp4_mtp.py | 2 +- .../e2e/models/test_qwen3_next_models.py | 2 +- .../models/test_qwen3_next_models_extra.py | 2 +- .../e2e/models/test_qwen3_next_models_mtp.py | 6 +- .../test_deepseek_v3_cutedsl_4gpu.py | 2 +- .../e2e/models_large/test_ling_2_6_flash.py | 2 +- .../e2e/models_large/test_qwen35.py | 2 +- test/registered/ep/test_deepep_large.py | 4 +- test/registered/gb300/test_qwen35_fp8_dp.py | 2 +- test/registered/gb300/test_qwen35_fp8_tp.py | 2 +- .../registered/hicache/test_qwen35_hicache.py | 2 +- .../kv_canary/test_self_e2e_bench_speed.py | 4 +- test/registered/mock_model/test_e2e_pd.py | 2 +- .../musa/test_llm_server_smoke_musa.py | 2 +- ...test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py | 2 +- .../test_npu_glm4_6v_flash_1p_mmmu.py | 2 +- .../test_npu_glm4_7_flash_1p_aime25.py | 2 +- .../test_npu_glm4_7_flash_1p_gsm8k.py | 2 +- ...1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py | 3 +- .../glm5_2/test_npu_glm_5_2_w4a8_16p_gpqa.py | 2 +- ...est_npu_glm5_top64_pruned_bf16_8p_gsm8k.py | 2 +- ..._k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py | 2 +- .../kimi_k3/test_npu_kimi_k3_w4a8_32p_gpqa.py | 2 +- ..._w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py | 2 +- ...max_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py | 2 +- ...est_npu_moonlight_16b_a3b_bf16_1p_gsm8k.py | 2 +- ...qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py | 2 +- ..._qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py | 2 +- ...n3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py | 2 +- .../test_npu_qwen3_32b_bf16_8p_gpqa.py | 2 +- ...wen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py | 2 +- ...3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa_a2.py | 2 +- .../test_npu_qwen3_5_9b_bf16_1p_gsm8k.py | 2 +- .../test_npu_qwen3_6_27b_1p_gpqa.py | 4 +- ...n3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py | 2 +- .../test_npu_qwen3_6_35b_a3b_1p_aime26.py | 2 +- ...a3b_1p_in64k_out1k_prefix90_50ms_aime26.py | 4 +- ...ext_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py | 2 +- ...npu_qwen3_omni_30b_a3b_thinking_1p_mmmu.py | 2 +- .../test_npu_qwen3_vl_8b_bf16_2p_gsm8k.py | 2 +- .../test_npu_piecewise_graph_prefill.py | 2 +- .../test_npu_deepep_auto_qwen3_480b.py | 2 +- .../test_npu_deepep_auto_qwen3_next.py | 2 +- .../test_npu_deepep_low_latency_qwen3_480b.py | 2 +- .../test_npu_deepep_low_latency_qwen3_next.py | 2 +- .../quant/test_npu_w4a4_quantization.py | 2 +- .../test_npu_basic_sanity_eagle3.py | 4 +- ..._v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py | 2 +- ...pseek_v4_flash_w8a8_8p_in32k_out1k_50ms.py | 2 +- ...epseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py | 2 +- ...u_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms.py | 3 +- ..._qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py | 2 +- ...n3_6_27b_1p_in1024x1024_30_out1024_50ms.py | 2 +- ...u_qwen3_6_27b_1p_in1080p_30_out256_50ms.py | 2 +- ...wen3_6_27b_2p_in64k_out1k_prefix90_50ms.py | 4 +- ...u_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms.py | 2 +- ...pu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py | 2 +- ...u_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py | 2 +- ...pu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py | 2 +- ...pu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py | 2 +- ...6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py | 4 +- ...pu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py | 2 +- ...npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py | 2 +- ..._6_35b_a3b_1p_in64k_out1k_prefix90_50ms.py | 4 +- .../npu/test_npu_memory_consumption.py | 2 +- test/registered/pp/test_pp_parallel_compat.py | 2 +- .../test_int8_mamba_checkpoint_e2e.py | 4 +- .../test_mamba2_extra_buffer_kl.py | 2 +- .../test_unified_radix_cache_hicache_pp_kl.py | 6 +- .../test_unified_radix_cache_kl_cp.py | 2 +- .../test_unified_radix_cache_kl_mamba.py | 6 +- .../test_unified_radix_cache_kl_swa.py | 2 +- .../rl/test_return_routed_experts.py | 2 +- test/registered/spec/dflash/test_dflash.py | 2 +- test/registered/spec/test_spec_mixed_chunk.py | 2 +- .../server_args/test_declared_fallbacks.py | 90 +------- .../unit/server_args/test_server_args.py | 7 + .../test_unified_prefill_cuda_graph_gate.py | 3 +- test/registered/unit/test_runtime_context.py | 33 ++- .../unit/test_server_args_migration.py | 86 ++++++- test/registered/vlm/test_vlm_tp4.py | 2 +- 213 files changed, 625 insertions(+), 892 deletions(-) diff --git a/.claude/skills/cookbook-add-model/references/authoring-reference.md b/.claude/skills/cookbook-add-model/references/authoring-reference.md index 9ee9a7bba..d98d2d4fe 100644 --- a/.claude/skills/cookbook-add-model/references/authoring-reference.md +++ b/.claude/skills/cookbook-add-model/references/authoring-reference.md @@ -178,7 +178,7 @@ schemas (full reference in the `_playground.jsx` header): | `pdDisagg` | Mode + transfer backend (+ optional per-backend env via `envWhen` hw-gate) + IB device + optional `router{port, command}` | Model supports prefill/decode disaggregation. A `modes[]` entry may carry `flags` / `env` that only that role needs (`prefill`'s `--load-balance-method`, `decode`'s `--disaggregation-decode-polling-interval`, ...); they are emitted only while that role is selected, and a flag whose head the base cell already sets is replaced rather than duplicated. Put role-specific settings here, NOT in the cells — a cell carries one recipe, and the role is a Playground overlay. In `router.command` the ports MUST be `{{PREFILL_PORT}}` / `{{DECODE_PORT}}` / `{{ROUTER_PORT}}`: the engine substitutes them from its own `PD_PORTS` (prefill 30000, decode **30100**), so a literal port silently points the router at something the generated decode command never binds. When a PD role is active and `router` is set, the playground shows the router (SGLang Model Gateway) launch command as a companion block and retargets the cURL modal to `router.port`. | | `hicache` | Enable + storage + write policy | Model is large enough that hierarchical KV cache matters. | | `hisparse` | Enable + host-ratio select; whole card gated on the live PD-Disagg mode being `decode` | DSA-style model (DeepSeek-V3.2 / V4, GLM-5) that supports decode-side hierarchical sparse attention. | -| `flagSelects` | A config-declared **list** of single-selects, each `{ id, title, stripPrefixes, options }` (option = `{ id, label, flags?, hide?, disable?, disableReason? }`); a flagless option is the "none"/accuracy-safe choice | A titled single-select that picks one value of a flag family the other axes don't model — e.g. KV-cache dtype (`--kv-cache-dtype`), mamba scheduler strategy (`--mamba-scheduler-strategy`). Generic: no engine change to add another. | +| `flagSelects` | A config-declared **list** of single-selects, each `{ id, title, stripPrefixes, options }` (option = `{ id, label, flags?, hide?, disable?, disableReason? }`); a flagless option is the "none"/accuracy-safe choice | A titled single-select that picks one value of a flag family the other axes don't model — e.g. KV-cache dtype (`--kv-cache-dtype`), mamba scheduler strategy (`--mamba-radix-cache-strategy`). Generic: no engine change to add another. | **Per-chip constraints**: any chip entry in any axis can be wrapped with `hide` / `disable` constraint objects: diff --git a/.claude/skills/cookbook-migrate-model/SKILL.md b/.claude/skills/cookbook-migrate-model/SKILL.md index eb1d514e8..ae9b55dad 100644 --- a/.claude/skills/cookbook-migrate-model/SKILL.md +++ b/.claude/skills/cookbook-migrate-model/SKILL.md @@ -73,7 +73,7 @@ your dispatch prompt, or ask for it. change in a migration PR. Model-specific features are config DATA consumed by generic axis handlers (MegaMoE precedent), so they need NO engine change. A **titled single-select that strips a flag family** — KV Cache DType - (`--kv-cache-dtype`), mamba (`--mamba-scheduler-strategy`), … — is already + (`--kv-cache-dtype`), mamba (`--mamba-radix-cache-strategy`), … — is already covered by the merged generic **`flagSelects`** axis: declare it in the config (a list of `{ id, title, stripPrefixes, options }`; see the Qwen3.5 mamba example), **no engine PR**. Only a genuinely new control *shape* that @@ -129,7 +129,7 @@ mention — and a model-specific control is **config data, not engine code** (MegaMoE W4A4 is all DSv4 config on the existing `moe` axis). It's pure config whenever it fits an existing axis's data schema. A **titled single-select that strips a flag family** (Nemotron3's "KV Cache DType", -mamba `--mamba-scheduler-strategy`, …) fits the merged generic **`flagSelects`** +mamba `--mamba-radix-cache-strategy`, …) fits the merged generic **`flagSelects`** axis — so it too is config-only (declare a `flagSelects` list). Only a control whose *shape* `flagSelects` still can't express would need a ONE-TIME generic primitive (never a model-named handler) on a separate PRIOR engine PR, keeping diff --git a/.claude/skills/cookbook-migrate-model/references/dimension-mapping.md b/.claude/skills/cookbook-migrate-model/references/dimension-mapping.md index acf87415d..3e34ae790 100644 --- a/.claude/skills/cookbook-migrate-model/references/dimension-mapping.md +++ b/.claude/skills/cookbook-migrate-model/references/dimension-mapping.md @@ -15,7 +15,7 @@ this file is about the *mapping decisions*. | toggle that **couples** with other parts of the command (changes TP/mem/EP), OR one the legacy page labels with **operating-point words** | `strategies` | The Playground applies pure flag diffs — it cannot do coupled changes. Example: Qwen3.5's MTP toggle bumps TP on three H100 combos → strategies `low-latency` (MTP on) / `high-throughput` (MTP off). **Naming counts like coupling**: GLM-5.1's / Kimi-K2.6's `dpattention` adds only `--dp N --enable-dp-attention` (uncoupled), but its options are subtitled "Low Latency" / "High Throughput" — the page's own named operating-point split → strategies; a flag-only spec toggle riding alongside it stays a Playground axis and bakes per its legacy default. GPU-count radios (GLM-4.7, MiniMax-M2.5/2.7) → budget-tier strategies with the legacy SUPPORT matrix preserved by which cells exist. Strategy count follows the page's operating points: 1 → `balanced`, 2 → `low-latency`+`high-throughput`, 3 → the full trio (§4). | | toggle that only adds/removes its own flags | Playground axis (+ bake, EXCEPT parsers and accuracy-degrading flags) | **Parsers (`--reasoning-parser` / `--tool-call-parser`) are NEVER baked into cells** — Deployment commands ship without them regardless of the legacy default or the measured command; the `parsers` axis adds them on top (DSv4 convention; cells mirror the legacy generator's parsers-OFF output). Accuracy-degrading toggles are never baked either — §2 caveats (axis-only, accuracy-safe cells). Other flag-only toggles: legacy default ON → bake into cells AND declare the axis so users can strip (red strikethrough); default OFF → keep cells clean, axis preset only. MTP/EAGLE presets → `speculative` axis; dp-attention → a strategy when the legacy page labels it as the operating-point split or when coupled (see the row above), else `attention.dpAttn`. **EVERY legacy control survives as an interactive control** (a dimension or a Playground axis), never a tips-only mention — but a model-specific control is **config DATA, not engine code**: the axis handler reads options/flags/env/gating straight from `config.playgroundFeatures` (MegaMoE W4A4 is entirely DSv4 config data on the existing `moe` axis — no per-model engine edit). A control that fits an existing axis's data schema is therefore pure config, full stop. A **titled single-select that strips a flag family** (e.g. Nemotron3's "KV Cache DType" `--kv-cache-dtype`) is covered by the merged generic **`flagSelects`** axis → **config-only**: declare a `flagSelects` list of `{ id, title, stripPrefixes, options }` (see the Qwen3.5 mamba example), **no engine PR**. Only a control whose *shape* `flagSelects` still can't express would need a new ONE-TIME generic primitive (never a model-named handler) on a prior engine PR; the backward-compat reasoning (opt-in per key, not in the opt-out set) is in engine-axis.md. | | per-combo hidden option (e.g. spec hidden on Xeon) | absent cells | Don't create cells for combos the legacy widget couldn't produce; the engine greys them automatically. `# Error:` pseudo-commands → no cell + explanation in §2 tips and/or a chip `disable`/`disableReason`. | -| coupled secondary knob (e.g. mamba cache V1/V2) | cells + Playground axis | Bake the correct value per cell following the legacy coupling (Qwen3.5: MTP ⇒ `--mamba-scheduler-strategy extra_buffer` on NVIDIA; AMD/Xeon ⇒ V1/no flag) and document the coupling in §2 tips — AND surface the knob as a Playground axis like every other legacy feature (row above; add the axis when none fits). Baking alone is NOT enough — the every-feature rule supersedes the pilot's cells+prose-only treatment of Qwen3.5's mamba knob (retrofit pending). The mamba knob is the same single-select shape as KV Cache DType, so it rides the merged generic **`flagSelects`** axis — Qwen3.6 / Qwen3-Coder-Next declare it purely in config (a `flagSelects` block), **no engine PR**. | +| coupled secondary knob (e.g. mamba cache V1/V2) | cells + Playground axis | Bake the correct value per cell following the legacy coupling (Qwen3.5: MTP ⇒ `--mamba-radix-cache-strategy extra_buffer` on NVIDIA; AMD/Xeon ⇒ V1/no flag) and document the coupling in §2 tips — AND surface the knob as a Playground axis like every other legacy feature (row above; add the axis when none fits). Baking alone is NOT enough — the every-feature rule supersedes the pilot's cells+prose-only treatment of Qwen3.5's mamba knob (retrofit pending). The mamba knob is the same single-select shape as KV Cache DType, so it rides the merged generic **`flagSelects`** axis — Qwen3.6 / Qwen3-Coder-Next declare it purely in config (a `flagSelects` block), **no engine PR**. | ## 2. Command rewrite table (the ONLY allowed normalizations) diff --git a/.claude/skills/sglang-runtime-context/SKILL.md b/.claude/skills/sglang-runtime-context/SKILL.md index 05193dcf8..e59fff1fc 100644 --- a/.claude/skills/sglang-runtime-context/SKILL.md +++ b/.claude/skills/sglang-runtime-context/SKILL.md @@ -544,10 +544,12 @@ ONE thread — do not design for TBO threads that don't exist. `ServerArgs.override` nor `ServerArgs.derive` exists, and nothing in the package calls either form. Rerouting a writer to the bags means flipping **all its readers in the same commit** (no transitional dual-write). -4. **Legacy-accessor ratchet** (`test_legacy_global_ratchet.py`): `get_global_server_args` - call sites must not grow. The replacement for a *decision* is a bag leaf, a named - accessor, or the owning runner's stamp — not `get_server_args().field`, which the - read ratchet below pins at zero. `runtime_context.get_server_args()` is only for the +4. **The legacy accessor is retired** (`test_runtime_context.py`): every + `get_global_server_args()` call now raises, because it answered with the record -- + a caller reading a field resolution had decided got a stale value and no error. + The replacement for a *decision* is a bag leaf, a named accessor, or the owning + runner's stamp — not `get_server_args().field`, which the read ratchet below pins + at zero. `runtime_context.get_server_args()` is only for the whole-object shapes (dumps, provenance, a hand-off to a callee that takes a config). 5. **Global config read ratchet** (`test_global_config_read_ratchet.py`): baselines are **0** for both the direct `get_server_args().field` and the alias form (function-local @@ -622,7 +624,7 @@ Key source files: `python/sglang/srt/runtime_context.py` (the container, every t `declare_late_resolution`), `python/sglang/srt/server_args.py` (`NS` metadata, `Arg(..., resolvable=True)`, `__setattr__` strict guard), and the guardrail tests under `test/registered/unit/` (`test_server_args_mutation_ratchet.py`, -`test_global_config_read_ratchet.py`, `test_legacy_global_ratchet.py`, +`test_global_config_read_ratchet.py`, `test_module_state_ratchet.py`, `test_server_args_namespaces.py`, `test_runtime_context.py` — the last one doubles as executable documentation of every tier's semantics). diff --git a/benchmark/bench_linear_attention/bench_int8_checkpoint_reuse.py b/benchmark/bench_linear_attention/bench_int8_checkpoint_reuse.py index 12d5f26e6..9e30f177a 100644 --- a/benchmark/bench_linear_attention/bench_int8_checkpoint_reuse.py +++ b/benchmark/bench_linear_attention/bench_int8_checkpoint_reuse.py @@ -20,7 +20,7 @@ Method, per K in ``--num-prefixes``: Run the server twice and compare (same flags, toggle int8): python -m sglang.launch_server --model-path --tp 4 \ - --trust-remote-code --mamba-scheduler-strategy extra_buffer \ + --trust-remote-code --mamba-radix-cache-strategy extra_buffer \ --max-mamba-cache-size 256 [--enable-int8-mamba-checkpoint] --port 30000 python benchmark/bench_linear_attention/bench_int8_checkpoint_reuse.py \ @@ -28,7 +28,7 @@ Run the server twice and compare (same flags, toggle int8): NOTE: prefix-tokens must cross the mamba cache chunk granularity (typically ~512), otherwise nothing is cacheable and reuse is 0 by construction (not a regression). -Use ``--mamba-scheduler-strategy extra_buffer`` on the server: ``no_buffer`` only +Use ``--mamba-radix-cache-strategy extra_buffer`` on the server: ``no_buffer`` only snapshots state at the full-sequence leaf, so a divergent-suffix probe never reuses. """ diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx index f61150d1a..61fc53f6d 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx @@ -392,7 +392,7 @@ python3 -m sglang.launch_server \ The default configuration is `--speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4`. Find the best values for your workload with [bench_speculative.py](https://github.com/sgl-project/sglang/blob/main/scripts/playground/bench_speculative.py). The minimum viable config is `--speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2`. -For large batch sizes (>48), increase `--max-running-requests` beyond the default of 48 for MTP. Also set `--cuda-graph-bs` to include your target batch sizes (default captured sizes for speculative decoding: 48). +For large batch sizes (>48), increase `--max-running-requests` beyond the default of 48 for MTP. Also set `--cuda-graph-bs-decode` to include your target batch sizes (default captured sizes for speculative decoding: 48). diff --git a/docs/cookbook/autoregressive/Meituan/LongCat-2.0.mdx b/docs/cookbook/autoregressive/Meituan/LongCat-2.0.mdx index b3db0f884..e50cf1033 100644 --- a/docs/cookbook/autoregressive/Meituan/LongCat-2.0.mdx +++ b/docs/cookbook/autoregressive/Meituan/LongCat-2.0.mdx @@ -102,7 +102,7 @@ import { Playground } from "/src/snippets/_playground.jsx"; - **Remote code.** Use `--trust-remote-code` for the Hugging Face checkpoint. - **Topology.** The 8x B300 recipe uses TP=8 and EP=8. H200, B200, and H20 use a 2-node 16 GPU layout with TP=16 and EP=16; the command panel injects the multi-node rank flags for you. -- **LongCat sparse attention.** Keep `--nsa-prefill-backend fa3` with `--chunked-prefill-size 2048` for the model-card-aligned prefill path. +- **LongCat sparse attention.** Keep `--dsa-prefill-backend fa3` with `--chunked-prefill-size 2048` for the model-card-aligned prefill path. - **Memory.** The recipe uses `--kv-cache-dtype bfloat16` and starts at `--mem-fraction-static 0.92`. Tune memory only after the generated command launches cleanly on your cluster. - **Weight loading.** `--model-loader-extra-config '{"enable_multithread_load":true,"num_threads":12}'` loads checkpoint shards in parallel and reduces startup time. - **FP8 backend selection.** Do not pass `--fp8-gemm-runner-backend` manually. SGLang selects the correct backend for the LongCat FP8 scale layout. diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx index 927522010..06153c0ea 100644 --- a/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx +++ b/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx @@ -260,7 +260,7 @@ checkpoint's calibration scales automatically. is NVFP4-packed; the BF16 and FP8 checkpoints keep a dense head). The Ascend comparison in #35629 used a 910C with BF16 target weights, `--tp-size 2 --attention-backend ascend --mamba-ssm-dtype bfloat16 - --mamba-scheduler-strategy extra_buffer`, and disabled RadixCache for both + --mamba-radix-cache-strategy extra_buffer`, and disabled RadixCache for both baseline and DFlash2 to exclude cache warm-up and prefix reuse. The DFlash2 run added the three flags shown above. Accuracy used zero-shot GSM8K with greedy sampling, `max_new_tokens=2048`, diff --git a/docs/docs/advanced_features/cuda_graph_for_multi_modal_encoder.mdx b/docs/docs/advanced_features/cuda_graph_for_multi_modal_encoder.mdx index 1ee463979..b1c1aed28 100644 --- a/docs/docs/advanced_features/cuda_graph_for_multi_modal_encoder.mdx +++ b/docs/docs/advanced_features/cuda_graph_for_multi_modal_encoder.mdx @@ -61,14 +61,17 @@ SGLANG_VIT_ENABLE_CUDA_GRAPH=1 \ python3 -m sglang.launch_server \ --model Qwen/Qwen3-VL-8B-Instruct ``` -Or you can run CUDA Graph for ViT together with Piecewise CUDA Graph feature by both setting env variable `SGLANG_VIT_ENABLE_CUDA_GRAPH=1` and setting `--enable-piecewise-cuda-graph`, for example: +To run the ViT graph together with Piecewise CUDA Graph over the decoder's +prefill, ask for the prefill backend explicitly. Qwen3-VL is on neither +multimodal allowlist, so the default backend resolves prefill to `disabled` for +it; naming a backend locks the choice and skips that rule: ```shell Command SGLANG_VIT_ENABLE_CUDA_GRAPH=1 \ python3 -m sglang.launch_server \ --model Qwen/Qwen3-VL-8B-Instruct \ - --piecewise-cuda-graph-max-tokens 4096 \ - --enable-piecewise-cuda-graph \ - --piecewise-cuda-graph-compiler eager + --cuda-graph-backend-prefill tc_piecewise \ + --cuda-graph-max-bs-prefill 4096 \ + --cuda-graph-tc-compiler eager ``` ## Known supported models diff --git a/docs/docs/advanced_features/pd_disaggregation.mdx b/docs/docs/advanced_features/pd_disaggregation.mdx index 685cbd956..e7a1a5754 100644 --- a/docs/docs/advanced_features/pd_disaggregation.mdx +++ b/docs/docs/advanced_features/pd_disaggregation.mdx @@ -568,7 +568,7 @@ python3 -m sglang.launch_server \ --disaggregation-bootstrap-port 8996 \ --base-gpu-id 8 \ --disable-radix-cache \ - --cuda-graph-bs 1 2 4 8 10 12 14 16 \ + --cuda-graph-bs-decode 1 2 4 8 10 12 14 16 \ --speculative-draft-model-quantization unquant \ --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ --enable-multi-layer-eagle \ @@ -681,9 +681,8 @@ do --port 8001 --trust-remote-code --nnodes 1 --node-rank 0 --tp-size 16 --dp-size 16 \ --mem-fraction-static 0.8 --max-running-requests 448 --attention-backend ascend --device npu --quantization modelslim \ --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency --enable-dp-lm-head \ - --cuda-graph-bs 2 4 6 8 10 12 14 16 18 20 22 24 26 28 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ - --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ - --prefill-round-robin-balance --disable-shared-experts-fusion --dtype bfloat16 --tokenizer-worker-num 4 \ + --cuda-graph-bs-decode 2 4 6 8 10 12 14 16 18 20 22 24 26 28 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 8192 \ + --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --disable-shared-experts-fusion --dtype bfloat16 --tokenizer-worker-num 4 \ --load-balance-method round_robin NODE_RANK=$i break diff --git a/docs/docs/advanced_features/piecewise_cuda_graph.mdx b/docs/docs/advanced_features/piecewise_cuda_graph.mdx index d1c3e7dd8..4f03665e8 100644 --- a/docs/docs/advanced_features/piecewise_cuda_graph.mdx +++ b/docs/docs/advanced_features/piecewise_cuda_graph.mdx @@ -10,7 +10,7 @@ Standard CUDA graphs capture the entire model forward pass as a single graph. Th Piecewise CUDA Graph (PCG) solves this by splitting the model's computation graph into pieces (roughly one per layer) at "split points" (e.g., MoE dispatch ops). Each piece is captured as a separate CUDA graph for a set of pre-defined token lengths. At runtime, the input is padded to the nearest captured size, and each piece is replayed. This eliminates kernel launch overhead for prefill/extend while still supporting dynamic shapes. -Recently we **enabled PCG by default**, which means that the old `--enable-piecewise-cuda-graph` flag is deprecated. Use `--disable-piecewise-cuda-graph` to turn it off. +PCG is **enabled by default**. Pass `--cuda-graph-backend-prefill=disabled` to turn it off. ## Usage @@ -26,7 +26,7 @@ python3 -m sglang.launch_server \ ```bash python3 -m sglang.launch_server \ --model-path meta-llama/Llama-3.1-8B-Instruct \ - --disable-piecewise-cuda-graph + --cuda-graph-backend-prefill=disabled ``` ### Custom capture sizes @@ -34,7 +34,7 @@ python3 -m sglang.launch_server \ ```bash python3 -m sglang.launch_server \ --model-path meta-llama/Llama-3.1-8B-Instruct \ - --piecewise-cuda-graph-max-tokens 2048 + --cuda-graph-max-bs-prefill 2048 ``` ### Server Args @@ -54,41 +54,31 @@ python3 -m sglang.launch_server \ - --disable-piecewise-cuda-graph - False - Disable PCG for extend/prefill. + --cuda-graph-backend-prefill + None (auto) + Backend for the prefill phase. Choices: full, breakable, tc_piecewise, disabled. Pass disabled to turn PCG off for extend/prefill, or tc_piecewise to force it on, skipping all auto-disable conditions (testing only). - --enforce-piecewise-cuda-graph - False - Force-enable PCG, skipping all auto-disable conditions. For testing only. - - - --piecewise-cuda-graph-max-tokens + --cuda-graph-max-bs-prefill None (auto) Maximum token count to capture. Defaults to chunked_prefill_size (non-MLA) or 2048 (MLA). - --piecewise-cuda-graph-tokens + --cuda-graph-bs-prefill None (auto) Explicit list of token lengths to capture. Auto-generated if not set. - --piecewise-cuda-graph-compiler + --cuda-graph-tc-compiler "eager" Compiler backend for the captured subgraphs. Choices: eager, inductor. - - --enable-piecewise-cuda-graph - — - Deprecated. PCG is now enabled by default. Use --enforce-piecewise-cuda-graph to skip auto-disable conditions. - ## Bug Report -PCG is enabled by default but is still in an experimental stage. Since PCG relies on `torch.compile` to trace the model's forward pass, most bugs are introduced by torch compile tracing failures (e.g., untraceable ops, dynamic control flow, or graph breaks). If you encounter any issues related to PCG, please disable it by adding `--disable-piecewise-cuda-graph` to your launch command and report the bug at [GitHub Issues](https://github.com/sgl-project/sglang/issues/new/choose). We greatly appreciate your help in improving this feature. +PCG is enabled by default but is still in an experimental stage. Since PCG relies on `torch.compile` to trace the model's forward pass, most bugs are introduced by torch compile tracing failures (e.g., untraceable ops, dynamic control flow, or graph breaks). If you encounter any issues related to PCG, please disable it by adding `--cuda-graph-backend-prefill=disabled` to your launch command and report the bug at [GitHub Issues](https://github.com/sgl-project/sglang/issues/new/choose). We greatly appreciate your help in improving this feature. ### For Users @@ -96,11 +86,11 @@ If you see an error message like the following during server startup, it is a PC ``` Piecewise CUDA Graph is enabled by default as an experimental feature. -To work around this error, add --disable-piecewise-cuda-graph to your launch command. +To work around this error, add --cuda-graph-backend-prefill=disabled to your launch command. Please report this issue at https://github.com/sgl-project/sglang/issues/new/choose ``` -To work around it, add `--disable-piecewise-cuda-graph` to your launch command. When filing a bug report, please include: +To work around it, add `--cuda-graph-backend-prefill=disabled` to your launch command. When filing a bug report, please include: 1. The full error traceback 2. Model name and quantization method 3. Launch command with all arguments @@ -228,7 +218,7 @@ The default capture schedule is auto-generated with increasing granularity: -For the auto-generated schedule, sizes are capped at `--piecewise-cuda-graph-max-tokens`. The default cap is `chunked_prefill_size` for non-MLA models and `2048` for MLA backend models. If `--max-total-tokens` is set, the cap is further limited to not exceed it. Additionally, Llama-2 models are auto-capped at 4096 tokens as a temporary workaround. +For the auto-generated schedule, sizes are capped at `--cuda-graph-max-bs-prefill`. The default cap is `chunked_prefill_size` for non-MLA models and `2048` for MLA backend models. If `--max-total-tokens` is set, the cap is further limited to not exceed it. Additionally, Llama-2 models are auto-capped at 4096 tokens as a temporary workaround. ## Compatibility @@ -247,7 +237,7 @@ PCG is auto-disabled in the following scenarios. We are actively working on expa - PD disaggregation - Expert distribution recorder / EPLB -Use `--enforce-piecewise-cuda-graph` to skip all auto-disable checks (for testing/debugging only). +Use `--cuda-graph-backend-prefill=tc_piecewise` to skip all auto-disable checks (for testing/debugging only). ## Code Reference diff --git a/docs/docs/advanced_features/server_arguments.mdx b/docs/docs/advanced_features/server_arguments.mdx index fccc06f5a..b28326f9b 100644 --- a/docs/docs/advanced_features/server_arguments.mdx +++ b/docs/docs/advanced_features/server_arguments.mdx @@ -798,12 +798,6 @@ Please consult the documentation below and [server_args.py](https://github.com/s `False` bool flag (set to enable) - - `--stream-output` - [Deprecated] Use --incremental-streaming-output instead. - — - Type: str - `--enable-streaming-session` Enable streaming session mode and StreamingSession wrapper. @@ -943,12 +937,6 @@ Please consult the documentation below and [server_args.py](https://github.com/s The buckets of end-to-end request latency, specified as a list of floats. None List[float] - - - `--collect-tokens-histogram` - Collect prompt/generation tokens histogram. - False - bool flag (set to enable) `--prompt-tokens-buckets` @@ -1522,18 +1510,6 @@ Please consult the documentation below and [server_args.py](https://github.com/s `None` Type: str - - `--nsa-prefill-backend` - [Deprecated] Use --dsa-prefill-backend instead. - Auto - flashmla_sparse, flashmla_kv, flashmla_auto, fa3, tilelang, aiter, trtllm - - - `--nsa-decode-backend` - [Deprecated] Use --dsa-decode-backend instead. - Auto - flashmla_sparse, flashmla_kv, flashmla_auto, fa3, tilelang, aiter, trtllm - @@ -1662,12 +1638,6 @@ Please consult the documentation below and [server_args.py](https://github.com/s `None` Type: int - - `--speculative-dflash-draft-window-size` - — - — - Type: int - @@ -2594,30 +2564,12 @@ Please consult the documentation below and [server_args.py](https://github.com/s `32` Type: int - - `--cuda-graph-max-bs-decode` - Deprecated alias for --cuda-graph-max-bs-decode. - `None` - Type: int - - - `--cuda-graph-bs` - Deprecated alias for --cuda-graph-bs-decode. - `None` - List[int] - `--disable-cuda-graph` Deprecated. Use --cuda-graph-backend-decode=disabled and/or --cuda-graph-backend-prefill=disabled. False bool flag (set to enable) - - `--enable-breakable-cuda-graph` - Deprecated alias for --cuda-graph-backend-prefill=breakable. - False - bool flag (set to enable) - `--disable-prefill-cuda-graph` Disable the prefill-phase CUDA graph. Convenience for --cuda-graph-backend-prefill=disabled. @@ -2630,36 +2582,6 @@ Please consult the documentation below and [server_args.py](https://github.com/s `False` bool flag (set to enable) - - `--disable-piecewise-cuda-graph` - Deprecated alias for --cuda-graph-backend-prefill=disabled. - False - bool flag (set to enable) - - - `--enforce-piecewise-cuda-graph` - Deprecated alias for --cuda-graph-backend-prefill=tc_piecewise. Explicitly setting the prefill backend now skips the auto-disable cascade automatically. - False - bool flag (set to enable) - - - `--piecewise-cuda-graph-tokens` - Deprecated alias for --cuda-graph-bs-prefill. - `None` - List[int] - - - `--piecewise-cuda-graph-compiler` - Deprecated alias for --cuda-graph-tc-compiler. - eager - eager, inductor - - - `--piecewise-cuda-graph-max-tokens` - Deprecated alias for --cuda-graph-max-bs-prefill. - 4096 - Type: int - `--enable-p2p-check` @@ -3485,12 +3407,6 @@ Please consult the documentation below and [server_args.py](https://github.com/s - - `--prefill-round-robin-balance` - Note: Note: --prefill-round-robin-balance is deprecated now. - `None` - N/A - diff --git a/docs/docs/advanced_features/speculative_decoding.mdx b/docs/docs/advanced_features/speculative_decoding.mdx index b0a058a6f..004e9ef75 100644 --- a/docs/docs/advanced_features/speculative_decoding.mdx +++ b/docs/docs/advanced_features/speculative_decoding.mdx @@ -561,7 +561,7 @@ Relevant parameters: None - --speculative-dflash-draft-window-size + --speculative-draft-window-size Draft KV sliding-window size. Must be >= speculative-num-draft-tokens when set. None @@ -895,7 +895,7 @@ Below is a comprehensive list of all speculative decoding parameters available i DFlash-only alias of --speculative-num-draft-tokens - --speculative-dflash-draft-window-size + --speculative-draft-window-size int None DFlash-only draft KV sliding-window size diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1.mdx index 46898890a..3a5e16157 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_r1.mdx @@ -169,14 +169,13 @@ do --enable-dp-attention \ --deepep-mode low_latency \ --enable-dp-lm-head \ - --cuda-graph-bs 2 4 6 8 10 12 14 16 18 20 22 24 26 \ + --cuda-graph-bs-decode 2 4 6 8 10 12 14 16 18 20 22 24 26 \ --watchdog-timeout 9000 \ --context-length 8192 \ --speculative-algorithm NEXTN \ --speculative-num-steps 2 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 3 \ - --prefill-round-robin-balance \ --disable-shared-experts-fusion \ --dtype bfloat16 \ --tokenizer-worker-num 4 \ @@ -296,7 +295,7 @@ python3 -m sglang.launch_server \ --device npu \ --quantization modelslim \ --watchdog-timeout 9000 \ - --cuda-graph-bs 4 8 12 14 \ + --cuda-graph-bs-decode 4 8 12 14 \ --mem-fraction-static 0.9 \ --max-running-requests 224 \ --context-length 8188 \ @@ -481,7 +480,7 @@ do --deepep-mode low_latency \ --enable-dp-lm-head \ --moe-dense-tp 1 \ - --cuda-graph-bs 2 4 6 \ + --cuda-graph-bs-decode 2 4 6 \ --watchdog-timeout 9000 \ --context-length 8192 \ --speculative-algorithm NEXTN \ @@ -489,7 +488,6 @@ do --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ --tokenizer-worker-num 4 \ - --prefill-round-robin-balance \ --disable-shared-experts-fusion \ --dtype bfloat16 \ --load-balance-method round_robin \ @@ -689,7 +687,7 @@ do --enable-dp-attention \ --deepep-mode low_latency \ --moe-dense-tp 1 \ - --cuda-graph-bs 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 \ + --cuda-graph-bs-decode 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 \ --watchdog-timeout 9000 \ --context-length 8192 \ --speculative-algorithm NEXTN \ @@ -697,7 +695,6 @@ do --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 2 \ --tokenizer-worker-num 4 \ - --prefill-round-robin-balance \ --disable-shared-experts-fusion \ --dtype bfloat16 \ --load-balance-method round_robin \ @@ -893,7 +890,7 @@ do --deepep-mode low_latency \ --enable-dp-lm-head \ --moe-dense-tp 1 \ - --cuda-graph-bs 2 4 6 \ + --cuda-graph-bs-decode 2 4 6 \ --watchdog-timeout 9000 \ --context-length 8192 \ --speculative-algorithm NEXTN \ @@ -901,7 +898,6 @@ do --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ --tokenizer-worker-num 4 \ - --prefill-round-robin-balance \ --disable-shared-experts-fusion \ --dtype bfloat16 \ --load-balance-method round_robin \ @@ -1097,7 +1093,7 @@ do --deepep-mode low_latency \ --enable-dp-lm-head \ --moe-dense-tp 1 \ - --cuda-graph-bs 2 4 6 \ + --cuda-graph-bs-decode 2 4 6 \ --watchdog-timeout 9000 \ --context-length 8192 \ --speculative-algorithm NEXTN \ @@ -1105,7 +1101,6 @@ do --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ --tokenizer-worker-num 4 \ - --prefill-round-robin-balance \ --disable-shared-experts-fusion \ --dtype bfloat16 \ --load-balance-method round_robin \ @@ -1300,14 +1295,13 @@ do --deepep-mode low_latency \ --enable-dp-lm-head \ --moe-dense-tp 1 \ - --cuda-graph-bs 2 4 6 \ + --cuda-graph-bs-decode 2 4 6 \ --watchdog-timeout 9000 \ --speculative-algorithm NEXTN \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ --tokenizer-worker-num 4 \ - --prefill-round-robin-balance \ --disable-shared-experts-fusion \ --dtype bfloat16 \ --load-balance-method round_robin \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash.mdx index 63a3b6628..c4e5fed01 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/deepseek_v4_flash.mdx @@ -185,7 +185,7 @@ do --quantization modelslim \ --enable-dp-lm-head \ --kv-cache-dtype bfloat16 \ - --cuda-graph-bs 1 2 4 8 16 24 36 40 48 56 \ + --cuda-graph-bs-decode 1 2 4 8 16 24 36 40 48 56 \ --speculative-algorithm EAGLE \ --speculative-num-steps 2 \ --speculative-eagle-topk 1 \ @@ -329,7 +329,7 @@ python3 -m sglang.launch_server \ --enable-dp-lm-head \ --kv-cache-dtype auto \ --skip-server-warmup \ - --cuda-graph-bs 1 2 4 8 \ + --cuda-graph-bs-decode 1 2 4 8 \ --speculative-algorithm EAGLE \ --speculative-num-steps 2 \ --speculative-eagle-topk 1 \ @@ -455,7 +455,7 @@ python3 -m sglang.launch_server \ --enable-dp-lm-head \ --kv-cache-dtype auto \ --skip-server-warmup \ - --cuda-graph-bs 1 2 4 8 10 \ + --cuda-graph-bs-decode 1 2 4 8 10 \ --speculative-algorithm EAGLE \ --speculative-num-steps 2 \ --speculative-eagle-topk 1 \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1.mdx index 04472d42f..d6a713988 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_1.mdx @@ -307,12 +307,11 @@ do --served-model-name glm-5 \ --moe-a2a-backend deepep \ --deepep-mode low_latency \ - --cuda-graph-bs 1 2 3 \ + --cuda-graph-bs-decode 1 2 3 \ --disaggregation-transfer-backend ascend \ --watchdog-timeout 9000 \ --context-length 180000 \ --tokenizer-worker-num 16 \ - --prefill-round-robin-balance \ --disable-shared-experts-fusion \ --dtype bfloat16 \ --load-balance-method round_robin \ @@ -507,12 +506,11 @@ do --served-model-name glm-5 \ --moe-a2a-backend deepep \ --deepep-mode low_latency \ - --cuda-graph-bs 1 2 3 \ + --cuda-graph-bs-decode 1 2 3 \ --disaggregation-transfer-backend ascend \ --watchdog-timeout 9000 \ --context-length 180000 \ --tokenizer-worker-num 4 \ - --prefill-round-robin-balance \ --disable-shared-experts-fusion \ --dtype bfloat16 \ --load-balance-method round_robin \ @@ -719,12 +717,11 @@ do --served-model-name glm-5 \ --moe-a2a-backend deepep \ --deepep-mode low_latency \ - --cuda-graph-bs 1 2 3 \ + --cuda-graph-bs-decode 1 2 3 \ --disaggregation-transfer-backend ascend \ --watchdog-timeout 9000 \ --context-length 180000 \ --tokenizer-worker-num 16 \ - --prefill-round-robin-balance \ --disable-shared-experts-fusion \ --dtype bfloat16 \ --load-balance-method round_robin \ @@ -921,12 +918,11 @@ do --served-model-name glm-5 \ --moe-a2a-backend deepep \ --deepep-mode low_latency \ - --cuda-graph-bs 1 2 3 \ + --cuda-graph-bs-decode 1 2 3 \ --disaggregation-transfer-backend ascend \ --watchdog-timeout 9000 \ --context-length 180000 \ --tokenizer-worker-num 16 \ - --prefill-round-robin-balance \ --disable-shared-experts-fusion \ --dtype bfloat16 \ --load-balance-method round_robin \ @@ -1120,12 +1116,11 @@ do --served-model-name glm-5 \ --moe-a2a-backend deepep \ --deepep-mode low_latency \ - --cuda-graph-bs 1 2 3 \ + --cuda-graph-bs-decode 1 2 3 \ --disaggregation-transfer-backend ascend \ --watchdog-timeout 9000 \ --context-length 180000 \ --tokenizer-worker-num 4 \ - --prefill-round-robin-balance \ --disable-shared-experts-fusion \ --dtype bfloat16 \ --load-balance-method round_robin \ @@ -1330,12 +1325,11 @@ do --served-model-name glm-5 \ --moe-a2a-backend deepep \ --deepep-mode low_latency \ - --cuda-graph-bs 1 2 3 4 5 6 \ + --cuda-graph-bs-decode 1 2 3 4 5 6 \ --disaggregation-transfer-backend ascend \ --watchdog-timeout 9000 \ --context-length 180000 \ --tokenizer-worker-num 32 \ - --prefill-round-robin-balance \ --disable-shared-experts-fusion \ --dtype bfloat16 \ --load-balance-method round_robin \ @@ -1540,12 +1534,11 @@ do --served-model-name glm-5 \ --moe-a2a-backend deepep \ --deepep-mode low_latency \ - --cuda-graph-bs 1 2 3 4 5 6 7 8 9 10 \ + --cuda-graph-bs-decode 1 2 3 4 5 6 7 8 9 10 \ --disaggregation-transfer-backend ascend \ --watchdog-timeout 9000 \ --context-length 180000 \ --tokenizer-worker-num 4 \ - --prefill-round-robin-balance \ --disable-shared-experts-fusion \ --dtype bfloat16 \ --load-balance-method round_robin \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_2.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_2.mdx index 50334f60d..deacbeeb9 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_2.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/glm_5_2.mdx @@ -171,12 +171,11 @@ do --served-model-name glm-5 \ --moe-a2a-backend deepep \ --deepep-mode low_latency \ - --cuda-graph-max-bs 8 \ + --cuda-graph-max-bs-decode 8 \ --disaggregation-transfer-backend ascend \ --watchdog-timeout 9000 \ --context-length 180000 \ --tokenizer-worker-num 8 \ - --prefill-round-robin-balance \ --disable-shared-experts-fusion \ --dtype bfloat16 \ --load-balance-method round_robin \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6.mdx index 56bee38e7..c19432fcb 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/kimi_k2_6.mdx @@ -123,7 +123,7 @@ do --dp-size 32 \ --moe-a2a-backend deepep \ --deepep-mode auto \ - --cuda-graph-bs 1 \ + --cuda-graph-bs-decode 1 \ --disable-radix-cache \ --speculative-algorithm EAGLE3 \ --speculative-draft-model-path $DRAFT_MODEL_PATH \ @@ -305,7 +305,7 @@ do --sampling-backend ascend \ --moe-a2a-backend deepep \ --deepep-mode auto \ - --cuda-graph-bs 1 2 4 6 8 16 \ + --cuda-graph-bs-decode 1 2 4 6 8 16 \ --speculative-algorithm EAGLE3 \ --speculative-draft-model-path $DRAFT_MODEL_PATH \ --speculative-num-steps 3 \ @@ -504,7 +504,7 @@ do --sampling-backend ascend \ --moe-a2a-backend deepep \ --deepep-mode auto \ - --cuda-graph-bs 1 2 4 6 8 16 \ + --cuda-graph-bs-decode 1 2 4 6 8 16 \ --speculative-algorithm EAGLE3 \ --speculative-draft-model-path $DRAFT_MODEL_PATH \ --speculative-num-steps 3 \ @@ -702,7 +702,7 @@ do --sampling-backend ascend \ --moe-a2a-backend deepep \ --deepep-mode auto \ - --cuda-graph-bs 16 \ + --cuda-graph-bs-decode 16 \ --reasoning-parser kimi_k2 \ --tool-call-parser kimi_k2 \ --speculative-algorithm EAGLE3 \ @@ -896,7 +896,7 @@ do --sampling-backend ascend \ --moe-a2a-backend deepep \ --deepep-mode auto \ - --cuda-graph-bs 1 2 4 6 8 \ + --cuda-graph-bs-decode 1 2 4 6 8 \ --reasoning-parser kimi_k2 \ --tool-call-parser kimi_k2 \ --speculative-algorithm EAGLE3 \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_flash.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_flash.mdx index 1a5d4f6cb..1b7bf98d4 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_flash.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/mimo_v2_flash.mdx @@ -129,7 +129,7 @@ do --disaggregation-transfer-backend ascend \ --disable-radix-cache \ --disable-cuda-graph \ - --disable-piecewise-cuda-graph \ + --cuda-graph-backend-prefill=disabled \ --dp-size 2 break fi @@ -159,7 +159,7 @@ do --max-running-requests 64 \ --mem-fraction-static 0.8 \ --swa-full-tokens-ratio 0.3 \ - --cuda-graph-bs 1 2 4 8 12 16 20 24 28 32 \ + --cuda-graph-bs-decode 1 2 4 8 12 16 20 24 28 32 \ --disaggregation-transfer-backend ascend \ --speculative-algorithm EAGLE \ --speculative-num-steps 3 \ @@ -314,7 +314,7 @@ do --disaggregation-transfer-backend ascend \ --disable-radix-cache \ --disable-cuda-graph \ - --disable-piecewise-cuda-graph \ + --cuda-graph-backend-prefill=disabled \ --dp-size 2 break fi @@ -344,7 +344,7 @@ do --max-running-requests 32 \ --mem-fraction-static 0.8 \ --swa-full-tokens-ratio 0.3 \ - --cuda-graph-bs 1 2 4 8 12 16 \ + --cuda-graph-bs-decode 1 2 4 8 12 16 \ --disaggregation-transfer-backend ascend \ --speculative-algorithm EAGLE \ --speculative-num-steps 3 \ @@ -499,7 +499,7 @@ do --disaggregation-transfer-backend ascend \ --disable-radix-cache \ --disable-cuda-graph \ - --disable-piecewise-cuda-graph \ + --cuda-graph-backend-prefill=disabled \ --dp-size 2 break fi @@ -529,7 +529,7 @@ do --max-running-requests 64 \ --mem-fraction-static 0.8 \ --swa-full-tokens-ratio 0.3 \ - --cuda-graph-bs 1 2 4 8 12 16 20 24 28 32 \ + --cuda-graph-bs-decode 1 2 4 8 12 16 20 24 28 32 \ --disaggregation-transfer-backend ascend \ --speculative-algorithm EAGLE \ --speculative-num-steps 3 \ @@ -684,7 +684,7 @@ do --disaggregation-transfer-backend ascend \ --disable-radix-cache \ --disable-cuda-graph \ - --disable-piecewise-cuda-graph \ + --cuda-graph-backend-prefill=disabled \ --dp-size 2 break fi @@ -714,7 +714,7 @@ do --max-running-requests 64 \ --mem-fraction-static 0.8 \ --swa-full-tokens-ratio 0.3 \ - --cuda-graph-bs 1 2 4 8 12 16 20 24 28 32 \ + --cuda-graph-bs-decode 1 2 4 8 12 16 20 24 28 32 \ --disaggregation-transfer-backend ascend \ --speculative-algorithm EAGLE \ --speculative-num-steps 3 \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5.mdx index e842e3cd3..83415f179 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/minimax_m2_5.mdx @@ -105,7 +105,7 @@ python3 -m sglang.launch_server \ --max-running-requests 18 \ --chunked-prefill-size -1 \ --max-prefill-tokens 32768 \ - --cuda-graph-bs 2 4 6 8 10 12 14 16 18 24 \ + --cuda-graph-bs-decode 2 4 6 8 10 12 14 16 18 24 \ --moe-a2a-backend deepep \ --deepep-mode auto \ --quantization modelslim \ @@ -212,7 +212,7 @@ python3 -m sglang.launch_server \ --prefill-max-requests 10 \ --chunked-prefill-size 67072 \ --max-prefill-tokens 67000 \ - --cuda-graph-bs 2 4 8 12 16 18 20 22 24 26 \ + --cuda-graph-bs-decode 2 4 8 12 16 18 20 22 24 26 \ --moe-a2a-backend ascend_fuseep \ --deepep-mode auto \ --quantization modelslim \ @@ -324,7 +324,7 @@ python3 -m sglang.launch_server \ --prefill-max-requests 4 \ --chunked-prefill-size 160000 \ --max-prefill-tokens 80000 \ - --cuda-graph-bs 2 4 6 8 \ + --cuda-graph-bs-decode 2 4 6 8 \ --moe-a2a-backend ascend_fuseep \ --deepep-mode auto \ --quantization modelslim \ @@ -437,7 +437,7 @@ python3 -m sglang.launch_server \ --prefill-max-requests 3 \ --chunked-prefill-size -1 \ --max-prefill-tokens 8192 \ - --cuda-graph-bs 1 2 3 4 5 6 \ + --cuda-graph-bs-decode 1 2 3 4 5 6 \ --moe-a2a-backend ascend_fuseep \ --deepep-mode auto \ --quantization modelslim \ @@ -541,7 +541,7 @@ python3 -m sglang.launch_server \ --enable-prefill-delayer \ --chunked-prefill-size 196608 \ --max-prefill-tokens 8192 \ - --cuda-graph-bs 1 2 4 8 12 16 20 \ + --cuda-graph-bs-decode 1 2 4 8 12 16 20 \ --moe-a2a-backend ascend_fuseep \ --fuseep-mode 2 \ --quantization modelslim \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_235b_a22b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_235b_a22b.mdx index b73f67f46..846f0107a 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_235b_a22b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_235b_a22b.mdx @@ -101,7 +101,7 @@ python3 -m sglang.launch_server \ --enable-dp-lm-head \ --tp 16 \ --mem-fraction-static 0.78 \ - --cuda-graph-bs 1 \ + --cuda-graph-bs-decode 1 \ --reasoning-parser qwen3 \ --tool-call-parser qwen25 ``` @@ -210,7 +210,7 @@ python3 -m sglang.launch_server \ --enable-dp-attention \ --enable-dp-lm-head \ --mem-fraction-static 0.8 \ - --cuda-graph-bs 1 2 4 8 16 20 24 26 27 \ + --cuda-graph-bs-decode 1 2 4 8 16 20 24 26 27 \ --reasoning-parser qwen3 \ --tool-call-parser qwen25 ``` diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_30b_a3b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_30b_a3b.mdx index 9324eaa3c..8f2d87a30 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_30b_a3b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_30b_a3b.mdx @@ -109,7 +109,7 @@ python3 -m sglang.launch_server \ --enable-dp-attention \ --dp-size 2 \ --mem-fraction-static 0.85 \ - --cuda-graph-bs 1 2 4 8 16 20 24 28 32 36 40 44 48 52 56 60 64 68 72 76 80 84 \ + --cuda-graph-bs-decode 1 2 4 8 16 20 24 28 32 36 40 44 48 52 56 60 64 68 72 76 80 84 \ --dtype bfloat16 \ --reasoning-parser qwen3 \ --tool-call-parser qwen @@ -211,7 +211,7 @@ python3 -m sglang.launch_server \ --speculative-num-draft-tokens 4 \ --tp-size 2 \ --mem-fraction-static 0.87 \ - --cuda-graph-bs 1 5 15 40 70 100 120 130 140 146 150 154 156 158 160 162 \ + --cuda-graph-bs-decode 1 5 15 40 70 100 120 130 140 146 150 154 156 158 160 162 \ --dtype bfloat16 \ --reasoning-parser qwen3 \ --tool-call-parser qwen @@ -311,7 +311,7 @@ python3 -m sglang.launch_server \ --speculative-num-draft-tokens 4 \ --tp-size 2 \ --mem-fraction-static 0.87 \ - --cuda-graph-bs 1 5 15 40 70 100 120 130 140 146 150 154 156 158 160 162 \ + --cuda-graph-bs-decode 1 5 15 40 70 100 120 130 140 146 150 154 156 158 160 162 \ --dtype bfloat16 \ --reasoning-parser qwen3 \ --tool-call-parser qwen @@ -411,7 +411,7 @@ python3 -m sglang.launch_server \ --max-prefill-tokens 35000 \ --tp-size 2 \ --mem-fraction-static 0.6 \ - --cuda-graph-bs 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 \ + --cuda-graph-bs-decode 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 \ --dtype bfloat16 \ --reasoning-parser qwen3 \ --tool-call-parser qwen diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_32b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_32b.mdx index 62d332d09..9f9be89c0 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_32b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_32b.mdx @@ -100,7 +100,7 @@ python3 -m sglang.launch_server \ --speculative-num-draft-tokens 5 \ --tp-size 16 \ --mem-fraction-static 0.72 \ - --cuda-graph-bs 1 \ + --cuda-graph-bs-decode 1 \ --dtype bfloat16 \ --reasoning-parser qwen3 \ --tool-call-parser qwen @@ -199,7 +199,7 @@ python3 -m sglang.launch_server \ --speculative-num-draft-tokens 4 \ --tp-size 4 \ --mem-fraction-static 0.845 \ - --cuda-graph-bs 16 32 64 72 88 90 92 94 96 97 98 99 100 101 \ + --cuda-graph-bs-decode 16 32 64 72 88 90 92 94 96 97 98 99 100 101 \ --dtype bfloat16 \ --reasoning-parser qwen3 \ --tool-call-parser qwen @@ -298,7 +298,7 @@ python3 -m sglang.launch_server \ --speculative-num-draft-tokens 4 \ --tp-size 4 \ --mem-fraction-static 0.845 \ - --cuda-graph-bs 16 32 64 72 88 90 92 94 96 97 98 99 100 101 \ + --cuda-graph-bs-decode 16 32 64 72 88 90 92 94 96 97 98 99 100 101 \ --dtype bfloat16 \ --reasoning-parser qwen3 \ --tool-call-parser qwen diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_5_397b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_5_397b.mdx index 75724ee1a..f7736146f 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_5_397b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_5_397b.mdx @@ -111,7 +111,7 @@ python3 -m sglang.launch_server \ --trust-remote-code \ --max-running-requests 16 \ --mem-fraction-static 0.6 \ - --cuda-graph-bs 2 3 4 5 6 8 10 12 14 16 \ + --cuda-graph-bs-decode 2 3 4 5 6 8 10 12 14 16 \ --quantization modelslim \ --enable-multimodal \ --moe-a2a-backend deepep \ @@ -330,7 +330,7 @@ python3 -m sglang.launch_server \ --max-prefill-tokens 131072 \ --max-mamba-cache-size 320 \ --prefill-max-requests 10 \ - --mamba-scheduler-strategy extra_buffer \ + --mamba-radix-cache-strategy extra_buffer \ --trust-remote-code \ --max-running-requests 64 \ --mem-fraction-static 0.6 \ @@ -451,7 +451,7 @@ python3 -m sglang.launch_server \ --max-running-requests 48 \ --mem-fraction-static 0.8 \ --max-total-tokens 210000 \ - --cuda-graph-bs 2 4 6 8 10 12 \ + --cuda-graph-bs-decode 2 4 6 8 10 12 \ --quantization modelslim \ --enable-multimodal \ --moe-a2a-backend deepep \ @@ -569,7 +569,7 @@ python3 -m sglang.launch_server \ --max-running-requests 144 \ --mem-fraction-static 0.8 \ --max-total-tokens 635000 \ - --cuda-graph-bs 2 4 6 8 12 14 16 18 20 24 26 28 30 32 34 36 \ + --cuda-graph-bs-decode 2 4 6 8 12 14 16 18 20 24 26 28 30 32 34 36 \ --quantization modelslim \ --enable-multimodal \ --moe-a2a-backend deepep \ @@ -686,7 +686,7 @@ python3 -m sglang.launch_server \ --trust-remote-code \ --max-running-requests 160 \ --mem-fraction-static 0.8 \ - --cuda-graph-bs 2 4 6 8 10 12 14 16 18 20 \ + --cuda-graph-bs-decode 2 4 6 8 10 12 14 16 18 20 \ --quantization modelslim \ --enable-multimodal \ --moe-a2a-backend deepep \ @@ -803,7 +803,7 @@ python3 -m sglang.launch_server \ --trust-remote-code \ --max-running-requests 432 \ --mem-fraction-static 0.8 \ - --cuda-graph-bs 2 4 6 8 12 16 20 24 28 32 36 40 44 48 50 52 54 \ + --cuda-graph-bs-decode 2 4 6 8 12 16 20 24 28 32 36 40 44 48 50 52 54 \ --quantization modelslim \ --enable-multimodal \ --moe-a2a-backend deepep \ @@ -921,7 +921,7 @@ python3 -m sglang.launch_server \ --max-running-requests 16 \ --mem-fraction-static 0.6 \ --max-total-tokens 1065000 \ - --cuda-graph-bs 2 4 6 8 10 12 14 16 \ + --cuda-graph-bs-decode 2 4 6 8 10 12 14 16 \ --quantization modelslim \ --enable-multimodal \ --moe-a2a-backend deepep \ @@ -1039,7 +1039,7 @@ python3 -m sglang.launch_server \ --max-running-requests 32 \ --mem-fraction-static 0.6 \ --max-total-tokens 1065000 \ - --cuda-graph-bs 2 4 6 8 12 14 16 \ + --cuda-graph-bs-decode 2 4 6 8 12 14 16 \ --quantization modelslim \ --enable-multimodal \ --moe-a2a-backend deepep \ @@ -1147,7 +1147,7 @@ python3 -m sglang.launch_server \ --chunked-prefill-size -1 \ --max-prefill-tokens 65536 \ --max-mamba-cache-size 640 \ - --mamba-scheduler-strategy extra_buffer \ + --mamba-radix-cache-strategy extra_buffer \ --trust-remote-code \ --max-running-requests 128 \ --mem-fraction-static 0.6 \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_27b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_27b.mdx index 5c52399c7..a481effa1 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_27b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_27b.mdx @@ -96,7 +96,7 @@ python3 -m sglang.launch_server \ --max-running-requests 60 \ --max-mamba-cache-size 60 \ --mem-fraction-static 0.74 \ - --cuda-graph-bs 2 4 8 14 16 24 26 32 36 37 40 42 44 45 46 50 52 60 \ + --cuda-graph-bs-decode 2 4 8 14 16 24 26 32 36 37 40 42 44 45 46 50 52 60 \ --enable-multimodal \ --mm-attention-backend ascend_attn \ --dtype bfloat16 \ @@ -194,7 +194,7 @@ python3 -m sglang.launch_server \ --max-running-requests 38 \ --max-mamba-cache-size 38 \ --mem-fraction-static 0.7 \ - --cuda-graph-bs 1 2 4 8 10 12 16 20 24 28 30 32 35 38 \ + --cuda-graph-bs-decode 1 2 4 8 10 12 16 20 24 28 30 32 35 38 \ --enable-prefill-delayer \ --prefill-delayer-queue-min-ratio 0.45 \ --prefill-delayer-max-delay-ms 5500 \ @@ -292,12 +292,12 @@ python3 -m sglang.launch_server \ --device npu \ --chunked-prefill-size 32768 \ --max-prefill-tokens 32768 \ - --mamba-scheduler-strategy extra_buffer \ + --mamba-radix-cache-strategy extra_buffer \ --trust-remote-code \ --max-running-requests 20 \ --max-mamba-cache-size 160 \ --mem-fraction-static 0.82 \ - --cuda-graph-bs 1 2 5 10 15 17 19 20 \ + --cuda-graph-bs-decode 1 2 5 10 15 17 19 20 \ --dtype bfloat16 \ --mamba-ssm-dtype bfloat16 \ --speculative-algorithm NEXTN \ @@ -397,7 +397,7 @@ python3 -m sglang.launch_server \ --max-running-requests 64 \ --max-mamba-cache-size 74 \ --mem-fraction-static 0.7 \ - --cuda-graph-bs 2 8 16 32 40 45 50 54 \ + --cuda-graph-bs-decode 2 8 16 32 40 45 50 54 \ --enable-multimodal \ --quantization modelslim \ --mm-attention-backend ascend_attn \ @@ -491,7 +491,7 @@ python3 -m sglang.launch_server \ --max-running-requests 6 \ --max-mamba-cache-size 16 \ --mem-fraction-static 0.6 \ - --cuda-graph-bs 1 2 4 5 6 \ + --cuda-graph-bs-decode 1 2 4 5 6 \ --quantization modelslim \ --dtype bfloat16 \ --mamba-ssm-dtype bfloat16 \ @@ -587,7 +587,7 @@ python3 -m sglang.launch_server \ --max-running-requests 6 \ --max-mamba-cache-size 7 \ --mem-fraction-static 0.63 \ - --cuda-graph-bs 1 2 4 5 6 \ + --cuda-graph-bs-decode 1 2 4 5 6 \ --enable-multimodal \ --quantization modelslim \ --mm-attention-backend ascend_attn \ @@ -682,7 +682,7 @@ python3 -m sglang.launch_server \ --max-running-requests 37 \ --max-mamba-cache-size 74 \ --mem-fraction-static 0.7 \ - --cuda-graph-bs 1 2 3 4 6 8 10 12 14 16 18 20 21 23 24 25 26 27 28 29 30 31 33 35 37 \ + --cuda-graph-bs-decode 1 2 3 4 6 8 10 12 14 16 18 20 21 23 24 25 26 27 28 29 30 31 33 35 37 \ --quantization modelslim \ --dtype bfloat16 \ --mamba-ssm-dtype bfloat16 \ @@ -779,7 +779,7 @@ python3 -m sglang.launch_server \ --max-running-requests 28 \ --max-mamba-cache-size 50 \ --mem-fraction-static 0.7 \ - --cuda-graph-bs 2 4 6 \ + --cuda-graph-bs-decode 2 4 6 \ --enable-multimodal \ --quantization modelslim \ --mm-attention-backend ascend_attn \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_35b_a3b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_35b_a3b.mdx index 6bf66c156..037ff132c 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_35b_a3b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_6_35b_a3b.mdx @@ -103,7 +103,7 @@ python3 -m sglang.launch_server \ --max-running-requests 120 \ --max-mamba-cache-size 120 \ --mem-fraction-static 0.85 \ - --cuda-graph-bs 4 16 32 48 64 110 165 \ + --cuda-graph-bs-decode 4 16 32 48 64 110 165 \ --enable-multimodal \ --mm-attention-backend ascend_attn \ --dtype bfloat16 \ @@ -205,7 +205,7 @@ python3 -m sglang.launch_server \ --max-running-requests 42 \ --max-mamba-cache-size 42 \ --mem-fraction-static 0.75 \ - --cuda-graph-bs 4 8 16 24 48 64 80 \ + --cuda-graph-bs-decode 4 8 16 24 48 64 80 \ --enable-multimodal \ --mm-attention-backend ascend_attn \ --dtype bfloat16 \ @@ -308,7 +308,7 @@ python3 -m sglang.launch_server \ --max-running-requests 3 \ --max-mamba-cache-size 3 \ --mem-fraction-static 0.9 \ - --cuda-graph-bs 1 2 3 \ + --cuda-graph-bs-decode 1 2 3 \ --enable-multimodal \ --mm-attention-backend ascend_attn \ --dtype bfloat16 \ @@ -402,11 +402,11 @@ python3 -m sglang.launch_server \ --max-prefill-tokens 65536 \ --trust-remote-code \ --enable-prefill-delayer \ - --mamba-scheduler-strategy extra_buffer \ + --mamba-radix-cache-strategy extra_buffer \ --max-running-requests 103 \ --max-mamba-cache-size 85 \ --mem-fraction-static 0.85 \ - --cuda-graph-bs 2 4 8 16 32 48 64 80 96 103 \ + --cuda-graph-bs-decode 2 4 8 16 32 48 64 80 96 103 \ --enable-multimodal \ --mm-attention-backend ascend_attn \ --dtype bfloat16 \ @@ -507,7 +507,7 @@ python3 -m sglang.launch_server \ --max-running-requests 1 \ --max-mamba-cache-size 6 \ --mem-fraction-static 0.65 \ - --cuda-graph-bs 1 \ + --cuda-graph-bs-decode 1 \ --enable-multimodal \ --mm-attention-backend ascend_attn \ --dtype bfloat16 \ @@ -606,7 +606,7 @@ python3 -m sglang.launch_server \ --max-running-requests 122 \ --max-mamba-cache-size 122 \ --mem-fraction-static 0.9 \ - --cuda-graph-bs 4 16 32 64 96 116 120 122 \ + --cuda-graph-bs-decode 4 16 32 64 96 116 120 122 \ --enable-multimodal \ --mm-attention-backend ascend_attn \ --dtype bfloat16 \ @@ -705,7 +705,7 @@ python3 -m sglang.launch_server \ --max-running-requests 10 \ --max-mamba-cache-size 20 \ --mem-fraction-static 0.65 \ - --cuda-graph-bs 2 4 8 12 14 16 \ + --cuda-graph-bs-decode 2 4 8 12 14 16 \ --enable-multimodal \ --mm-attention-backend ascend_attn \ --dtype bfloat16 \ @@ -800,11 +800,11 @@ python3 -m sglang.launch_server \ --max-total-tokens 470784 \ --max-prefill-tokens 65536 \ --trust-remote-code \ - --mamba-scheduler-strategy extra_buffer \ + --mamba-radix-cache-strategy extra_buffer \ --max-running-requests 40 \ --max-mamba-cache-size 200 \ --mem-fraction-static 0.9 \ - --cuda-graph-bs 2 8 16 24 32 36 40 \ + --cuda-graph-bs-decode 2 8 16 24 32 36 40 \ --enable-multimodal \ --mm-attention-backend ascend_attn \ --dtype bfloat16 \ @@ -906,7 +906,7 @@ python3 -m sglang.launch_server \ --max-running-requests 1 \ --max-mamba-cache-size 6 \ --mem-fraction-static 0.68 \ - --cuda-graph-bs 1 \ + --cuda-graph-bs-decode 1 \ --enable-multimodal \ --mm-attention-backend ascend_attn \ --dtype bfloat16 \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_8b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_8b.mdx index 82696fb69..2ae6bf832 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_8b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_8b.mdx @@ -95,7 +95,7 @@ python3 -m sglang.launch_server \ --chunked-prefill-size 16384 \ --tp-size 1 \ --mem-fraction-static 0.85 \ - --cuda-graph-bs 8 12 24 36 48 51 55 60 63 64 66 68 70 \ + --cuda-graph-bs-decode 8 12 24 36 48 51 55 60 63 64 66 68 70 \ --dtype bfloat16 \ --speculative-draft-model-quantization unquant \ --speculative-algorithm EAGLE3 \ @@ -192,7 +192,7 @@ python3 -m sglang.launch_server \ --chunked-prefill-size -1 \ --tp-size 2 \ --mem-fraction-static 0.894 \ - --cuda-graph-bs 1 \ + --cuda-graph-bs-decode 1 \ --dtype bfloat16 \ --speculative-draft-model-quantization unquant \ --speculative-algorithm EAGLE3 \ @@ -289,7 +289,7 @@ python3 -m sglang.launch_server \ --chunked-prefill-size -1 \ --tp-size 2 \ --mem-fraction-static 0.894 \ - --cuda-graph-bs 1 5 15 16 \ + --cuda-graph-bs-decode 1 5 15 16 \ --dtype bfloat16 \ --speculative-draft-model-quantization unquant \ --speculative-algorithm EAGLE3 \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_next_80b_a3b_instruct.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_next_80b_a3b_instruct.mdx index 15bd5d67e..d147cc65e 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_next_80b_a3b_instruct.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/best-practices/qwen3_next_80b_a3b_instruct.mdx @@ -116,7 +116,7 @@ python3 -m sglang.launch_server \ --speculative-draft-model-quantization unquant \ --chunked-prefill-size -1 \ --max-running-requests 2 \ - --cuda-graph-bs 2 \ + --cuda-graph-bs-decode 2 \ --mamba-ssm-dtype bfloat16 \ --speculative-draft-model-path $DRAFT_MODEL_PATH \ --reasoning-parser qwen3 \ @@ -235,7 +235,7 @@ python3 -m sglang.launch_server \ --enable-dp-lm-head \ --moe-a2a-backend deepep \ --deepep-mode auto \ - --cuda-graph-bs 1 2 3 4 5 6 7 8 10 12 14 16 18 20 22 24 26 28 30 32 40 44 48 52 56 60 64 72 80 88 96 104 112 120 128 136 144 150 \ + --cuda-graph-bs-decode 1 2 3 4 5 6 7 8 10 12 14 16 18 20 22 24 26 28 30 32 40 44 48 52 56 60 64 72 80 88 96 104 112 120 128 136 144 150 \ --reasoning-parser qwen3 \ --tool-call-parser qwen3_coder ``` @@ -349,7 +349,7 @@ python3 -m sglang.launch_server \ --speculative-draft-model-quantization unquant \ --chunked-prefill-size -1 \ --max-running-requests 16 \ - --cuda-graph-bs 2 4 8 \ + --cuda-graph-bs-decode 2 4 8 \ --mamba-ssm-dtype bfloat16 \ --speculative-draft-model-path $DRAFT_MODEL_PATH \ --reasoning-parser qwen3 \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_r1.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_r1.mdx index e5833c23e..1c27bd712 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_r1.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_r1.mdx @@ -27,7 +27,7 @@ v0.5.16 or a later version. | Expert Parallelism | `--ep-size 16 \`
`--moe-a2a-backend deepep \`
`--deepep-mode auto` | | PD Disaggregation | `--disaggregation-mode prefill \`
`--disaggregation-transfer-backend ascend` | | Quantization | `--quantization modelslim` | -| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs 4 8 20 21 22` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs-decode` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs-decode 4 8 20 21 22` | | Speculative Decoding | `--speculative-algorithm NEXTN \`
`--speculative-num-steps 2 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 3` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | | DP LM Head | `--enable-dp-lm-head` | diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v3_2.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v3_2.mdx index 0a1363732..544e54885 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v3_2.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v3_2.mdx @@ -30,7 +30,7 @@ recommended to use v0.5.16 or a later version. | PD Disaggregation | `--disaggregation-mode prefill \`
`--disaggregation-transfer-backend ascend` | | Quantization | `--quantization modelslim` | | Speculative Decoding | `--speculative-algorithm NEXTN \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4` | -| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs 1 2 3 4 5 6 7 8` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs-decode` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs-decode 1 2 3 4 5 6 7 8` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | | DP LM Head | `--enable-dp-lm-head` | diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v4_flash.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v4_flash.mdx index 545bfbd18..2a334e454 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v4_flash.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/deepseek_v4_flash.mdx @@ -27,7 +27,7 @@ recommended to use v0.5.16 or a later version. | Expert Parallelism | `--ep-size 16 \`
`--moe-a2a-backend deepep \`
`--deepep-mode auto` | | PD Disaggregation | `--disaggregation-mode prefill \`
`--disaggregation-transfer-backend ascend` | | Quantization | `--quantization modelslim` | -| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs 1 2 4 8 16 24 36 40 48 56` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs-decode` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs-decode 1 2 4 8 16 24 36 40 48 56` | | Speculative Decoding | `--speculative-algorithm EAGLE \`
`--speculative-num-steps 2 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 3` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | | DP LM Head | `--enable-dp-lm-head` | diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_1.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_1.mdx index 5d8ec0fa9..29e59b653 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_1.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_1.mdx @@ -31,7 +31,7 @@ v0.5.16 or a later version. | PD Disaggregation | `--disaggregation-mode prefill \`
`--disaggregation-transfer-backend ascend` | | Quantization | `--quantization modelslim` | | Chunked Prefill | auto based on device memory, or set explicit value;
disable with `--chunked-prefill-size -1`; e.g., `--chunked-prefill-size 16384` | -| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs 1 2 3 4 5 6` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs-decode` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs-decode 1 2 3 4 5 6` | | Speculative Decoding | `--speculative-algorithm NEXTN \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4 \`
`--speculative-draft-model-quantization unquant` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | | DP LM Head | `--enable-dp-lm-head` | diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_2.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_2.mdx index 8b82e41fa..e29cd64d5 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_2.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/glm_5_2.mdx @@ -23,7 +23,7 @@ multi-node deployment, prefill-decode disaggregation, feature configuration, and | PD Disaggregation | `--disaggregation-mode prefill \`
`--disaggregation-transfer-backend ascend` | | Quantization | `--quantization modelslim` | | Chunked Prefill | auto based on device memory, or set explicit value;
disable with `--chunked-prefill-size -1`; e.g., `--chunked-prefill-size 16384` | -| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs`; e.g., `--cuda-graph-bs 16` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs-decode` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs-decode 16` | | Speculative Decoding | `--speculative-algorithm NEXTN \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4 \`
`--speculative-draft-model-quantization unquant` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | | DP LM Head | `--enable-dp-lm-head` | @@ -220,7 +220,7 @@ python3 -m sglang.launch_server \ --mem-fraction-static 0.7 \ --port 8000 \ --served-model-name glm-5 \ - --cuda-graph-bs 16 \ + --cuda-graph-bs-decode 16 \ --quantization modelslim \ --speculative-draft-model-quantization unquant \ --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 \ @@ -302,7 +302,7 @@ do --mem-fraction-static 0.8 \ --port 8000 \ --served-model-name glm-5 \ - --cuda-graph-max-bs 32 \ + --cuda-graph-max-bs-decode 32 \ --moe-a2a-backend deepep \ --deepep-mode auto \ --speculative-draft-model-quantization unquant \ @@ -421,8 +421,8 @@ do --port 8003 --trust-remote-code --nnodes 1 --node-rank $i --tp-size 16 --dp-size 16 --ep-size 16 \ --mem-fraction-static 0.8 --max-running-requests 128 --attention-backend ascend --device npu --quantization modelslim \ --served-model-name glm-5 --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency \ - --cuda-graph-max-bs 4 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 180000 \ - --tokenizer-worker-num 4 --prefill-round-robin-balance --disable-shared-experts-fusion --dtype bfloat16 --load-balance-method round_robin \ + --cuda-graph-max-bs-decode 4 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 180000 \ + --tokenizer-worker-num 4 --disable-shared-experts-fusion --dtype bfloat16 --load-balance-method round_robin \ --speculative-draft-model-quantization unquant \ --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 NODE_RANK=$i diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/hy3.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/hy3.mdx index 0ad0b420a..b7909ed90 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/hy3.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/hy3.mdx @@ -21,7 +21,7 @@ multi-node (Atlas 800I A2) PD mixed mode and speculative decoding. | Feature | Example usage | |--------------------------|----------------------------------------------------------------------------------------------------------------------------------------| | Tensor Parallelism | `--tp-size 16` | -| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs`; e.g. `--cuda-graph-bs 4 8 16 20 24 28 32 36 40` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs-decode`; e.g. `--cuda-graph-bs-decode 4 8 16 20 24 28 32 36 40` | | Speculative Decoding | `--speculative-algorithm EAGLE \`
`--speculative-num-steps 2 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 3` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | | Reasoning Mode | `extra_body={"chat_template_kwargs": {"reasoning_effort": "high"}}` — deep chain-of-thought;
`"reasoning_effort": "low"` — short thinking chain;
`"reasoning_effort": "no_think"` — direct response, no thinking | @@ -209,7 +209,7 @@ python3 -m sglang.launch_server \ --base-gpu-id 0 \ --prefill-max-requests 40 \ --max-running-requests 40 \ - --cuda-graph-bs 4 8 16 20 24 28 32 36 40 \ + --cuda-graph-bs-decode 4 8 16 20 24 28 32 36 40 \ --speculative-algorithm EAGLE \ --speculative-num-steps 2 \ --speculative-eagle-topk 1 \ @@ -286,7 +286,7 @@ for i in "${!NODE_IPS[@]}"; do --base-gpu-id 0 \ --prefill-max-requests 40 \ --max-running-requests 40 \ - --cuda-graph-bs 4 8 16 20 24 28 32 36 40 \ + --cuda-graph-bs-decode 4 8 16 20 24 28 32 36 40 \ --speculative-algorithm EAGLE \ --speculative-num-steps 2 \ --speculative-eagle-topk 1 \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/kimi_k2_6.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/kimi_k2_6.mdx index 35d50fb65..0b7837b06 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/kimi_k2_6.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/kimi_k2_6.mdx @@ -30,7 +30,7 @@ v0.5.16 or a later version. | PD Disaggregation | `--disaggregation-mode prefill \`
`--disaggregation-transfer-backend ascend` | | Quantization | `--quantization modelslim` | | Chunked Prefill | auto based on device memory, or set explicit value;
disable with `--chunked-prefill-size -1`; e.g., `--chunked-prefill-size 32768` | -| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs 1 2 4 8 12 16 24 32 48 64 96 120` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs-decode` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs-decode 1 2 4 8 12 16 24 32 48 64 96 120` | | Speculative Decoding | `--speculative-algorithm EAGLE3 \`
`--speculative-draft-model-path /path/to/draft-model-weights \`
`--speculative-num-steps 4 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 5 \`
`--speculative-draft-model-quantization unquant` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | | DP LM Head | `--enable-dp-lm-head` | diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/mimo_v2_flash.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/mimo_v2_flash.mdx index e42eeeec9..f6ff9e4c5 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/mimo_v2_flash.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/mimo_v2_flash.mdx @@ -26,7 +26,7 @@ v0.5.16 or a later version. | Expert Parallelism | `--moe-a2a-backend deepep \`
`--deepep-mode low_latency` | | PD Disaggregation | `--disaggregation-mode prefill \`
`--disaggregation-transfer-backend ascend` | | Quantization | `--quantization modelslim` | -| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs`; e.g., `--cuda-graph-bs 1 2 4 8 12 16 20 24 28 32` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs-decode` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs-decode 1 2 4 8 12 16 20 24 28 32` | | Speculative Decoding | `--speculative-algorithm EAGLE \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4 \`
`--enable-multi-layer-eagle` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0` | | DP LM Head | `--enable-dp-lm-head` | diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/minimax_m2_5.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/minimax_m2_5.mdx index 296405276..b8fd627de 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/minimax_m2_5.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/minimax_m2_5.mdx @@ -28,7 +28,7 @@ v0.5.16 or a later version. | Expert Parallelism | `--ep-size 16 \`
`--moe-a2a-backend ascend_fuseep \`
`--deepep-mode auto` | | Quantization | `--quantization modelslim` | | Chunked Prefill | auto based on device memory, or set explicit value;
disable with `--chunked-prefill-size -1`; e.g., `--chunked-prefill-size 8192` | -| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs 8 16 24 32 48 64 80` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs-decode` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs-decode 8 16 24 32 48 64 80` | | Speculative Decoding | `--speculative-algorithm EAGLE3 \`
`--speculative-draft-model-path /path/to/draft-model-weights \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4 \`
`--speculative-draft-model-quantization unquant` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_235b_a22b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_235b_a22b.mdx index f75e7ed6b..b826ab416 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_235b_a22b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_235b_a22b.mdx @@ -30,7 +30,7 @@ v0.5.16 or a later version. | PD Disaggregation | `--disaggregation-mode prefill \`
`--disaggregation-transfer-backend ascend` | | Quantization | `--quantization modelslim` | | Chunked Prefill | auto based on device memory, or set explicit value;
disable with `--chunked-prefill-size -1`; e.g., `--chunked-prefill-size 94208` | -| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs 1 2 4 8 16 20 24 26 27` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs-decode` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs-decode 1 2 4 8 16 20 24 26 27` | | Speculative Decoding | `--speculative-algorithm EAGLE3 \`
`--speculative-draft-model-path /path/to/draft-model-weights \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4 \`
`--speculative-draft-model-quantization unquant` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | | DP LM Head | `--enable-dp-lm-head` | diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_30b_a3b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_30b_a3b.mdx index f13f56614..0c71b5731 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_30b_a3b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_30b_a3b.mdx @@ -26,7 +26,7 @@ v0.5.16 or a later version. | Tensor Parallelism | `--tp-size 2` | | Data Parallelism | `--dp-size 2` | | Quantization | `--quantization modelslim` | -| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs 1 5 15 40 70 100 120 130 140 146 150 154 156 158 160 162` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs-decode` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs-decode 1 5 15 40 70 100 120 130 140 146 150 154 156 158 160 162` | | Speculative Decoding | `--speculative-algorithm EAGLE3 \`
`--speculative-draft-model-path /path/to/draft-model-weights \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4 \`
`--speculative-draft-model-quantization unquant` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_32b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_32b.mdx index 835fdd765..543e00d53 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_32b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_32b.mdx @@ -24,7 +24,7 @@ version. |-------------------------------|-----------------------------------------------------------------------------------------------| | Tensor Parallelism | `--tp-size 4` | | Quantization | `--quantization modelslim` | -| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs 1 4 8 16` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs-decode` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs-decode 1 4 8 16` | | Speculative Decoding | `--speculative-algorithm EAGLE3 \`
`--speculative-draft-model-path /path/to/draft-model-weights \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4 \`
`--speculative-draft-model-quantization unquant` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_5_397b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_5_397b.mdx index 6a7181bfa..cdd8cbfef 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_5_397b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_5_397b.mdx @@ -28,7 +28,7 @@ use v0.5.16 or a later version. | Data Parallelism | `--dp-size 8` | | Expert Parallelism | `--ep-size 16 \`
`--moe-a2a-backend deepep \`
`--deepep-mode auto` | | Quantization | `--quantization modelslim` | -| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs 2 4 6 8 10 12 14 16 18 20` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs-decode` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs-decode 2 4 6 8 10 12 14 16 18 20` | | Speculative Decoding | `--speculative-algorithm NEXTN \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4 \`
`--speculative-draft-model-quantization unquant` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | | DP LM Head | `--enable-dp-lm-head` | diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_6_27b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_6_27b.mdx index b4366bce5..ad1d6c29c 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_6_27b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_6_27b.mdx @@ -26,7 +26,7 @@ v0.5.16 or a later version. | Tensor Parallelism | `--tp-size 2` | | Quantization | `--quantization modelslim` | | Chunked Prefill | auto based on device memory, or set explicit value;
disable with `--chunked-prefill-size -1`; e.g., `--chunked-prefill-size 32768` | -| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs 2 8 16 32 48` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs-decode` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs-decode 2 8 16 32 48` | | Speculative Decoding | `--speculative-algorithm NEXTN \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_6_35b_a3b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_6_35b_a3b.mdx index 95d268049..ef794bbf0 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_6_35b_a3b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_6_35b_a3b.mdx @@ -26,7 +26,7 @@ version**. |-------------------------------|-----------------------------------------------------------------------------------------------| | Tensor Parallelism | `--tp-size 2` | | Chunked Prefill | auto based on device memory, or set explicit value;
disable with `--chunked-prefill-size -1`; e.g., `--chunked-prefill-size 16384` | -| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs 4 8 16 24 32 48 64 80 96 112 120` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs-decode` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs-decode 4 8 16 24 32 48 64 80 96 112 120` | | Speculative Decoding | `--speculative-algorithm NEXTN \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_8_max.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_8_max.mdx index b90657af5..89728e74f 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_8_max.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_8_max.mdx @@ -28,7 +28,7 @@ source. | Expert Parallelism | `--moe-a2a-backend deepep \`
`--deepep-mode auto` | | Quantization | `--quantization modelslim` | | Chunked Prefill | auto based on device memory, or set explicit value;
disable with `--chunked-prefill-size -1`; e.g., `--chunked-prefill-size 8192` | -| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs 16` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs-decode` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs-decode 16` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | | DP LM Head | `--enable-dp-lm-head` | @@ -215,7 +215,7 @@ do --enable-dp-attention --dp-size 4 --enable-dp-lm-head \ --mem-fraction-static 0.8 \ --chunked-prefill-size 8192 \ - --cuda-graph-bs 16 \ + --cuda-graph-bs-decode 16 \ --disable-radix-cache \ --max-running-requests 64 \ --host 0.0.0.0 \ diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_8b.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_8b.mdx index 3b0826358..8b5533162 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_8b.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_8b.mdx @@ -26,7 +26,7 @@ version. | Tensor Parallelism | `--tp-size 2` | | Quantization | `--quantization modelslim` | | Chunked Prefill | auto based on device memory, or set explicit value;
disable with `--chunked-prefill-size -1`; e.g., `--chunked-prefill-size 8192` | -| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs 1 2 4 6 9 10 15 16` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs-decode` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs-decode 1 2 4 6 9 10 15 16` | | Speculative Decoding | `--speculative-algorithm EAGLE3 \`
`--speculative-draft-model-path /path/to/draft-model-weights \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4 \`
`--speculative-draft-model-quantization unquant` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | diff --git a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_next_80b_a3b_instruct.mdx b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_next_80b_a3b_instruct.mdx index b1740ecac..699eece1e 100644 --- a/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_next_80b_a3b_instruct.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/model-deployment/tutorials/qwen3_next_80b_a3b_instruct.mdx @@ -27,7 +27,7 @@ v0.5.16 or a later version. | Data Parallelism | `--dp-size 2` | | Expert Parallelism | `--ep-size 4 \`
`--moe-a2a-backend deepep \`
`--deepep-mode auto` | | Quantization | `--quantization modelslim` | -| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs 2 4 8` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs-decode` or `--cuda-graph-max-bs-decode`; e.g., `--cuda-graph-bs-decode 2 4 8` | | Speculative Decoding | `--speculative-algorithm NEXTN \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4 \`
`--speculative-draft-model-quantization unquant \`
`--speculative-draft-model-path /path/to/draft-model-weights` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | | DP LM Head | `--enable-dp-lm-head` | diff --git a/docs/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning.mdx b/docs/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning.mdx index b4874b473..2b5c45fa0 100644 --- a/docs/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/optimization/parameter_tuning.mdx @@ -755,9 +755,9 @@ see [Quantization on Ascend](/docs/hardware-platforms/ascend-npus/optimization/q Breaks large prefill computations into smaller, fixed-size chunks for better scheduling interleaving with decode batches. This prevents long prefill requests from blocking decode latency. Use `-1` to disable chunked prefill on dedicated prefill nodes. Has partial compatibility with most features because chunk boundaries introduce scheduling complexity. -### NPU Graph (`--cuda-graph-bs`) +### NPU Graph (`--cuda-graph-bs-decode`) -Captures the compute graph on NPU and replays it to eliminate kernel launch overhead, analogous to CUDA Graph on NVIDIA GPUs. Internally uses `torch.npu.NPUGraph`. Most effective for decode with stable, predictable batch sizes (via `--cuda-graph-bs`). Has partial compatibility with most features because graph capture requires fixed control flow and tensor shapes. +Captures the compute graph on NPU and replays it to eliminate kernel launch overhead, analogous to CUDA Graph on NVIDIA GPUs. Internally uses `torch.npu.NPUGraph`. Most effective for decode with stable, predictable batch sizes (via `--cuda-graph-bs-decode`). Has partial compatibility with most features because graph capture requires fixed control flow and tensor shapes. `--enable-torch-compile` is incompatible with NPU Graph. When torch.compile is enabled, NPU Graph must be disabled via `--disable-cuda-graph`. diff --git a/docs/docs/hardware-platforms/ascend-npus/reference/support_features.mdx b/docs/docs/hardware-platforms/ascend-npus/reference/support_features.mdx index 419c0014a..d54a7f70b 100644 --- a/docs/docs/hardware-platforms/ascend-npus/reference/support_features.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/reference/support_features.mdx @@ -816,12 +816,6 @@ click [Server Arguments](../../../advanced_features/server_arguments). List[float] A2, A3 - - `--collect-tokens-histogram` - `False` - bool flag
(set to enable) - A2, A3 - `--prompt-tokens-buckets` `None` @@ -1498,7 +1492,7 @@ non-default speculative acceptance thresholds or deterministic inference. A2, A3 - `--speculative-dflash-draft-window-size`
`--speculative-draft-window-size` + `--speculative-draft-window-size` `None` Type: int A2, A3 @@ -2073,7 +2067,7 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD `--cuda-graph-backend-prefill` `None` - `disabled` + `disabled`, `tc_piecewise`
(`tc_piecewise` currently supports Llama-3.1-8B-Instruct and Qwen2.5-7B-Instruct) A2, A3 @@ -2113,7 +2107,7 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD A2, A3 - `--cuda-graph-bs` + `--cuda-graph-bs-decode` `None` List[int] A2, A3 @@ -2257,19 +2251,13 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD A2, A3 - `--enforce-piecewise-cuda-graph` - `False` - bool flag
(set to enable);
Currently, Llama-3.1-8B-Instruct and Qwen2.5-7B-Instruct models are supported. - A2, A3 - - - `--piecewise-cuda-graph-tokens` + `--cuda-graph-bs-prefill` `None` Type: JSON
list A2, A3 - `--piecewise-cuda-graph-compiler` + `--cuda-graph-tc-compiler` `eager` `eager` A2, A3 @@ -2281,7 +2269,7 @@ If the value is int8, you must also set the environment variable:DEEP_NORMAL_MOD A2, A3 - `--piecewise-cuda-graph-max-tokens` + `--cuda-graph-max-bs-prefill` `None` Type: int A2, A3 diff --git a/docs/docs/sglang-diffusion/disaggregation.mdx b/docs/docs/sglang-diffusion/disaggregation.mdx index b8d70d5a6..593b0df38 100644 --- a/docs/docs/sglang-diffusion/disaggregation.mdx +++ b/docs/docs/sglang-diffusion/disaggregation.mdx @@ -223,7 +223,7 @@ sglang serve \ --model-path ./zai-org/GLM-Image/vision_language_encoder/ \ --tokenizer-path ./zai-org/GLM-Image/processor/ \ --enable-multimodal \ - --cuda-graph-max-bs 28 \ + --cuda-graph-max-bs-decode 28 \ --device npu \ --attention-backend ascend \ --disable-fast-image-processor \ diff --git a/docs/docs/sglang-diffusion/models_with_ar.mdx b/docs/docs/sglang-diffusion/models_with_ar.mdx index 1d5c38167..8b734f82e 100644 --- a/docs/docs/sglang-diffusion/models_with_ar.mdx +++ b/docs/docs/sglang-diffusion/models_with_ar.mdx @@ -124,7 +124,7 @@ export HCCL_HOST_SOCKET_PORT_RANGE="23000-23199" export HCCL_NPU_SOCKET_PORT_RANGE="23200-23399" sglang serve --model-path /path/to/zai-org/GLM-Image/vision_language_encoder/ \ --tokenizer-path /path/to/zai-org/GLM-Image/processor/ --enable-multimodal \ ---cuda-graph-bs 1 --device npu --attention-backend ascend --image-processor-backend pil \ +--cuda-graph-bs-decode 1 --device npu --attention-backend ascend --image-processor-backend pil \ --tp-size 4 --port ${PORT} --mem-fraction-static 0.4 ``` Second terminal with diffusion server: diff --git a/docs/src/snippets/configs/Qwen/qwen3.8-27b.jsx b/docs/src/snippets/configs/Qwen/qwen3.8-27b.jsx index bd0a13d0a..0ac0d0135 100644 --- a/docs/src/snippets/configs/Qwen/qwen3.8-27b.jsx +++ b/docs/src/snippets/configs/Qwen/qwen3.8-27b.jsx @@ -589,7 +589,7 @@ export const config = { // RTX 5090 32GB. NVFP4 is the only checkpoint that fits (FP8 does not // boot — total_rest_memory negative at every mem-fraction, measured — // and BF16 does not fit). Published operating point is ONE request in - // flight; --cuda-graph-max-bs 1 also protects the token pool (default + // flight; --cuda-graph-max-bs-decode 1 also protects the token pool (default // capture set costs 39,247 -> 37,347 and K 8 -> 7). The `warn` below // carries the user-facing guidance for raising concurrency. match: { hw: "rtx5090", variant: "default", quant: "nvfp4-bf16-head", nodes: "single" }, @@ -598,7 +598,7 @@ export const config = { // multi-user deployment unaware. warn: "This recipe serves ONE request at a time: --max-running-requests 1 " + - "and --cuda-graph-max-bs 1 pin it to the validated single-stream " + + "and --cuda-graph-max-bs-decode 1 pin it to the validated single-stream " + "envelope. To handle more concurrent requests, raise both flags " + "together and re-derive --mamba-full-memory-ratio (and mem-fraction) " + "with the [Mamba ratio calculator](#mamba-ratio-calculator) — on this " + @@ -611,7 +611,7 @@ export const config = { "--mem-fraction-static 0.9", "--attention-backend flashinfer", "--max-running-requests 1", - "--cuda-graph-max-bs 1", + "--cuda-graph-max-bs-decode 1", "--reasoning-parser qwen3", "--tool-call-parser qwen3_coder", "--host {{HOST_IP}}", @@ -624,7 +624,7 @@ export const config = { // RTX 5090 32GB. NVFP4 is the only checkpoint that fits (FP8 does not // boot — total_rest_memory negative at every mem-fraction, measured — // and BF16 does not fit). Published operating point is ONE request in - // flight; --cuda-graph-max-bs 1 also protects the token pool (default + // flight; --cuda-graph-max-bs-decode 1 also protects the token pool (default // capture set costs 39,247 -> 37,347 and K 8 -> 7). The `warn` below // carries the user-facing guidance for raising concurrency. match: { hw: "rtx5090", variant: "default", quant: "nvfp4-fp4-head", nodes: "single" }, @@ -633,7 +633,7 @@ export const config = { // multi-user deployment unaware. warn: "This recipe serves ONE request at a time: --max-running-requests 1 " + - "and --cuda-graph-max-bs 1 pin it to the validated single-stream " + + "and --cuda-graph-max-bs-decode 1 pin it to the validated single-stream " + "envelope. To handle more concurrent requests, raise both flags " + "together and re-derive --mamba-full-memory-ratio (and mem-fraction) " + "with the [Mamba ratio calculator](#mamba-ratio-calculator) — on this " + @@ -646,7 +646,7 @@ export const config = { "--mem-fraction-static 0.9", "--attention-backend flashinfer", "--max-running-requests 1", - "--cuda-graph-max-bs 1", + "--cuda-graph-max-bs-decode 1", "--reasoning-parser qwen3", "--tool-call-parser qwen3_coder", "--host {{HOST_IP}}", diff --git a/docs/src/snippets/configs/Qwen/qwen3.8.jsx b/docs/src/snippets/configs/Qwen/qwen3.8.jsx index b7b5b4e8d..b0f0ce4fc 100644 --- a/docs/src/snippets/configs/Qwen/qwen3.8.jsx +++ b/docs/src/snippets/configs/Qwen/qwen3.8.jsx @@ -233,8 +233,8 @@ export const config = { // ----- Card: "PD Disaggregation" ----- // Role flags follow the P/D bundle's own prefill and decode workers. Two // flags those recipes carry are deliberately not emitted: - // --prefill-round-robin-balance is a DeprecatedAction on current SGLang and - // does nothing, and --mamba-track-interval is context-dependent (the source + // --prefill-round-robin-balance no longer exists on current SGLang (it was + // a deprecated no-op), and --mamba-track-interval is context-dependent (the source // recipes set it equal to their context cap) so a fixed value here would be // wrong for cells serving the native window. pdDisagg: { diff --git a/docs/src/snippets/configs/inclusionAI/ling-3.0-flash.jsx b/docs/src/snippets/configs/inclusionAI/ling-3.0-flash.jsx index abd21f443..2fee8f55b 100644 --- a/docs/src/snippets/configs/inclusionAI/ling-3.0-flash.jsx +++ b/docs/src/snippets/configs/inclusionAI/ling-3.0-flash.jsx @@ -114,7 +114,7 @@ sgl-eval run gsm8k \\ hicache: { defaultBackend: "mooncake", requiredFlags: [ - "--mamba-scheduler-strategy extra_buffer", + "--mamba-radix-cache-strategy extra_buffer", "--enable-cache-report", ], backends: [ @@ -618,7 +618,7 @@ sgl-eval run gsm8k \\ "--hicache-storage-backend mooncake", "--hicache-io-backend direct", "--hicache-mem-layout page_first_direct", - "--mamba-scheduler-strategy extra_buffer", + "--mamba-radix-cache-strategy extra_buffer", "--enable-cache-report", "--hicache-storage-prefetch-policy wait_complete", "--hicache-storage-backend-extra-config '{\"hicache_storage_pass_prefix_keys\":true}'", @@ -647,7 +647,7 @@ sgl-eval run gsm8k \\ "--hicache-storage-backend mooncake", "--hicache-io-backend direct", "--hicache-mem-layout page_first_direct", - "--mamba-scheduler-strategy extra_buffer", + "--mamba-radix-cache-strategy extra_buffer", "--enable-cache-report", "--hicache-storage-prefetch-policy wait_complete", "--hicache-storage-backend-extra-config '{\"hicache_storage_pass_prefix_keys\":true}'", @@ -675,7 +675,7 @@ sgl-eval run gsm8k \\ "--hicache-storage-backend mooncake", "--hicache-io-backend direct", "--hicache-mem-layout page_first_direct", - "--mamba-scheduler-strategy extra_buffer", + "--mamba-radix-cache-strategy extra_buffer", "--enable-cache-report", "--hicache-storage-prefetch-policy wait_complete", "--hicache-storage-backend-extra-config '{\"hicache_storage_pass_prefix_keys\":true}'", @@ -704,7 +704,7 @@ sgl-eval run gsm8k \\ "--hicache-storage-backend mooncake", "--hicache-io-backend direct", "--hicache-mem-layout page_first_direct", - "--mamba-scheduler-strategy extra_buffer", + "--mamba-radix-cache-strategy extra_buffer", "--enable-cache-report", "--hicache-storage-prefetch-policy wait_complete", "--hicache-storage-backend-extra-config '{\"hicache_storage_pass_prefix_keys\":true}'", diff --git a/docs/src/snippets/configs/meituan-longcat/longcat-2.0.jsx b/docs/src/snippets/configs/meituan-longcat/longcat-2.0.jsx index 5122556b8..a5eb37519 100644 --- a/docs/src/snippets/configs/meituan-longcat/longcat-2.0.jsx +++ b/docs/src/snippets/configs/meituan-longcat/longcat-2.0.jsx @@ -134,7 +134,7 @@ sgl-eval run gsm8k \\ "--max-running-requests 64", "--mem-fraction-static 0.92", "--chunked-prefill-size 2048", - "--nsa-prefill-backend fa3", + "--dsa-prefill-backend fa3", "--kv-cache-dtype bfloat16", "--model-loader-extra-config '{\"enable_multithread_load\":true,\"num_threads\":12}'", "--host {{HOST_IP}}", @@ -154,7 +154,7 @@ sgl-eval run gsm8k \\ "--max-running-requests 64", "--mem-fraction-static 0.92", "--chunked-prefill-size 2048", - "--nsa-prefill-backend fa3", + "--dsa-prefill-backend fa3", "--kv-cache-dtype bfloat16", "--model-loader-extra-config '{\"enable_multithread_load\":true,\"num_threads\":12}'", "--host {{HOST_IP}}", @@ -173,7 +173,7 @@ sgl-eval run gsm8k \\ "--max-running-requests 64", "--mem-fraction-static 0.92", "--chunked-prefill-size 2048", - "--nsa-prefill-backend fa3", + "--dsa-prefill-backend fa3", "--kv-cache-dtype bfloat16", "--model-loader-extra-config '{\"enable_multithread_load\":true,\"num_threads\":12}'", "--host {{HOST_IP}}", @@ -192,7 +192,7 @@ sgl-eval run gsm8k \\ "--max-running-requests 64", "--mem-fraction-static 0.92", "--chunked-prefill-size 2048", - "--nsa-prefill-backend fa3", + "--dsa-prefill-backend fa3", "--kv-cache-dtype bfloat16", "--model-loader-extra-config '{\"enable_multithread_load\":true,\"num_threads\":12}'", "--host {{HOST_IP}}", diff --git a/docs/src/snippets/configs/moonshotai/kimi-k3.jsx b/docs/src/snippets/configs/moonshotai/kimi-k3.jsx index da32d9af0..4bca32497 100644 --- a/docs/src/snippets/configs/moonshotai/kimi-k3.jsx +++ b/docs/src/snippets/configs/moonshotai/kimi-k3.jsx @@ -1183,7 +1183,7 @@ export const config = { "--kv-cache-dtype fp8_e4m3", "--dtype bfloat16", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 256", + "--cuda-graph-max-bs-decode 256", "--reasoning-parser kimi_k3", "--tool-call-parser kimi_k3", "--host {{HOST_IP}}", @@ -1210,7 +1210,7 @@ export const config = { "--kv-cache-dtype fp8_e4m3", "--dtype bfloat16", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 256", + "--cuda-graph-max-bs-decode 256", "--reasoning-parser kimi_k3", "--tool-call-parser kimi_k3", "--host {{HOST_IP}}", @@ -1959,7 +1959,7 @@ export const config = { "--kv-cache-dtype fp8_e4m3", "--dtype bfloat16", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 256", + "--cuda-graph-max-bs-decode 256", "--reasoning-parser kimi_k3", "--tool-call-parser kimi_k3", "--disaggregation-mode decode", @@ -1987,7 +1987,7 @@ export const config = { "--kv-cache-dtype fp8_e4m3", "--dtype bfloat16", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 256", + "--cuda-graph-max-bs-decode 256", "--reasoning-parser kimi_k3", "--tool-call-parser kimi_k3", "--disaggregation-mode decode", @@ -2379,7 +2379,7 @@ export const config = { "--enable-dp-lm-head", "--mem-fraction-static 0.78", "--chunked-prefill-size 16384", - "--cuda-graph-bs 2 4 8 16", + "--cuda-graph-bs-decode 2 4 8 16", "--max-running-requests 64", "--max-mamba-cache-size 64", "--moe-a2a-backend deepep", diff --git a/docs/src/snippets/configs/openbmb/minicpm5-2b-benchmarks.jsx b/docs/src/snippets/configs/openbmb/minicpm5-2b-benchmarks.jsx index 905aee9be..559bc8e04 100644 --- a/docs/src/snippets/configs/openbmb/minicpm5-2b-benchmarks.jsx +++ b/docs/src/snippets/configs/openbmb/minicpm5-2b-benchmarks.jsx @@ -4,7 +4,7 @@ // RTX 5090 32GB (`zijiexia-5090-1gpu`, `lmsysorg/sglang:dev` @ 30705c004c) with // `python3 -m sglang.bench_serving`, random 1024/1024, against the exact cell // command on this page — including `--mem-fraction-static 0.75 -// --cuda-graph-max-bs 128`, which is what keeps concurrency 64 and 128 +// --cuda-graph-max-bs-decode 128`, which is what keeps concurrency 64 and 128 // graph-backed (see the model page's Configuration Tips). // // `tokens_per_sec_per_gpu` is total (input+output) per GPU = the measured diff --git a/docs/src/snippets/configs/openbmb/minicpm5-2b.jsx b/docs/src/snippets/configs/openbmb/minicpm5-2b.jsx index 7e8ffe5a0..0e49e5d50 100644 --- a/docs/src/snippets/configs/openbmb/minicpm5-2b.jsx +++ b/docs/src/snippets/configs/openbmb/minicpm5-2b.jsx @@ -206,7 +206,7 @@ export const config = { "--reasoning-parser qwen3", "--tool-call-parser minicpm5", "--mem-fraction-static 0.75", - "--cuda-graph-max-bs 128", + "--cuda-graph-max-bs-decode 128", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -225,7 +225,7 @@ export const config = { "--reasoning-parser qwen3", "--tool-call-parser minicpm5", "--mem-fraction-static 0.30", - "--cuda-graph-max-bs 128", + "--cuda-graph-max-bs-decode 128", "--host {{HOST_IP}}", "--port {{PORT}}", ], diff --git a/docs/src/snippets/configs/zai-org/glm-5.2.jsx b/docs/src/snippets/configs/zai-org/glm-5.2.jsx index 0f9606ce8..7655359b4 100644 --- a/docs/src/snippets/configs/zai-org/glm-5.2.jsx +++ b/docs/src/snippets/configs/zai-org/glm-5.2.jsx @@ -772,7 +772,7 @@ sgl-eval run aime25 \\ "--chunked-prefill-size 8192", "--mem-fraction-static 0.85", "--max-running-requests 16", - "--cuda-graph-max-bs 16", + "--cuda-graph-max-bs-decode 16", "--max-prefill-tokens 8192", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -837,7 +837,7 @@ sgl-eval run aime25 \\ "--chunked-prefill-size 8192", "--mem-fraction-static 0.85", "--max-running-requests 16", - "--cuda-graph-max-bs 16", + "--cuda-graph-max-bs-decode 16", "--max-prefill-tokens 8192", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -927,7 +927,7 @@ sgl-eval run aime25 \\ "--dsa-decode-backend tilelang", "--chunked-prefill-size 32768", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 128", + "--cuda-graph-max-bs-decode 128", "--max-running-requests 80", "--watchdog-timeout 1200", "--host {{HOST_IP}}", @@ -944,7 +944,7 @@ sgl-eval run aime25 \\ "--dsa-prefill-backend tilelang", "--dsa-decode-backend tilelang", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 256", + "--cuda-graph-max-bs-decode 256", "--max-running-requests 256", "--watchdog-timeout 1200", "--host {{HOST_IP}}", @@ -978,7 +978,7 @@ sgl-eval run aime25 \\ "--dsa-decode-backend tilelang", "--chunked-prefill-size 32768", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 128", + "--cuda-graph-max-bs-decode 128", "--max-running-requests 80", "--watchdog-timeout 1200", "--host {{HOST_IP}}", @@ -995,7 +995,7 @@ sgl-eval run aime25 \\ "--dsa-prefill-backend tilelang", "--dsa-decode-backend tilelang", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 256", + "--cuda-graph-max-bs-decode 256", "--max-running-requests 256", "--watchdog-timeout 1200", "--host {{HOST_IP}}", @@ -1043,7 +1043,7 @@ sgl-eval run aime25 \\ "--dsa-decode-backend tilelang", "--chunked-prefill-size 32768", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 128", + "--cuda-graph-max-bs-decode 128", "--max-running-requests 80", "--watchdog-timeout 1200", "--host {{HOST_IP}}", @@ -1062,7 +1062,7 @@ sgl-eval run aime25 \\ "--dsa-prefill-backend tilelang", "--dsa-decode-backend tilelang", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 256", + "--cuda-graph-max-bs-decode 256", "--max-running-requests 256", "--watchdog-timeout 1200", "--host {{HOST_IP}}", @@ -1094,7 +1094,7 @@ sgl-eval run aime25 \\ "--speculative-num-draft-tokens 4", "--chunked-prefill-size 131072", "--mem-fraction-static 0.80", - "--cuda-graph-max-bs 160", + "--cuda-graph-max-bs-decode 160", "--max-running-requests 160", "--watchdog-timeout 1800", "--host {{HOST_IP}}", @@ -1128,7 +1128,7 @@ sgl-eval run aime25 \\ "--dsa-decode-backend tilelang", "--chunked-prefill-size 32768", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 128", + "--cuda-graph-max-bs-decode 128", "--max-running-requests 80", "--watchdog-timeout 1200", "--host {{HOST_IP}}", @@ -1145,7 +1145,7 @@ sgl-eval run aime25 \\ "--dsa-prefill-backend tilelang", "--dsa-decode-backend tilelang", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 256", + "--cuda-graph-max-bs-decode 256", "--max-running-requests 256", "--watchdog-timeout 1200", "--host {{HOST_IP}}", @@ -1179,7 +1179,7 @@ sgl-eval run aime25 \\ "--dsa-decode-backend tilelang", "--chunked-prefill-size 32768", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 128", + "--cuda-graph-max-bs-decode 128", "--max-running-requests 80", "--watchdog-timeout 1200", "--host {{HOST_IP}}", @@ -1196,7 +1196,7 @@ sgl-eval run aime25 \\ "--dsa-prefill-backend tilelang", "--dsa-decode-backend tilelang", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 256", + "--cuda-graph-max-bs-decode 256", "--max-running-requests 256", "--watchdog-timeout 1200", "--host {{HOST_IP}}", @@ -1230,7 +1230,7 @@ sgl-eval run aime25 \\ "--dsa-decode-backend tilelang", "--chunked-prefill-size 32768", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 128", + "--cuda-graph-max-bs-decode 128", "--max-running-requests 80", "--watchdog-timeout 1200", "--host {{HOST_IP}}", @@ -1247,7 +1247,7 @@ sgl-eval run aime25 \\ "--dsa-prefill-backend tilelang", "--dsa-decode-backend tilelang", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 256", + "--cuda-graph-max-bs-decode 256", "--max-running-requests 256", "--watchdog-timeout 1200", "--host {{HOST_IP}}", diff --git a/docs/src/snippets/configs/zai-org/glm-5.3-flash.jsx b/docs/src/snippets/configs/zai-org/glm-5.3-flash.jsx index 561f96a4c..ffcbd4385 100644 --- a/docs/src/snippets/configs/zai-org/glm-5.3-flash.jsx +++ b/docs/src/snippets/configs/zai-org/glm-5.3-flash.jsx @@ -416,7 +416,7 @@ sgl-eval run gsm8k \\ "--reasoning-parser glm45", "--tool-call-parser glm47", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 32", + "--cuda-graph-max-bs-decode 32", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -472,7 +472,7 @@ sgl-eval run gsm8k \\ "--reasoning-parser glm45", "--tool-call-parser glm47", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 32", + "--cuda-graph-max-bs-decode 32", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -518,7 +518,7 @@ sgl-eval run gsm8k \\ "--reasoning-parser glm45", "--tool-call-parser glm47", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 32", + "--cuda-graph-max-bs-decode 32", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -564,7 +564,7 @@ sgl-eval run gsm8k \\ "--reasoning-parser glm45", "--tool-call-parser glm47", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 32", + "--cuda-graph-max-bs-decode 32", "--host {{HOST_IP}}", "--port {{PORT}}", ], diff --git a/docs/src/snippets/configs/zai-org/glm-5.3.jsx b/docs/src/snippets/configs/zai-org/glm-5.3.jsx index 8b6ff04a0..da002e0cd 100644 --- a/docs/src/snippets/configs/zai-org/glm-5.3.jsx +++ b/docs/src/snippets/configs/zai-org/glm-5.3.jsx @@ -716,7 +716,7 @@ sgl-eval run aime25 \\ "--chunked-prefill-size 8192", "--mem-fraction-static 0.85", "--max-running-requests 16", - "--cuda-graph-max-bs 16", + "--cuda-graph-max-bs-decode 16", "--max-prefill-tokens 8192", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -758,7 +758,7 @@ sgl-eval run aime25 \\ "--chunked-prefill-size 8192", "--mem-fraction-static 0.85", "--max-running-requests 16", - "--cuda-graph-max-bs 16", + "--cuda-graph-max-bs-decode 16", "--max-prefill-tokens 8192", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -825,7 +825,7 @@ sgl-eval run aime25 \\ "--dsa-decode-backend tilelang", "--chunked-prefill-size 32768", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 128", + "--cuda-graph-max-bs-decode 128", "--max-running-requests 80", "--watchdog-timeout 1200", "--host {{HOST_IP}}", @@ -842,7 +842,7 @@ sgl-eval run aime25 \\ "--dsa-prefill-backend tilelang", "--dsa-decode-backend tilelang", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 256", + "--cuda-graph-max-bs-decode 256", "--max-running-requests 256", "--watchdog-timeout 1200", "--host {{HOST_IP}}", @@ -876,7 +876,7 @@ sgl-eval run aime25 \\ "--dsa-decode-backend tilelang", "--chunked-prefill-size 32768", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 128", + "--cuda-graph-max-bs-decode 128", "--max-running-requests 80", "--watchdog-timeout 1200", "--host {{HOST_IP}}", @@ -893,7 +893,7 @@ sgl-eval run aime25 \\ "--dsa-prefill-backend tilelang", "--dsa-decode-backend tilelang", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 256", + "--cuda-graph-max-bs-decode 256", "--max-running-requests 256", "--watchdog-timeout 1200", "--host {{HOST_IP}}", @@ -927,7 +927,7 @@ sgl-eval run aime25 \\ "--dsa-decode-backend tilelang", "--chunked-prefill-size 32768", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 128", + "--cuda-graph-max-bs-decode 128", "--max-running-requests 80", "--watchdog-timeout 1200", "--host {{HOST_IP}}", @@ -944,7 +944,7 @@ sgl-eval run aime25 \\ "--dsa-prefill-backend tilelang", "--dsa-decode-backend tilelang", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 256", + "--cuda-graph-max-bs-decode 256", "--max-running-requests 256", "--watchdog-timeout 1200", "--host {{HOST_IP}}", @@ -978,7 +978,7 @@ sgl-eval run aime25 \\ "--dsa-decode-backend tilelang", "--chunked-prefill-size 32768", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 128", + "--cuda-graph-max-bs-decode 128", "--max-running-requests 80", "--watchdog-timeout 1200", "--host {{HOST_IP}}", @@ -995,7 +995,7 @@ sgl-eval run aime25 \\ "--dsa-prefill-backend tilelang", "--dsa-decode-backend tilelang", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 256", + "--cuda-graph-max-bs-decode 256", "--max-running-requests 256", "--watchdog-timeout 1200", "--host {{HOST_IP}}", @@ -1029,7 +1029,7 @@ sgl-eval run aime25 \\ "--dsa-decode-backend tilelang", "--chunked-prefill-size 32768", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 128", + "--cuda-graph-max-bs-decode 128", "--max-running-requests 80", "--watchdog-timeout 1200", "--host {{HOST_IP}}", @@ -1046,7 +1046,7 @@ sgl-eval run aime25 \\ "--dsa-prefill-backend tilelang", "--dsa-decode-backend tilelang", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 256", + "--cuda-graph-max-bs-decode 256", "--max-running-requests 256", "--watchdog-timeout 1200", "--host {{HOST_IP}}", diff --git a/python/sglang/kernels/aot/tests/test_fused_qk_norm_rope.py b/python/sglang/kernels/aot/tests/test_fused_qk_norm_rope.py index 57419b63b..001188743 100644 --- a/python/sglang/kernels/aot/tests/test_fused_qk_norm_rope.py +++ b/python/sglang/kernels/aot/tests/test_fused_qk_norm_rope.py @@ -6,7 +6,6 @@ from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.rotary_embedding import get_rope from sglang.srt.server_args import ( ServerArgs, - get_global_server_args, set_global_server_args_for_scheduler, ) from sglang.srt.utils import ( diff --git a/python/sglang/multimodal_gen/test/server/ascend/test_glm_image_distributed.py b/python/sglang/multimodal_gen/test/server/ascend/test_glm_image_distributed.py index d8fad5fe1..65aa25d9c 100644 --- a/python/sglang/multimodal_gen/test/server/ascend/test_glm_image_distributed.py +++ b/python/sglang/multimodal_gen/test/server/ascend/test_glm_image_distributed.py @@ -120,7 +120,7 @@ class _GlmDistributedCluster: "--disable-fast-image-processor", "--tp-size", "1", - "--cuda-graph-bs", + "--cuda-graph-bs-decode", "2", "--base-gpu-id", "0", diff --git a/python/sglang/multimodal_gen/test/single_test_file/test_ar_models.py b/python/sglang/multimodal_gen/test/single_test_file/test_ar_models.py index 768aafc40..f09351681 100644 --- a/python/sglang/multimodal_gen/test/single_test_file/test_ar_models.py +++ b/python/sglang/multimodal_gen/test/single_test_file/test_ar_models.py @@ -71,7 +71,7 @@ class ARCluster(DisaggCluster): "--tokenizer-path", os.path.join(local_model, "processor"), "--enable-multimodal", - "--cuda-graph-bs", + "--cuda-graph-bs-decode", "1", "--image-processor-backend", "pil", diff --git a/python/sglang/multimodal_gen/test/unit/test_disagg_trace.py b/python/sglang/multimodal_gen/test/unit/test_disagg_trace.py index 12225f1c6..168375643 100644 --- a/python/sglang/multimodal_gen/test/unit/test_disagg_trace.py +++ b/python/sglang/multimodal_gen/test/unit/test_disagg_trace.py @@ -29,10 +29,9 @@ from sglang.multimodal_gen.runtime.disaggregation.scheduler_mixin import ( ) from sglang.multimodal_gen.runtime.disaggregation.transport.codec import pack_tensors from sglang.multimodal_gen.runtime.pipelines_core import Req -from sglang.srt import server_args as srt_server_args_module from sglang.srt.observability import trace as srt_trace from sglang.srt.observability.trace import TraceNullContext, TraceReqContext -from sglang.srt.runtime_context import reset_context +from sglang.srt.runtime_context import get_server_args, reset_context from sglang.srt.server_args import set_global_server_args_for_scheduler try: @@ -63,7 +62,7 @@ def _enable_minimal_otel() -> None: @contextmanager def _srt_trace_server_args(): try: - prev_server_args = srt_server_args_module.get_global_server_args() + prev_server_args = get_server_args() except ValueError: # nothing published yet prev_server_args = None # publish resolves what it is handed, so a stand-in cannot go through it. diff --git a/python/sglang/srt/arg_groups/arg_utils.py b/python/sglang/srt/arg_groups/arg_utils.py index fe535ff40..8b3a63f6c 100644 --- a/python/sglang/srt/arg_groups/arg_utils.py +++ b/python/sglang/srt/arg_groups/arg_utils.py @@ -39,7 +39,6 @@ annotation is equivalent to ``Arg(help=that_string)``. from __future__ import annotations -import copy import dataclasses import functools import types @@ -57,20 +56,6 @@ from typing import ( A = Annotated -class _NoFallback: - """Sentinel for ``Arg.fallback``: this field declares none. - - ``None`` cannot serve, because ``None`` is what a field *holds* when the - operator did not type it -- the state a fallback answers for. - """ - - def __repr__(self) -> str: # pragma: no cover - debugging aid - return "" - - -NO_FALLBACK = _NoFallback() - - @dataclasses.dataclass(frozen=True) class Arg: """CLI argument metadata attached to a dataclass field via ``Annotated``.""" @@ -96,11 +81,13 @@ class Arg: # What the field means when nobody said anything -- the bottom of the read # chain: override, decision, input, then this. Not the dataclass default, # which stays `None` because that is how the record spells "not typed". + # `None` here means the field declares no fallback, which is the same + # answer the read chain gives without one. # # Only a value fixed for the life of the configuration belongs here. One # that depends on the machine, on another field, or on anything impure is a # decision, and decisions stay in a hook where their order is visible. - fallback: Any = NO_FALLBACK + fallback: Any = None @dataclasses.dataclass(frozen=True) @@ -232,7 +219,20 @@ def fallbacks_of(cls) -> dict: out = {} for field in dataclasses.fields(cls): _, arg = _unwrap_annotated(hints.get(field.name, field.type)) - if arg is not None and arg.fallback is not NO_FALLBACK: + if arg is not None and arg.fallback is not None: + # Two things `with_fallback` relies on and cannot check itself, + # asserted where a new declaration passes through. This function is + # cached, so a mutable fallback would hand one shared object to + # every reader; and a field whose dataclass default is not `None` + # can never reach the fallback, which makes the declaration dead. + assert not isinstance(arg.fallback, (list, dict, set)), ( + f"{cls.__name__}.{field.name}: a mutable fallback would be " + "shared by every reader -- use a scalar" + ) + assert field.default is None, ( + f"{cls.__name__}.{field.name}: declares a fallback but defaults " + f"to {field.default!r}, so the fallback is unreachable" + ) out[field.name] = arg.fallback return out @@ -246,17 +246,12 @@ def with_fallback(cls, name: str, value: Any) -> Any: on `if cfg.swa_full_tokens_ratio is None`, and a fallback answering there would make that branch dead. `test_declared_fallbacks.py` pins both halves. - A container fallback is copied, for the reason a dataclass spells this - `default_factory`. + A mutable fallback would need copying per read, for the reason a dataclass + spells this `default_factory`. Every declared one is a scalar. """ if value is not None: return value - fallback = fallbacks_of(cls).get(name, NO_FALLBACK) - if fallback is NO_FALLBACK: - return value - if isinstance(fallback, (list, dict, set)): - return copy.deepcopy(fallback) - return fallback + return fallbacks_of(cls).get(name, value) # --------------------------------------------------------------------------- diff --git a/python/sglang/srt/arg_groups/argparse_actions.py b/python/sglang/srt/arg_groups/argparse_actions.py index 06b98ceb4..6b2b31e80 100644 --- a/python/sglang/srt/arg_groups/argparse_actions.py +++ b/python/sglang/srt/arg_groups/argparse_actions.py @@ -29,18 +29,45 @@ def print_deprecated_warning(message: str): logger.warning(f"\033[1;33m{message}\033[0m") +# Retiring a flag comes in four shapes, and which one you need depends on what +# the flag was and what replaced it: +# +# * the flag is gone and there is no automatic translation +# -> `DeprecatedAction` with `error_message=`, which stops the launch and +# names the replacement; +# * an old boolean whose field survives, possibly renamed +# -> `DeprecatedStoreTrueAction`; +# * an old boolean replaced by one *value* of a new valued flag +# -> `DeprecatedStoreConstAction` with `const_value=`; +# * an old valued flag replaced by a renamed valued flag +# -> `DeprecatedAliasStoreAction`. +# +# Only the second has a registration today (`--disable-cuda-graph`). The other +# three are kept because the shapes recur -- this package has retired flags of +# every one of them -- and the fiddly parts (`nargs=0` on a boolean, where the +# const goes, warn-and-continue versus `parser.error`) are what a +# reimplementation gets wrong. Pass `new_flag=` so the warning tells the +# operator what to switch to; that pointer is the whole point. + + class DeprecatedAction(argparse.Action): + """A retired flag with no automatic translation: stop and say so. + + `error_message` should name the replacement, because a bare "unrecognized + arguments" leaves the operator guessing. Without one it warns and continues, + which suits a flag that has become a no-op rather than a rename. + """ + def __init__(self, option_strings, dest, error_message=None, nargs=0, **kwargs): self.error_message = error_message - super(DeprecatedAction, self).__init__( - option_strings, dest, nargs=nargs, **kwargs - ) + super().__init__(option_strings, dest, nargs=nargs, **kwargs) def __call__(self, parser, namespace, values, option_string=None): if self.error_message is not None: parser.error(self.error_message) print_deprecated_warning( - f"The command line argument '{option_string}' is deprecated and will be removed in future versions." + f"The command line argument '{option_string}' is deprecated and " + "will be removed in future versions." ) @@ -71,10 +98,12 @@ class DeprecatedStoreTrueAction(argparse.Action): class DeprecatedStoreConstAction(argparse.Action): - """Deprecated boolean flag that stores a fixed string/value into ``dest`` - and prints a warning. Used to translate a legacy boolean flag into a - setting on the new per-phase config dict (e.g. - ``--disable-piecewise-cuda-graph`` -> ``cuda_graph_backend_prefill="disabled"``).""" + """An old boolean whose replacement is one *value* of a valued flag. + + The bool-to-enum migration: the operator passes no value, and the action + writes the fixed one `const_value` names onto the new field. `nargs=0` + because the old spelling took no argument. + """ def __init__( self, @@ -93,13 +122,14 @@ class DeprecatedStoreConstAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): replacement = f" Use '{self.new_flag}' instead." if self.new_flag else "" print_deprecated_warning( - f"'{option_string}' is deprecated and will be removed in a future release.{replacement}" + f"'{option_string}' is deprecated and will be removed in a future " + f"release.{replacement}" ) setattr(namespace, self.dest, self.const_value) class DeprecatedAliasStoreAction(argparse.Action): - """Deprecated alias that stores its value and prints a warning.""" + """An old valued flag renamed: keep the value, move it to the new dest.""" def __init__(self, option_strings, dest, new_flag=None, **kwargs): self.new_flag = new_flag @@ -108,6 +138,7 @@ class DeprecatedAliasStoreAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): replacement = f" Use '{self.new_flag}' instead." if self.new_flag else "" print_deprecated_warning( - f"'{option_string}' is deprecated and will be removed in a future release.{replacement}" + f"'{option_string}' is deprecated and will be removed in a future " + f"release.{replacement}" ) setattr(namespace, self.dest, values) diff --git a/python/sglang/srt/arg_groups/attention_hook.py b/python/sglang/srt/arg_groups/attention_hook.py index f772f05e4..d61d27d1a 100644 --- a/python/sglang/srt/arg_groups/attention_hook.py +++ b/python/sglang/srt/arg_groups/attention_hook.py @@ -175,6 +175,11 @@ def handle_attention_backend_compatibility(server_args: Any): # AMD platforms backends if resolved_view(server_args).attention_backend == "aiter": if model_config.context_len > 8192: + # The record, via the input snapshot rather than the field: a + # hook may not read a field off the record (the guard in + # `test_resolution_reads_the_declarations.py`), and what this + # needs is the input anyway -- whether the operator asked for a + # memory fraction, not the value in effect. explicit_mem_fraction = ( getattr(server_args, "_raw_input", None) or {} ).get("mem_fraction_static") is not None diff --git a/python/sglang/srt/arg_groups/cuda_graph_hook.py b/python/sglang/srt/arg_groups/cuda_graph_hook.py index 2004ddcfb..bb0f22772 100644 --- a/python/sglang/srt/arg_groups/cuda_graph_hook.py +++ b/python/sglang/srt/arg_groups/cuda_graph_hook.py @@ -40,8 +40,8 @@ def parse_cuda_graph_config(server_args: Any): Precedence (highest first): explicit JSON > convenience > legacy > defaults. Also populates server_args._cuda_graph_config_locked — the set of (phase, key) tuples that came from non-default sources; the - auto-disable cascade respects this lock (the old - --enforce-piecewise-cuda-graph semantics generalized). + auto-disable cascade respects this lock (an explicitly supplied prefill + backend skips the cascade, whichever value it is). """ cfg = resolving_view(server_args) raw_input = cfg.cuda_graph_config @@ -107,8 +107,8 @@ def apply_cuda_graph_compatibility(server_args: Any): """Auto-disable prefill cuda graph for incompatible configs. Rules are split per backend — TcPiecewise and Breakable have different constraints. Skipped when the user explicitly set the - prefill backend (this folds in the old - --enforce-piecewise-cuda-graph contract). + prefill backend, whichever value they chose (the contract the removed + --enforce-piecewise-cuda-graph used to spell). """ cfg = resolving_view(server_args) diff --git a/python/sglang/srt/arg_groups/model_overrides/inkling.py b/python/sglang/srt/arg_groups/model_overrides/inkling.py index 10182e766..8cf128289 100644 --- a/python/sglang/srt/arg_groups/model_overrides/inkling.py +++ b/python/sglang/srt/arg_groups/model_overrides/inkling.py @@ -48,7 +48,7 @@ def _inkling_overrides(server_args: Any, hf_config: Any) -> dict: overrides["mamba_full_memory_ratio"] = 0.1 # Inkling requires the extra-buffer mamba strategy (inkling.py asserts # enable_mamba_extra_buffer()); the generic "auto" resolution does not cover - # Inkling, so pin it here. Yields to an explicit --mamba-scheduler-strategy. + # Inkling, so pin it here. Yields to an explicit --mamba-radix-cache-strategy. # # Compared against the unresolved token rather than the class default: the # default only answers "unset" while nothing has declared the field first, diff --git a/python/sglang/srt/arg_groups/moe_hook.py b/python/sglang/srt/arg_groups/moe_hook.py index b1146dd32..945bf328e 100644 --- a/python/sglang/srt/arg_groups/moe_hook.py +++ b/python/sglang/srt/arg_groups/moe_hook.py @@ -429,7 +429,7 @@ def validate_deepep_v2_dispatch_token_budget(server_args: Any) -> None: "SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK: " f"required={graph_tokens}, capacity={capacity} " f"(requests={graph_bs}, tokens/request={tokens_per_req}). Raise " - "the environment value or lower --cuda-graph-max-bs." + "the environment value or lower --cuda-graph-max-bs-decode." ) diff --git a/python/sglang/srt/kv_canary/single_forward_manager/manager.py b/python/sglang/srt/kv_canary/single_forward_manager/manager.py index 304773ead..7773eb232 100644 --- a/python/sglang/srt/kv_canary/single_forward_manager/manager.py +++ b/python/sglang/srt/kv_canary/single_forward_manager/manager.py @@ -132,7 +132,7 @@ class SingleForwardManager: if bs > self._write_req_capacity: raise RuntimeError( f"kv-canary: forward_batch.batch_size={bs} exceeds pre-allocated " - f"write_req_capacity={self._write_req_capacity}; raise --cuda-graph-max-bs " + f"write_req_capacity={self._write_req_capacity}; raise --cuda-graph-max-bs-decode " f"or check CanaryLaunchCapacities.from_args" ) if num_tokens > self._write_entry_capacity: diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index 9e07ac3e4..323d4c309 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -1375,7 +1375,7 @@ class CommunicateWithAllReduceAndLayerNormFn: # - During CP extend: zigzag split guarantees all CP ranks have non-zero tokens, # so no rank hits this path while others proceed to the allgather. # - During decode: moe_cp allgather is skipped (guarded by is_context_parallel_extend). - # - CUDA graph warmup: not applicable when --disable-piecewise-cuda-graph is used. + # - CUDA graph warmup: not applicable when --cuda-graph-backend-prefill=disabled is used. if hidden_states.shape[0] == 0: return hidden_states, residual diff --git a/python/sglang/srt/layers/moe/mega_moe.py b/python/sglang/srt/layers/moe/mega_moe.py index 0ecf88497..1f3f83344 100644 --- a/python/sglang/srt/layers/moe/mega_moe.py +++ b/python/sglang/srt/layers/moe/mega_moe.py @@ -224,8 +224,8 @@ def _run_mega_routed( assert num_tokens <= num_max_tokens_per_rank, ( f"mega MoE: num_tokens={num_tokens} exceeds cap " f"SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=" - f"{num_max_tokens_per_rank}; raise the env var or shrink " - f"cuda_graph_max_bs / chunked_prefill_size accordingly" + f"{num_max_tokens_per_rank}; raise the env var or lower " + f"--cuda-graph-max-bs-decode / --chunked-prefill-size accordingly" ) buf = _get_mega_moe_symm_buffer( diff --git a/python/sglang/srt/layers/radix_attention.py b/python/sglang/srt/layers/radix_attention.py index 64bd108e6..15aabe904 100644 --- a/python/sglang/srt/layers/radix_attention.py +++ b/python/sglang/srt/layers/radix_attention.py @@ -314,7 +314,7 @@ def _unified_attention_with_output_impl( sinks: Optional[torch.Tensor] = None, attn_sink: Optional[torch.Tensor] = None, # MLA / TRT-LLM / NSA paths pass these through RadixAttention.forward(**kwargs); - # they must appear in the schema when --enforce-piecewise-cuda-graph is on. + # they must appear in the schema when --cuda-graph-backend-prefill=tc_piecewise is on. cos_sin_cache: Optional[torch.Tensor] = None, is_neox: Optional[bool] = None, llama_4_scaling: Optional[torch.Tensor] = None, diff --git a/python/sglang/srt/mem_cache/storage/nixl/README.md b/python/sglang/srt/mem_cache/storage/nixl/README.md index 4d3edbacf..32e6cbec3 100644 --- a/python/sglang/srt/mem_cache/storage/nixl/README.md +++ b/python/sglang/srt/mem_cache/storage/nixl/README.md @@ -206,7 +206,7 @@ Important details from this validation: - Use a real `.toml` file path with `--hicache-storage-backend-extra-config`. - For this validated path, the storage directory was provided through `SGLANG_HICACHE_NIXL_BACKEND_STORAGE_DIR`. -- Use `--mamba-scheduler-strategy extra_buffer` to support page sizes larger than 1. +- Use `--mamba-radix-cache-strategy extra_buffer` to support page sizes larger than 1. Example TOML file: @@ -237,7 +237,7 @@ export SGLANG_HICACHE_NIXL_BACKEND_STORAGE_DIR=/tmp/sglang_nixl_e2e_storage --disable-cuda-graph \ --hicache-storage-backend nixl \ --hicache-storage-backend-extra-config @/tmp/nixl.config.toml \ - --mamba-scheduler-strategy extra_buffer + --mamba-radix-cache-strategy extra_buffer ``` Expected behavior for this validated setup: diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index 7b5b75fb7..6c22e5517 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -193,7 +193,6 @@ from sglang.srt.server_args import ( # noqa: F401 (re-export) CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACKENDS, ServerArgs, add_chunked_prefix_cache_attention_backend, - get_global_server_args, ) from sglang.srt.speculative.adaptive_spec_params import ( resolve_candidate_steps_from_config, diff --git a/python/sglang/srt/runtime_context.py b/python/sglang/srt/runtime_context.py index 8249119fe..e6687cf9f 100644 --- a/python/sglang/srt/runtime_context.py +++ b/python/sglang/srt/runtime_context.py @@ -26,8 +26,8 @@ user's raw input, kept **read-only** for debug and reproduction; what resolution decided lives in the declarations (``resolution_result``) and, for business code, in the namespace bags below -- never on this object's fields. The context owns the storage: publishing goes through ``RuntimeContext.set_server_args`` (the legacy -``set_global_server_args_for_scheduler`` / ``get_global_server_args`` are thin -shims over this slot). +``set_global_server_args_for_scheduler`` is a thin shim over this slot; +``get_global_server_args`` is retired and raises). ``get_exec()`` / ``get_memory()`` / ``get_schedule()`` / ``get_device()`` / ``get_model()`` / ``get_spec()`` / ``get_lora()`` / ``get_mm()`` / diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 9576e14a2..f970c14ef 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -41,16 +41,13 @@ import logging import tempfile import uuid from contextlib import contextmanager -from typing import Any +from typing import Any, NoReturn from sglang.kernels.ops.kv_canary.consts import RealKvHashMode from sglang.srt.arg_groups.arg_utils import ( add_cli_args_from_dataclass, ) from sglang.srt.arg_groups.argparse_actions import ( - DeprecatedAction, - DeprecatedAliasStoreAction, - DeprecatedStoreConstAction, DeprecatedStoreTrueAction, ) from sglang.srt.arg_groups.model_override_base import ep_joiner_of, ep_scale_joiner_of @@ -61,13 +58,8 @@ from sglang.srt.arg_groups.overrides import ( ) from sglang.srt.environ import envs from sglang.srt.function_call.function_call_parser import FunctionCallParser -from sglang.srt.model_executor.cuda_graph_config import Backend from sglang.srt.parser.reasoning_parser import ReasoningParser -from sglang.srt.runtime_context import ( - get_context, - get_platform, - publish, -) +from sglang.srt.runtime_context import get_platform, publish from sglang.srt.speculative.decoupled_spec_io import DecoupledSpecIpcConfig from sglang.srt.utils.network import NetworkAddress, get_free_port, wait_port_available @@ -210,8 +202,9 @@ class ServerArgs: A few arguments cannot use the annotation style and must be registered manually in ``add_cli_args``: - - **Deprecated flags** that redirect to another field via - ``DeprecatedAction`` / ``DeprecatedAliasStoreAction`` / etc. + - **Deprecated flags** that redirect to another field via one of the + ``Deprecated*Action`` classes in ``arg_groups/argparse_actions.py`` + (that module's header says which shape fits which migration). - **Dynamic choices** computed at runtime (e.g. ``reasoning_parser`` whose choices come from a plugin registry). - The ``--config`` meta-argument (not a dataclass field). @@ -434,175 +427,14 @@ class ServerArgs: ) # --- Deprecated argument registrations --- - parser.add_argument( - "--enable-expert-distribution-metrics", - action=DeprecatedAction, - error_message=( - "--enable-expert-distribution-metrics is no longer supported. Use " - "--expert-balancedness-report-mode with one of: off, server_log, " - "prometheus, both." - ), - help=( - "Removed. Use --expert-balancedness-report-mode with one of: " - "off, server_log, prometheus, both." - ), - ) - parser.add_argument( - "--stream-output", - action=DeprecatedStoreTrueAction, - dest="incremental_streaming_output", - new_flag="--incremental-streaming-output", - help="[Deprecated] Use --incremental-streaming-output instead.", - ) - parser.add_argument( - "--prefill-round-robin-balance", - action=DeprecatedAction, - help="Note: --prefill-round-robin-balance is deprecated now.", - ) - parser.add_argument( - "--collect-tokens-histogram", - action=DeprecatedAction, - help="Deprecated. Token histograms are now automatically collected when --enable-metrics is set.", - ) - parser.add_argument( - "--nsa-prefill-backend", - dest="dsa_prefill_backend", - action=DeprecatedAliasStoreAction, - new_flag="--dsa-prefill-backend", - default=argparse.SUPPRESS, - type=str, - choices=[ - "flashmla_sparse", - "flashmla_sparse_q8", - "flashmla_kv", - "flashmla_auto", - "flashinfer_sparse_mla", - "fa3", - "tilelang", - "aiter", - "trtllm", - ], - help="[Deprecated] Use --dsa-prefill-backend instead.", - ) - parser.add_argument( - "--nsa-decode-backend", - dest="dsa_decode_backend", - action=DeprecatedAliasStoreAction, - new_flag="--dsa-decode-backend", - default=argparse.SUPPRESS, - type=str, - choices=[ - "flashmla_sparse", - "flashmla_sparse_q8", - "flashmla_kv", - "flashmla_auto", - "flashinfer_sparse_mla", - "fa3", - "tilelang", - "aiter", - "trtllm", - ], - help="[Deprecated] Use --dsa-decode-backend instead.", - ) - parser.add_argument( - "--speculative-dflash-draft-window-size", - type=int, - dest="speculative_draft_window_size", - action=DeprecatedAliasStoreAction, - new_flag="--speculative-draft-window-size", - help=argparse.SUPPRESS, - ) - parser.add_argument( - "--mamba-scheduler-strategy", - dest="mamba_radix_cache_strategy", - type=str, - action=DeprecatedAliasStoreAction, - new_flag="--mamba-radix-cache-strategy", - default=ServerArgs.mamba_radix_cache_strategy, - help="Deprecated alias for --mamba-radix-cache-strategy.", - ) - parser.add_argument( - "--cuda-graph-max-bs", - type=int, - action=DeprecatedAliasStoreAction, - new_flag="--cuda-graph-max-bs-decode", - dest="cuda_graph_max_bs_decode", - help="Deprecated alias for --cuda-graph-max-bs-decode.", - ) - parser.add_argument( - "--cuda-graph-bs", - type=int, - nargs="+", - action=DeprecatedAliasStoreAction, - new_flag="--cuda-graph-bs-decode", - dest="cuda_graph_bs_decode", - help="Deprecated alias for --cuda-graph-bs-decode.", - ) + # `disable_cuda_graph` is `no_cli=True`, so this deprecated spelling is + # its only command-line entry point. parser.add_argument( "--disable-cuda-graph", action=DeprecatedStoreTrueAction, new_flag="--cuda-graph-backend-{decode,prefill}=disabled", help="Deprecated. Use --cuda-graph-backend-{decode,prefill}=disabled instead.", ) - parser.add_argument( - "--enable-breakable-cuda-graph", - action=DeprecatedStoreConstAction, - dest="cuda_graph_backend_prefill", - const_value=Backend.BREAKABLE, - new_flag="--cuda-graph-backend-prefill=breakable", - help="Deprecated alias for --cuda-graph-backend-prefill=breakable.", - ) - parser.add_argument( - "--disable-piecewise-cuda-graph", - action=DeprecatedStoreConstAction, - dest="cuda_graph_backend_prefill", - const_value=Backend.DISABLED, - new_flag="--cuda-graph-backend-prefill=disabled", - help="Deprecated alias for --cuda-graph-backend-prefill=disabled.", - ) - parser.add_argument( - "--enforce-piecewise-cuda-graph", - action=DeprecatedStoreConstAction, - dest="cuda_graph_backend_prefill", - const_value=Backend.TC_PIECEWISE, - new_flag="--cuda-graph-backend-prefill=tc_piecewise", - help="Deprecated alias for --cuda-graph-backend-prefill=tc_piecewise. " - "Explicitly setting the prefill backend now skips the auto-disable " - "cascade automatically.", - ) - parser.add_argument( - "--piecewise-cuda-graph-tokens", - type=int, - nargs="+", - action=DeprecatedAliasStoreAction, - new_flag="--cuda-graph-bs-prefill", - dest="cuda_graph_bs_prefill", - help="Deprecated alias for --cuda-graph-bs-prefill.", - ) - parser.add_argument( - "--piecewise-cuda-graph-compiler", - type=str, - choices=["eager", "inductor"], - action=DeprecatedAliasStoreAction, - new_flag="--cuda-graph-tc-compiler", - dest="cuda_graph_tc_compiler", - help="Deprecated alias for --cuda-graph-tc-compiler.", - ) - parser.add_argument( - "--piecewise-cuda-graph-max-tokens", - type=int, - action=DeprecatedAliasStoreAction, - new_flag="--cuda-graph-max-bs-prefill", - dest="cuda_graph_max_bs_prefill", - help="Deprecated alias for --cuda-graph-max-bs-prefill.", - ) - parser.add_argument( - "--enable-gdn-replayssm-spec", - dest="enable_linear_replayssm_spec", - action=DeprecatedStoreTrueAction, - new_flag="--enable-linear-replayssm-spec", - help="[Deprecated] Use --enable-linear-replayssm-spec instead.", - ) parser.add_argument( "--enable-flashinfer-allreduce-fusion", action="store_true", @@ -648,6 +480,11 @@ class ServerArgs: # the record exists to remember, and the decision it meant to record # belongs in the stash, where it carries a source and does not destroy # the input it was derived from. + # Underscore names are mostly the record's own bookkeeping -- + # `_input_frozen`, `_raw_input`, `_resolved_overrides`, the memo slots + # -- which resolution writes on purpose. A *field* spelled that way is + # still configuration, so the test cannot be on spelling alone or that + # one leaf stays writable on a read-only record. if not name.startswith("_") or name in _underscore_field_names(): if getattr(self, "_input_frozen", False): raise AttributeError( @@ -770,11 +607,6 @@ def m3_fp8_attn_gemm_enabled(args) -> bool: ) -# NOTE: The process-wide ServerArgs is owned by the runtime context -# (sglang.srt.runtime_context). The two functions below are LEGACY shims kept -# for the existing call-sites; they publish/read the same live object by -# reference. Do not add new call-sites. -# Imports are in-function so the two modules stay cycle-free at import time. @functools.lru_cache(maxsize=1) def _underscore_field_names() -> frozenset: """Real dataclass fields whose names start with an underscore. @@ -792,6 +624,12 @@ def _underscore_field_names() -> frozenset: ) +# NOTE: The process-wide ServerArgs is owned by the runtime context +# (sglang.srt.runtime_context). The two publish functions below are LEGACY +# shims kept for the existing call-sites; they hand over the same live object +# by reference. Do not add new call-sites. The third function is retired and +# only raises. +# Imports are in-function so the two modules stay cycle-free at import time. def set_global_server_args_for_scheduler(server_args: ServerArgs): """Legacy publish shim (role=scheduler) — prefer ``runtime_context.publish(server_args, role=...)`` in new code.""" @@ -806,11 +644,23 @@ def set_global_server_args_for_tokenizer(server_args: ServerArgs): publish(server_args, role="tokenizer") -def get_global_server_args() -> ServerArgs: - """Legacy accessor shim — prefer ``get_server_args()`` from - ``sglang.srt.runtime_context`` in new code.""" +def get_global_server_args() -> NoReturn: + """Retired. It raises, because what it used to return is the problem: the + record answers with the operator's *input*, so a caller reading a field + resolution decided got a stale value and no error. - return get_context().server_args + The name survives so that a caller importing it from this module lands on + a message instead of an ImportError. Annotated ``NoReturn`` so a type + checker rejects the call rather than accepting the attribute access after + it. The message lives once, in the exception. + """ + raise RuntimeError( + "get_global_server_args() is retired. Read the value that is in effect " + "from its namespace bag -- `get_exec().kernel.attention_backend`, " + "`get_schedule().max_running_requests`, and so on " + "(sglang.srt.runtime_context). For the operator's raw input, which is a " + "different question, `get_server_args()` still answers it." + ) @contextmanager diff --git a/python/sglang/test/ascend/test_npu_logging.py b/python/sglang/test/ascend/test_npu_logging.py index 6ef799f96..4be642968 100644 --- a/python/sglang/test/ascend/test_npu_logging.py +++ b/python/sglang/test/ascend/test_npu_logging.py @@ -25,7 +25,7 @@ class TestNPULoggingBase(CustomTestCase): [Test Target] --log-requests; --log-requests-level; --log-requests-target; --uvicorn-access-log-exclude-prefixes; --enable-metrics; --enable-metrics-for-all-scheduler; --bucket-time-to-first-token; --bucket-inter-token-latency; --bucket-e2e-request-latency; - --collect-tokens-histogram; --prompt-tokens-buckets; --generation-tokens-buckets; + --prompt-tokens-buckets; --generation-tokens-buckets; --tokenizer-metrics-custom-labels-header; --tokenizer-metrics-allowed-custom-labels; --gc-warning-threshold-secs """ diff --git a/python/sglang/test/chunked_prefill_test_utils.py b/python/sglang/test/chunked_prefill_test_utils.py index d1a86b0d7..45b79636c 100644 --- a/python/sglang/test/chunked_prefill_test_utils.py +++ b/python/sglang/test/chunked_prefill_test_utils.py @@ -34,7 +34,7 @@ KV_CANARY_ARGS: List[str] = [ "partial", "--kv-canary-sweep-interval", "100", - "--disable-piecewise-cuda-graph", + "--cuda-graph-backend-prefill=disabled", ] diff --git a/python/sglang/test/kv_canary/utils.py b/python/sglang/test/kv_canary/utils.py index 92ec31e93..72f78c177 100644 --- a/python/sglang/test/kv_canary/utils.py +++ b/python/sglang/test/kv_canary/utils.py @@ -17,7 +17,7 @@ def build_canary_server_args( args = [ "--kv-canary", kv_canary_mode.value, - "--disable-piecewise-cuda-graph", + "--cuda-graph-backend-prefill=disabled", "--context-length", "16384", *extra_server_args, diff --git a/python/sglang/test/mock_model/utils.py b/python/sglang/test/mock_model/utils.py index 65cb2dbdc..591d4e7bf 100644 --- a/python/sglang/test/mock_model/utils.py +++ b/python/sglang/test/mock_model/utils.py @@ -21,7 +21,7 @@ _MOCK_MODEL_SERVER_ARGS_NO_CANARY: list[str] = [ "dummy", "--sampling-backend", "token_oracle", - "--disable-piecewise-cuda-graph", + "--cuda-graph-backend-prefill=disabled", ] diff --git a/scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/2p1d-ep16-mtp.yaml b/scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/2p1d-ep16-mtp.yaml index bdf327cc9..4b6c233a3 100644 --- a/scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/2p1d-ep16-mtp.yaml +++ b/scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/2p1d-ep16-mtp.yaml @@ -70,7 +70,7 @@ runtime: # (no chunking). Largest buffer = max_total_num_tokens * 256 B, so the # ceiling is 16,777,216 tokens; MTP runs at 7,000,000 (the validated value, # well under the ceiling and below every MTP leg's natural pool). - decode_extra_flags: "--max-total-tokens 7000000 --cuda-graph-bs 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 --prefill-round-robin-balance" + decode_extra_flags: "--max-total-tokens 7000000 --cuda-graph-bs-decode 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128" prefill_extra_env: MORI_MAX_DISPATCH_TOKENS_PREFILL: 8192 MORI_MAX_DISPATCH_TOKENS_DECODE: 256 diff --git a/scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/2p1d-ep16.yaml b/scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/2p1d-ep16.yaml index 89ee62d9a..7c097b426 100644 --- a/scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/2p1d-ep16.yaml +++ b/scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/2p1d-ep16.yaml @@ -69,7 +69,7 @@ runtime: # region over 4 GiB and mori registers each KV buffer as one region # (no chunking). Largest buffer = max_total_num_tokens * 256 B, so the # ceiling is 16,777,216 tokens; 16,000,000 leaves headroom. - decode_extra_flags: "--max-total-tokens 16000000 --cuda-graph-bs 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 --prefill-round-robin-balance" + decode_extra_flags: "--max-total-tokens 16000000 --cuda-graph-bs-decode 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128" prefill_extra_env: MORI_MAX_DISPATCH_TOKENS_PREFILL: 8192 MORI_MAX_DISPATCH_TOKENS_DECODE: 256 diff --git a/scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/2p1d-ep16-mtp.yaml b/scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/2p1d-ep16-mtp.yaml index 7765bd3c4..4d591c134 100644 --- a/scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/2p1d-ep16-mtp.yaml +++ b/scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/2p1d-ep16-mtp.yaml @@ -70,7 +70,7 @@ runtime: # (no chunking). Largest buffer = max_total_num_tokens * 256 B, so the # ceiling is 16,777,216 tokens; MTP runs at 7,000,000 (the validated value, # well under the ceiling and below every MTP leg's natural pool). - decode_extra_flags: "--max-total-tokens 7000000 --cuda-graph-bs 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 --prefill-round-robin-balance" + decode_extra_flags: "--max-total-tokens 7000000 --cuda-graph-bs-decode 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128" prefill_extra_env: MORI_MAX_DISPATCH_TOKENS_PREFILL: 8192 MORI_MAX_DISPATCH_TOKENS_DECODE: 256 diff --git a/scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/2p1d-ep16.yaml b/scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/2p1d-ep16.yaml index 8b086b1d1..21fec826f 100644 --- a/scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/2p1d-ep16.yaml +++ b/scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/2p1d-ep16.yaml @@ -69,7 +69,7 @@ runtime: # region over 4 GiB and mori registers each KV buffer as one region # (no chunking). Largest buffer = max_total_num_tokens * 256 B, so the # ceiling is 16,777,216 tokens; 16,000,000 leaves headroom. - decode_extra_flags: "--max-total-tokens 16000000 --cuda-graph-bs 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 --prefill-round-robin-balance" + decode_extra_flags: "--max-total-tokens 16000000 --cuda-graph-bs-decode 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128" prefill_extra_env: MORI_MAX_DISPATCH_TOKENS_PREFILL: 8192 MORI_MAX_DISPATCH_TOKENS_DECODE: 256 diff --git a/scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/2p1d-ep16-mtp.yaml b/scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/2p1d-ep16-mtp.yaml index 25bc577d3..63ec5d78f 100644 --- a/scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/2p1d-ep16-mtp.yaml +++ b/scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/2p1d-ep16-mtp.yaml @@ -61,7 +61,7 @@ runtime: decode_max_running_requests: 1024 common_extra_flags: "--moe-dense-tp-size 1 --enable-dp-lm-head --decode-log-interval 100 --watchdog-timeout 3600 --load-balance-method round_robin --dist-timeout 3600" prefill_extra_flags: "--context-length 9217 --max-total-tokens 262144" - decode_extra_flags: "--disable-cuda-graph --prefill-round-robin-balance" + decode_extra_flags: "--disable-cuda-graph" prefill_extra_env: MORI_MAX_DISPATCH_TOKENS_PREFILL: 8192 MORI_MAX_DISPATCH_TOKENS_DECODE: 256 diff --git a/scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/2p1d-ep16.yaml b/scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/2p1d-ep16.yaml index 4d1759d4a..cb85d2737 100644 --- a/scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/2p1d-ep16.yaml +++ b/scripts/ci/slurm/recipes/mi355x-fp4/glm52/1k1k/2p1d-ep16.yaml @@ -61,7 +61,7 @@ runtime: decode_max_running_requests: 1024 common_extra_flags: "--moe-dense-tp-size 1 --enable-dp-lm-head --decode-log-interval 100 --watchdog-timeout 3600 --load-balance-method round_robin --dist-timeout 3600" prefill_extra_flags: "--context-length 9217 --max-total-tokens 262144" - decode_extra_flags: "--disable-cuda-graph --prefill-round-robin-balance" + decode_extra_flags: "--disable-cuda-graph" prefill_extra_env: MORI_MAX_DISPATCH_TOKENS_PREFILL: 8192 MORI_MAX_DISPATCH_TOKENS_DECODE: 256 diff --git a/scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/2p1d-ep16-mtp.yaml b/scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/2p1d-ep16-mtp.yaml index 653c86eeb..689943e93 100644 --- a/scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/2p1d-ep16-mtp.yaml +++ b/scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/2p1d-ep16-mtp.yaml @@ -70,7 +70,7 @@ runtime: # (no chunking). Largest buffer = max_total_num_tokens * 256 B, so the # ceiling is 16,777,216 tokens; MTP runs at 7,000,000 (the validated value, # well under the ceiling and below every MTP leg's natural pool). - decode_extra_flags: "--max-total-tokens 7000000 --cuda-graph-bs 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 --prefill-round-robin-balance" + decode_extra_flags: "--max-total-tokens 7000000 --cuda-graph-bs-decode 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128" prefill_extra_env: MORI_MAX_DISPATCH_TOKENS_PREFILL: 8192 MORI_MAX_DISPATCH_TOKENS_DECODE: 256 diff --git a/scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/2p1d-ep16.yaml b/scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/2p1d-ep16.yaml index a71da7fc1..98f2fd228 100644 --- a/scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/2p1d-ep16.yaml +++ b/scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/2p1d-ep16.yaml @@ -69,7 +69,7 @@ runtime: # region over 4 GiB and mori registers each KV buffer as one region # (no chunking). Largest buffer = max_total_num_tokens * 256 B, so the # ceiling is 16,777,216 tokens; 16,000,000 leaves headroom. - decode_extra_flags: "--max-total-tokens 16000000 --cuda-graph-bs 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 --prefill-round-robin-balance" + decode_extra_flags: "--max-total-tokens 16000000 --cuda-graph-bs-decode 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128" prefill_extra_env: MORI_MAX_DISPATCH_TOKENS_PREFILL: 8192 MORI_MAX_DISPATCH_TOKENS_DECODE: 256 diff --git a/scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/2p1d-ep16-mtp.yaml b/scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/2p1d-ep16-mtp.yaml index f5928008a..e00b406ae 100644 --- a/scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/2p1d-ep16-mtp.yaml +++ b/scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/2p1d-ep16-mtp.yaml @@ -70,7 +70,7 @@ runtime: # (no chunking). Largest buffer = max_total_num_tokens * 256 B, so the # ceiling is 16,777,216 tokens; MTP runs at 7,000,000 (the validated value, # well under the ceiling and below every MTP leg's natural pool). - decode_extra_flags: "--max-total-tokens 7000000 --cuda-graph-bs 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 --prefill-round-robin-balance" + decode_extra_flags: "--max-total-tokens 7000000 --cuda-graph-bs-decode 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128" prefill_extra_env: MORI_MAX_DISPATCH_TOKENS_PREFILL: 8192 MORI_MAX_DISPATCH_TOKENS_DECODE: 256 diff --git a/scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/2p1d-ep16.yaml b/scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/2p1d-ep16.yaml index c0591de6f..f88165431 100644 --- a/scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/2p1d-ep16.yaml +++ b/scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/2p1d-ep16.yaml @@ -69,7 +69,7 @@ runtime: # region over 4 GiB and mori registers each KV buffer as one region # (no chunking). Largest buffer = max_total_num_tokens * 256 B, so the # ceiling is 16,777,216 tokens; 16,000,000 leaves headroom. - decode_extra_flags: "--max-total-tokens 16000000 --cuda-graph-bs 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 --prefill-round-robin-balance" + decode_extra_flags: "--max-total-tokens 16000000 --cuda-graph-bs-decode 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128" prefill_extra_env: MORI_MAX_DISPATCH_TOKENS_PREFILL: 8192 MORI_MAX_DISPATCH_TOKENS_DECODE: 256 diff --git a/scripts/ci/slurm/recipes/mi355x-fp8/kimik26/1k1k/2p1d-ep16-mtp-mxfp4.yaml b/scripts/ci/slurm/recipes/mi355x-fp8/kimik26/1k1k/2p1d-ep16-mtp-mxfp4.yaml index 96d19d371..28eb39729 100644 --- a/scripts/ci/slurm/recipes/mi355x-fp8/kimik26/1k1k/2p1d-ep16-mtp-mxfp4.yaml +++ b/scripts/ci/slurm/recipes/mi355x-fp8/kimik26/1k1k/2p1d-ep16-mtp-mxfp4.yaml @@ -84,7 +84,7 @@ runtime: decode_max_running_requests: 1024 common_extra_flags: "--moe-dense-tp-size 1 --enable-dp-lm-head --decode-log-interval 100 --watchdog-timeout 3600 --load-balance-method round_robin" prefill_extra_flags: "--context-length 9217 --max-total-tokens 262144" - decode_extra_flags: "--disable-cuda-graph --prefill-round-robin-balance" + decode_extra_flags: "--disable-cuda-graph" prefill_extra_env: MORI_MAX_DISPATCH_TOKENS_PREFILL: 8192 MORI_MAX_DISPATCH_TOKENS_DECODE: 256 diff --git a/scripts/ci/slurm/recipes/mi355x-fp8/kimik26/1k1k/2p1d-ep16-mxfp4.yaml b/scripts/ci/slurm/recipes/mi355x-fp8/kimik26/1k1k/2p1d-ep16-mxfp4.yaml index d3f73abe0..5a355e8c0 100644 --- a/scripts/ci/slurm/recipes/mi355x-fp8/kimik26/1k1k/2p1d-ep16-mxfp4.yaml +++ b/scripts/ci/slurm/recipes/mi355x-fp8/kimik26/1k1k/2p1d-ep16-mxfp4.yaml @@ -84,7 +84,7 @@ runtime: decode_max_running_requests: 1024 common_extra_flags: "--moe-dense-tp-size 1 --enable-dp-lm-head --decode-log-interval 100 --watchdog-timeout 3600 --load-balance-method round_robin" prefill_extra_flags: "--context-length 9217 --max-total-tokens 262144" - decode_extra_flags: "--disable-cuda-graph --prefill-round-robin-balance" + decode_extra_flags: "--disable-cuda-graph" prefill_extra_env: MORI_MAX_DISPATCH_TOKENS_PREFILL: 8192 MORI_MAX_DISPATCH_TOKENS_DECODE: 256 diff --git a/scripts/playground/bench_speculative.py b/scripts/playground/bench_speculative.py index 6638a1796..c515ef4e0 100644 --- a/scripts/playground/bench_speculative.py +++ b/scripts/playground/bench_speculative.py @@ -188,7 +188,7 @@ def main(args, server_args): other_args.extend( [ - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", batch_size, "--mem-fraction-static", server_args.mem_fraction_static, diff --git a/scripts/playground/disaggregation/cli-so.py b/scripts/playground/disaggregation/cli-so.py index 7ccafc7ed..d68eb7c26 100644 --- a/scripts/playground/disaggregation/cli-so.py +++ b/scripts/playground/disaggregation/cli-so.py @@ -31,4 +31,4 @@ response = requests.post( print(response.json()) -# python3 -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --trust-remote-code --disaggregation-mode prefill --tp 2 --disaggregation-ib-device mlx5_roce0,mlx5_roce1 --speculative-algorithm EAGLE --speculative-draft-model-path lmsys/sglang-EAGLE-llama2-chat-7B --speculative-num-steps 3 --speculative-eagle-topk 4 --speculative-num-draft-tokens 16 --cuda-graph-max-bs 8 --host 127.0.0.1 --port 8100 +# python3 -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --trust-remote-code --disaggregation-mode prefill --tp 2 --disaggregation-ib-device mlx5_roce0,mlx5_roce1 --speculative-algorithm EAGLE --speculative-draft-model-path lmsys/sglang-EAGLE-llama2-chat-7B --speculative-num-steps 3 --speculative-eagle-topk 4 --speculative-num-draft-tokens 16 --cuda-graph-max-bs-decode 8 --host 127.0.0.1 --port 8100 diff --git a/test/manual/4-gpu-models/test_qwen35_fp4_triton.py b/test/manual/4-gpu-models/test_qwen35_fp4_triton.py index 5d7535acc..983f7caf8 100644 --- a/test/manual/4-gpu-models/test_qwen35_fp4_triton.py +++ b/test/manual/4-gpu-models/test_qwen35_fp4_triton.py @@ -21,7 +21,7 @@ class TestQwen35FP4(CustomTestCase): "4", "--chunked-prefill-size", "2048", - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer", "--mamba-track-interval", "128", diff --git a/test/manual/4-gpu-models/test_qwen35_models_archived.py b/test/manual/4-gpu-models/test_qwen35_models_archived.py index f0443c482..cb25d9f36 100644 --- a/test/manual/4-gpu-models/test_qwen35_models_archived.py +++ b/test/manual/4-gpu-models/test_qwen35_models_archived.py @@ -37,7 +37,7 @@ class TestQwen35FP4(CustomTestCase): "4", "--chunked-prefill-size", "2048", - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer", "--mamba-track-interval", "128", @@ -103,7 +103,7 @@ class TestQwen35FP4MTP(ReasoningTokenUsageMixin, CustomTestCase): "4", "--chunked-prefill-size", "2048", - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer", "--mamba-track-interval", "128", diff --git a/test/manual/4-gpu-models/test_qwen3_next_models.py b/test/manual/4-gpu-models/test_qwen3_next_models.py index 2e3c4ec60..72fd50275 100644 --- a/test/manual/4-gpu-models/test_qwen3_next_models.py +++ b/test/manual/4-gpu-models/test_qwen3_next_models.py @@ -20,7 +20,7 @@ class TestQwen3Next( "4", "--chunked-prefill-size", "1024", - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer", "--mamba-track-interval", "2", @@ -45,7 +45,7 @@ class TestQwen3NextLazyExtraBuffer( "4", "--chunked-prefill-size", "1024", - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer_lazy", "--mamba-track-interval", "2", diff --git a/test/manual/4-gpu-models/test_qwen3_next_models_mtp_archived.py b/test/manual/4-gpu-models/test_qwen3_next_models_mtp_archived.py index 3f6142aee..625f49285 100644 --- a/test/manual/4-gpu-models/test_qwen3_next_models_mtp_archived.py +++ b/test/manual/4-gpu-models/test_qwen3_next_models_mtp_archived.py @@ -34,7 +34,7 @@ class TestQwen3NextMTP(GSM8KMixin, KLDivergenceMixin, DefaultServerBase): "4", "--chunked-prefill-size", "2048", - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "no_buffer", "--disable-radix-cache", ] diff --git a/test/manual/chunked_prefill/test_e2e_hybrid_swa.py b/test/manual/chunked_prefill/test_e2e_hybrid_swa.py index 6d2e5c72a..fcab29c78 100644 --- a/test/manual/chunked_prefill/test_e2e_hybrid_swa.py +++ b/test/manual/chunked_prefill/test_e2e_hybrid_swa.py @@ -14,7 +14,7 @@ class TestChunkedFeatureHybridSWA(ChunkedTestBase): feature_args = [ "--mem-fraction-static", "0.70", - "--disable-piecewise-cuda-graph", + "--cuda-graph-backend-prefill=disabled", ] diff --git a/test/manual/ep/test_eplb.py b/test/manual/ep/test_eplb.py index 2034c1b13..5dd97364e 100755 --- a/test/manual/ep/test_eplb.py +++ b/test/manual/ep/test_eplb.py @@ -73,7 +73,6 @@ class _BaseTestDynamicEPLB(CustomTestCase): "--expert-distribution-recorder-buffer-size", "50", # TODO pr-chain: enable later - # "--enable-expert-distribution-metrics", # TODO auto determine these flags "--expert-distribution-recorder-mode", "stat", diff --git a/test/manual/lora/run_spec_lora_matrix.py b/test/manual/lora/run_spec_lora_matrix.py index 54c32403b..c38114b8f 100644 --- a/test/manual/lora/run_spec_lora_matrix.py +++ b/test/manual/lora/run_spec_lora_matrix.py @@ -14,7 +14,7 @@ without spec, then with spec) and runs four checks: 2. distinct — each adapter's output differs from base (LoRA really applied) 3. mixed — a batch interleaving every adapter matches the solo outputs (crossed verify segments serve a request the wrong adapter) - 4. eager — a batch wider than --cuda-graph-max-bs still matches + 4. eager — a batch wider than --cuda-graph-max-bs-decode still matches (exercises the non-cuda-graph target-verify path) It also reports per-adapter accept length from each response's @@ -99,7 +99,7 @@ CONFIGS = { common_args=[ "--mem-fraction-static=0.7", "--max-lora-rank=128", - "--cuda-graph-max-bs=2", + "--cuda-graph-max-bs-decode=2", ], tp=1, ), @@ -813,7 +813,7 @@ def main() -> int: "--wide-batch", type=int, default=16, - help="batch size for the eager-path check; keep it above --cuda-graph-max-bs", + help="batch size for the eager-path check; keep it above --cuda-graph-max-bs-decode", ) parser.add_argument( "--lora-backend", help="override the config's LoRA kernel backend" diff --git a/test/manual/models/test_granite_moe_hybrid.py b/test/manual/models/test_granite_moe_hybrid.py index a8e498523..add4a0bcb 100644 --- a/test/manual/models/test_granite_moe_hybrid.py +++ b/test/manual/models/test_granite_moe_hybrid.py @@ -24,7 +24,7 @@ class TestGraniteMoeHybridExtraBuffer( other_args = [ "--mem-fraction-static", "0.8", - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer", ] diff --git a/test/manual/models/test_nvidia_nemotron_nano_v2.py b/test/manual/models/test_nvidia_nemotron_nano_v2.py index 770f4f544..17ed3836a 100644 --- a/test/manual/models/test_nvidia_nemotron_nano_v2.py +++ b/test/manual/models/test_nvidia_nemotron_nano_v2.py @@ -34,7 +34,7 @@ class TestNvidiaNemotronNanoV2BF16ExtraBuffer( "256", "--mem-fraction-static", "0.8", - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer", ] @@ -88,7 +88,7 @@ class TestNvidiaNemotronNanoV2SpeculativeDecodingExtraBuffer( model = NVIDIA_NEMOTRON_NANO_V2_MODEL gsm8k_accuracy_thres = 0.87 other_args = SPECULATIVE_DECODING_OTHER_ARGS + [ - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer", ] diff --git a/test/manual/nightly/test_vlms_piecewise_cuda_graph.py b/test/manual/nightly/test_vlms_piecewise_cuda_graph.py index ac9b22f4a..83f911f59 100644 --- a/test/manual/nightly/test_vlms_piecewise_cuda_graph.py +++ b/test/manual/nightly/test_vlms_piecewise_cuda_graph.py @@ -135,11 +135,11 @@ class TestVLMPiecewiseCudaGraph(CustomTestCase): api_key=self.api_key, other_args=[ "--trust-remote-code", - "--piecewise-cuda-graph-max-tokens", + "--cuda-graph-max-bs-prefill", "8192", - "--enforce-piecewise-cuda-graph", + "--cuda-graph-backend-prefill=tc_piecewise", "--tp=8", - "--piecewise-cuda-graph-compiler=eager", + "--cuda-graph-tc-compiler=eager", "--disable-radix-cache", "--log-level", log_level, diff --git a/test/manual/nightly/test_vlms_vit_cuda_graph.py b/test/manual/nightly/test_vlms_vit_cuda_graph.py index 19472a145..e86dc06c2 100644 --- a/test/manual/nightly/test_vlms_vit_cuda_graph.py +++ b/test/manual/nightly/test_vlms_vit_cuda_graph.py @@ -140,14 +140,14 @@ class TestVLMViTCudaGraph(CustomTestCase): other_args=[ "--mm-attention-backend", "fa3", - "--enforce-piecewise-cuda-graph", - "--piecewise-cuda-graph-max-tokens", + "--cuda-graph-backend-prefill=tc_piecewise", + "--cuda-graph-max-bs-prefill", "8192", "--chunked-prefill-size", "8192", "--disable-radix-cache", "--disable-overlap-schedule", - "--piecewise-cuda-graph-compiler", + "--cuda-graph-tc-compiler", "eager", ], env=process_env, diff --git a/test/manual/piecewise_cuda_graph/test_piecewise_cuda_graph_support_1_gpu_archived.py b/test/manual/piecewise_cuda_graph/test_piecewise_cuda_graph_support_1_gpu_archived.py index eb806da22..66e8ae784 100644 --- a/test/manual/piecewise_cuda_graph/test_piecewise_cuda_graph_support_1_gpu_archived.py +++ b/test/manual/piecewise_cuda_graph/test_piecewise_cuda_graph_support_1_gpu_archived.py @@ -31,7 +31,7 @@ class TestPiecewiseCudaGraphInternVL25(CustomTestCase): cls.base_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, other_args=[ - "--enforce-piecewise-cuda-graph", + "--cuda-graph-backend-prefill=tc_piecewise", "--disable-radix-cache", ], ) diff --git a/test/manual/piecewise_cudagraph/test_disaggregation_piecewise_cuda_graph.py b/test/manual/piecewise_cudagraph/test_disaggregation_piecewise_cuda_graph.py index 9f2e7bfa0..4246c977e 100644 --- a/test/manual/piecewise_cudagraph/test_disaggregation_piecewise_cuda_graph.py +++ b/test/manual/piecewise_cudagraph/test_disaggregation_piecewise_cuda_graph.py @@ -38,7 +38,7 @@ class TestDisaggregationPiecewiseCudaGraph(PDDisaggregationServerBase): "prefill", "--tp", "1", - "--enforce-piecewise-cuda-graph", + "--cuda-graph-backend-prefill=tc_piecewise", ] prefill_args += cls.transfer_backend + cls.rdma_devices cls.process_prefill = popen_launch_pd_server( diff --git a/test/manual/test_dsa_alias_cli_registry_env.py b/test/manual/test_dsa_alias_cli_registry_env.py index 066b871c2..fb0c0bd04 100644 --- a/test/manual/test_dsa_alias_cli_registry_env.py +++ b/test/manual/test_dsa_alias_cli_registry_env.py @@ -41,12 +41,6 @@ class TestDSAChoicesAndFields(unittest.TestCase): self.assertIn("tilelang", choices) self.assertIn("flashinfer_sparse_mla", choices) - def test_nsa_choices_match_dsa_choices(self): - self.assertEqual( - self.actions["--nsa-prefill-backend"].choices, - self.actions["--dsa-prefill-backend"].choices, - ) - def test_serverargs_has_dsa_fields(self): sa = self.ServerArgs self.assertTrue(hasattr(sa, "dsa_prefill_backend")) @@ -94,63 +88,6 @@ class TestCLICanonicalFlags(unittest.TestCase): args = self._parse(["--attention-backend", "dsa"]) self.assertEqual(args.attention_backend, "dsa") - -class TestCLIDeprecatedFlags(unittest.TestCase): - """Deprecated flags write to canonical attributes and emit logger warning.""" - - def setUp(self): - import logging - - from sglang.srt.server_args import ServerArgs - - self.parser = argparse.ArgumentParser() - ServerArgs.add_cli_args(self.parser) - - # Capture log output to detect deprecation warnings - self.log_records = [] - handler = ( - logging.handlers_collector(self.log_records) - if hasattr(logging, "handlers_collector") - else None - ) - - def _parse(self, extra_args): - return self.parser.parse_args(["--model", "dummy"] + extra_args) - - def _parse_capture_warnings(self, extra_args): - """Parse and capture both warnings.warn and logger output.""" - import io - import logging - - log_stream = io.StringIO() - handler = logging.StreamHandler(log_stream) - handler.setLevel(logging.WARNING) - root = logging.getLogger() - root.addHandler(handler) - try: - args = self._parse(extra_args) - finally: - root.removeHandler(handler) - return args, log_stream.getvalue() - - def test_nsa_prefill_backend_deprecated_writes_to_dsa(self): - args, log_output = self._parse_capture_warnings( - ["--nsa-prefill-backend", "fa3"] - ) - self.assertEqual(args.dsa_prefill_backend, "fa3") - self.assertIn( - "deprecated", - log_output.lower(), - f"Expected deprecation warning in log; got: {log_output!r}", - ) - - def test_nsa_decode_backend_deprecated_writes_to_dsa(self): - args, log_output = self._parse_capture_warnings( - ["--nsa-decode-backend", "tilelang"] - ) - self.assertEqual(args.dsa_decode_backend, "tilelang") - self.assertIn("deprecated", log_output.lower()) - def test_attention_backend_nsa_still_accepted(self): """attention_backend='nsa' still parses without error (registry handles the deprecation).""" args = self._parse(["--attention-backend", "nsa"]) diff --git a/test/manual/test_w4a8_deepseek_v3.py b/test/manual/test_w4a8_deepseek_v3.py index 679e3d663..4440d4792 100644 --- a/test/manual/test_w4a8_deepseek_v3.py +++ b/test/manual/test_w4a8_deepseek_v3.py @@ -63,7 +63,7 @@ class TestDeepseekV3W4Afp8Mtp(CustomTestCase): "--trust-remote-code", "--ep-size", "8", - "--cuda-graph-bs", + "--cuda-graph-bs-decode", "256", "--disable-radix-cache", "--speculative-algorithm", @@ -130,7 +130,7 @@ class TestDeepseekV3W4Afp8DeepepNormal(CustomTestCase): "--trust-remote-code", "--ep-size", "8", - "--cuda-graph-bs", + "--cuda-graph-bs-decode", "256", "--disable-radix-cache", "--moe-a2a-backend", @@ -185,7 +185,7 @@ class TestDeepseekV3W4Afp8DeepepAutoMtp(CustomTestCase): "--trust-remote-code", "--ep-size", "8", - "--cuda-graph-bs", + "--cuda-graph-bs-decode", "256", "--disable-radix-cache", "--moe-a2a-backend", diff --git a/test/registered/amd/accuracy/mi45x/test_deepseek_v4_flash_eval_mi45x.py b/test/registered/amd/accuracy/mi45x/test_deepseek_v4_flash_eval_mi45x.py index f0b84774c..2000b0256 100644 --- a/test/registered/amd/accuracy/mi45x/test_deepseek_v4_flash_eval_mi45x.py +++ b/test/registered/amd/accuracy/mi45x/test_deepseek_v4_flash_eval_mi45x.py @@ -77,7 +77,7 @@ MI45X_DEEPSEEK_V4_FLASH_MODELS = [ "deepseek-v4", "--chunked-prefill-size", "8192", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "256", "--max-running-requests", "256", diff --git a/test/registered/amd/test_deepseek_r1_mxfp4_8gpu.py b/test/registered/amd/test_deepseek_r1_mxfp4_8gpu.py index a6a980e06..1763f0d7e 100644 --- a/test/registered/amd/test_deepseek_r1_mxfp4_8gpu.py +++ b/test/registered/amd/test_deepseek_r1_mxfp4_8gpu.py @@ -40,10 +40,10 @@ class TestDeepseekR1MXFP4(CustomTestCase): "131072", "--model-loader-extra-config", '{"enable_multithread_load": true}', - "--enforce-piecewise-cuda-graph", - "--piecewise-cuda-graph-compiler", + "--cuda-graph-backend-prefill=tc_piecewise", + "--cuda-graph-tc-compiler", "eager", - "--piecewise-cuda-graph-max-tokens", + "--cuda-graph-max-bs-prefill", "8192", ] cls.process = popen_launch_server( diff --git a/test/registered/amd/test_deepseek_v4_flash_fp8_tbo.py b/test/registered/amd/test_deepseek_v4_flash_fp8_tbo.py index 39a5ebcf5..de3bc6651 100644 --- a/test/registered/amd/test_deepseek_v4_flash_fp8_tbo.py +++ b/test/registered/amd/test_deepseek_v4_flash_fp8_tbo.py @@ -105,7 +105,7 @@ class TestDeepseekV4FlashFp8Tbo(CustomTestCase): "fp8_e4m3", "--max-running-requests", "512", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "512", "--page-size", "256", diff --git a/test/registered/amd/test_deepseek_v4_pro_fp4_dspark.py b/test/registered/amd/test_deepseek_v4_pro_fp4_dspark.py index 944f7493e..d3a9d4fee 100644 --- a/test/registered/amd/test_deepseek_v4_pro_fp4_dspark.py +++ b/test/registered/amd/test_deepseek_v4_pro_fp4_dspark.py @@ -145,7 +145,7 @@ class TestDeepseekV4DSparkUnifiedKVGSM8K(CustomTestCase): "fp8_e4m3", "--chunked-prefill-size", "65536", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "512", "--max-running-requests", "512", diff --git a/test/registered/amd/test_deepseek_v4_pro_fp4_tbo.py b/test/registered/amd/test_deepseek_v4_pro_fp4_tbo.py index 65995474e..48fc8f306 100644 --- a/test/registered/amd/test_deepseek_v4_pro_fp4_tbo.py +++ b/test/registered/amd/test_deepseek_v4_pro_fp4_tbo.py @@ -95,7 +95,7 @@ class TestDeepseekV4ProFp4Tbo(CustomTestCase): "fp8_e4m3", "--max-running-requests", "512", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "512", "--page-size", "256", diff --git a/test/registered/amd/test_deepseek_v4_pro_fp4_tbo_mtp.py b/test/registered/amd/test_deepseek_v4_pro_fp4_tbo_mtp.py index 90713f77d..ff73c6d14 100644 --- a/test/registered/amd/test_deepseek_v4_pro_fp4_tbo_mtp.py +++ b/test/registered/amd/test_deepseek_v4_pro_fp4_tbo_mtp.py @@ -93,7 +93,7 @@ class TestDeepseekV4ProFp4TboMTP(CustomTestCase): "4", "--max-running-requests", "512", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "512", "--page-size", "256", diff --git a/test/registered/attention/test_qwen35_deterministic.py b/test/registered/attention/test_qwen35_deterministic.py index 6e8c19a88..60b9fee3c 100644 --- a/test/registered/attention/test_qwen35_deterministic.py +++ b/test/registered/attention/test_qwen35_deterministic.py @@ -30,7 +30,7 @@ class TestQwen35Fa3Deterministic(TestDeterministicBase): "--attention-backend", "fa3", "--skip-server-warmup", - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer", "--enable-flashinfer-allreduce-fusion", "--tokenizer-worker-num", diff --git a/test/registered/core/test_basic_sanity_dflash.py b/test/registered/core/test_basic_sanity_dflash.py index d220752a4..40048d8c2 100644 --- a/test/registered/core/test_basic_sanity_dflash.py +++ b/test/registered/core/test_basic_sanity_dflash.py @@ -63,12 +63,12 @@ class TestBasicSanityDFlash( "DFLASH", "--speculative-draft-model-path", DEFAULT_DRAFT_MODEL_DFLASH, - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "4", "--mem-fraction-static", "0.7", "--enable-metrics", - "--disable-piecewise-cuda-graph", + "--cuda-graph-backend-prefill=disabled", ], env={"SGLANG_ENABLE_METRICS_DEVICE_TIMER": "1"}, ) diff --git a/test/registered/core/test_basic_sanity_dspark.py b/test/registered/core/test_basic_sanity_dspark.py index 4cc8f79c5..9e5fad7ab 100644 --- a/test/registered/core/test_basic_sanity_dspark.py +++ b/test/registered/core/test_basic_sanity_dspark.py @@ -81,7 +81,7 @@ class TestBasicSanityDSpark( "--page-size", "1", "--enable-metrics", - "--disable-piecewise-cuda-graph", + "--cuda-graph-backend-prefill=disabled", ], env={ "SGLANG_ENABLE_METRICS_DEVICE_TIMER": "1", diff --git a/test/registered/core/test_basic_sanity_eagle3.py b/test/registered/core/test_basic_sanity_eagle3.py index efe2f05e7..a45430c43 100644 --- a/test/registered/core/test_basic_sanity_eagle3.py +++ b/test/registered/core/test_basic_sanity_eagle3.py @@ -75,7 +75,7 @@ class TestBasicSanityEagle3( "--mem-fraction-static", "0.7", "--enable-metrics", - "--disable-piecewise-cuda-graph", + "--cuda-graph-backend-prefill=disabled", ], env={"SGLANG_ENABLE_METRICS_DEVICE_TIMER": "1"}, ) diff --git a/test/registered/cp/test_dsa_prefill_cp.py b/test/registered/cp/test_dsa_prefill_cp.py index 7660ca29b..8671a9a97 100644 --- a/test/registered/cp/test_dsa_prefill_cp.py +++ b/test/registered/cp/test_dsa_prefill_cp.py @@ -42,7 +42,7 @@ class TestDSACPInterleave(CustomTestCase): "4", "--mem-frac", "0.85", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--max-running-requests", "32", diff --git a/test/registered/cp/test_gqa_prefill_cp.py b/test/registered/cp/test_gqa_prefill_cp.py index fafa40e29..69ea296d7 100644 --- a/test/registered/cp/test_gqa_prefill_cp.py +++ b/test/registered/cp/test_gqa_prefill_cp.py @@ -44,7 +44,7 @@ class TestGQACP2TP2EP2(CustomTestCase): "--max-running-requests", "32", "--trust-remote-code", - "--disable-piecewise-cuda-graph", + "--cuda-graph-backend-prefill=disabled", "--model-loader-extra-config", '{"enable_multithread_load": true, "num_threads": 64}', ], @@ -102,7 +102,7 @@ class TestGQACPTP2CP2EP4(CustomTestCase): "--max-running-requests", "32", "--trust-remote-code", - "--disable-piecewise-cuda-graph", + "--cuda-graph-backend-prefill=disabled", "--model-loader-extra-config", '{"enable_multithread_load": true, "num_threads": 64}', ], @@ -162,7 +162,7 @@ class TestGQACPCP4EP4(CustomTestCase): "--max-running-requests", "32", "--trust-remote-code", - "--disable-piecewise-cuda-graph", + "--cuda-graph-backend-prefill=disabled", "--model-loader-extra-config", '{"enable_multithread_load": true, "num_threads": 64}', ], diff --git a/test/registered/cpu/test_cpu_graph.py b/test/registered/cpu/test_cpu_graph.py index 55be85eb7..c9511f3b7 100644 --- a/test/registered/cpu/test_cpu_graph.py +++ b/test/registered/cpu/test_cpu_graph.py @@ -34,7 +34,7 @@ class TestCPUGraph(CustomTestCase): "--enable-torch-compile", "--torch-compile-max-bs", "2", - "--cuda-graph-bs", + "--cuda-graph-bs-decode", "2", ], min_throughput=7, @@ -62,7 +62,7 @@ class TestCPUGraph(CustomTestCase): "--trust-remote-code", "--disable-overlap-schedule", "--enable-torch-compile", - "--cuda-graph-bs", + "--cuda-graph-bs-decode", "2", "--tp", f"{n_numa_node}", diff --git a/test/registered/cuda_graph/breakable/test_breakable_cuda_graph.py b/test/registered/cuda_graph/breakable/test_breakable_cuda_graph.py index d669c9570..9c9faa1eb 100644 --- a/test/registered/cuda_graph/breakable/test_breakable_cuda_graph.py +++ b/test/registered/cuda_graph/breakable/test_breakable_cuda_graph.py @@ -4,7 +4,7 @@ Two test classes: - TestBreakableCUDAGraphBasic / TestCopyOutput / TestBreakGraphHelper: unit tests for the core capture / replay mechanism (simple tensor ops). - TestBreakableCudaGraph: integration test — spin up Qwen3-8B with - --enable-breakable-cuda-graph and check mgsm_en accuracy. + --cuda-graph-backend-prefill=breakable and check mgsm_en accuracy. """ import unittest @@ -364,7 +364,7 @@ class TestBreakGraphHelper(CustomTestCase): class TestBreakableCudaGraph(CustomTestCase): - """Integration: Qwen3-8B with --enable-breakable-cuda-graph on mgsm_en.""" + """Integration: Qwen3-8B with --cuda-graph-backend-prefill=breakable on mgsm_en.""" @classmethod def setUpClass(cls): diff --git a/test/registered/cuda_graph/piecewise/test_pcg_with_speculative_decoding_extra.py b/test/registered/cuda_graph/piecewise/test_pcg_with_speculative_decoding_extra.py index fd6cd41cc..4b16fef8c 100644 --- a/test/registered/cuda_graph/piecewise/test_pcg_with_speculative_decoding_extra.py +++ b/test/registered/cuda_graph/piecewise/test_pcg_with_speculative_decoding_extra.py @@ -21,7 +21,7 @@ class TestPCGWithMTP(PCGSpecBase, unittest.TestCase): "--trust-remote-code", "--quantization", "fp8", - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer", "--speculative-algorithm", "NEXTN", diff --git a/test/registered/dcp/test_dsv31_dcp8_gsm8k.py b/test/registered/dcp/test_dsv31_dcp8_gsm8k.py index e147fdce4..dc075a85e 100644 --- a/test/registered/dcp/test_dsv31_dcp8_gsm8k.py +++ b/test/registered/dcp/test_dsv31_dcp8_gsm8k.py @@ -77,7 +77,7 @@ _COMMON_SERVER_ARGS = [ "256", "--attention-backend", "flashinfer", - "--disable-piecewise-cuda-graph", + "--cuda-graph-backend-prefill=disabled", "--log-level", "info", "--log-requests", diff --git a/test/registered/debug_utils/test_nightly_precision_regression.py b/test/registered/debug_utils/test_nightly_precision_regression.py index d0db92ced..ca3730989 100644 --- a/test/registered/debug_utils/test_nightly_precision_regression.py +++ b/test/registered/debug_utils/test_nightly_precision_regression.py @@ -608,7 +608,7 @@ def _run_server_and_dump( "--mem-fraction-static", "0.9", "--disable-cuda-graph", - "--disable-piecewise-cuda-graph", + "--cuda-graph-backend-prefill=disabled", "--disable-radix-cache", "--flashinfer-allreduce-fusion-backend", PRECISION_FUSION_BACKEND, diff --git a/test/registered/disaggregation/test_disaggregation_hybrid_attention.py b/test/registered/disaggregation/test_disaggregation_hybrid_attention.py index dc91c1c50..b97c51c2c 100644 --- a/test/registered/disaggregation/test_disaggregation_hybrid_attention.py +++ b/test/registered/disaggregation/test_disaggregation_hybrid_attention.py @@ -114,7 +114,7 @@ class TestDisaggregationHybridAttentionGDNExtraBuffer(PDDisaggregationServerBase cls.bootstrap_port, "--tp", "4", - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer", ] prefill_args += cls.transfer_backend + cls.rdma_devices @@ -137,7 +137,7 @@ class TestDisaggregationHybridAttentionGDNExtraBuffer(PDDisaggregationServerBase "4", "--base-gpu-id", "4", - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer", ] decode_args += cls.transfer_backend + cls.rdma_devices @@ -342,7 +342,7 @@ class TestDisaggregationHybridAttentionMambaExtraBuffer(PDDisaggregationServerBa cls.bootstrap_port, "--tp", "4", - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer", ] prefill_args += cls.transfer_backend + cls.rdma_devices @@ -365,7 +365,7 @@ class TestDisaggregationHybridAttentionMambaExtraBuffer(PDDisaggregationServerBa "4", "--base-gpu-id", "4", - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer", ] decode_args += cls.transfer_backend + cls.rdma_devices diff --git a/test/registered/dllm/test_dllm_batching_fdfo.py b/test/registered/dllm/test_dllm_batching_fdfo.py index 556eae9b5..44c1e30e0 100644 --- a/test/registered/dllm/test_dllm_batching_fdfo.py +++ b/test/registered/dllm/test_dllm_batching_fdfo.py @@ -41,7 +41,7 @@ class TestBatchingFDFO(CustomTestCase): "--dllm-algorithm", "LowConfidence", "--dllm-fdfo", - "--cuda-graph-bs", + "--cuda-graph-bs-decode", "1", "2", "3", diff --git a/test/registered/dllm/test_dllm_fdfo_joint_threshold.py b/test/registered/dllm/test_dllm_fdfo_joint_threshold.py index 8e99b2464..15dd2f4c4 100644 --- a/test/registered/dllm/test_dllm_fdfo_joint_threshold.py +++ b/test/registered/dllm/test_dllm_fdfo_joint_threshold.py @@ -44,7 +44,7 @@ class TestBatchingFDFOJointThreshold(CustomTestCase): "flashinfer", "--dllm-algorithm", "JointThreshold", - "--cuda-graph-bs", + "--cuda-graph-bs-decode", "1", ] # FDFO is the default; the sync arm must opt out explicitly. diff --git a/test/registered/e2e/models/test_deepseek_v4_flash_fp4_b200.py b/test/registered/e2e/models/test_deepseek_v4_flash_fp4_b200.py index ebdbab507..4958d2f2c 100644 --- a/test/registered/e2e/models/test_deepseek_v4_flash_fp4_b200.py +++ b/test/registered/e2e/models/test_deepseek_v4_flash_fp4_b200.py @@ -192,7 +192,7 @@ class TestDSV4FlashFP4BreakableCudaGraphB200( DEEPEP_CONFIG, "--chunked-prefill-size", "4096", - "--piecewise-cuda-graph-max-tokens", + "--cuda-graph-max-bs-prefill", "1024", "--mem-fraction-static", "0.80", diff --git a/test/registered/e2e/models/test_qwen35_fp4_mtp.py b/test/registered/e2e/models/test_qwen35_fp4_mtp.py index f514bad44..bb3191b31 100644 --- a/test/registered/e2e/models/test_qwen35_fp4_mtp.py +++ b/test/registered/e2e/models/test_qwen35_fp4_mtp.py @@ -24,7 +24,7 @@ MTP_BASE_ARGS = [ "4", "--chunked-prefill-size", "2048", - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer", "--mamba-track-interval", "128", diff --git a/test/registered/e2e/models/test_qwen3_next_models.py b/test/registered/e2e/models/test_qwen3_next_models.py index 1b30d363e..57e2c804a 100644 --- a/test/registered/e2e/models/test_qwen3_next_models.py +++ b/test/registered/e2e/models/test_qwen3_next_models.py @@ -16,7 +16,7 @@ _COMMON_ARGS = [ "4", "--chunked-prefill-size", "2048", - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer_lazy", "--attention-backend", "triton", diff --git a/test/registered/e2e/models/test_qwen3_next_models_extra.py b/test/registered/e2e/models/test_qwen3_next_models_extra.py index 0c4b9f5ad..d32e358fe 100644 --- a/test/registered/e2e/models/test_qwen3_next_models_extra.py +++ b/test/registered/e2e/models/test_qwen3_next_models_extra.py @@ -15,7 +15,7 @@ _COMMON_ARGS = [ "4", "--chunked-prefill-size", "2048", - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer_lazy", "--attention-backend", "triton", diff --git a/test/registered/e2e/models/test_qwen3_next_models_mtp.py b/test/registered/e2e/models/test_qwen3_next_models_mtp.py index b2edc5ee0..0ecf55e01 100644 --- a/test/registered/e2e/models/test_qwen3_next_models_mtp.py +++ b/test/registered/e2e/models/test_qwen3_next_models_mtp.py @@ -29,7 +29,7 @@ def _mtp_args(*, strategy, steps, topk, draft_tokens, track_interval): "4", "--chunked-prefill-size", "2048", - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", strategy, "--mamba-track-interval", str(track_interval), @@ -62,7 +62,7 @@ class TestQwen3NextMTPTopk( "4", "--chunked-prefill-size", "2048", - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer", "--mamba-track-interval", "128", @@ -94,7 +94,7 @@ class TestQwen3NextMTPV2(GSM8KMixin, KLDivergenceMixin, DefaultServerBase): "4", "--chunked-prefill-size", "2048", - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer", "--mamba-track-interval", "128", diff --git a/test/registered/e2e/models_large/test_deepseek_v3_cutedsl_4gpu.py b/test/registered/e2e/models_large/test_deepseek_v3_cutedsl_4gpu.py index 1aeb69b8b..46b3e6aaf 100644 --- a/test/registered/e2e/models_large/test_deepseek_v3_cutedsl_4gpu.py +++ b/test/registered/e2e/models_large/test_deepseek_v3_cutedsl_4gpu.py @@ -109,7 +109,7 @@ class TestDummyWithSBO(CustomTestCase): "256", "--chunked-prefill-size", "1024", - "--cuda-graph-bs", + "--cuda-graph-bs-decode", "64", "--tp", "4", diff --git a/test/registered/e2e/models_large/test_ling_2_6_flash.py b/test/registered/e2e/models_large/test_ling_2_6_flash.py index 4ee6bf3f0..0088fcf45 100644 --- a/test/registered/e2e/models_large/test_ling_2_6_flash.py +++ b/test/registered/e2e/models_large/test_ling_2_6_flash.py @@ -26,7 +26,7 @@ class TestLing26Flash(GSM8KMixin, DefaultServerBase): "--tp-size", "4", "--trust-remote-code", - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer", "--mem-fraction-static", "0.75", diff --git a/test/registered/e2e/models_large/test_qwen35.py b/test/registered/e2e/models_large/test_qwen35.py index 04d47321a..24a222859 100644 --- a/test/registered/e2e/models_large/test_qwen35.py +++ b/test/registered/e2e/models_large/test_qwen35.py @@ -37,7 +37,7 @@ class TestQwen35(unittest.TestCase): "--speculative-num-steps=3", "--speculative-eagle-topk=1", "--speculative-num-draft-tokens=4", - "--mamba-scheduler-strategy=extra_buffer", + "--mamba-radix-cache-strategy=extra_buffer", ] variants = [ diff --git a/test/registered/ep/test_deepep_large.py b/test/registered/ep/test_deepep_large.py index ac7017bc5..568433bae 100644 --- a/test/registered/ep/test_deepep_large.py +++ b/test/registered/ep/test_deepep_large.py @@ -51,7 +51,7 @@ class TestDeepseek(CustomTestCase): "dynamic", "--eplb-algorithm", "deepseek", - "--cuda-graph-bs", + "--cuda-graph-bs-decode", "256", "--max-running-requests", "2048", @@ -112,7 +112,7 @@ class TestDeepseekMTP(CustomTestCase): "dynamic", "--eplb-algorithm", "deepseek", - "--cuda-graph-bs", + "--cuda-graph-bs-decode", "64", # TODO: increase it to 128 when TBO is supported in draft_extend "--max-running-requests", "512", diff --git a/test/registered/gb300/test_qwen35_fp8_dp.py b/test/registered/gb300/test_qwen35_fp8_dp.py index 481e787bf..e7c5e31da 100644 --- a/test/registered/gb300/test_qwen35_fp8_dp.py +++ b/test/registered/gb300/test_qwen35_fp8_dp.py @@ -18,7 +18,7 @@ COMMON_ARGS = [ "--enable-flashinfer-allreduce-fusion", "--attention-backend=trtllm_mha", "--mem-fraction-static=0.8", - "--mamba-scheduler-strategy=extra_buffer", + "--mamba-radix-cache-strategy=extra_buffer", "--enable-multimodal", "--enable-metrics", "--nccl-port", diff --git a/test/registered/gb300/test_qwen35_fp8_tp.py b/test/registered/gb300/test_qwen35_fp8_tp.py index 13b2acd74..2e6f17b9d 100644 --- a/test/registered/gb300/test_qwen35_fp8_tp.py +++ b/test/registered/gb300/test_qwen35_fp8_tp.py @@ -18,7 +18,7 @@ COMMON_ARGS = [ "--enable-flashinfer-allreduce-fusion", "--attention-backend=trtllm_mha", "--mem-fraction-static=0.8", - "--mamba-scheduler-strategy=extra_buffer", + "--mamba-radix-cache-strategy=extra_buffer", "--enable-multimodal", "--enable-metrics", "--nccl-port", diff --git a/test/registered/hicache/test_qwen35_hicache.py b/test/registered/hicache/test_qwen35_hicache.py index 4e6e6f8b9..8d14cbf94 100644 --- a/test/registered/hicache/test_qwen35_hicache.py +++ b/test/registered/hicache/test_qwen35_hicache.py @@ -51,7 +51,7 @@ class TestQwen35WithHiCache(CustomTestCase): "120000", "--chunked-prefill-size", "2048", - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer", "--mamba-track-interval", "128", diff --git a/test/registered/kv_canary/test_self_e2e_bench_speed.py b/test/registered/kv_canary/test_self_e2e_bench_speed.py index b1fc8c8e5..e0e4d7e4d 100644 --- a/test/registered/kv_canary/test_self_e2e_bench_speed.py +++ b/test/registered/kv_canary/test_self_e2e_bench_speed.py @@ -36,11 +36,11 @@ _PROFILE_NO_GRAPH_STEPS = 3 def _make_server_args( *, canary_on: bool, disable_cuda_graph: bool = False ) -> ServerArgs: - # install_canary asserts --disable-piecewise-cuda-graph; pass on both sides for apples-to-apples. + # install_canary asserts --cuda-graph-backend-prefill=disabled; pass on both sides for apples-to-apples. extra = [ "--model-path", _QWEN3_MODEL, - "--disable-piecewise-cuda-graph", + "--cuda-graph-backend-prefill=disabled", ] if disable_cuda_graph: extra.append("--disable-cuda-graph") diff --git a/test/registered/mock_model/test_e2e_pd.py b/test/registered/mock_model/test_e2e_pd.py index 1ecb535fc..bb1c84f8c 100644 --- a/test/registered/mock_model/test_e2e_pd.py +++ b/test/registered/mock_model/test_e2e_pd.py @@ -27,7 +27,7 @@ register_amd_ci(est_time=165, stage="extra-a", runner_config="2-gpu-large-amd") # full graph silently bypasses the only path that exercises that invariant # end-to-end. # -# --disable-piecewise-cuda-graph is REQUIRED by canary: install_canary +# --cuda-graph-backend-prefill=disabled is REQUIRED by canary: install_canary # (api.py) asserts it, and the SingleForwardManager design depends on it. # mock_model_server_args() already passes it; do not remove it. _NUM_PROMPTS = 32 diff --git a/test/registered/musa/test_llm_server_smoke_musa.py b/test/registered/musa/test_llm_server_smoke_musa.py index 783c847f1..e663d6dd2 100644 --- a/test/registered/musa/test_llm_server_smoke_musa.py +++ b/test/registered/musa/test_llm_server_smoke_musa.py @@ -34,7 +34,7 @@ class TestMusaDeepSeekV2LiteChatServerSmoke(DefaultServerBase): "1", "--chunked-prefill-size", "-1", - "--disable-piecewise-cuda-graph", + "--cuda-graph-backend-prefill=disabled", "--context-length", "4096", "--max-total-tokens", diff --git a/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py b/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py index 03d5237e4..1389d7db4 100644 --- a/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py +++ b/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py @@ -93,7 +93,7 @@ DEEPSEEK_V4_FLASH_W8A8_DSPARK_8P_OTHER_ARGS = [ "--speculative-dspark-block-size", 5, "--skip-server-warmup", - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 1, 2, 4, diff --git a/test/registered/npu/accuracy/glm4_6v_flash/test_npu_glm4_6v_flash_1p_mmmu.py b/test/registered/npu/accuracy/glm4_6v_flash/test_npu_glm4_6v_flash_1p_mmmu.py index 8b122225f..83e2c3c57 100644 --- a/test/registered/npu/accuracy/glm4_6v_flash/test_npu_glm4_6v_flash_1p_mmmu.py +++ b/test/registered/npu/accuracy/glm4_6v_flash/test_npu_glm4_6v_flash_1p_mmmu.py @@ -41,7 +41,7 @@ OTHER_ARGS = [ "--trust-remote-code", "--mem-fraction-static", 0.5, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 1, 2, 4, diff --git a/test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py b/test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py index 670274c63..0a34a061a 100644 --- a/test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py +++ b/test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py @@ -40,7 +40,7 @@ OTHER_ARGS = [ "--trust-remote-code", "--mem-fraction-static", 0.75, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 1, 2, 4, diff --git a/test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_gsm8k.py b/test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_gsm8k.py index a0a14628e..3041313f4 100644 --- a/test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_gsm8k.py +++ b/test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_gsm8k.py @@ -36,7 +36,7 @@ OTHER_ARGS = [ "--trust-remote-code", "--mem-fraction-static", 0.75, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 1, 2, 4, diff --git a/test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py b/test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py index 705ab29f6..502e521f9 100644 --- a/test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py +++ b/test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py @@ -127,7 +127,7 @@ GLM_5_1_PD_SEP_DECODE_ARGS = [ "deepep", "--deepep-mode", "low_latency", - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 1, 2, 3, @@ -139,7 +139,6 @@ GLM_5_1_PD_SEP_DECODE_ARGS = [ 180000, "--tokenizer-worker-num", 16, - "--prefill-round-robin-balance", "--disable-shared-experts-fusion", "--dtype", "bfloat16", diff --git a/test/registered/npu/accuracy/glm5_2/test_npu_glm_5_2_w4a8_16p_gpqa.py b/test/registered/npu/accuracy/glm5_2/test_npu_glm_5_2_w4a8_16p_gpqa.py index 044ae58c5..2cc5a452a 100644 --- a/test/registered/npu/accuracy/glm5_2/test_npu_glm_5_2_w4a8_16p_gpqa.py +++ b/test/registered/npu/accuracy/glm5_2/test_npu_glm_5_2_w4a8_16p_gpqa.py @@ -56,7 +56,7 @@ GLM_5_2_W4A8_16P_TWO_NODE_OTHER_ARGS = [ 135000, "--served-model-name", "glm-5.2-w4a8", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", 4, "--max-running-requests", 32, diff --git a/test/registered/npu/accuracy/glm5_top64_pruned/test_npu_glm5_top64_pruned_bf16_8p_gsm8k.py b/test/registered/npu/accuracy/glm5_top64_pruned/test_npu_glm5_top64_pruned_bf16_8p_gsm8k.py index 3e08a489b..bde14d585 100644 --- a/test/registered/npu/accuracy/glm5_top64_pruned/test_npu_glm5_top64_pruned_bf16_8p_gsm8k.py +++ b/test/registered/npu/accuracy/glm5_top64_pruned/test_npu_glm5_top64_pruned_bf16_8p_gsm8k.py @@ -40,7 +40,7 @@ OTHER_ARGS = [ "deepep", "--deepep-mode", "auto", - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 16, ] diff --git a/test/registered/npu/accuracy/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py b/test/registered/npu/accuracy/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py index 0490b38be..defc373d5 100644 --- a/test/registered/npu/accuracy/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py +++ b/test/registered/npu/accuracy/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py @@ -63,7 +63,7 @@ OTHER_ARGS = [ "deepep", "--deepep-mode", "auto", - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 1, "--disable-radix-cache", "--speculative-algorithm", diff --git a/test/registered/npu/accuracy/kimi_k3/test_npu_kimi_k3_w4a8_32p_gpqa.py b/test/registered/npu/accuracy/kimi_k3/test_npu_kimi_k3_w4a8_32p_gpqa.py index 319537dec..ef943a0a2 100644 --- a/test/registered/npu/accuracy/kimi_k3/test_npu_kimi_k3_w4a8_32p_gpqa.py +++ b/test/registered/npu/accuracy/kimi_k3/test_npu_kimi_k3_w4a8_32p_gpqa.py @@ -70,7 +70,7 @@ KIMI_K3_W4A8_32P_OTHER_ARGS = [ 0.72, "--chunked-prefill-size", 8192, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 1, 4, 16, diff --git a/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py b/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py index 617b5ccc9..1096a5552 100644 --- a/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py +++ b/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py @@ -50,7 +50,7 @@ MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_OTHER_ARGS = [ 67072, "--max-prefill-token", 67000, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 2, 4, 8, diff --git a/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py b/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py index 825b28f66..bd9d19385 100644 --- a/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py +++ b/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py @@ -53,7 +53,7 @@ MINIMAX_M2_5_HIGH_THROUGHPUT_OTHER_ARGS = [ 196608, "--max-prefill-token", 8192, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 1, 2, 4, diff --git a/test/registered/npu/accuracy/moonshotai_moonlight_16b_a3b/test_npu_moonlight_16b_a3b_bf16_1p_gsm8k.py b/test/registered/npu/accuracy/moonshotai_moonlight_16b_a3b/test_npu_moonlight_16b_a3b_bf16_1p_gsm8k.py index 08a971e0b..9e702cbd3 100644 --- a/test/registered/npu/accuracy/moonshotai_moonlight_16b_a3b/test_npu_moonlight_16b_a3b_bf16_1p_gsm8k.py +++ b/test/registered/npu/accuracy/moonshotai_moonlight_16b_a3b/test_npu_moonlight_16b_a3b_bf16_1p_gsm8k.py @@ -38,7 +38,7 @@ MODEL_OTHER_ARGS = [ 4096, "--max-prefill-tokens", 16384, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 1, 2, 4, diff --git a/test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py b/test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py index b7e342248..d00fa662f 100644 --- a/test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py +++ b/test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py @@ -49,7 +49,7 @@ QWEN3_8B_OTHER_ARGS = [ 1, "--mem-fraction-static", 0.85, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 8, 12, 24, diff --git a/test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py b/test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py index 534d7488e..f1181658a 100644 --- a/test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py +++ b/test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py @@ -47,7 +47,7 @@ QWEN3_8B_OTHER_ARGS = [ 2, "--mem-fraction-static", 0.894, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 1, 5, 15, diff --git a/test/registered/npu/accuracy/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py b/test/registered/npu/accuracy/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py index 54f0ddc81..4406259b0 100644 --- a/test/registered/npu/accuracy/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py +++ b/test/registered/npu/accuracy/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py @@ -63,7 +63,7 @@ QWEN3_30B_A3B_OTHER_ARGS = [ 2, "--mem-fraction-static", 0.87, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 1, 5, 15, diff --git a/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py b/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py index 437907bfe..c4b3918c7 100644 --- a/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py +++ b/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py @@ -59,7 +59,7 @@ QWEN3_32B_OTHER_ARGS = [ 16, "--mem-fraction-static", 0.72, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 64, "--dtype", "bfloat16", diff --git a/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py b/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py index 7fc8e9fec..570927d75 100644 --- a/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py +++ b/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py @@ -62,7 +62,7 @@ QWEN3_32B_OTHER_ARGS = [ 4, "--mem-fraction-static", 0.845, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 16, 32, 64, diff --git a/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa_a2.py b/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa_a2.py index a0aaae51a..6dde1bf10 100644 --- a/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa_a2.py +++ b/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa_a2.py @@ -62,7 +62,7 @@ QWEN3_32B_OTHER_ARGS = [ 4, "--mem-fraction-static", 0.845, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 16, 32, 64, diff --git a/test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py b/test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py index 1aa88934e..66d9820f7 100644 --- a/test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py +++ b/test/registered/npu/accuracy/qwen3_5_9b/test_npu_qwen3_5_9b_bf16_1p_gsm8k.py @@ -39,7 +39,7 @@ QWEN3_5_9B_OTHER_ARGS = [ "--trust-remote-code", "--mem-fraction-static", 0.7, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 16, "--enable-multimodal", "--mm-attention-backend", diff --git a/test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py b/test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py index a799c97b9..ae048613f 100644 --- a/test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py +++ b/test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py @@ -43,7 +43,7 @@ QWEN3_6_27B_64K_PREFIX_OTHER_ARGS = [ 32768, "--max-prefill-tokens", 32768, - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer", "--trust-remote-code", "--max-running-requests", @@ -52,7 +52,7 @@ QWEN3_6_27B_64K_PREFIX_OTHER_ARGS = [ 120, "--mem-fraction-static", 0.8, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 1, 2, 4, diff --git a/test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py b/test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py index 8ecdc892d..f1b2624b1 100644 --- a/test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py +++ b/test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py @@ -48,7 +48,7 @@ QWEN3_6_27B_3K5_1K5_OTHER_ARGS = [ 74, "--mem-fraction-static", 0.7, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 2, 8, 16, diff --git a/test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_aime26.py b/test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_aime26.py index a6cf52913..c9035467a 100644 --- a/test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_aime26.py +++ b/test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_aime26.py @@ -45,7 +45,7 @@ QWEN3_6_35B_A3B_3K5_1K5_OTHER_ARGS = [ 4, "--mem-fraction-static", 0.7, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 1, 2, 3, diff --git a/test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py b/test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py index ce81a32b0..bc1c5cb78 100644 --- a/test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py +++ b/test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py @@ -43,7 +43,7 @@ QWEN3_6_35B_A3B_64K_PREFIX_OTHER_ARGS = [ "--max-prefill-tokens", 65536, "--trust-remote-code", - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer", "--max-running-requests", 40, @@ -51,7 +51,7 @@ QWEN3_6_35B_A3B_64K_PREFIX_OTHER_ARGS = [ 200, "--mem-fraction-static", 0.9, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 2, 8, 16, diff --git a/test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py b/test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py index 7acc6e50d..68d57b758 100644 --- a/test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py +++ b/test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py @@ -80,7 +80,7 @@ QWEN3_NEXT_80B_A3B_OTHER_ARGS = [ -1, "--max-running-requests", 16, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 2, 4, 8, diff --git a/test/registered/npu/accuracy/qwen3_omni_30b_a3b_thinking/test_npu_qwen3_omni_30b_a3b_thinking_1p_mmmu.py b/test/registered/npu/accuracy/qwen3_omni_30b_a3b_thinking/test_npu_qwen3_omni_30b_a3b_thinking_1p_mmmu.py index 7a473229d..c0b63144e 100644 --- a/test/registered/npu/accuracy/qwen3_omni_30b_a3b_thinking/test_npu_qwen3_omni_30b_a3b_thinking_1p_mmmu.py +++ b/test/registered/npu/accuracy/qwen3_omni_30b_a3b_thinking/test_npu_qwen3_omni_30b_a3b_thinking_1p_mmmu.py @@ -63,7 +63,7 @@ OTHER_ARGS = [ 2, "--mem-fraction-static", 0.87, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 1, 5, 15, diff --git a/test/registered/npu/accuracy/qwen3_vl_8b/test_npu_qwen3_vl_8b_bf16_2p_gsm8k.py b/test/registered/npu/accuracy/qwen3_vl_8b/test_npu_qwen3_vl_8b_bf16_2p_gsm8k.py index ae7487597..437cf440f 100644 --- a/test/registered/npu/accuracy/qwen3_vl_8b/test_npu_qwen3_vl_8b_bf16_2p_gsm8k.py +++ b/test/registered/npu/accuracy/qwen3_vl_8b/test_npu_qwen3_vl_8b_bf16_2p_gsm8k.py @@ -40,7 +40,7 @@ QWEN3_VL_8B_OTHER_ARGS = [ "qwen", "--reasoning-parser", "qwen3", - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 8, 16, 32, diff --git a/test/registered/npu/basic_function/optimization_debug/test_npu_piecewise_graph_prefill.py b/test/registered/npu/basic_function/optimization_debug/test_npu_piecewise_graph_prefill.py index c3230817b..1db72e5fb 100644 --- a/test/registered/npu/basic_function/optimization_debug/test_npu_piecewise_graph_prefill.py +++ b/test/registered/npu/basic_function/optimization_debug/test_npu_piecewise_graph_prefill.py @@ -27,7 +27,7 @@ class TestPiecewiseGraphPrefillCorrectness(GSM8KAscendMixin, CustomTestCase): 0.8, "--attention-backend", "ascend", - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 128, "--cuda-graph-backend-prefill=tc_piecewise", "--cuda-graph-bs-prefill", diff --git a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py index d7adbac6e..840271f16 100644 --- a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py +++ b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py @@ -57,7 +57,7 @@ class TestDeepEpQwen(GSM8KAscendMixin, TestMMLU, CustomTestCase): "--enable-dp-lm-head", "--mem-fraction-static", 0.7, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 16, 20, 24, diff --git a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_next.py b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_next.py index 76f3870e5..f7470caa1 100644 --- a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_next.py +++ b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_next.py @@ -41,7 +41,7 @@ class TestQwen3Next(GSM8KAscendMixin, TestMMLU, CustomTestCase): "--watchdog-timeout", 9000, "--disable-radix-cache", - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 2, 4, 6, diff --git a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_480b.py b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_480b.py index bdfcd4f03..ed82f9d00 100644 --- a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_480b.py +++ b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_480b.py @@ -58,7 +58,7 @@ class TestDeepEpQwen(GSM8KAscendMixin, TestMMLU, CustomTestCase): "--enable-dp-lm-head", "--mem-fraction-static", 0.7, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 16, 20, 24, diff --git a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_next.py b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_next.py index 51f905a04..cc4204e97 100644 --- a/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_next.py +++ b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_next.py @@ -41,7 +41,7 @@ class TestQwen3Next(GSM8KAscendMixin, TestMMLU, CustomTestCase): "--watchdog-timeout", 9000, "--disable-radix-cache", - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 2, 4, 6, diff --git a/test/registered/npu/basic_function/quant/test_npu_w4a4_quantization.py b/test/registered/npu/basic_function/quant/test_npu_w4a4_quantization.py index 42c6db12e..2d2ac2332 100644 --- a/test/registered/npu/basic_function/quant/test_npu_w4a4_quantization.py +++ b/test/registered/npu/basic_function/quant/test_npu_w4a4_quantization.py @@ -29,7 +29,7 @@ class TestAscendW4A4(GSM8KAscendMixin, CustomTestCase): "4", "--mem-fraction-static", "0.8", - "--cuda-graph-bs", + "--cuda-graph-bs-decode", "64", "--disable-radix-cache", ] diff --git a/test/registered/npu/basic_function/speculative_inference/test_npu_basic_sanity_eagle3.py b/test/registered/npu/basic_function/speculative_inference/test_npu_basic_sanity_eagle3.py index 979a01bee..7b06d57e5 100644 --- a/test/registered/npu/basic_function/speculative_inference/test_npu_basic_sanity_eagle3.py +++ b/test/registered/npu/basic_function/speculative_inference/test_npu_basic_sanity_eagle3.py @@ -56,11 +56,11 @@ class TestBasicSanityEagle3( "1", "--speculative-num-draft-tokens", "2", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "4", "--mem-fraction-static", "0.7", - "--disable-piecewise-cuda-graph", + "--cuda-graph-backend-prefill=disabled", ], ) diff --git a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py index ce287f020..a8dd19da0 100644 --- a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py +++ b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py @@ -171,7 +171,7 @@ DEEPSEEK_V4_FLASH_W8A8_1P1D_DECODE_ARGS = [ "--enable-dp-lm-head", "--kv-cache-dtype", "bfloat16", - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 1, 2, 4, diff --git a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in32k_out1k_50ms.py b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in32k_out1k_50ms.py index 053239207..1e5f71d91 100644 --- a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in32k_out1k_50ms.py +++ b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in32k_out1k_50ms.py @@ -80,7 +80,7 @@ DEEPSEEK_V4_FLASH_W8A8_8P_OTHER_ARGS = [ "--kv-cache-dtype", "bfloat16", "--skip-server-warmup", - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 1, 2, 4, diff --git a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py index 7c698dd89..5fc3efb96 100644 --- a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py +++ b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py @@ -79,7 +79,7 @@ DEEPSEEK_V4_FLASH_W8A8_8P_OTHER_ARGS = [ "--kv-cache-dtype", "bfloat16", "--skip-server-warmup", - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 1, 2, 4, diff --git a/test/registered/npu/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms.py b/test/registered/npu/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms.py index dbb9f16f4..a8577e708 100644 --- a/test/registered/npu/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms.py +++ b/test/registered/npu/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms.py @@ -127,7 +127,7 @@ GLM_5_1_PD_SEP_DECODE_ARGS = [ "deepep", "--deepep-mode", "low_latency", - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 1, 2, 3, @@ -139,7 +139,6 @@ GLM_5_1_PD_SEP_DECODE_ARGS = [ 180000, "--tokenizer-worker-num", 16, - "--prefill-round-robin-balance", "--disable-shared-experts-fusion", "--dtype", "bfloat16", diff --git a/test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py index 69f487b4e..e4f1d74e6 100644 --- a/test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py +++ b/test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py @@ -53,7 +53,7 @@ QWEN3_5_397B_A17B_3K5_1K5_OTHER_ARGS = [ 432, "--mem-fraction-static", 0.8, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 2, 4, 6, diff --git a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py index 63daf043b..9d2af6fc3 100644 --- a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py +++ b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py @@ -49,7 +49,7 @@ QWEN3_6_27B_1024_OTHER_ARGS = [ 60, "--mem-fraction-static", 0.74, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 2, 4, 8, diff --git a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py index bd90e0ffe..d308201c2 100644 --- a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py +++ b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py @@ -49,7 +49,7 @@ QWEN3_6_27B_1080P_OTHER_ARGS = [ 38, "--mem-fraction-static", 0.70, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 1, 2, 4, diff --git a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms.py b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms.py index 1ab4e74f4..9ce555d28 100644 --- a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms.py +++ b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms.py @@ -38,7 +38,7 @@ QWEN3_6_27B_64K_PREFIX_OTHER_ARGS = [ 32768, "--max-prefill-tokens", 32768, - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer", "--trust-remote-code", "--max-running-requests", @@ -47,7 +47,7 @@ QWEN3_6_27B_64K_PREFIX_OTHER_ARGS = [ 160, "--mem-fraction-static", 0.82, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 1, 2, 5, diff --git a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms.py index dec90357e..5d85d10b7 100644 --- a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms.py +++ b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms.py @@ -48,7 +48,7 @@ QWEN3_6_27B_3K5_1K5_OTHER_ARGS = [ 74, "--mem-fraction-static", 0.7, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 2, 8, 16, diff --git a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py index aff01ed9f..8bab4e2b1 100644 --- a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py +++ b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py @@ -41,7 +41,7 @@ QWEN3_6_27B_64K_1K_OTHER_ARGS = [ 16, "--mem-fraction-static", 0.6, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 1, 2, 4, diff --git a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py index 062273708..f123e90bc 100644 --- a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py +++ b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py @@ -48,7 +48,7 @@ QWEN3_6_27B_128K_OTHER_ARGS = [ 7, "--mem-fraction-static", 0.63, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 1, 2, 4, diff --git a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py index 5d8eb24ac..965c0a2ec 100644 --- a/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py +++ b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py @@ -49,7 +49,7 @@ QWEN3_6_27B_16K_1k_OTHER_ARGS = [ 74, "--mem-fraction-static", 0.70, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 1, 2, 3, diff --git a/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py b/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py index 6a9600b4a..d59d1367a 100644 --- a/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py +++ b/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py @@ -51,7 +51,7 @@ QWEN3_6_35B_A3B_128K_1K_OTHER_ARGS = [ 3, "--mem-fraction-static", 0.9, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 1, 2, 3, diff --git a/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py b/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py index 70f1f4174..244bf2f9d 100644 --- a/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py +++ b/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py @@ -40,7 +40,7 @@ QWEN3_6_35B_A3B_128K_PREFIX_OTHER_ARGS = [ 65536, "--trust-remote-code", "--enable-prefill-delayer", - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer", "--max-running-requests", 103, @@ -48,7 +48,7 @@ QWEN3_6_35B_A3B_128K_PREFIX_OTHER_ARGS = [ 85, "--mem-fraction-static", 0.85, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 2, 4, 8, diff --git a/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py index e23edefeb..5fea46022 100644 --- a/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py +++ b/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py @@ -47,7 +47,7 @@ QWEN3_6_35B_A3B_3K5_1K5_OTHER_ARGS = [ 122, "--mem-fraction-static", 0.9, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 4, 16, 32, diff --git a/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py b/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py index 034014d00..e9f80f9f9 100644 --- a/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py +++ b/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py @@ -46,7 +46,7 @@ QWEN3_6_35B_A3B_64K_1K_OTHER_ARGS = [ 20, "--mem-fraction-static", 0.65, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 2, 4, 8, diff --git a/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms.py b/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms.py index c803d6fbc..f517f33ac 100644 --- a/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms.py +++ b/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms.py @@ -40,7 +40,7 @@ QWEN3_6_35B_A3B_64K_PREFIX_OTHER_ARGS = [ "--max-prefill-tokens", 65536, "--trust-remote-code", - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer", "--max-running-requests", 40, @@ -48,7 +48,7 @@ QWEN3_6_35B_A3B_64K_PREFIX_OTHER_ARGS = [ 200, "--mem-fraction-static", 0.9, - "--cuda-graph-bs", + "--cuda-graph-bs-decode", 2, 8, 16, diff --git a/test/registered/npu/test_npu_memory_consumption.py b/test/registered/npu/test_npu_memory_consumption.py index 51229b61e..3728ba97a 100644 --- a/test/registered/npu/test_npu_memory_consumption.py +++ b/test/registered/npu/test_npu_memory_consumption.py @@ -55,7 +55,7 @@ class TestMemoryConsumptionAscend(CustomTestCase): "2", "--mem-fraction-static", "0.8", - "--cuda-graph-bs", + "--cuda-graph-bs-decode", "1", "--max-total-tokens", "1024", diff --git a/test/registered/pp/test_pp_parallel_compat.py b/test/registered/pp/test_pp_parallel_compat.py index 9719d3c57..ceb431b43 100644 --- a/test/registered/pp/test_pp_parallel_compat.py +++ b/test/registered/pp/test_pp_parallel_compat.py @@ -43,7 +43,7 @@ class _Qwen3MoePPCompatMixin: "--max-running-requests", "32", "--trust-remote-code", - "--disable-piecewise-cuda-graph", + "--cuda-graph-backend-prefill=disabled", "--model-loader-extra-config", '{"enable_multithread_load": true, "num_threads": 64}', ], diff --git a/test/registered/radix_cache/test_int8_mamba_checkpoint_e2e.py b/test/registered/radix_cache/test_int8_mamba_checkpoint_e2e.py index 82fedc0b1..0a2f76e0b 100644 --- a/test/registered/radix_cache/test_int8_mamba_checkpoint_e2e.py +++ b/test/registered/radix_cache/test_int8_mamba_checkpoint_e2e.py @@ -12,7 +12,7 @@ path is exercised: * test_gsm8k — end-to-end task accuracy holds. NOTE: the int8 checkpoint is only engaged when a cached prefix is reused FROM the -int8 pool, which requires ``--mamba-scheduler-strategy extra_buffer`` — the default +int8 pool, which requires ``--mamba-radix-cache-strategy extra_buffer`` — the default ``no_buffer`` only snapshots the recurrent state at the full-sequence leaf, so a fixed-prefix / divergent-question workload reuses ~0 mamba state and the int8 path would never fire. @@ -72,7 +72,7 @@ class TestInt8MambaCheckpointE2E(KLDivergenceMixin, DefaultServerBase): "--mem-fraction-static", "0.7", "--enable-int8-mamba-checkpoint", - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer", ] diff --git a/test/registered/radix_cache/test_mamba2_extra_buffer_kl.py b/test/registered/radix_cache/test_mamba2_extra_buffer_kl.py index 9e6ded12d..79ad99bdb 100644 --- a/test/registered/radix_cache/test_mamba2_extra_buffer_kl.py +++ b/test/registered/radix_cache/test_mamba2_extra_buffer_kl.py @@ -48,7 +48,7 @@ class TestMamba2ExtraBufferKL(KLDivergenceMixin, DefaultServerBase): "256", "--mem-fraction-static", "0.8", - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer", # The 512-token decode turns must cross a track boundary for the tree # to hold decode-seeded states; halve the default interval (must stay diff --git a/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_hicache_pp_kl.py b/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_hicache_pp_kl.py index 6e2e7c921..24b0b1056 100644 --- a/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_hicache_pp_kl.py +++ b/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_hicache_pp_kl.py @@ -86,7 +86,7 @@ class TestUnifiedQwen3HiCachePP(UnifiedRadixTreeTestMixin, CustomTestCase): str(cls.max_running_requests), "--max-total-tokens", "14000", - "--disable-piecewise-cuda-graph", + "--cuda-graph-backend-prefill=disabled", "--model-loader-extra-config", '{"enable_multithread_load": true, "num_threads": 64}', "--enable-hierarchical-cache", @@ -130,11 +130,11 @@ class TestUnifiedQwen3HiCachePPL3(AccuracyTwoPassMixin, CustomTestCase): "2", "--mem-fraction-static", "0.8", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--max-total-tokens", "14000", - "--disable-piecewise-cuda-graph", + "--cuda-graph-backend-prefill=disabled", "--model-loader-extra-config", '{"enable_multithread_load": true, "num_threads": 64}', "--enable-hierarchical-cache", diff --git a/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_cp.py b/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_cp.py index 7e1b96986..5f7a774ff 100644 --- a/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_cp.py +++ b/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_cp.py @@ -50,7 +50,7 @@ class TestUnifiedQwen3HiCacheCP(UnifiedRadixTreeTestMixin, CustomTestCase): str(cls.max_running_requests), "--max-total-tokens", "14000", # loadback trigger - "--disable-piecewise-cuda-graph", + "--cuda-graph-backend-prefill=disabled", "--model-loader-extra-config", '{"enable_multithread_load": true, "num_threads": 64}', "--enable-hierarchical-cache", diff --git a/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_mamba.py b/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_mamba.py index cfd5c9524..68a4e3e53 100644 --- a/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_mamba.py +++ b/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_mamba.py @@ -56,7 +56,7 @@ class TestUnifiedMambaRadixCache(UnifiedRadixTreeTestMixin, CustomTestCase): str(MAMBA_CHUNKED_PREFILL_SIZE), "--mem-fraction-static", "0.85", - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer", "--mamba-track-interval", str(MAMBA_TRACK_INTERVAL), @@ -101,7 +101,7 @@ class TestUnifiedMambaHiCache(UnifiedRadixTreeTestMixin, CustomTestCase): str(MAMBA_CHUNKED_PREFILL_SIZE), "--mem-fraction-static", "0.85", - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer", "--mamba-track-interval", str(MAMBA_TRACK_INTERVAL), @@ -160,7 +160,7 @@ class TestUnifiedMambaHiCacheL3(AccuracyTwoPassMixin, CustomTestCase): str(MAMBA_CHUNKED_PREFILL_SIZE), "--mem-fraction-static", "0.85", - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer", "--mamba-track-interval", str(MAMBA_TRACK_INTERVAL), diff --git a/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_swa.py b/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_swa.py index 78a9aaf54..d55629cec 100644 --- a/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_swa.py +++ b/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_swa.py @@ -35,7 +35,7 @@ class TestUnifiedSWARadixCache(UnifiedRadixTreeTestMixin, CustomTestCase): "2", "--mem-fraction-static", "0.7", - "--disable-piecewise-cuda-graph", + "--cuda-graph-backend-prefill=disabled", ], env={"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"}, ) diff --git a/test/registered/rl/test_return_routed_experts.py b/test/registered/rl/test_return_routed_experts.py index 08533a2ea..fc54a43f3 100644 --- a/test/registered/rl/test_return_routed_experts.py +++ b/test/registered/rl/test_return_routed_experts.py @@ -196,7 +196,7 @@ class TestReturnRoutedExperts(CustomTestCase): 2, "--enable-return-routed-experts", "--disable-cuda-graph", - "--disable-piecewise-cuda-graph", + "--cuda-graph-backend-prefill=disabled", *other_args, ], ) diff --git a/test/registered/spec/dflash/test_dflash.py b/test/registered/spec/dflash/test_dflash.py index ed87bd15f..b412baa91 100644 --- a/test/registered/spec/dflash/test_dflash.py +++ b/test/registered/spec/dflash/test_dflash.py @@ -68,7 +68,7 @@ class TestDFlashServerBase( # private pools on 32GB CI cards. "--mem-fraction-static", "0.7", - "--cuda-graph-bs", + "--cuda-graph-bs-decode", *[str(i) for i in range(1, cls.max_running_requests + 1)], ] if cls.disable_overlap: diff --git a/test/registered/spec/test_spec_mixed_chunk.py b/test/registered/spec/test_spec_mixed_chunk.py index 45e63e0b2..8cd1b0916 100644 --- a/test/registered/spec/test_spec_mixed_chunk.py +++ b/test/registered/spec/test_spec_mixed_chunk.py @@ -125,7 +125,7 @@ class TestDSparkMixedChunk(GSM8KMixin, CustomTestCase): "0.7", "--page-size", "1", - "--disable-piecewise-cuda-graph", + "--cuda-graph-backend-prefill=disabled", ], ) diff --git a/test/registered/unit/server_args/test_declared_fallbacks.py b/test/registered/unit/server_args/test_declared_fallbacks.py index 23970d1a2..f0601c095 100644 --- a/test/registered/unit/server_args/test_declared_fallbacks.py +++ b/test/registered/unit/server_args/test_declared_fallbacks.py @@ -10,8 +10,16 @@ purpose: * the **record** still holds `None` -- so a model family asking "did anyone set this?" still gets an answer, and the wire format is unchanged; -* the **views** a resolution pass reads still answer `None` -- so a family's - `if cfg.x is None` fires and its declaration lands; +* `resolving_view` -- the view a resolution pass decides on -- still answers + `None`, so a family's `if cfg.x is None` fires and its declaration lands. + This is the half that carries the design: `model_overrides/inkling.py` and + `deepseek_v4.py` are the only `is None` readers of either declared field, and + both read this view. Pinned by the family-shaped test below rather than by + asserting the view directly, because the shape is what has to keep working. + `resolved_view` is a separate class and is not pinned here -- nothing reads a + declared field through it, and `with_fallback` is called from exactly one + place (`resolution_result`), which is what makes both views answer `None` + without either of them knowing about fallbacks; * the **effective** surface -- `resolution_result`, the projection, and the config bags every runtime reader goes through -- answers with the fallback. @@ -20,18 +28,8 @@ would make `if cfg.swa_full_tokens_ratio is None` in `model_overrides/inkling.py never fire, and the family's 0.1 would be silently replaced by the generic 0.8. """ -import dataclasses import unittest -from typing import Optional, get_args, get_type_hints -from sglang.srt.arg_groups.arg_utils import ( - NO_FALLBACK, - A, - Arg, - fallbacks_of, - with_fallback, -) -from sglang.srt.arg_groups.model_override_base import resolved_view from sglang.srt.arg_groups.overrides import ( declare_resolution, resolution_result, @@ -69,11 +67,6 @@ class TestAFallbackIsWhatNobodySaid(CustomTestCase): server_args = _resolved(swa_full_tokens_ratio=0.25) self.assertEqual(resolution_result(server_args, "swa_full_tokens_ratio"), 0.25) - def test_a_value_the_operator_typed_that_equals_the_fallback_is_still_input(self): - server_args = _resolved(swa_full_tokens_ratio=0.8) - self.assertEqual(server_args.swa_full_tokens_ratio, 0.8) - self.assertEqual(resolution_result(server_args, "swa_full_tokens_ratio"), 0.8) - def test_a_decision_wins(self): server_args = _resolved() declare_resolution(server_args, "a_model_family", swa_full_tokens_ratio=0.1) @@ -108,14 +101,6 @@ class TestAPassDecidingStillSeesUnset(CustomTestCase): would never be declared. """ - def test_the_resolving_view_answers_none(self): - server_args = _resolved() - self.assertIsNone(resolving_view(server_args).swa_full_tokens_ratio) - - def test_the_resolved_view_answers_none(self): - server_args = _resolved() - self.assertIsNone(resolved_view(server_args).mamba_full_memory_ratio) - def test_a_family_that_tests_is_none_still_fires(self): server_args = _resolved() cfg = resolving_view(server_args) @@ -125,60 +110,5 @@ class TestAPassDecidingStillSeesUnset(CustomTestCase): self.assertEqual(declared, {"swa_full_tokens_ratio": 0.1}) -class TestTheDeclarationIsTheOnlyPlaceTheValueLives(CustomTestCase): - def test_the_declared_set_is_what_the_record_carries(self): - self.assertEqual( - fallbacks_of(ServerArgs), - {"swa_full_tokens_ratio": 0.8, "mamba_full_memory_ratio": 0.9}, - ) - - def test_a_fallback_field_is_optional_and_defaults_to_none(self): - # `None` is what the fallback answers for. A field that defaults to - # anything else can never reach it, so the declaration would be dead. - hints = get_type_hints(ServerArgs, include_extras=True) - for name in fallbacks_of(ServerArgs): - field = next(f for f in dataclasses.fields(ServerArgs) if f.name == name) - with self.subTest(field=name): - self.assertIsNone(field.default, f"{name} must default to None") - inner = get_args(hints[name])[0] - self.assertIn( - type(None), - get_args(inner), - f"{name} must be Optional[...] to hold its unset state", - ) - - def test_the_help_text_does_not_restate_the_value(self): - # The value used to be written twice: as a literal in the hook that - # filled it, and as prose in the help. Two copies drift. - hints = get_type_hints(ServerArgs, include_extras=True) - for name, value in fallbacks_of(ServerArgs).items(): - arg = next(a for a in get_args(hints[name])[1:] if isinstance(a, Arg)) - with self.subTest(field=name): - self.assertNotIn(str(value), arg.help) - - -class TestWithFallback(CustomTestCase): - def test_a_container_fallback_is_copied_per_read(self): - @dataclasses.dataclass - class Cfg: - paths: A[Optional[list], Arg(help="x", fallback=[])] = None - - first = with_fallback(Cfg, "paths", None) - first.append("mutated") - self.assertEqual(with_fallback(Cfg, "paths", None), []) - - def test_a_field_without_a_declaration_is_untouched(self): - self.assertIsNone(with_fallback(ServerArgs, "tokenizer_path", None)) - - def test_a_non_dataclass_has_no_fallbacks(self): - self.assertEqual(fallbacks_of(int), {}) - - def test_the_sentinel_is_not_none(self): - # `None` cannot mark "declares no fallback": it is the state a - # fallback exists to answer for. - self.assertIsNot(NO_FALLBACK, None) - self.assertNotEqual(Arg().fallback, None) - - if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index e8db519fb..e6fbffd19 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -254,12 +254,19 @@ class TestPrepareServerArgs(CustomTestCase): resolution_result(inherited, "speculative_draft_model_quantization"), "modelopt_fp4", ) + # The provenance bit, not the public field: `from_server_args` reads + # `cfg._speculative_draft_quantization_explicitly_set` to tell an + # inherited draft quantization from one the operator asked for, and + # resolution decided the value without consuming that evidence. self.assertFalse( resolution_result( inherited, "_speculative_draft_quantization_explicitly_set" ) ) + # And across the hop that matters: the scheduler and the draft worker + # rebuild the record from its fields and resolve again, so the bit has + # to survive `asdict` and come back the same the second time. reconstructed = ServerArgs(**dataclasses.asdict(inherited)) handle_missing_default_values(reconstructed) diff --git a/test/registered/unit/server_args/test_unified_prefill_cuda_graph_gate.py b/test/registered/unit/server_args/test_unified_prefill_cuda_graph_gate.py index c951f5747..41b5da611 100644 --- a/test/registered/unit/server_args/test_unified_prefill_cuda_graph_gate.py +++ b/test/registered/unit/server_args/test_unified_prefill_cuda_graph_gate.py @@ -20,8 +20,7 @@ captured store would silently write wrong slots. The old gate only rejected `TC_PIECEWISE`, but the generic prefill default is `BREAKABLE` -- so the DEFAULT unified invocation was broken; it only ever -worked when `--disable-piecewise-cuda-graph` (a deprecated alias for -`--cuda-graph-backend-prefill=disabled`) happened to be passed. +worked when `--cuda-graph-backend-prefill=disabled` happened to be passed. Pinned: the default is auto-disabled with a warning (unified boots out of the box), an EXPLICIT prefill backend still raises (never silently override a diff --git a/test/registered/unit/test_runtime_context.py b/test/registered/unit/test_runtime_context.py index 0b8a26bd9..851416ea5 100644 --- a/test/registered/unit/test_runtime_context.py +++ b/test/registered/unit/test_runtime_context.py @@ -11,6 +11,7 @@ import pathlib as _pathlib import shutil import tempfile import unittest +import warnings from unittest.mock import patch import sglang as _sglang @@ -246,10 +247,33 @@ class TestServerArgsOwnership(_IsolatedServerArgs): # Identity, not equality: the slot holds the very object published. sentinel = ServerArgs(model_path="dummy") server_args_module.set_global_server_args_for_scheduler(sentinel) - self.assertIs(server_args_module.get_global_server_args(), sentinel) self.assertIs(get_server_args(), sentinel) self.assertIs(get_context().server_args, sentinel) + def test_the_retired_accessor_raises_and_names_the_replacement(self): + """`get_global_server_args` is retired: it answered with the record, + so a caller reading a field resolution had decided got a stale value + and no error at all. + + `RuntimeError` unconditionally, not a warning first: a + `DeprecationWarning` is filtered by default outside `__main__`, so no + production caller would have seen it, and under + `-W error::DeprecationWarning` it would have changed the exception a + caller catches. The message has to name where to read instead, since + the answer differs by what the caller wanted. + """ + with self.assertRaises(RuntimeError) as cm: + server_args_module.get_global_server_args() + message = str(cm.exception) + self.assertIn("runtime_context", message) + self.assertIn("get_server_args()", message) + + # And the type does not change when warnings are errors. + with warnings.catch_warnings(): + warnings.simplefilter("error") + with self.assertRaises(RuntimeError): + server_args_module.get_global_server_args() + def test_tokenizer_alias_is_distinct_role_shim(self): # Deliberately NOT an alias: the two legacy setters publish with # different process roles (scheduler vs tokenizer). @@ -260,10 +284,9 @@ class TestServerArgsOwnership(_IsolatedServerArgs): def test_pre_publish_error_verbatim(self): reset_context() - for accessor in (get_server_args, server_args_module.get_global_server_args): - with self.assertRaises(ValueError) as cm: - accessor() - self.assertEqual(str(cm.exception), "Global server args is not set yet!") + with self.assertRaises(ValueError) as cm: + get_server_args() + self.assertEqual(str(cm.exception), "Global server args is not set yet!") def test_republish_overwrite_allowed(self): first = ServerArgs(model_path="dummy") diff --git a/test/registered/unit/test_server_args_migration.py b/test/registered/unit/test_server_args_migration.py index 2a3d16c72..00808302c 100644 --- a/test/registered/unit/test_server_args_migration.py +++ b/test/registered/unit/test_server_args_migration.py @@ -151,10 +151,88 @@ class TestServerArgsAnnotatedCli(CustomTestCase): ] ) - def test_deprecated_flags_still_work(self): - """Deprecated flags set the correct dest field.""" - sa = self._parse(["--stream-output"]) - self.assertTrue(sa.incremental_streaming_output) + def test_the_deprecated_flag_still_reaches_its_replacement(self): + """`--disable-cuda-graph` is the one deprecated alias left standing. + + It is worth a test because it is not a rename. The flag writes + `disable_cuda_graph`, which is `no_cli=True` -- so this spelling is its + only way in -- and the cuda-graph hook is what turns that into the two + phase backends the replacement flag sets directly. Asserting the + backends, against the baseline of not passing it, covers that second + half; asserting the dest alone would not. + """ + from sglang.srt.arg_groups.cuda_graph_hook import parse_cuda_graph_config + from sglang.srt.model_executor.cuda_graph_config import ( + Backend, + default_prefill_backend, + ) + + def backends(argv): + sa = self._parse(argv) + parse_cuda_graph_config(sa) + config = resolution_result(sa, "cuda_graph_config") + return sa.disable_cuda_graph, config.decode.backend, config.prefill.backend + + # Not a literal: the prefill default is BREAKABLE on CUDA and + # TC_PIECEWISE elsewhere, and this file runs on the CPU runner. + self.assertEqual(backends([]), (False, Backend.FULL, default_prefill_backend())) + self.assertEqual( + backends(["--disable-cuda-graph"]), + (True, Backend.DISABLED, Backend.DISABLED), + ) + + def test_each_deprecation_shape_does_its_job(self): + """The four `Deprecated*Action` shapes, on a throwaway parser. + + Only `DeprecatedStoreTrueAction` has a registration in `ServerArgs` + today (`--disable-cuda-graph`, covered above). The other three are kept + for the next flag that needs them, which makes this the only thing + standing between them and silent rot -- and it doubles as the worked + example of which shape to reach for. + """ + from sglang.srt.arg_groups.argparse_actions import ( + DeprecatedAction, + DeprecatedAliasStoreAction, + DeprecatedStoreConstAction, + DeprecatedStoreTrueAction, + ) + + def parser_with(**kwargs): + parser = argparse.ArgumentParser() + parser.add_argument("--old", **kwargs) + return parser + + # An old boolean whose field survives. + args = parser_with( + action=DeprecatedStoreTrueAction, dest="new", new_flag="--new" + ).parse_args(["--old"]) + self.assertIs(args.new, True) + + # An old boolean replaced by one value of a valued flag. + args = parser_with( + action=DeprecatedStoreConstAction, + dest="backend", + const_value="disabled", + new_flag="--backend", + ).parse_args(["--old"]) + self.assertEqual(args.backend, "disabled") + + # An old valued flag, renamed: the value survives the move. + args = parser_with( + action=DeprecatedAliasStoreAction, dest="new", new_flag="--new" + ).parse_args(["--old", "fa3"]) + self.assertEqual(args.new, "fa3") + + # Retired outright: stop, and say what to use instead. + parser = parser_with( + action=DeprecatedAction, dest="gone", error_message="use --new instead" + ) + with self.assertRaises(SystemExit): + parser.parse_args(["--old"]) + + # Without a message it is a no-op that warns rather than a hard stop. + args = parser_with(action=DeprecatedAction, dest="gone").parse_args(["--old"]) + self.assertIsNone(args.gone) def test_combined_parse(self): """Multiple option types parsed together in one invocation.""" diff --git a/test/registered/vlm/test_vlm_tp4.py b/test/registered/vlm/test_vlm_tp4.py index 34d051269..a2b8f5098 100644 --- a/test/registered/vlm/test_vlm_tp4.py +++ b/test/registered/vlm/test_vlm_tp4.py @@ -39,7 +39,7 @@ class TestVLMTP4(CustomTestCase): "--mem-fraction-static", "0.8", "--trust-remote-code", - "--mamba-scheduler-strategy", + "--mamba-radix-cache-strategy", "extra_buffer", "--mamba-track-interval", "128",