diff --git a/benchmark/deepseek_v3/README.md b/benchmark/deepseek_v3/README.md index cf6a569cb..269959628 100644 --- a/benchmark/deepseek_v3/README.md +++ b/benchmark/deepseek_v3/README.md @@ -177,19 +177,19 @@ There is one B200 node with 4 (for FP4) GPUs or 8 (for FP4 or FP8) GPUs. Both F If using 4 GPUs: ```bash -python3 -m sglang.launch_server --model-path nvidia/DeepSeek-R1-0528-FP4-V2 --host 0.0.0.0 --port 8000 --tensor-parallel-size=4 --cuda-graph-max-bs 256 --max-running-requests 256 --mem-fraction-static 0.85 --ep-size 4 --scheduler-recv-interval 30 --enable-symm-mem --stream-interval 10 +python3 -m sglang.launch_server --model-path nvidia/DeepSeek-R1-0528-FP4-V2 --host 0.0.0.0 --port 8000 --tensor-parallel-size=4 --cuda-graph-max-bs-decode 256 --max-running-requests 256 --mem-fraction-static 0.85 --ep-size 4 --scheduler-recv-interval 30 --enable-symm-mem --stream-interval 10 ``` If using 8 GPUs: ```bash -python3 -m sglang.launch_server --model-path nvidia/DeepSeek-R1-0528-FP4-V2 --host 0.0.0.0 --port 8000 --tensor-parallel-size=8 --cuda-graph-max-bs 256 --max-running-requests 256 --mem-fraction-static 0.85 --ep-size 8 --scheduler-recv-interval 30 --enable-symm-mem --stream-interval 10 +python3 -m sglang.launch_server --model-path nvidia/DeepSeek-R1-0528-FP4-V2 --host 0.0.0.0 --port 8000 --tensor-parallel-size=8 --cuda-graph-max-bs-decode 256 --max-running-requests 256 --mem-fraction-static 0.85 --ep-size 8 --scheduler-recv-interval 30 --enable-symm-mem --stream-interval 10 ``` #### FP8 ```bash -SGLANG_ENABLE_JIT_DEEPGEMM=false python3 -m sglang.launch_server --model-path=deepseek-ai/DeepSeek-R1-0528 --host=0.0.0.0 --port=8000 --tensor-parallel-size=8 --cuda-graph-max-bs 128 --max-running-requests 128 --mem-fraction-static 0.82 --kv-cache-dtype fp8_e4m3 --chunked-prefill-size 32768 --max-prefill-tokens 32768 --scheduler-recv-interval 30 --stream-interval 30 --fp8-gemm-backend flashinfer_trtllm +SGLANG_ENABLE_JIT_DEEPGEMM=false python3 -m sglang.launch_server --model-path=deepseek-ai/DeepSeek-R1-0528 --host=0.0.0.0 --port=8000 --tensor-parallel-size=8 --cuda-graph-max-bs-decode 128 --max-running-requests 128 --mem-fraction-static 0.82 --kv-cache-dtype fp8_e4m3 --chunked-prefill-size 32768 --max-prefill-tokens 32768 --scheduler-recv-interval 30 --stream-interval 30 --fp8-gemm-backend flashinfer_trtllm ``` ### Example: Serving with two H200\*8 nodes and docker diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx index 97b5bf04f..6db67ff32 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx @@ -497,7 +497,7 @@ sglang_args=$(echo serve \ --dsa-decode-backend flashmla_sparse \ --enable-dsa-prefill-context-parallel \ --dsa-prefill-cp-mode round-robin-split \ - --cuda-graph-max-bs 128 \ + --cuda-graph-max-bs-decode 128 \ --max-running-requests 128 \ --trust-remote-code --host "0.0.0.0" --port 30000 \ --log-requests \ diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx index b1bd86793..5bbcd399c 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx @@ -174,7 +174,7 @@ Both Instruct repos ship as **FP4 MoE experts + FP8 attention / dense** (one mix **Concurrency & DeepEP dispatch buffer** -Must hold: `max-running-requests × MTP_draft_tokens ≤ SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK`. Violating it blows DeepEP's dispatch buffer at steady-state load (`deep_ep.cpp:1105`). When tuning, move `--cuda-graph-max-bs`, `--max-running-requests`, and the env together. +Must hold: `max-running-requests × MTP_draft_tokens ≤ SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK`. Violating it blows DeepEP's dispatch buffer at steady-state load (`deep_ep.cpp:1105`). When tuning, move `--cuda-graph-max-bs-decode`, `--max-running-requests`, and the env together. The generator currently picks values on the **conservative** side (mirroring an internal stress-test matrix). They run safely out of the box but likely leave throughput on the table — please tune them up toward your actual workload's peak concurrency and report findings back so the defaults can be revised. diff --git a/docs_new/docs/advanced_features/hyperparameter_tuning.mdx b/docs_new/docs/advanced_features/hyperparameter_tuning.mdx index 6a52d5a36..327e2efce 100644 --- a/docs_new/docs/advanced_features/hyperparameter_tuning.mdx +++ b/docs_new/docs/advanced_features/hyperparameter_tuning.mdx @@ -64,10 +64,10 @@ If you encounter out-of-memory (OOM) errors, you can adjust the following parame - If OOM occurs during decoding, try lowering `--max-running-requests`. - You can also reduce `--mem-fraction-static` to a smaller value, such as 0.8 or 0.7. This decreases the memory usage of the KV cache memory pool and helps prevent OOM errors during both prefill and decoding. However, it limits maximum concurrency and reduces peak throughput. -### Tune `--cuda-graph-max-bs` +### Tune `--cuda-graph-max-bs-decode` By default, CUDA graph is enabled only for small batch sizes (e.g., less than 160 or 256). However, for some models, especially at large tensor parallelism sizes, CUDA graph can be useful for batch sizes up to 512 or 768. -Therefore, it may be beneficial to increase `--cuda-graph-max-bs` to a larger value. +Therefore, it may be beneficial to increase `--cuda-graph-max-bs-decode` to a larger value. Note that CUDA graph consumes more memory, so you may need to reduce `--mem-fraction-static` at the same time. ### Tune `--dp-size` and `--tp-size` diff --git a/docs_new/docs/advanced_features/lora.ipynb b/docs_new/docs/advanced_features/lora.ipynb index 8e6e6d0a0..310c7eba1 100644 --- a/docs_new/docs/advanced_features/lora.ipynb +++ b/docs_new/docs/advanced_features/lora.ipynb @@ -228,7 +228,7 @@ "server_process, port = launch_server_cmd(\"\"\"\n", " python3 -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n", " --enable-lora \\\n", - " --cuda-graph-max-bs 2 \\\n", + " --cuda-graph-max-bs-decode 2 \\\n", " --max-loras-per-batch 2 \\\n", " --max-lora-rank 256\n", " --lora-target-modules all\n", @@ -434,7 +434,7 @@ "server_process, port = launch_server_cmd(\"\"\"\n", " python3 -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n", " --enable-lora \\\n", - " --cuda-graph-max-bs 8 \\\n", + " --cuda-graph-max-bs-decode 8 \\\n", " --max-loras-per-batch 3 \\\n", " --max-lora-rank 256 \\\n", " --lora-target-modules all \\\n", diff --git a/docs_new/docs/advanced_features/lora.mdx b/docs_new/docs/advanced_features/lora.mdx index 3ed6b4430..9fe011b24 100644 --- a/docs_new/docs/advanced_features/lora.mdx +++ b/docs_new/docs/advanced_features/lora.mdx @@ -165,7 +165,7 @@ server_process, port = launch_server_cmd( """ python3 -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ --enable-lora \ - --cuda-graph-max-bs 2 \ + --cuda-graph-max-bs-decode 2 \ --max-loras-per-batch 2 \ --max-lora-rank 256 --lora-target-modules all @@ -314,7 +314,7 @@ server_process, port = launch_server_cmd( """ python3 -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ --enable-lora \ - --cuda-graph-max-bs 8 \ + --cuda-graph-max-bs-decode 8 \ --max-loras-per-batch 3 \ --max-lora-rank 256 \ --lora-target-modules all \ diff --git a/docs_new/docs/advanced_features/server_arguments.mdx b/docs_new/docs/advanced_features/server_arguments.mdx index 8667cafdf..065d1d1f1 100644 --- a/docs_new/docs/advanced_features/server_arguments.mdx +++ b/docs_new/docs/advanced_features/server_arguments.mdx @@ -2489,7 +2489,7 @@ Please consult the documentation below and [server_args.py](https://github.com/s Type: int - `--cuda-graph-max-bs` + `--cuda-graph-max-bs-decode` Deprecated alias for --cuda-graph-max-bs-decode. `None` Type: int diff --git a/docs_new/docs/advanced_features/speculative_decoding.ipynb b/docs_new/docs/advanced_features/speculative_decoding.ipynb index c24cac402..2cd02c0e8 100644 --- a/docs_new/docs/advanced_features/speculative_decoding.ipynb +++ b/docs_new/docs/advanced_features/speculative_decoding.ipynb @@ -36,7 +36,7 @@ "\n", "You can find the best combinations of these parameters with [bench_speculative.py](https://github.com/sgl-project/sglang/blob/main/scripts/playground/bench_speculative.py).\n", "\n", - "In the documentation below, we set `--cuda-graph-max-bs` to be a small value for faster engine startup. For your own workloads, please tune the above parameters together with `--cuda-graph-max-bs`, `--max-running-requests`, `--mem-fraction-static` for the best performance. " + "In the documentation below, we set `--cuda-graph-max-bs-decode` to be a small value for faster engine startup. For your own workloads, please tune the above parameters together with `--cuda-graph-max-bs-decode`, `--max-running-requests`, `--mem-fraction-static` for the best performance. " ] }, { @@ -69,7 +69,7 @@ "server_process, port = launch_server_cmd(\"\"\"\n", "python3 -m sglang.launch_server --model meta-llama/Llama-2-7b-chat-hf --speculative-algorithm EAGLE \\\n", " --speculative-draft-model-path lmsys/sglang-EAGLE-llama2-chat-7B --speculative-num-steps 3 \\\n", - " --speculative-eagle-topk 4 --speculative-num-draft-tokens 16 --cuda-graph-max-bs 8 --log-level warning\n", + " --speculative-eagle-topk 4 --speculative-num-draft-tokens 16 --cuda-graph-max-bs-decode 8 --log-level warning\n", "\"\"\")\n", "\n", "wait_for_server(f\"http://localhost:{port}\")" @@ -181,7 +181,7 @@ "python3 -m sglang.launch_server --model meta-llama/Meta-Llama-3-8B-Instruct --speculative-algorithm EAGLE \\\n", " --speculative-draft-model-path lmsys/sglang-EAGLE-LLaMA3-Instruct-8B --speculative-num-steps 5 \\\n", " --speculative-eagle-topk 8 --speculative-num-draft-tokens 64 --speculative-token-map thunlp/LLaMA3-Instruct-8B-FR-Spec/freq_32768.pt \\\n", - " --mem-fraction 0.7 --cuda-graph-max-bs 2 --dtype float16 --log-level warning\n", + " --mem-fraction 0.7 --cuda-graph-max-bs-decode 2 --dtype float16 --log-level warning\n", "\"\"\")\n", "\n", "wait_for_server(f\"http://localhost:{port}\")" @@ -235,7 +235,7 @@ "python3 -m sglang.launch_server --model meta-llama/Llama-3.1-8B-Instruct --speculative-algorithm EAGLE3 \\\n", " --speculative-draft-model-path jamesliu1/sglang-EAGLE3-Llama-3.1-Instruct-8B --speculative-num-steps 5 \\\n", " --speculative-eagle-topk 8 --speculative-num-draft-tokens 32 --mem-fraction 0.6 \\\n", - " --cuda-graph-max-bs 2 --dtype float16 --log-level warning\n", + " --cuda-graph-max-bs-decode 2 --dtype float16 --log-level warning\n", "\"\"\")\n", "\n", "wait_for_server(f\"http://localhost:{port}\")" diff --git a/docs_new/docs/advanced_features/speculative_decoding.mdx b/docs_new/docs/advanced_features/speculative_decoding.mdx index 5efa8d32f..af009ae83 100644 --- a/docs_new/docs/advanced_features/speculative_decoding.mdx +++ b/docs_new/docs/advanced_features/speculative_decoding.mdx @@ -242,7 +242,7 @@ python3 -m sglang.launch_server \ --speculative-eagle-topk 4 \ --speculative-num-draft-tokens 16 \ --mem-fraction-static 0.7 \ - --cuda-graph-max-bs 8 \ + --cuda-graph-max-bs-decode 8 \ --log-level warning ``` @@ -326,7 +326,7 @@ python3 -m sglang.launch_server \ --speculative-num-draft-tokens 16 \ --speculative-token-map thunlp/LLaMA3-Instruct-8B-FR-Spec/freq_32768.pt \ --mem-fraction-static 0.7 \ - --cuda-graph-max-bs 8 \ + --cuda-graph-max-bs-decode 8 \ --dtype float16 \ --log-level warning ``` @@ -365,7 +365,7 @@ python3 -m sglang.launch_server \ --speculative-eagle-topk 4 \ --speculative-num-draft-tokens 16 \ --mem-fraction-static 0.7 \ - --cuda-graph-max-bs 8 \ + --cuda-graph-max-bs-decode 8 \ --dtype float16 \ --log-level warning ``` @@ -405,7 +405,7 @@ python3 -m sglang.launch_server \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 2 \ --mem-fraction-static 0.7 \ - --cuda-graph-max-bs 8 \ + --cuda-graph-max-bs-decode 8 \ --log-level warning ``` @@ -557,7 +557,7 @@ python3 -m sglang.launch_server \ --speculative-eagle-topk 2 \ --speculative-num-draft-tokens 7 \ --mem-fraction-static 0.7 \ - --cuda-graph-max-bs 8 \ + --cuda-graph-max-bs-decode 8 \ --log-level warning ``` @@ -600,7 +600,7 @@ python3 -m sglang.launch_server \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 5 \ --mem-fraction-static 0.7 \ - --cuda-graph-max-bs 8 \ + --cuda-graph-max-bs-decode 8 \ --log-level warning ``` @@ -695,7 +695,7 @@ python3 -m sglang.launch_server \ --speculative-num-draft-tokens 16 \ --speculative-ngram-max-bfs-breadth 10 \ --mem-fraction-static 0.7 \ - --cuda-graph-max-bs 8 \ + --cuda-graph-max-bs-decode 8 \ --log-level warning ``` @@ -972,10 +972,10 @@ Below is a comprehensive list of all speculative decoding parameters available i ```bash Command # Fewer CUDA graph captures = less memory reserved ---cuda-graph-max-bs 4 # or even 2 for tight memory situations +--cuda-graph-max-bs-decode 4 # or even 2 for tight memory situations ``` -- If omitted, `--cuda-graph-max-bs` is auto-selected based on GPU memory and TP size, and can be much larger on high-memory GPUs. +- If omitted, `--cuda-graph-max-bs-decode` is auto-selected based on GPU memory and TP size, and can be much larger on high-memory GPUs. ### Step 3: Reduce draft tree size @@ -1008,13 +1008,13 @@ python3 -m sglang.launch_server \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ --speculative-num-draft-tokens 4 \ - --cuda-graph-max-bs 2 \ + --cuda-graph-max-bs-decode 2 \ --mem-fraction-static 0.5 \ --max-running-requests 4 \ --log-level warning ``` -Then gradually increase `--speculative-num-draft-tokens`, `--speculative-eagle-topk`, and `--cuda-graph-max-bs`. Increase `--mem-fraction-static` last, only after the run is stable. +Then gradually increase `--speculative-num-draft-tokens`, `--speculative-eagle-topk`, and `--cuda-graph-max-bs-decode`. Increase `--mem-fraction-static` last, only after the run is stable. --- diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_glm5.2_examples.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_glm5.2_examples.mdx index e560bc30d..0a54b9dc8 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_glm5.2_examples.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_glm5.2_examples.mdx @@ -245,7 +245,7 @@ do --mem-fraction-static 0.8 \ --port 8000 \ --served-model-name glm-5 \ - --cuda-graph-max-bs 32 \ + --cuda-graph-max-bs-decode 32 \ --moe-a2a-backend deepep \ --deepep-mode auto \ --speculative-draft-model-quantization unquant \ @@ -352,7 +352,7 @@ do --port 8003 --trust-remote-code --nnodes 1 --node-rank $i --tp-size 16 --dp-size 16 --ep-size 16 \ --mem-fraction-static 0.8 --max-running-requests 128 --attention-backend ascend --device npu --quantization modelslim \ --served-model-name glm-5 --moe-a2a-backend deepep --enable-dp-attention --deepep-mode low_latency \ - --cuda-graph-max-bs 4 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 180000 \ + --cuda-graph-max-bs-decode 4 --disaggregation-transfer-backend ascend --watchdog-timeout 9000 --context-length 180000 \ --tokenizer-worker-num 4 --prefill-round-robin-balance --disable-shared-experts-fusion --dtype bfloat16 --load-balance-method round_robin \ --speculative-draft-model-quantization unquant \ --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_optimization.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_optimization.mdx index c3ae0204c..ff4fef1dc 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_optimization.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_optimization.mdx @@ -238,7 +238,7 @@ These arguments and environment variables are critical for tuning decode perform Enabled - `--cuda-graph-max-bs` + `--cuda-graph-max-bs-decode` Caps the maximum batch size for which CUDA Graphs are captured. Larger values cover more batch sizes but increase graph capture time and memory overhead. If your `max-running-requests` is high but typical batch sizes are lower, use a smaller value to reduce capture overhead. Tune based on your observed batch size distribution during serving. `4` diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_support_features.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_support_features.mdx index 302bf6d84..161e2b0d0 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_support_features.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_support_features.mdx @@ -2072,7 +2072,7 @@ click [Server Arguments](../../advanced_features/server_arguments). A2, A3 - `--cuda-graph-max-bs` + `--cuda-graph-max-bs-decode` `None` Type: int A2, A3 diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/deepseek_v3_2.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/deepseek_v3_2.mdx index 6bfb401a4..02e324ea9 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/deepseek_v3_2.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/deepseek_v3_2.mdx @@ -168,7 +168,7 @@ do --chunked-prefill-size -1 \ --max-prefill-tokens 68000 \ --max-running-requests 32 \ - --cuda-graph-max-bs 4 \ + --cuda-graph-max-bs-decode 4 \ --moe-a2a-backend deepep \ --deepep-mode low_latency \ --quantization modelslim \ @@ -363,7 +363,7 @@ do --chunked-prefill-size -1 \ --max-prefill-tokens 68000 \ --max-running-requests 32 \ - --cuda-graph-max-bs 4 \ + --cuda-graph-max-bs-decode 4 \ --moe-a2a-backend deepep \ --deepep-mode low_latency \ --quantization modelslim \ @@ -558,7 +558,7 @@ do --chunked-prefill-size -1 \ --max-prefill-tokens 68000 \ --max-running-requests 32 \ - --cuda-graph-max-bs 4 \ + --cuda-graph-max-bs-decode 4 \ --moe-a2a-backend deepep \ --deepep-mode low_latency \ --quantization modelslim \ diff --git a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/glm5_1.mdx b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/glm5_1.mdx index 3656098a4..52790e77f 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/best_practice/glm5_1.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/best_practice/glm5_1.mdx @@ -112,7 +112,7 @@ do --trust-remote-code \ --mem-fraction-static 0.65 \ --served-model-name glm-5 \ - --cuda-graph-max-bs 16 \ + --cuda-graph-max-bs-decode 16 \ --max-running-requests 256 \ --quantization modelslim \ --speculative-draft-model-quantization unquant \ diff --git a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/deepseek_r1.mdx b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/deepseek_r1.mdx index 018ba26d4..c3823826f 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/deepseek_r1.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/deepseek_r1.mdx @@ -27,7 +27,7 @@ v0.5.13 or a later version. | Expert Parallelism | `--ep-size 16 \`
`--moe-a2a-backend deepep \`
`--deepep-mode auto` | | PD Disaggregation | `--disaggregation-mode prefill \`
`--disaggregation-transfer-backend ascend` | | Quantization | `--quantization modelslim` | -| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs`; e.g. `--cuda-graph-bs 4 8 20 21 22` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs-decode`; e.g. `--cuda-graph-bs 4 8 20 21 22` | | Speculative Decoding | `--speculative-algorithm NEXTN \`
`--speculative-num-steps 2 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 3` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | | DP LM Head | `--enable-dp-lm-head` | diff --git a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/deepseek_v3_2.mdx b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/deepseek_v3_2.mdx index c91dd99a9..c133d4255 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/deepseek_v3_2.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/deepseek_v3_2.mdx @@ -30,7 +30,7 @@ recommended to use v0.5.13 or a later version. | PD Disaggregation | `--disaggregation-mode prefill \`
`--disaggregation-transfer-backend ascend` | | Quantization | `--quantization modelslim` | | Speculative Decoding | `--speculative-algorithm NEXTN \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4` | -| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs`; e.g. `--cuda-graph-bs 1 2 3 4 5 6 7 8` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs-decode`; e.g. `--cuda-graph-bs 1 2 3 4 5 6 7 8` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | | DP LM Head | `--enable-dp-lm-head` | diff --git a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/glm_5_1.mdx b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/glm_5_1.mdx index e36c331a6..db45d23b8 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/glm_5_1.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/glm_5_1.mdx @@ -31,7 +31,7 @@ v0.5.13 or a later version. | PD Disaggregation | `--disaggregation-mode prefill \`
`--disaggregation-transfer-backend ascend` | | Quantization | `--quantization modelslim` | | Chunked Prefill | auto based on device memory, or set explicit value;
disable with `--chunked-prefill-size -1`; e.g. `--chunked-prefill-size 16384` | -| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs`; e.g. `--cuda-graph-bs 1 2 3 4 5 6` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs-decode`; e.g. `--cuda-graph-bs 1 2 3 4 5 6` | | Speculative Decoding | `--speculative-algorithm NEXTN \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4 \`
`--speculative-draft-model-quantization unquant` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | | DP LM Head | `--enable-dp-lm-head` | diff --git a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/kimi_k2_6.mdx b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/kimi_k2_6.mdx index 9d9ef5516..bbad01462 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/kimi_k2_6.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/kimi_k2_6.mdx @@ -29,7 +29,7 @@ v0.5.13 or a later version. | PD Disaggregation | `--disaggregation-mode prefill \`
`--disaggregation-transfer-backend ascend` | | Quantization | `--quantization modelslim` | | Chunked Prefill | auto based on device memory, or set explicit value;
disable with `--chunked-prefill-size -1`; e.g. `--chunked-prefill-size 32768` | -| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs`; e.g. `--cuda-graph-bs 1 2 4 8 12 16 24 32 48 64 96 120` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs-decode`; e.g. `--cuda-graph-bs 1 2 4 8 12 16 24 32 48 64 96 120` | | Speculative Decoding | `--speculative-algorithm EAGLE3 \`
`--speculative-draft-model-path /path/to/draft-model-weights \`
`--speculative-num-steps 4 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 5 \`
`--speculative-draft-model-quantization unquant` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | | DP LM Head | `--enable-dp-lm-head` | diff --git a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/minimax_m2_5.mdx b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/minimax_m2_5.mdx index 7ac83c7f4..8a8da9849 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/minimax_m2_5.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/minimax_m2_5.mdx @@ -29,7 +29,7 @@ v0.5.13 or a later version. | PD Disaggregation | `--disaggregation-mode prefill \`
`--disaggregation-transfer-backend ascend` | | Quantization | `--quantization modelslim` | | Chunked Prefill | auto based on device memory, or set explicit value;
disable with `--chunked-prefill-size -1`; e.g. `--chunked-prefill-size 8192` | -| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs`; e.g. `--cuda-graph-bs 8 16 24 32 48 64 80` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs-decode`; e.g. `--cuda-graph-bs 8 16 24 32 48 64 80` | | Speculative Decoding | `--speculative-algorithm EAGLE3 \`
`--speculative-draft-model-path /path/to/draft-model-weights \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4 \`
`--speculative-draft-model-quantization unquant` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | diff --git a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3-8b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3-8b.mdx index 71bf1b5e3..140b85ca3 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3-8b.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3-8b.mdx @@ -26,7 +26,7 @@ version. | Tensor Parallelism | `--tp-size 2` | | Quantization | `--quantization modelslim` | | Chunked Prefill | auto based on device memory, or set explicit value;
disable with `--chunked-prefill-size -1`; e.g. `--chunked-prefill-size 8192` | -| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs`; e.g. `--cuda-graph-bs 1 2 4 6 9 10 15 16` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs-decode`; e.g. `--cuda-graph-bs 1 2 4 6 9 10 15 16` | | Speculative Decoding | `--speculative-algorithm EAGLE3 \`
`--speculative-draft-model-path /path/to/draft-model-weights \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4 \`
`--speculative-draft-model-quantization unquant` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | diff --git a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_235b_a22b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_235b_a22b.mdx index bad84e227..a24e5ca3e 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_235b_a22b.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_235b_a22b.mdx @@ -30,7 +30,7 @@ v0.5.13 or a later version. | PD Disaggregation | `--disaggregation-mode prefill \`
`--disaggregation-transfer-backend ascend` | | Quantization | `--quantization modelslim` | | Chunked Prefill | auto based on device memory, or set explicit value;
disable with `--chunked-prefill-size -1`; e.g. `--chunked-prefill-size 94208` | -| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs`; e.g. `--cuda-graph-bs 1 2 4 8 16 20 24 26 27` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs-decode`; e.g. `--cuda-graph-bs 1 2 4 8 16 20 24 26 27` | | Speculative Decoding | `--speculative-algorithm EAGLE3 \`
`--speculative-draft-model-path /path/to/draft-model-weights \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4 \`
`--speculative-draft-model-quantization unquant` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | | DP LM Head | `--enable-dp-lm-head` | diff --git a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_30b_a3b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_30b_a3b.mdx index 9666ccd81..0f362ed65 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_30b_a3b.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_30b_a3b.mdx @@ -26,7 +26,7 @@ v0.5.13 or a later version. | Tensor Parallelism | `--tp-size 2` | | Data Parallelism | `--dp-size 2` | | Quantization | `--quantization modelslim` | -| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs`; e.g. `--cuda-graph-bs 1 5 15 40 70 100 120 130 140 146 150 154 156 158 160 162` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs-decode`; e.g. `--cuda-graph-bs 1 5 15 40 70 100 120 130 140 146 150 154 156 158 160 162` | | Speculative Decoding | `--speculative-algorithm EAGLE3 \`
`--speculative-draft-model-path /path/to/draft-model-weights \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4 \`
`--speculative-draft-model-quantization unquant` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | diff --git a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_32b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_32b.mdx index 302e7c19a..7b0f505b2 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_32b.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_32b.mdx @@ -24,7 +24,7 @@ version. |-------------------------------|-----------------------------------------------------------------------------------------------| | Tensor Parallelism | `--tp-size 4` | | Quantization | `--quantization modelslim` | -| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs`; e.g. `--cuda-graph-bs 1 4 8 16` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs-decode`; e.g. `--cuda-graph-bs 1 4 8 16` | | Speculative Decoding | `--speculative-algorithm EAGLE3 \`
`--speculative-draft-model-path /path/to/draft-model-weights \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4 \`
`--speculative-draft-model-quantization unquant` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | diff --git a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_5_397b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_5_397b.mdx index 0ee319f7f..ad16aa888 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_5_397b.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_5_397b.mdx @@ -28,7 +28,7 @@ use v0.5.13 or a later version. | Data Parallelism | `--dp-size 8` | | Expert Parallelism | `--ep-size 16 \`
`--moe-a2a-backend deepep \`
`--deepep-mode auto` | | Quantization | `--quantization modelslim` | -| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs`; e.g. `--cuda-graph-bs 2 4 6 8 10 12 14 16 18 20` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs-decode`; e.g. `--cuda-graph-bs 2 4 6 8 10 12 14 16 18 20` | | Speculative Decoding | `--speculative-algorithm NEXTN \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4 \`
`--speculative-draft-model-quantization unquant` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | | DP LM Head | `--enable-dp-lm-head` | diff --git a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_6_27b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_6_27b.mdx index ea51dd415..9bc4b663a 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_6_27b.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_6_27b.mdx @@ -26,7 +26,7 @@ v0.5.13 or a later version. | Tensor Parallelism | `--tp-size 2` | | Quantization | `--quantization modelslim` | | Chunked Prefill | auto based on device memory, or set explicit value;
disable with `--chunked-prefill-size -1`; e.g. `--chunked-prefill-size 32768` | -| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs`; e.g. `--cuda-graph-bs 2 8 16 32 48` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs-decode`; e.g. `--cuda-graph-bs 2 8 16 32 48` | | Speculative Decoding | `--speculative-algorithm NEXTN \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | diff --git a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_6_35b_a3b.mdx b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_6_35b_a3b.mdx index 413d8ecf8..ab825b17b 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_6_35b_a3b.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_6_35b_a3b.mdx @@ -26,7 +26,7 @@ v0.5.13 or a later version. |-------------------------------|-----------------------------------------------------------------------------------------------| | Tensor Parallelism | `--tp-size 2` | | Chunked Prefill | auto based on device memory, or set explicit value;
disable with `--chunked-prefill-size -1`; e.g. `--chunked-prefill-size 16384` | -| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs`; e.g. `--cuda-graph-bs 4 8 16 24 32 48 64 80 96 112 120` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs-decode`; e.g. `--cuda-graph-bs 4 8 16 24 32 48 64 80 96 112 120` | | Speculative Decoding | `--speculative-algorithm NEXTN \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | diff --git a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_next_80b_a3b_instruct.mdx b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_next_80b_a3b_instruct.mdx index 327cdf5b0..105649359 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_next_80b_a3b_instruct.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/model-tutorials/qwen3_next_80b_a3b_instruct.mdx @@ -27,7 +27,7 @@ v0.5.13 or a later version. | Data Parallelism | `--dp-size 2` | | Expert Parallelism | `--ep-size 4 \`
`--moe-a2a-backend deepep \`
`--deepep-mode auto` | | Quantization | `--quantization modelslim` | -| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs`; e.g. `--cuda-graph-bs 2 4 8` | +| NPU Graph | enabled by default; disable with `--disable-cuda-graph`;
control range via `--cuda-graph-bs` or `--cuda-graph-max-bs-decode`; e.g. `--cuda-graph-bs 2 4 8` | | Speculative Decoding | `--speculative-algorithm NEXTN \`
`--speculative-num-steps 3 \`
`--speculative-eagle-topk 1 \`
`--speculative-num-draft-tokens 4 \`
`--speculative-draft-model-quantization unquant \`
`--speculative-draft-model-path /path/to/draft-model-weights` | | Overlap Schedule | `export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` | | DP LM Head | `--enable-dp-lm-head` | diff --git a/docs_new/docs/references/faq.mdx b/docs_new/docs/references/faq.mdx index a2cb48695..f4f1b52f1 100644 --- a/docs_new/docs/references/faq.mdx +++ b/docs_new/docs/references/faq.mdx @@ -23,7 +23,7 @@ This error may result from kernel errors or out-of-memory issues: ### The server hangs - If the server hangs during initialization or running, it can be memory issues (out of memory), network issues (nccl errors), or other bugs in sglang. - If it is out of memory, you might see that `avail mem` is very low during the initialization or right after initialization. In this case, - you can try to decrease `--mem-fraction-static`, decrease `--cuda-graph-max-bs`, or decrease `--chunked-prefill-size`. + you can try to decrease `--mem-fraction-static`, decrease `--cuda-graph-max-bs-decode`, or decrease `--chunked-prefill-size`. - Other bugs, please file an issue on GitHub. diff --git a/docs_new/docs/references/multi_node_deployment/lws_pd/lws_pd_deploy.mdx b/docs_new/docs/references/multi_node_deployment/lws_pd/lws_pd_deploy.mdx index 14eac03fd..380b7c545 100644 --- a/docs_new/docs/references/multi_node_deployment/lws_pd/lws_pd_deploy.mdx +++ b/docs_new/docs/references/multi_node_deployment/lws_pd/lws_pd_deploy.mdx @@ -384,7 +384,7 @@ spec: - "32768" - --disaggregation-ib-device - "mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3" - - --cuda-graph-max-bs + - --cuda-graph-max-bs-decode - "64" - --max-running-requests - "2048" @@ -519,7 +519,7 @@ spec: - --disaggregation-ib-device # should modify according your rdma env - "mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3" - - --cuda-graph-max-bs + - --cuda-graph-max-bs-decode - "64" - --max-running-requests - "2048" diff --git a/docs_new/docs/references/multi_node_deployment/rbg_pd/deepseekv32_pd.mdx b/docs_new/docs/references/multi_node_deployment/rbg_pd/deepseekv32_pd.mdx index fbc63eb1b..c0fa4228e 100644 --- a/docs_new/docs/references/multi_node_deployment/rbg_pd/deepseekv32_pd.mdx +++ b/docs_new/docs/references/multi_node_deployment/rbg_pd/deepseekv32_pd.mdx @@ -285,7 +285,7 @@ spec: - "2048" - --tp-size - "8" # Size of Tensor Parallelism - - --cuda-graph-max-bs + - --cuda-graph-max-bs-decode - "16" - --dist-init-addr - $(LWS_LEADER_ADDRESS):20102 @@ -356,7 +356,7 @@ spec: - mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_6,mlx5_7 - --max-running-requests - "4096" - - --cuda-graph-max-bs + - --cuda-graph-max-bs-decode - "16" - --tp-size - "8" # Size of Tensor Parallelism diff --git a/docs_new/src/snippets/autoregressive/deepseek-r1-advanced-deployment.jsx b/docs_new/src/snippets/autoregressive/deepseek-r1-advanced-deployment.jsx index f24213932..d367be0df 100644 --- a/docs_new/src/snippets/autoregressive/deepseek-r1-advanced-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/deepseek-r1-advanced-deployment.jsx @@ -83,7 +83,7 @@ const lookupData = { "parameters": { "model_path": "nvidia/DeepSeek-R1-0528-FP4-v2", "tensor_parallel_size": 4, - "cuda_graph_max_bs": 256, + "cuda_graph_max_bs_decode": 256, "max_running_requests": 256, "mem_fraction_static": 0.85, "ep_size": 4, @@ -100,7 +100,7 @@ const lookupData = { "parameters": { "model_path": "nvidia/DeepSeek-R1-0528-FP4-v2", "tensor_parallel_size": 4, - "cuda_graph_max_bs": 256, + "cuda_graph_max_bs_decode": 256, "max_running_requests": 256, "mem_fraction_static": 0.85, "ep_size": 4, @@ -117,7 +117,7 @@ const lookupData = { "parameters": { "model_path": "nvidia/DeepSeek-R1-0528-FP4-v2", "tensor_parallel_size": 8, - "cuda_graph_max_bs": 256, + "cuda_graph_max_bs_decode": 256, "max_running_requests": 256, "mem_fraction_static": 0.85, "kv_cache_dtype": "fp8_e4m3", @@ -136,7 +136,7 @@ const lookupData = { "parameters": { "model_path": "nvidia/DeepSeek-R1-0528-FP4-v2", "tensor_parallel_size": 8, - "cuda_graph_max_bs": 256, + "cuda_graph_max_bs_decode": 256, "max_running_requests": 256, "mem_fraction_static": 0.85, "kv_cache_dtype": "fp8_e4m3", @@ -156,7 +156,7 @@ const lookupData = { "env_vars": "SGLANG_ENABLE_JIT_DEEPGEMM=false", "model_path": "deepseek-ai/DeepSeek-R1-0528", "tensor_parallel_size": 8, - "cuda_graph_max_bs": 128, + "cuda_graph_max_bs_decode": 128, "max_running_requests": 128, "mem_fraction_static": 0.82, "kv_cache_dtype": "fp8_e4m3", @@ -176,7 +176,7 @@ const lookupData = { "env_vars": "SGLANG_ENABLE_JIT_DEEPGEMM=false", "model_path": "deepseek-ai/DeepSeek-R1-0528", "tensor_parallel_size": 8, - "cuda_graph_max_bs": 128, + "cuda_graph_max_bs_decode": 128, "max_running_requests": 128, "mem_fraction_static": 0.82, "kv_cache_dtype": "fp8_e4m3", @@ -258,7 +258,7 @@ const lookupData = { "tensor_parallel_size": 8, "disable_radix_cache": true, "max_running_requests": 256, - "cuda_graph_max_bs": 256, + "cuda_graph_max_bs_decode": 256, "chunked_prefill_size": 32768, "max_prefill_tokens": 32768, "mem_fraction_static": 0.82, @@ -278,7 +278,7 @@ const lookupData = { "tensor_parallel_size": 8, "disable_radix_cache": true, "max_running_requests": 512, - "cuda_graph_max_bs": 512, + "cuda_graph_max_bs_decode": 512, "chunked_prefill_size": 32768, "max_prefill_tokens": 32768, "mem_fraction_static": 0.82, @@ -298,7 +298,7 @@ const lookupData = { "trust_remote_code": true, "tensor_parallel_size": 8, "mem_fraction_static": 0.8, - "cuda_graph_max_bs": 128, + "cuda_graph_max_bs_decode": 128, "chunked_prefill_size": 131072, "num_continuous_decode_steps": 4, "max_prefill_tokens": 131072, @@ -318,7 +318,7 @@ const lookupData = { "trust_remote_code": true, "tensor_parallel_size": 8, "mem_fraction_static": 0.8, - "cuda_graph_max_bs": 512, + "cuda_graph_max_bs_decode": 512, "chunked_prefill_size": 131072, "num_continuous_decode_steps": 4, "max_prefill_tokens": 131072, @@ -338,7 +338,7 @@ const lookupData = { "trust_remote_code": true, "tensor_parallel_size": 8, "mem_fraction_static": 0.8, - "cuda_graph_max_bs": 128, + "cuda_graph_max_bs_decode": 128, "chunked_prefill_size": 131072, "num_continuous_decode_steps": 4, "max_prefill_tokens": 131072, @@ -358,7 +358,7 @@ const lookupData = { "trust_remote_code": true, "tensor_parallel_size": 8, "mem_fraction_static": 0.8, - "cuda_graph_max_bs": 512, + "cuda_graph_max_bs_decode": 512, "chunked_prefill_size": 131072, "num_continuous_decode_steps": 4, "max_prefill_tokens": 131072, @@ -382,7 +382,7 @@ const lookupData = { "chunked_prefill_size": 196608, "num_continuous_decode_steps": 4, "max_prefill_tokens": 196608, - "cuda_graph_max_bs": 128, + "cuda_graph_max_bs_decode": 128, "attention_backend": "aiter", "kv_cache_dtype": "fp8_e4m3" } @@ -402,7 +402,7 @@ const lookupData = { "chunked_prefill_size": 196608, "num_continuous_decode_steps": 4, "max_prefill_tokens": 196608, - "cuda_graph_max_bs": 512, + "cuda_graph_max_bs_decode": 512, "attention_backend": "aiter", "kv_cache_dtype": "fp8_e4m3" } @@ -422,7 +422,7 @@ const lookupData = { "chunked_prefill_size": 196608, "num_continuous_decode_steps": 4, "max_prefill_tokens": 196608, - "cuda_graph_max_bs": 128, + "cuda_graph_max_bs_decode": 128, "attention_backend": "aiter", "kv_cache_dtype": "fp8_e4m3" } @@ -442,7 +442,7 @@ const lookupData = { "chunked_prefill_size": 196608, "num_continuous_decode_steps": 4, "max_prefill_tokens": 196608, - "cuda_graph_max_bs": 512, + "cuda_graph_max_bs_decode": 512, "attention_backend": "aiter", "kv_cache_dtype": "fp8_e4m3" } @@ -463,7 +463,7 @@ const fieldToFlag = { tensor_parallel_size: 'tp', data_parallel_size: 'dp', ep_size: 'ep-size', - cuda_graph_max_bs: 'cuda-graph-max-bs', + cuda_graph_max_bs_decode: 'cuda-graph-max-bs-decode', max_running_requests: 'max-running-requests', mem_fraction_static: 'mem-fraction-static', kv_cache_dtype: 'kv-cache-dtype', diff --git a/docs_new/src/snippets/autoregressive/deepseek-v32-deployment.jsx b/docs_new/src/snippets/autoregressive/deepseek-v32-deployment.jsx index 2502e85d7..4e4ece37b 100644 --- a/docs_new/src/snippets/autoregressive/deepseek-v32-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/deepseek-v32-deployment.jsx @@ -209,7 +209,7 @@ export const DeepSeekV32Deployment = () => { cmd += ' \\\n --trust-remote-code'; cmd += ' \\\n --dsa-prefill-backend tilelang'; cmd += ' \\\n --dsa-decode-backend tilelang'; - cmd += ' \\\n --cuda-graph-max-bs 64'; + cmd += ' \\\n --cuda-graph-max-bs-decode 64'; } // Strategy configurations diff --git a/docs_new/src/snippets/autoregressive/llama4-maverick-deployment.jsx b/docs_new/src/snippets/autoregressive/llama4-maverick-deployment.jsx index 1457f7ccc..7df75205d 100644 --- a/docs_new/src/snippets/autoregressive/llama4-maverick-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/llama4-maverick-deployment.jsx @@ -80,7 +80,7 @@ export const Llama4MaverickDeployment = () => { cmd += ` --speculative-eagle-topk 1 \\\n`; cmd += ` --speculative-num-draft-tokens 4 \\\n`; cmd += ` --mem-fraction-static 0.75 \\\n`; - cmd += ` --cuda-graph-max-bs 2`; + cmd += ` --cuda-graph-max-bs-decode 2`; } cmd += ` \\\n --enable-multimodal`; diff --git a/docs_new/src/snippets/autoregressive/llama4-scout-deployment.jsx b/docs_new/src/snippets/autoregressive/llama4-scout-deployment.jsx index 14d4029f1..57b525c75 100644 --- a/docs_new/src/snippets/autoregressive/llama4-scout-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/llama4-scout-deployment.jsx @@ -81,7 +81,7 @@ export const Llama4ScoutDeployment = () => { cmd += ` --speculative-eagle-topk 1 \\\n`; cmd += ` --speculative-num-draft-tokens 4 \\\n`; cmd += ` --mem-fraction-static 0.75 \\\n`; - cmd += ` --cuda-graph-max-bs 2`; + cmd += ` --cuda-graph-max-bs-decode 2`; } cmd += ` \\\n --enable-multimodal`; diff --git a/docs_new/src/snippets/autoregressive/mimo-v25-deployment.jsx b/docs_new/src/snippets/autoregressive/mimo-v25-deployment.jsx index cc7e98a6d..d1923e535 100644 --- a/docs_new/src/snippets/autoregressive/mimo-v25-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/mimo-v25-deployment.jsx @@ -344,7 +344,7 @@ export const MiMoV25Deployment = () => { flags.push(" --mem-fraction-static 0.7"); flags.push(" --max-running-requests 128"); flags.push(" --chunked-prefill-size 32768"); - flags.push(" --cuda-graph-max-bs 64"); + flags.push(" --cuda-graph-max-bs-decode 64"); flags.push(" --page-size 64"); flags.push(" --swa-full-tokens-ratio 0.3"); flags.push(` --model-loader-extra-config '{"enable_multithread_load": true, "num_threads": 64}'`); diff --git a/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx b/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx index 7a0789e6d..821dbe033 100644 --- a/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx +++ b/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx @@ -438,7 +438,7 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 2", "--mem-fraction-static 0.92", - "--cuda-graph-max-bs 256", + "--cuda-graph-max-bs-decode 256", "--deepep-config '{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}'", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -458,7 +458,7 @@ sgl-eval run aime25 \\ "--enable-dp-attention", "--moe-a2a-backend megamoe", "--mem-fraction-static 0.835", - "--cuda-graph-max-bs 544", + "--cuda-graph-max-bs-decode 544", "--swa-full-tokens-ratio 0.075", "--chunked-prefill-size 65536", "--tokenizer-worker-num 8", @@ -565,7 +565,7 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 2", "--mem-fraction-static 0.92", - "--cuda-graph-max-bs 256", + "--cuda-graph-max-bs-decode 256", "--deepep-config '{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}'", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -585,7 +585,7 @@ sgl-eval run aime25 \\ "--enable-dp-attention", "--moe-a2a-backend megamoe", "--mem-fraction-static 0.835", - "--cuda-graph-max-bs 544", + "--cuda-graph-max-bs-decode 544", "--swa-full-tokens-ratio 0.075", "--chunked-prefill-size 65536", "--tokenizer-worker-num 8", @@ -789,7 +789,7 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 2", "--mem-fraction-static 0.78", - "--cuda-graph-max-bs 64", + "--cuda-graph-max-bs-decode 64", "--max-running-requests 128", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -811,7 +811,7 @@ sgl-eval run aime25 \\ "--enable-dp-attention", "--moe-a2a-backend megamoe", "--mem-fraction-static 0.78", - "--cuda-graph-max-bs 64", + "--cuda-graph-max-bs-decode 64", "--max-running-requests 256", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -959,7 +959,7 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 2", "--mem-fraction-static 0.9", - "--cuda-graph-max-bs 128", + "--cuda-graph-max-bs-decode 128", "--max-running-requests 256", "--deepep-config '{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}'", "--host {{HOST_IP}}", @@ -980,7 +980,7 @@ sgl-eval run aime25 \\ "--enable-dp-attention", "--moe-a2a-backend megamoe", "--mem-fraction-static 0.9", - "--cuda-graph-max-bs 128", + "--cuda-graph-max-bs-decode 128", "--max-running-requests 256", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -1068,7 +1068,7 @@ sgl-eval run aime25 \\ "--speculative-num-steps 1", "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 2", - "--cuda-graph-max-bs 128", + "--cuda-graph-max-bs-decode 128", "--max-running-requests 128", "--deepep-config '{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}'", "--host {{HOST_IP}}", @@ -1089,7 +1089,7 @@ sgl-eval run aime25 \\ "--dp 4", "--enable-dp-attention", "--moe-a2a-backend deepep", - "--cuda-graph-max-bs 128", + "--cuda-graph-max-bs-decode 128", "--max-running-requests 256", "--deepep-config '{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}'", "--host {{HOST_IP}}", @@ -1110,7 +1110,7 @@ sgl-eval run aime25 \\ "--dp 16", "--enable-dp-attention", "--moe-a2a-backend deepep", - "--cuda-graph-max-bs 8", + "--cuda-graph-max-bs-decode 8", "--max-running-requests 32", "--speculative-algorithm EAGLE", "--speculative-num-steps 3", @@ -1140,7 +1140,7 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 2", "--mem-fraction-static 0.88", - "--cuda-graph-max-bs 8", + "--cuda-graph-max-bs-decode 8", "--max-running-requests 32", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -1161,7 +1161,7 @@ sgl-eval run aime25 \\ "--enable-dp-attention", "--moe-a2a-backend deepep", "--mem-fraction-static 0.88", - "--cuda-graph-max-bs 128", + "--cuda-graph-max-bs-decode 128", "--max-running-requests 256", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -1330,7 +1330,7 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 4", "--mem-fraction-static 0.9", - "--cuda-graph-max-bs 8", + "--cuda-graph-max-bs-decode 8", "--max-running-requests 32", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -1350,7 +1350,7 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 2", "--mem-fraction-static 0.9", - "--cuda-graph-max-bs 8", + "--cuda-graph-max-bs-decode 8", "--max-running-requests 32", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -1385,7 +1385,7 @@ sgl-eval run aime25 \\ "--tp 4", "--moe-runner-backend marlin", "--mem-fraction-static 0.70", - "--cuda-graph-max-bs 32", + "--cuda-graph-max-bs-decode 32", "--host {{HOST_IP}}", "--port {{PORT}}", ], diff --git a/docs_new/src/snippets/configs/zai-org/glm-5.2.jsx b/docs_new/src/snippets/configs/zai-org/glm-5.2.jsx index bc0a66660..d3846488e 100644 --- a/docs_new/src/snippets/configs/zai-org/glm-5.2.jsx +++ b/docs_new/src/snippets/configs/zai-org/glm-5.2.jsx @@ -185,7 +185,7 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 6", "--mem-fraction-static 0.8", - "--cuda-graph-max-bs 32", + "--cuda-graph-max-bs-decode 32", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -205,7 +205,7 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 2", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 128", + "--cuda-graph-max-bs-decode 128", // Large chunked-prefill is the dominant balanced lever (prefill-bound at this // concurrency); max-running tracks KV capacity (~60-80 for 8K+1K reqs on 8xH200). "--chunked-prefill-size 32768", @@ -225,7 +225,7 @@ sgl-eval run aime25 \\ "--enable-dp-attention", "--moe-a2a-backend deepep", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 256", + "--cuda-graph-max-bs-decode 256", "--max-running-requests 256", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -247,7 +247,7 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 6", "--mem-fraction-static 0.8", - "--cuda-graph-max-bs 32", + "--cuda-graph-max-bs-decode 32", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -267,7 +267,7 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 2", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 128", + "--cuda-graph-max-bs-decode 128", // Large chunked-prefill is the dominant balanced lever (prefill-bound at this // concurrency); max-running tracks KV capacity (~89 for 8K+1K reqs on 8xB200). "--chunked-prefill-size 32768", @@ -287,7 +287,7 @@ sgl-eval run aime25 \\ "--enable-dp-attention", "--moe-a2a-backend deepep", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 256", + "--cuda-graph-max-bs-decode 256", "--max-running-requests 256", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -312,7 +312,7 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 6", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 32", + "--cuda-graph-max-bs-decode 32", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -332,7 +332,7 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 2", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 128", + "--cuda-graph-max-bs-decode 128", // Same prefill lever as H200/B200 balanced; max-running tracks the TP4 KV capacity. "--chunked-prefill-size 32768", "--max-running-requests 80", @@ -351,7 +351,7 @@ sgl-eval run aime25 \\ "--enable-dp-attention", "--moe-a2a-backend deepep", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 256", + "--cuda-graph-max-bs-decode 256", "--max-running-requests 256", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -375,7 +375,7 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 6", "--mem-fraction-static 0.8", - "--cuda-graph-max-bs 32", + "--cuda-graph-max-bs-decode 32", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -395,7 +395,7 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 2", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 128", + "--cuda-graph-max-bs-decode 128", "--chunked-prefill-size 32768", "--max-running-requests 80", "--host {{HOST_IP}}", @@ -413,7 +413,7 @@ sgl-eval run aime25 \\ "--enable-dp-attention", "--moe-a2a-backend deepep", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 256", + "--cuda-graph-max-bs-decode 256", "--max-running-requests 256", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -438,7 +438,7 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 6", "--mem-fraction-static 0.9", - "--cuda-graph-max-bs 32", + "--cuda-graph-max-bs-decode 32", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -455,7 +455,7 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 2", "--mem-fraction-static 0.9", - "--cuda-graph-max-bs 128", + "--cuda-graph-max-bs-decode 128", "--chunked-prefill-size 32768", "--max-running-requests 80", "--host {{HOST_IP}}", @@ -470,7 +470,7 @@ sgl-eval run aime25 \\ "--model-path {{MODEL_NAME}}", "--tp 8", "--mem-fraction-static 0.9", - "--cuda-graph-max-bs 256", + "--cuda-graph-max-bs-decode 256", "--max-running-requests 256", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -495,7 +495,7 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 6", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 32", + "--cuda-graph-max-bs-decode 32", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -512,7 +512,7 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 2", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 128", + "--cuda-graph-max-bs-decode 128", "--chunked-prefill-size 32768", "--max-running-requests 80", "--host {{HOST_IP}}", @@ -527,7 +527,7 @@ sgl-eval run aime25 \\ "--model-path {{MODEL_NAME}}", "--tp 16", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 256", + "--cuda-graph-max-bs-decode 256", "--max-running-requests 256", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -545,7 +545,7 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 6", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 32", + "--cuda-graph-max-bs-decode 32", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -562,7 +562,7 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 2", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 128", + "--cuda-graph-max-bs-decode 128", "--chunked-prefill-size 32768", "--max-running-requests 80", "--host {{HOST_IP}}", @@ -577,7 +577,7 @@ sgl-eval run aime25 \\ "--model-path {{MODEL_NAME}}", "--tp 16", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 256", + "--cuda-graph-max-bs-decode 256", "--max-running-requests 256", "--host {{HOST_IP}}", "--port {{PORT}}", @@ -595,7 +595,7 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 6", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 32", + "--cuda-graph-max-bs-decode 32", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -612,7 +612,7 @@ sgl-eval run aime25 \\ "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 2", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 128", + "--cuda-graph-max-bs-decode 128", "--chunked-prefill-size 32768", "--max-running-requests 80", "--host {{HOST_IP}}", @@ -627,7 +627,7 @@ sgl-eval run aime25 \\ "--model-path {{MODEL_NAME}}", "--tp 8", "--mem-fraction-static 0.85", - "--cuda-graph-max-bs 256", + "--cuda-graph-max-bs-decode 256", "--max-running-requests 256", "--host {{HOST_IP}}", "--port {{PORT}}", diff --git a/examples/runtime/engine/offline_batch_inference_eagle.py b/examples/runtime/engine/offline_batch_inference_eagle.py index a7a89ef5c..2176901ea 100644 --- a/examples/runtime/engine/offline_batch_inference_eagle.py +++ b/examples/runtime/engine/offline_batch_inference_eagle.py @@ -21,7 +21,7 @@ def main(): speculative_num_steps=3, speculative_eagle_topk=4, speculative_num_draft_tokens=16, - cuda_graph_max_bs=8, + cuda_graph_max_bs_decode=8, ) outputs = llm.generate(prompts, sampling_params) diff --git a/python/sglang/test/ascend/vlm_utils.py b/python/sglang/test/ascend/vlm_utils.py index cab91eeb8..d7ddde25f 100644 --- a/python/sglang/test/ascend/vlm_utils.py +++ b/python/sglang/test/ascend/vlm_utils.py @@ -17,7 +17,7 @@ class TestVLMModels(CustomTestCase): mmmu_accuracy = 0.00 other_args = [ "--trust-remote-code", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--enable-multimodal", "--mem-fraction-static", diff --git a/python/sglang/test/doc_patch.py b/python/sglang/test/doc_patch.py index 6da9d635f..f1fd61505 100644 --- a/python/sglang/test/doc_patch.py +++ b/python/sglang/test/doc_patch.py @@ -52,7 +52,7 @@ def launch_server_cmd(command: str, host: str = "0.0.0.0", port: int = None): f"--max-running-requests {DEFAULT_MAX_RUNNING_REQUESTS} " f"--max-total-tokens {DEFAULT_MAX_TOTAL_TOKENS} " f"--disable-cuda-graph " - f"--cuda-graph-max-bs 4" + f"--cuda-graph-max-bs-decode 4" ) full_command = f"{command} --port {port} {extra_flags}" diff --git a/python/sglang/test/gpt_oss_common.py b/python/sglang/test/gpt_oss_common.py index 3f9c6bc97..8f3dbe065 100644 --- a/python/sglang/test/gpt_oss_common.py +++ b/python/sglang/test/gpt_oss_common.py @@ -40,7 +40,7 @@ class BaseTestGptOss(CustomTestCase): }[(model_variant, quantization)] if model_variant == "20b": - other_args += ["--cuda-graph-max-bs", "600"] + other_args += ["--cuda-graph-max-bs-decode", "600"] # Respect SGLANG_USE_AITER if already set, otherwise default to "0" for HIP if _is_hip and "SGLANG_USE_AITER" not in os.environ: os.environ["SGLANG_USE_AITER"] = "0" diff --git a/python/sglang/test/kits/mmmu_vlm_kit.py b/python/sglang/test/kits/mmmu_vlm_kit.py index a2dd6a537..3ab45d5c2 100644 --- a/python/sglang/test/kits/mmmu_vlm_kit.py +++ b/python/sglang/test/kits/mmmu_vlm_kit.py @@ -367,7 +367,7 @@ class MMMUMultiModelTestBase(CustomTestCase): api_key=self.api_key, other_args=[ "--trust-remote-code", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "64", "--enable-multimodal", "--mem-fraction-static", diff --git a/python/sglang/test/kv_canary/consts.py b/python/sglang/test/kv_canary/consts.py index 152c9c178..3a16ca1cf 100644 --- a/python/sglang/test/kv_canary/consts.py +++ b/python/sglang/test/kv_canary/consts.py @@ -28,7 +28,7 @@ DSV4_POOL_SERVER_ARGS: Final[tuple[str, ...]] = ( "--enable-dp-attention", "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", "--max-running-requests", "256", diff --git a/python/sglang/test/server_fixtures/hybrid_attn_backend_fixture.py b/python/sglang/test/server_fixtures/hybrid_attn_backend_fixture.py index 0a184ea80..774020530 100644 --- a/python/sglang/test/server_fixtures/hybrid_attn_backend_fixture.py +++ b/python/sglang/test/server_fixtures/hybrid_attn_backend_fixture.py @@ -28,7 +28,7 @@ GSM_DATASET_PATH = None # Default server arguments shared across all hybrid-attn-backend tests DEFAULT_HYBRID_ATTN_SERVER_ARGS = [ "--trust-remote-code", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "8", "--prefill-attention-backend", "fa3", diff --git a/python/sglang/test/server_fixtures/mmmu_fixture.py b/python/sglang/test/server_fixtures/mmmu_fixture.py index 6e5c9b096..8070eff66 100644 --- a/python/sglang/test/server_fixtures/mmmu_fixture.py +++ b/python/sglang/test/server_fixtures/mmmu_fixture.py @@ -44,7 +44,7 @@ class MMMUServerBase(CustomTestCase): # Build server args with MMMU-specific settings server_args = [ "--trust-remote-code", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "64", "--enable-multimodal", "--mem-fraction-static", diff --git a/python/sglang/test/server_fixtures/ngram_fixture.py b/python/sglang/test/server_fixtures/ngram_fixture.py index 4caa15d69..2dfc18fdb 100644 --- a/python/sglang/test/server_fixtures/ngram_fixture.py +++ b/python/sglang/test/server_fixtures/ngram_fixture.py @@ -26,7 +26,7 @@ from sglang.test.test_utils import ( DEFAULT_NGRAM_SERVER_ARGS = [ "--trust-remote-code", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "8", "--speculative-algorithm", "NGRAM", diff --git a/python/sglang/test/server_fixtures/spec_eagle_fixture.py b/python/sglang/test/server_fixtures/spec_eagle_fixture.py index 10426ecbd..ddb6acb39 100644 --- a/python/sglang/test/server_fixtures/spec_eagle_fixture.py +++ b/python/sglang/test/server_fixtures/spec_eagle_fixture.py @@ -64,7 +64,7 @@ class SpecEagleServerBase(CustomTestCase): # bf16 rather than fp16: fp16 activations can overflow (-> Inf -> NaN) on # degenerate draft branches in verify and trip the CI NaN asserts. dtype = "bfloat16" - cuda_graph_max_bs = None + cuda_graph_max_bs_decode = None trust_remote_code = True # Launch with --enable-return-hidden-states so SpecHiddenStatesKit can probe # per-request hidden states; per-request gated, so other requests don't pay. @@ -109,8 +109,8 @@ class SpecEagleServerBase(CustomTestCase): args.append("--trust-remote-code") if cls.enable_return_hidden_states: args.append("--enable-return-hidden-states") - if cls.cuda_graph_max_bs is not None: - args += ["--cuda-graph-max-bs", str(cls.cuda_graph_max_bs)] + if cls.cuda_graph_max_bs_decode is not None: + args += ["--cuda-graph-max-bs-decode", str(cls.cuda_graph_max_bs_decode)] args += [str(a) for a in cls.extra_args] return args diff --git a/python/sglang/test/server_fixtures/standalone_fixture.py b/python/sglang/test/server_fixtures/standalone_fixture.py index 1c83005bd..12dcbfd13 100644 --- a/python/sglang/test/server_fixtures/standalone_fixture.py +++ b/python/sglang/test/server_fixtures/standalone_fixture.py @@ -49,7 +49,7 @@ class StandaloneServerBase: assert cls.attention_backend, f"{cls.__name__} must set `attention_backend`" args = [ "--trust-remote-code", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "8", "--speculative-algorithm", "STANDALONE", diff --git a/python/sglang/test/test_deterministic_utils.py b/python/sglang/test/test_deterministic_utils.py index 46c9d5ec2..c1bd308f9 100644 --- a/python/sglang/test/test_deterministic_utils.py +++ b/python/sglang/test/test_deterministic_utils.py @@ -12,7 +12,7 @@ from sglang.test.test_utils import ( DEFAULT_MODEL = "Qwen/Qwen3-8B" COMMON_SERVER_ARGS = [ "--trust-remote-code", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--enable-deterministic-inference", ] diff --git a/test/manual/attention/test_fa3.py b/test/manual/attention/test_fa3.py index 7b008f23f..1737a84fd 100644 --- a/test/manual/attention/test_fa3.py +++ b/test/manual/attention/test_fa3.py @@ -45,7 +45,7 @@ if OFFLINE_MODE: # Default server arguments shared across all tests DEFAULT_SERVER_ARGS = [ "--trust-remote-code", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "8", "--attention-backend", "fa3", @@ -144,7 +144,7 @@ class TestFlashAttention3SpeculativeDecode(BaseFlashAttentionTest): args = DEFAULT_SERVER_ARGS args.extend( [ - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "4", "--speculative-algorithm", "EAGLE3", @@ -178,7 +178,7 @@ class TestFlashAttention3SpeculativeDecodeTopk(BaseFlashAttentionTest): args = DEFAULT_SERVER_ARGS args.extend( [ - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "4", "--speculative-algorithm", "EAGLE3", @@ -210,7 +210,7 @@ class TestFlashAttention3MLASpeculativeDecode(BaseFlashAttentionTest): args = DEFAULT_SERVER_ARGS args.extend( [ - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "4", "--speculative-algorithm", "EAGLE", @@ -242,7 +242,7 @@ class TestFlashAttention3MLASpeculativeDecodeTopk(BaseFlashAttentionTest): args = DEFAULT_SERVER_ARGS args.extend( [ - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "4", "--speculative-algorithm", "EAGLE", diff --git a/test/manual/attention/test_local_attn.py b/test/manual/attention/test_local_attn.py index 78321ed96..4f00e2691 100644 --- a/test/manual/attention/test_local_attn.py +++ b/test/manual/attention/test_local_attn.py @@ -25,7 +25,7 @@ class TestFlashAttention3LocalAttn(CustomTestCase): @classmethod def get_server_args(cls): return [ - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "2", "--attention-backend", "fa3", diff --git a/test/manual/dsv4/test_b200_pro.py b/test/manual/dsv4/test_b200_pro.py index b6bc279b5..b246413b3 100644 --- a/test/manual/dsv4/test_b200_pro.py +++ b/test/manual/dsv4/test_b200_pro.py @@ -60,7 +60,7 @@ class TestB200ProBalanced(DSV4ProAime25TestBase): "2", "--mem-fraction-static", "0.82", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "64", "--max-running-requests", "128", @@ -83,7 +83,7 @@ class TestB200ProMaxThroughput(DSV4ProAime25TestBase): "deepep", "--mem-fraction-static", "0.82", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "64", "--max-running-requests", "256", @@ -108,7 +108,7 @@ class TestB200ProCP(DSV4ProAime25TestBase): "16384", "--mem-fraction-static", "0.78", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "256", "--max-running-requests", "256", diff --git a/test/manual/dsv4/test_b300_pro.py b/test/manual/dsv4/test_b300_pro.py index 4f8618b51..3a81d497f 100644 --- a/test/manual/dsv4/test_b300_pro.py +++ b/test/manual/dsv4/test_b300_pro.py @@ -62,7 +62,7 @@ class TestB300ProBalanced(DSV4ProAime25TestBase): "2", "--mem-fraction-static", "0.82", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "64", "--max-running-requests", "128", @@ -85,7 +85,7 @@ class TestB300ProMaxThroughput(DSV4ProAime25TestBase): "deepep", "--mem-fraction-static", "0.82", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "64", "--max-running-requests", "256", @@ -110,7 +110,7 @@ class TestB300ProCP(DSV4ProAime25TestBase): "16384", "--mem-fraction-static", "0.78", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "256", "--max-running-requests", "256", diff --git a/test/manual/dsv4/test_dsv4_flash_mtp_dp4.py b/test/manual/dsv4/test_dsv4_flash_mtp_dp4.py index 716a90ece..11e809bc9 100644 --- a/test/manual/dsv4/test_dsv4_flash_mtp_dp4.py +++ b/test/manual/dsv4/test_dsv4_flash_mtp_dp4.py @@ -37,7 +37,7 @@ DSV4_FLASH_ENV = { "SGLANG_DSV4_FP4_EXPERTS": "0", # MTP runs ~num_draft_tokens forward passes per step, so the deepep # dispatch input size scales by that factor. Default 256 (used by the - # plain server) overflows once cuda-graph-max-bs * num_draft_tokens + # plain server) overflows once cuda-graph-max-bs-decode * num_draft_tokens # > 256. 1024 covers bs=128 * 4 draft tokens with headroom. "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "1024", } @@ -66,7 +66,7 @@ class DSV4FlashMTPServerBase(CustomTestCase): "--enable-dp-attention", "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", "--max-running-requests", "256", diff --git a/test/manual/dsv4/test_dsv4_flash_sanity_dp4.py b/test/manual/dsv4/test_dsv4_flash_sanity_dp4.py index 9a4cad134..1546bbc54 100644 --- a/test/manual/dsv4/test_dsv4_flash_sanity_dp4.py +++ b/test/manual/dsv4/test_dsv4_flash_sanity_dp4.py @@ -65,7 +65,7 @@ class TestDSV4FlashTP4DP4( "--enable-dp-attention", "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", "--max-running-requests", "256", @@ -101,7 +101,7 @@ class TestDSV4FlashTP4EP( # No --enable-dp-attention by design: covers TP-attn path. "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", "--max-running-requests", "64", @@ -139,7 +139,7 @@ class TestDSV4FlashTP4DP4ChunkedPrefillLarge( "deepep", "--chunked-prefill-size", "16384", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", "--max-running-requests", "256", diff --git a/test/manual/dsv4/test_dsv4_pd_disagg_nixl.py b/test/manual/dsv4/test_dsv4_pd_disagg_nixl.py index 60dcb5431..0e5bd684e 100644 --- a/test/manual/dsv4/test_dsv4_pd_disagg_nixl.py +++ b/test/manual/dsv4/test_dsv4_pd_disagg_nixl.py @@ -61,7 +61,7 @@ class TestDSV4FlashPDDisaggNIXL(PDDisaggregationServerBase): "deepep", "--deepep-config", DEEPEP_CONFIG, - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", "--max-running-requests", "256", @@ -103,7 +103,7 @@ class TestDSV4FlashPDDisaggNIXL(PDDisaggregationServerBase): "deepep", "--deepep-config", DEEPEP_CONFIG, - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", "--max-running-requests", "256", diff --git a/test/manual/dsv4/test_dsv4_swa_radix_retract.py b/test/manual/dsv4/test_dsv4_swa_radix_retract.py index 7482eec62..95fe2a807 100644 --- a/test/manual/dsv4/test_dsv4_swa_radix_retract.py +++ b/test/manual/dsv4/test_dsv4_swa_radix_retract.py @@ -66,7 +66,7 @@ class TestDSV4FlashSWARadixRetract(CustomTestCase): "--enable-dp-attention", "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", "--max-running-requests", "256", diff --git a/test/manual/dsv4/test_gb300_pro.py b/test/manual/dsv4/test_gb300_pro.py index 54ca95983..f87b6e9b6 100644 --- a/test/manual/dsv4/test_gb300_pro.py +++ b/test/manual/dsv4/test_gb300_pro.py @@ -62,7 +62,7 @@ class TestGB300ProBalanced(DSV4ProAime25TestBase): "2", "--mem-fraction-static", "0.9", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", "--max-running-requests", "256", @@ -85,7 +85,7 @@ class TestGB300ProMaxThroughput(DSV4ProAime25TestBase): "deepep", "--mem-fraction-static", "0.9", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", "--max-running-requests", "256", @@ -110,7 +110,7 @@ class TestGB300ProCP(DSV4ProAime25TestBase): "16384", "--mem-fraction-static", "0.88", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "256", "--max-running-requests", "256", diff --git a/test/manual/dsv4/test_h200_fp8_flash.py b/test/manual/dsv4/test_h200_fp8_flash.py index b69834f9b..5e3b8809d 100644 --- a/test/manual/dsv4/test_h200_fp8_flash.py +++ b/test/manual/dsv4/test_h200_fp8_flash.py @@ -53,7 +53,7 @@ class TestH200Fp8FlashBalanced(DSV4FlashAime25TestBase): "1", "--speculative-num-draft-tokens", "2", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", "--max-running-requests", "128", @@ -77,7 +77,7 @@ class TestH200Fp8FlashMaxThroughput(DSV4FlashAime25TestBase): "--enable-dp-attention", "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", "--max-running-requests", "256", diff --git a/test/manual/dsv4/test_h200_fp8_pro.py b/test/manual/dsv4/test_h200_fp8_pro.py index cdf95ff60..3e724a4b7 100644 --- a/test/manual/dsv4/test_h200_fp8_pro.py +++ b/test/manual/dsv4/test_h200_fp8_pro.py @@ -44,7 +44,7 @@ class TestH200Fp8ProLowLatency(DSV4ProAime25TestBase): *multinode_args(2), "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "8", "--max-running-requests", "32", @@ -88,7 +88,7 @@ class TestH200Fp8ProBalanced(DSV4ProAime25TestBase): "2", "--mem-fraction-static", "0.88", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "8", "--max-running-requests", "32", @@ -114,7 +114,7 @@ class TestH200Fp8ProMaxThroughput(DSV4ProAime25TestBase): "deepep", "--mem-fraction-static", "0.88", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", "--max-running-requests", "256", diff --git a/test/manual/ep/test_eplb_mori.py b/test/manual/ep/test_eplb_mori.py index 1fef948af..2d6bf821d 100644 --- a/test/manual/ep/test_eplb_mori.py +++ b/test/manual/ep/test_eplb_mori.py @@ -69,7 +69,7 @@ common_args = [ "12288", "--attention-backend", "aiter", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", ] diff --git a/test/manual/ep/test_moe_deepep_eval_accuracy_large.py b/test/manual/ep/test_moe_deepep_eval_accuracy_large.py index 4781bb9ae..2db5881f3 100644 --- a/test/manual/ep/test_moe_deepep_eval_accuracy_large.py +++ b/test/manual/ep/test_moe_deepep_eval_accuracy_large.py @@ -32,7 +32,7 @@ class TestMoEDeepEPEvalAccuracyLarge(CustomTestCase): "8", "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", ], ) diff --git a/test/manual/ep/test_mooncake_expert_backup.py b/test/manual/ep/test_mooncake_expert_backup.py index c6cec9cbd..b846d6ad7 100644 --- a/test/manual/ep/test_mooncake_expert_backup.py +++ b/test/manual/ep/test_mooncake_expert_backup.py @@ -58,7 +58,7 @@ class TestBackup(CustomTestCase): "50", "--chunked-prefill-size", "512", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", "--max-running-requests", "512", diff --git a/test/manual/mla/test_flashmla_archived.py b/test/manual/mla/test_flashmla_archived.py index 39266d3c5..eb1bdcf1a 100644 --- a/test/manual/mla/test_flashmla_archived.py +++ b/test/manual/mla/test_flashmla_archived.py @@ -35,7 +35,7 @@ class TestFlashMLAAttnBackend(unittest.TestCase): if torch.cuda.is_available() and torch.version.cuda: other_args.extend( [ - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "2", "--attention-backend", "flashmla", diff --git a/test/manual/mla/test_mla_deepseek_v3.py b/test/manual/mla/test_mla_deepseek_v3.py index f6da4c379..4ddc1827c 100644 --- a/test/manual/mla/test_mla_deepseek_v3.py +++ b/test/manual/mla/test_mla_deepseek_v3.py @@ -22,7 +22,9 @@ class TestMLADeepseekV3(CustomTestCase): cls.base_url = DEFAULT_URL_FOR_TEST other_args = ["--trust-remote-code", "--chunked-prefill-size", "256"] if is_cuda(): - other_args.extend(["--enable-torch-compile", "--cuda-graph-max-bs", "2"]) + other_args.extend( + ["--enable-torch-compile", "--cuda-graph-max-bs-decode", "2"] + ) cls.process = popen_launch_server( cls.model, cls.base_url, @@ -59,7 +61,7 @@ class TestMLADeepseekV3DisableFusedFunc(CustomTestCase): cls.base_url = DEFAULT_URL_FOR_TEST other_args = ["--trust-remote-code", "--chunked-prefill-size", "256"] if is_cuda(): - other_args.extend(["--cuda-graph-max-bs", "2"]) + other_args.extend(["--cuda-graph-max-bs-decode", "2"]) cls.process = popen_launch_server( cls.model, cls.base_url, @@ -107,7 +109,7 @@ class TestMLADeepseekV3Fa3Fp8Kvcache(CustomTestCase): "fa3", "--mem-fraction-static", "0.8", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "2", ] ) @@ -145,7 +147,7 @@ class TestDeepseekV3MTP(CustomTestCase): cls.base_url = DEFAULT_URL_FOR_TEST other_args = [ "--trust-remote-code", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "2", "--disable-radix", "--enable-torch-compile", diff --git a/test/manual/mla/test_mla_flashinfer_archived.py b/test/manual/mla/test_mla_flashinfer_archived.py index 350e865db..f120a2022 100644 --- a/test/manual/mla/test_mla_flashinfer_archived.py +++ b/test/manual/mla/test_mla_flashinfer_archived.py @@ -31,7 +31,7 @@ class TestFlashinferMLA(CustomTestCase): other_args.extend( [ "--enable-torch-compile", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "4", "--attention-backend", "flashinfer", diff --git a/test/manual/mla/test_mla_int8_deepseek_v3_archived.py b/test/manual/mla/test_mla_int8_deepseek_v3_archived.py index 8f083fb6c..7b7ce3d0e 100644 --- a/test/manual/mla/test_mla_int8_deepseek_v3_archived.py +++ b/test/manual/mla/test_mla_int8_deepseek_v3_archived.py @@ -31,7 +31,7 @@ class TestMLADeepseekV3ChannelInt8(CustomTestCase): if torch.cuda.is_available() and torch.version.cuda: other_args.extend( [ - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "16", "--enable-torch-compile", "--torch-compile-max-bs", @@ -75,7 +75,7 @@ class TestMLADeepseekV3BlockInt8(CustomTestCase): if torch.cuda.is_available() and torch.version.cuda: other_args.extend( [ - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "16", "--enable-torch-compile", "--torch-compile-max-bs", diff --git a/test/manual/perf/test_bench_one_batch_1gpu.py b/test/manual/perf/test_bench_one_batch_1gpu.py index b70818f21..6d8ece897 100644 --- a/test/manual/perf/test_bench_one_batch_1gpu.py +++ b/test/manual/perf/test_bench_one_batch_1gpu.py @@ -20,7 +20,7 @@ class TestBenchOneBatch1GPU(CustomTestCase): def test_bs1_small(self): _, output_throughput, _ = run_bench_one_batch( - DEFAULT_SMALL_MODEL_NAME_FOR_TEST, ["--cuda-graph-max-bs", "2"] + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, ["--cuda-graph-max-bs-decode", "2"] ) self.assertGreater(output_throughput, 50) @@ -42,7 +42,7 @@ class TestBenchOneBatch1GPU(CustomTestCase): "1024", "--model-path", DEFAULT_MODEL_NAME_FOR_TEST, - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "2", ] diff --git a/test/manual/test_kv_events.py b/test/manual/test_kv_events.py index 4d8874bca..80a1f3e07 100644 --- a/test/manual/test_kv_events.py +++ b/test/manual/test_kv_events.py @@ -46,7 +46,7 @@ class TestKvEvents(CustomTestCase): '{"publisher": "zmq", "topic": "kv-events"}', "--max-total-tokens", 32, - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", 2, "--enable-dp-attention", "--dp-size", @@ -190,7 +190,7 @@ class TestKvEvents(CustomTestCase): '{"publisher": "zmq", "topic": "kv-events"}', "--max-total-tokens", 64, - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", 4, "--enable-dp-attention", "--dp-size", @@ -327,7 +327,7 @@ class TestKvEvents(CustomTestCase): "--max-running-requests", 4, "--disable-cuda-graph", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", 4, "--model-loader-extra-config", '{"enable_multithread_load": true, "num_threads": 64}', diff --git a/test/manual/test_mla_tp.py b/test/manual/test_mla_tp.py index 5684e7b50..b971f5948 100644 --- a/test/manual/test_mla_tp.py +++ b/test/manual/test_mla_tp.py @@ -21,7 +21,13 @@ class TestDeepseekTP2(CustomTestCase): other_args = ["--trust-remote-code"] if torch.cuda.is_available() and torch.version.cuda: other_args.extend( - ["--tp", "2", "--enable-torch-compile", "--cuda-graph-max-bs", "2"] + [ + "--tp", + "2", + "--enable-torch-compile", + "--cuda-graph-max-bs-decode", + "2", + ] ) cls.process = popen_launch_server( cls.model, diff --git a/test/manual/test_qwen3_235b.py b/test/manual/test_qwen3_235b.py index acae0bd1e..41cf71a47 100644 --- a/test/manual/test_qwen3_235b.py +++ b/test/manual/test_qwen3_235b.py @@ -84,7 +84,7 @@ class TestQwen3235BFP8(unittest.TestCase): ] MTP_ARGS = [ - "--cuda-graph-max-bs=32", + "--cuda-graph-max-bs-decode=32", "--max-running-requests=32", ] variants = [ diff --git a/test/manual/test_vertex_endpoint.py b/test/manual/test_vertex_endpoint.py index 42e48cb1b..a425ef117 100644 --- a/test/manual/test_vertex_endpoint.py +++ b/test/manual/test_vertex_endpoint.py @@ -26,7 +26,7 @@ class TestVertexEndpoint(CustomTestCase): cls.model, cls.base_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=["--cuda-graph-max-bs", 2], + other_args=["--cuda-graph-max-bs-decode", 2], ) @classmethod diff --git a/test/manual/vlm/test_anthropic_vision.py b/test/manual/vlm/test_anthropic_vision.py index 3cf0d3a86..9cb91fd90 100644 --- a/test/manual/vlm/test_anthropic_vision.py +++ b/test/manual/vlm/test_anthropic_vision.py @@ -44,7 +44,7 @@ class TestAnthropicVision(CustomTestCase): other_args=[ "--trust-remote-code", "--enable-multimodal", - "--cuda-graph-max-bs=4", + "--cuda-graph-max-bs-decode=4", ], ) cls.messages_url = cls.base_url + "/v1/messages" diff --git a/test/registered/8-gpu-models/test_gpt_oss_120b.py b/test/registered/8-gpu-models/test_gpt_oss_120b.py index 114d93781..ca003cfa1 100644 --- a/test/registered/8-gpu-models/test_gpt_oss_120b.py +++ b/test/registered/8-gpu-models/test_gpt_oss_120b.py @@ -26,14 +26,14 @@ class TestGptOss120B(unittest.TestCase): base_args = [ "--tp=8", "--trust-remote-code", - "--cuda-graph-max-bs=200", + "--cuda-graph-max-bs-decode=200", "--mem-fraction-static=0.93", ] # Lower batch size for EAGLE3 variants to avoid OOM base_args_eagle3 = [ "--tp=8", "--trust-remote-code", - "--cuda-graph-max-bs=100", + "--cuda-graph-max-bs-decode=100", "--mem-fraction-static=0.85", ] parser_args = [ diff --git a/test/registered/amd/accuracy/mi30x/test_deepseek_r1_eval_amd.py b/test/registered/amd/accuracy/mi30x/test_deepseek_r1_eval_amd.py index ee7748669..cdd8cf2a2 100644 --- a/test/registered/amd/accuracy/mi30x/test_deepseek_r1_eval_amd.py +++ b/test/registered/amd/accuracy/mi30x/test_deepseek_r1_eval_amd.py @@ -138,7 +138,7 @@ DEEPSEEK_R1_MODELS = [ "131072", "--mem-fraction-static", "0.70", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "8", "--enable-torch-compile", "--disable-cuda-graph", diff --git a/test/registered/amd/accuracy/mi30x/test_deepseek_v32_tc_eval_amd.py b/test/registered/amd/accuracy/mi30x/test_deepseek_v32_tc_eval_amd.py index b1b4df6ee..8e464e181 100644 --- a/test/registered/amd/accuracy/mi30x/test_deepseek_v32_tc_eval_amd.py +++ b/test/registered/amd/accuracy/mi30x/test_deepseek_v32_tc_eval_amd.py @@ -56,7 +56,7 @@ class TestDeepseekV32TC(CustomTestCase): "131072", "--mem-fraction-static", "0.70", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "8", "--enable-torch-compile", "--disable-cuda-graph", diff --git a/test/registered/amd/test_moriep_small.py b/test/registered/amd/test_moriep_small.py index 22c7e2e96..c5808a220 100644 --- a/test/registered/amd/test_moriep_small.py +++ b/test/registered/amd/test_moriep_small.py @@ -72,7 +72,7 @@ common_args = [ "32768", "--attention-backend", "aiter", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", ] diff --git a/test/registered/ascend/basic_function/runtime_opts/test_npu_tp4_bf16.py b/test/registered/ascend/basic_function/runtime_opts/test_npu_tp4_bf16.py index 85873ad7b..e33b928c2 100644 --- a/test/registered/ascend/basic_function/runtime_opts/test_npu_tp4_bf16.py +++ b/test/registered/ascend/basic_function/runtime_opts/test_npu_tp4_bf16.py @@ -39,7 +39,7 @@ class TestAscendTp4Bf16(CustomTestCase): "--attention-backend", "ascend", "--disable-radix-cache", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", 32, "--tp-size", 4, diff --git a/test/registered/ascend/llm_models/test_npu_llama_2_7b_communications_compression.py b/test/registered/ascend/llm_models/test_npu_llama_2_7b_communications_compression.py index 5a45eb3b0..ebc4472fb 100644 --- a/test/registered/ascend/llm_models/test_npu_llama_2_7b_communications_compression.py +++ b/test/registered/ascend/llm_models/test_npu_llama_2_7b_communications_compression.py @@ -25,7 +25,7 @@ class TestLlama(GSM8KAscendMixin, CustomTestCase): 32, "--attention-backend", "ascend", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", 32, "--tp-size", 2, diff --git a/test/registered/ascend/llm_models/test_npu_qwen3_30b.py b/test/registered/ascend/llm_models/test_npu_qwen3_30b.py index 181afafe2..3204aab07 100644 --- a/test/registered/ascend/llm_models/test_npu_qwen3_30b.py +++ b/test/registered/ascend/llm_models/test_npu_qwen3_30b.py @@ -28,7 +28,7 @@ class TestQwen330B(GSM8KAscendMixin, CustomTestCase): "--attention-backend", "ascend", "--disable-cuda-graph", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", 32, "--tp-size", 2, diff --git a/test/registered/ascend/llm_models/test_npu_qwen3_30b_attn_cp.py b/test/registered/ascend/llm_models/test_npu_qwen3_30b_attn_cp.py index 20f0099bb..201b85b50 100644 --- a/test/registered/ascend/llm_models/test_npu_qwen3_30b_attn_cp.py +++ b/test/registered/ascend/llm_models/test_npu_qwen3_30b_attn_cp.py @@ -37,7 +37,7 @@ class TestQwen330BAttnCP(GSM8KAscendMixin, CustomTestCase): "2", "--attn-cp-size", "2", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--enable-prefill-context-parallel", ] diff --git a/test/registered/ascend/llm_models/test_npu_qwen3_30b_w4a4.py b/test/registered/ascend/llm_models/test_npu_qwen3_30b_w4a4.py index b31f3c7e2..1b4dae5d7 100644 --- a/test/registered/ascend/llm_models/test_npu_qwen3_30b_w4a4.py +++ b/test/registered/ascend/llm_models/test_npu_qwen3_30b_w4a4.py @@ -26,7 +26,7 @@ class TestQwen317BGPTQInt8(GSM8KAscendMixin, CustomTestCase): "--attention-backend", "ascend", "--disable-cuda-graph", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", 32, "--tp-size", 2, diff --git a/test/registered/ascend/llm_models/test_npu_qwen3_8b_communications_quantization.py b/test/registered/ascend/llm_models/test_npu_qwen3_8b_communications_quantization.py index 5c23e336f..5f8df9550 100644 --- a/test/registered/ascend/llm_models/test_npu_qwen3_8b_communications_quantization.py +++ b/test/registered/ascend/llm_models/test_npu_qwen3_8b_communications_quantization.py @@ -25,7 +25,7 @@ class TestQwen38BCommQuantization(GSM8KAscendMixin, CustomTestCase): 32, "--attention-backend", "ascend", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", 32, "--tp-size", 2, diff --git a/test/registered/ascend/vlm_models/test_ascend_glm_4_5v.py b/test/registered/ascend/vlm_models/test_ascend_glm_4_5v.py index 9b7ba83ea..ec28a57e0 100644 --- a/test/registered/ascend/vlm_models/test_ascend_glm_4_5v.py +++ b/test/registered/ascend/vlm_models/test_ascend_glm_4_5v.py @@ -11,7 +11,7 @@ class TestGLM4Models(TestVLMModels): mmmu_accuracy = 0.2 other_args = [ "--trust-remote-code", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--enable-multimodal", "--mem-fraction-static", diff --git a/test/registered/ascend/vlm_models/test_npu_qwen2_5_vl_72b_instruct.py b/test/registered/ascend/vlm_models/test_npu_qwen2_5_vl_72b_instruct.py index 91774b254..8755c6a08 100644 --- a/test/registered/ascend/vlm_models/test_npu_qwen2_5_vl_72b_instruct.py +++ b/test/registered/ascend/vlm_models/test_npu_qwen2_5_vl_72b_instruct.py @@ -18,7 +18,7 @@ class TestQwen25VL72B(TestVLMModels): mmmu_accuracy = 0.2 other_args = [ "--trust-remote-code", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--enable-multimodal", "--mem-fraction-static", diff --git a/test/registered/ascend/vlm_models/test_npu_qwen3_vl_235b_a22b_instruct.py b/test/registered/ascend/vlm_models/test_npu_qwen3_vl_235b_a22b_instruct.py index 130c928de..bcd91e236 100644 --- a/test/registered/ascend/vlm_models/test_npu_qwen3_vl_235b_a22b_instruct.py +++ b/test/registered/ascend/vlm_models/test_npu_qwen3_vl_235b_a22b_instruct.py @@ -20,7 +20,7 @@ class TestQwen3VL235BA22B(TestVLMModels): mmmu_accuracy = 0.2 other_args = [ "--trust-remote-code", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--enable-multimodal", "--mem-fraction-static", diff --git a/test/registered/attention/test_triton_attention_backend.py b/test/registered/attention/test_triton_attention_backend.py index 17e1f2238..f5371ee68 100644 --- a/test/registered/attention/test_triton_attention_backend.py +++ b/test/registered/attention/test_triton_attention_backend.py @@ -32,7 +32,7 @@ class TestTritonAttnBackend(CustomTestCase): "--attention-backend", "triton", "--enable-torch-compile", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", 4, ], ) diff --git a/test/registered/backends/test_deepseek_r1_fp8_trtllm_backend.py b/test/registered/backends/test_deepseek_r1_fp8_trtllm_backend.py index 74e89060f..385caaa6d 100644 --- a/test/registered/backends/test_deepseek_r1_fp8_trtllm_backend.py +++ b/test/registered/backends/test_deepseek_r1_fp8_trtllm_backend.py @@ -31,7 +31,7 @@ class TestDeepseekR1Fp8Flashinfer(CustomTestCase): "8192", "--mem-fraction-static", "0.9", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", "--max-prefill-tokens", "8192", diff --git a/test/registered/backends/test_flashinfer_trtllm_gen_attn_backend.py b/test/registered/backends/test_flashinfer_trtllm_gen_attn_backend.py index 11aed30fa..0346fd500 100644 --- a/test/registered/backends/test_flashinfer_trtllm_gen_attn_backend.py +++ b/test/registered/backends/test_flashinfer_trtllm_gen_attn_backend.py @@ -28,7 +28,7 @@ class TestFlashinferTrtllmGenAttnBackend(CustomTestCase): other_args=[ "--attention-backend", "trtllm_mha", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "512", "--tp-size", "4", diff --git a/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py b/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py index d875c4a0c..d51377016 100644 --- a/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py +++ b/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py @@ -78,7 +78,7 @@ class FlashinferTrtllmGenMoeBackendBF16Base: "triton", "--moe-runner-backend", cls.backend, - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "512", "--tp-size", "4", @@ -261,7 +261,7 @@ class FlashinferTrtllmGenMoeBackendNvFp4OnlineBase: "triton", "--moe-runner-backend", cls.backend, - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", "--tp-size", "4", diff --git a/test/registered/backends/test_torch_compile.py b/test/registered/backends/test_torch_compile.py index e44a7fcba..cdde49ab7 100644 --- a/test/registered/backends/test_torch_compile.py +++ b/test/registered/backends/test_torch_compile.py @@ -32,7 +32,7 @@ class TestTorchCompile(CustomTestCase, MMLUMixin): cls.model, cls.base_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=["--enable-torch-compile", "--cuda-graph-max-bs", "4"], + other_args=["--enable-torch-compile", "--cuda-graph-max-bs-decode", "4"], ) @classmethod diff --git a/test/registered/core/test_basic_sanity.py b/test/registered/core/test_basic_sanity.py index 1a3d955e6..da84ca786 100644 --- a/test/registered/core/test_basic_sanity.py +++ b/test/registered/core/test_basic_sanity.py @@ -48,7 +48,7 @@ class TestBasicSanity( cls.base_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, other_args=[ - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "4", "--mem-fraction-static", "0.7", diff --git a/test/registered/core/test_basic_sanity_eagle3.py b/test/registered/core/test_basic_sanity_eagle3.py index d435cd0c2..fc2c99c60 100644 --- a/test/registered/core/test_basic_sanity_eagle3.py +++ b/test/registered/core/test_basic_sanity_eagle3.py @@ -70,7 +70,7 @@ class TestBasicSanityEagle3( "1", "--speculative-num-draft-tokens", "2", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "4", "--mem-fraction-static", "0.7", diff --git a/test/registered/core/test_srt_endpoint.py b/test/registered/core/test_srt_endpoint.py index 81010b6c5..4b55136fe 100644 --- a/test/registered/core/test_srt_endpoint.py +++ b/test/registered/core/test_srt_endpoint.py @@ -48,7 +48,7 @@ class TestSRTEndpoint(CustomTestCase): "--enable-custom-logit-processor", "--mem-fraction-static", "0.7", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "8", ), ) diff --git a/test/registered/cp/test_deepseek_v32_cp_single_node.py b/test/registered/cp/test_deepseek_v32_cp_single_node.py index 5b7105d83..22ca57172 100644 --- a/test/registered/cp/test_deepseek_v32_cp_single_node.py +++ b/test/registered/cp/test_deepseek_v32_cp_single_node.py @@ -44,7 +44,7 @@ class TestDeepseekV32CPInSeqSplit(CustomTestCase): "4", "--mem-frac", "0.85", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--max-running-requests", "32", @@ -110,7 +110,7 @@ class TestDeepseekV32CPRoundRobinSplit(CustomTestCase): "4", "--mem-frac", "0.85", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--max-running-requests", "32", diff --git a/test/registered/cp/test_deepseek_v3_cp_single_node.py b/test/registered/cp/test_deepseek_v3_cp_single_node.py index 74e0258d5..48af95363 100644 --- a/test/registered/cp/test_deepseek_v3_cp_single_node.py +++ b/test/registered/cp/test_deepseek_v3_cp_single_node.py @@ -42,7 +42,7 @@ class TestDeepseekV3CPInSeqSplit(CustomTestCase): "fa3", "--mem-frac", "0.7", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--max-running-requests", "32", diff --git a/test/registered/cp/test_gqa_prefill_cp_legacy.py b/test/registered/cp/test_gqa_prefill_cp_legacy.py index 925e54184..7b97da77f 100644 --- a/test/registered/cp/test_gqa_prefill_cp_legacy.py +++ b/test/registered/cp/test_gqa_prefill_cp_legacy.py @@ -37,7 +37,7 @@ class TestGQACP2TP2EP2(CustomTestCase): "--attn-cp-size", "2", "--enable-prefill-context-parallel", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--max-running-requests", "32", @@ -94,7 +94,7 @@ class TestGQACPTP2CP2EP4(CustomTestCase): "--attn-cp-size", "2", "--enable-prefill-context-parallel", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--max-running-requests", "32", diff --git a/test/registered/cp/test_gqa_preill_cp.py b/test/registered/cp/test_gqa_preill_cp.py index d19847ab5..f4a939d85 100644 --- a/test/registered/cp/test_gqa_preill_cp.py +++ b/test/registered/cp/test_gqa_preill_cp.py @@ -39,7 +39,7 @@ class TestGQACP2TP2EP2(CustomTestCase): "--enable-prefill-cp", "--cp-strategy", "zigzag", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--max-running-requests", "32", @@ -98,7 +98,7 @@ class TestGQACPTP2CP2EP4(CustomTestCase): "--enable-prefill-cp", "--cp-strategy", "zigzag", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--max-running-requests", "32", @@ -159,7 +159,7 @@ class TestGQACPCP4EP4(CustomTestCase): "deepep", "--attention-backend", "fa3", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--max-running-requests", "32", diff --git a/test/registered/cuda_graph/piecewise/test_pcg_with_speculative_decoding_extra.py b/test/registered/cuda_graph/piecewise/test_pcg_with_speculative_decoding_extra.py index 6a7d126bc..80425012d 100644 --- a/test/registered/cuda_graph/piecewise/test_pcg_with_speculative_decoding_extra.py +++ b/test/registered/cuda_graph/piecewise/test_pcg_with_speculative_decoding_extra.py @@ -69,7 +69,7 @@ class TestPCGWithNGRAM(PCGSpecBase, unittest.TestCase): "NGRAM", "--speculative-num-draft-tokens", "16", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "8", "--mem-fraction-static", "0.8", diff --git a/test/registered/dcp/test_dsv31_dcp8_gsm8k.py b/test/registered/dcp/test_dsv31_dcp8_gsm8k.py index d71206800..9e6670e16 100644 --- a/test/registered/dcp/test_dsv31_dcp8_gsm8k.py +++ b/test/registered/dcp/test_dsv31_dcp8_gsm8k.py @@ -73,7 +73,7 @@ _COMMON_SERVER_ARGS = [ "16384", "--max-running-requests", "256", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "256", "--attention-backend", "flashinfer", diff --git a/test/registered/disaggregation/test_disaggregation_basic.py b/test/registered/disaggregation/test_disaggregation_basic.py index 2c6d50415..045b5742b 100644 --- a/test/registered/disaggregation/test_disaggregation_basic.py +++ b/test/registered/disaggregation/test_disaggregation_basic.py @@ -240,7 +240,7 @@ class TestDisaggregationMooncakeSpec( "4", "--speculative-num-draft-tokens", "16", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "8", "--dtype=float16", ] diff --git a/test/registered/disaggregation/test_disaggregation_dsv4.py b/test/registered/disaggregation/test_disaggregation_dsv4.py index a60ef9fe2..deb430331 100644 --- a/test/registered/disaggregation/test_disaggregation_dsv4.py +++ b/test/registered/disaggregation/test_disaggregation_dsv4.py @@ -71,7 +71,7 @@ class TestDisaggregationDSV4(SpecDecodingMixin, PDDisaggregationServerBase, GSM8 "deepep", "--deepep-config", DEEPEP_CONFIG, - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", "--max-running-requests", "128", @@ -107,7 +107,7 @@ class TestDisaggregationDSV4(SpecDecodingMixin, PDDisaggregationServerBase, GSM8 "deepep", "--deepep-config", DEEPEP_CONFIG, - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", "--max-running-requests", "128", diff --git a/test/registered/ep/test_deepep_large.py b/test/registered/ep/test_deepep_large.py index ed143cfe3..fed39170c 100644 --- a/test/registered/ep/test_deepep_large.py +++ b/test/registered/ep/test_deepep_large.py @@ -176,7 +176,7 @@ class TestDeepseekV32TBO(CustomTestCase): "--enable-two-batch-overlap", "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "256", "--model-loader-extra-config", '{"enable_multithread_load": true, "num_threads": 64}', diff --git a/test/registered/ep/test_deepep_small.py b/test/registered/ep/test_deepep_small.py index b5a75a006..01edfc238 100644 --- a/test/registered/ep/test_deepep_small.py +++ b/test/registered/ep/test_deepep_small.py @@ -37,7 +37,7 @@ class TestPureDP(CustomTestCase): "4", "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", "--max-running-requests", "512", @@ -84,7 +84,7 @@ class TestHybridDPTP(CustomTestCase): "2", "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", "--max-running-requests", "256", @@ -126,7 +126,7 @@ class TestTP(CustomTestCase): "4", "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", "--max-running-requests", "128", @@ -175,7 +175,7 @@ class TestNoGatherdBuffer(CustomTestCase): "--enable-dp-lm-head", "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--max-running-requests", "512", @@ -223,7 +223,7 @@ class TestTBO(CustomTestCase): "--moe-a2a-backend", "deepep", "--enable-two-batch-overlap", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", "--max-running-requests", "512", @@ -270,7 +270,7 @@ class TestTBOWithTPAttn(CustomTestCase): "--moe-a2a-backend", "deepep", "--enable-two-batch-overlap", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", "--max-running-requests", "512", @@ -323,7 +323,7 @@ class TestTBOWithTPAttnAndDenseDP(CustomTestCase): "--moe-a2a-backend", "deepep", "--enable-two-batch-overlap", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", "--max-running-requests", "512", @@ -386,7 +386,7 @@ class TestMTP(CustomTestCase): "3", "--speculative-num-draft-tokens", "3", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--max-running-requests", "64", @@ -456,7 +456,7 @@ class TestMTPWithTBO(CustomTestCase): DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN, "--chunked-prefill-size", "256", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--max-running-requests", "128", @@ -530,7 +530,7 @@ class TestMTPWithTPAttnAndTBO(CustomTestCase): DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN, "--chunked-prefill-size", "256", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--max-running-requests", "128", diff --git a/test/registered/ep/test_mooncake_ep_small.py b/test/registered/ep/test_mooncake_ep_small.py index aa6b11425..0740e1596 100644 --- a/test/registered/ep/test_mooncake_ep_small.py +++ b/test/registered/ep/test_mooncake_ep_small.py @@ -53,7 +53,7 @@ class TestTP(CustomTestCase): "72", "--chunked-prefill-size", "512", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", "--max-running-requests", "512", diff --git a/test/registered/gb300/test_deepseek_v4_pro_fp4.py b/test/registered/gb300/test_deepseek_v4_pro_fp4.py index 96b687e88..3509a9d58 100644 --- a/test/registered/gb300/test_deepseek_v4_pro_fp4.py +++ b/test/registered/gb300/test_deepseek_v4_pro_fp4.py @@ -53,7 +53,7 @@ BALANCED_ARGS = [ "2", "--mem-fraction-static", "0.85", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", "--max-running-requests", "256", @@ -70,7 +70,7 @@ HIGH_THROUGHPUT_ARGS = [ "megamoe", "--mem-fraction-static", "0.9", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", "--max-running-requests", "256", diff --git a/test/registered/hicache/test_hicache_storage_mooncake_backend.py b/test/registered/hicache/test_hicache_storage_mooncake_backend.py index 949bfda2c..eec88bcaf 100644 --- a/test/registered/hicache/test_hicache_storage_mooncake_backend.py +++ b/test/registered/hicache/test_hicache_storage_mooncake_backend.py @@ -281,7 +281,7 @@ class TestMooncakeBackendQwen330BCP2( "--attn-cp-size": 2, "--enable-prefill-context-parallel": True, "--trust-remote-code": True, - "--cuda-graph-max-bs": 32, + "--cuda-graph-max-bs-decode": 32, "--max-running-requests": 32, "--max-total-tokens": 8192, "--model-loader-extra-config": ( diff --git a/test/registered/input_embedding/test_input_embeddings.py b/test/registered/input_embedding/test_input_embeddings.py index fc49562cd..f9ba44c0c 100644 --- a/test/registered/input_embedding/test_input_embeddings.py +++ b/test/registered/input_embedding/test_input_embeddings.py @@ -31,7 +31,7 @@ class TestInputEmbeds(CustomTestCase): cls.model, cls.base_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=["--disable-radix", "--cuda-graph-max-bs", 4], + other_args=["--disable-radix", "--cuda-graph-max-bs-decode", 4], ) cls.texts = [ "The capital of France is", diff --git a/test/registered/input_embedding/test_input_embeds_chunked.py b/test/registered/input_embedding/test_input_embeds_chunked.py index c26623068..cfaa353b0 100644 --- a/test/registered/input_embedding/test_input_embeds_chunked.py +++ b/test/registered/input_embedding/test_input_embeds_chunked.py @@ -94,7 +94,7 @@ class TestInputEmbedsChunkedAndRetract(CustomTestCase): "--disable-radix-cache", "--chunked-prefill-size", str(CHUNKED_PREFILL_SIZE), - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "4", ], ) diff --git a/test/registered/jit/test_deepseek_v4_compress_state_runtime_shapes.py b/test/registered/jit/test_deepseek_v4_compress_state_runtime_shapes.py index 58489668d..a66215863 100644 --- a/test/registered/jit/test_deepseek_v4_compress_state_runtime_shapes.py +++ b/test/registered/jit/test_deepseek_v4_compress_state_runtime_shapes.py @@ -53,7 +53,7 @@ # --chunked-prefill-size 4096 \ # --max-prefill-tokens 16384 \ # --max-running-requests 32 \ -# --cuda-graph-max-bs 16 \ +# --cuda-graph-max-bs-decode 16 \ # --enable-metrics \ # --disable-radix-cache # diff --git a/test/registered/kv_canary/test_self_e2e_pr_25015.py b/test/registered/kv_canary/test_self_e2e_pr_25015.py index d643c3c59..f040238ad 100644 --- a/test/registered/kv_canary/test_self_e2e_pr_25015.py +++ b/test/registered/kv_canary/test_self_e2e_pr_25015.py @@ -30,7 +30,7 @@ _SPEC_EAGLE_SERVER_ARGS = ( "token_oracle", "--speculative-algorithm", "EAGLE", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", str(_CUDA_GRAPH_MAX_BS), "--max-running-requests", "32", diff --git a/test/registered/kv_canary/test_self_e2e_pr_26329.py b/test/registered/kv_canary/test_self_e2e_pr_26329.py index 13effedf5..b7a8f695f 100644 --- a/test/registered/kv_canary/test_self_e2e_pr_26329.py +++ b/test/registered/kv_canary/test_self_e2e_pr_26329.py @@ -21,7 +21,7 @@ _EAGLE_CHUNKED_SERVER_ARGS = ( "EAGLE", "--chunked-prefill-size", str(_CHUNKED_PREFILL_SIZE), - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "1", "--max-running-requests", "4", diff --git a/test/registered/mla/test_flashmla.py b/test/registered/mla/test_flashmla.py index d5961cdad..955c859cf 100644 --- a/test/registered/mla/test_flashmla.py +++ b/test/registered/mla/test_flashmla.py @@ -32,7 +32,7 @@ class TestFlashMLAMTP(CustomTestCase): if torch.cuda.is_available() and torch.version.cuda: other_args.extend( [ - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "4", "--disable-radix", "--enable-torch-compile", diff --git a/test/registered/mla/test_mla_flashinfer.py b/test/registered/mla/test_mla_flashinfer.py index 5c000cc75..4dd4e8637 100644 --- a/test/registered/mla/test_mla_flashinfer.py +++ b/test/registered/mla/test_mla_flashinfer.py @@ -27,7 +27,7 @@ class TestFlashinferMLAMTP(CustomTestCase): if torch.cuda.is_available() and torch.version.cuda: other_args.extend( [ - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "4", "--enable-torch-compile", "--torch-compile-max-bs", diff --git a/test/registered/mla/test_mla_int8_deepseek_v3.py b/test/registered/mla/test_mla_int8_deepseek_v3.py index 46f9a8e1e..1add20c65 100644 --- a/test/registered/mla/test_mla_int8_deepseek_v3.py +++ b/test/registered/mla/test_mla_int8_deepseek_v3.py @@ -28,7 +28,7 @@ class TestDeepseekV3MTPChannelInt8(CustomTestCase): if torch.cuda.is_available() and torch.version.cuda: other_args.extend( [ - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "16", "--enable-torch-compile", "--torch-compile-max-bs", @@ -91,7 +91,7 @@ class TestDeepseekV3MTPBlockInt8(CustomTestCase): if torch.cuda.is_available() and torch.version.cuda: other_args.extend( [ - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "16", "--enable-torch-compile", "--torch-compile-max-bs", diff --git a/test/registered/model_loading/test_load_weights_from_remote_instance.py b/test/registered/model_loading/test_load_weights_from_remote_instance.py index 478653fc8..7e1f52efd 100644 --- a/test/registered/model_loading/test_load_weights_from_remote_instance.py +++ b/test/registered/model_loading/test_load_weights_from_remote_instance.py @@ -211,7 +211,7 @@ def init_process_dst( str(base_gpu_id), "--tp-size", str(tp_size), - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", 2, "--tokenizer-path", model_name, diff --git a/test/registered/model_loading/test_load_weights_from_remote_instance_npu.py b/test/registered/model_loading/test_load_weights_from_remote_instance_npu.py index 6c9a16d57..936cddcb9 100644 --- a/test/registered/model_loading/test_load_weights_from_remote_instance_npu.py +++ b/test/registered/model_loading/test_load_weights_from_remote_instance_npu.py @@ -222,7 +222,7 @@ def init_process_dst( str(base_gpu_id), "--tp-size", str(tp_size), - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", 2, "--tokenizer-path", model_name, diff --git a/test/registered/models_e2e/test_deepseek_v4_flash_fp4_b200.py b/test/registered/models_e2e/test_deepseek_v4_flash_fp4_b200.py index 6800b1776..6f5530e63 100644 --- a/test/registered/models_e2e/test_deepseek_v4_flash_fp4_b200.py +++ b/test/registered/models_e2e/test_deepseek_v4_flash_fp4_b200.py @@ -198,7 +198,7 @@ class TestDSV4FlashFP4BreakableCudaGraphB200( "1024", "--mem-fraction-static", "0.80", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "16", "--max-running-requests", "128", diff --git a/test/registered/models_e2e/test_deepseek_v4_flash_fp8_h200.py b/test/registered/models_e2e/test_deepseek_v4_flash_fp8_h200.py index b97cc879e..a44094ef0 100644 --- a/test/registered/models_e2e/test_deepseek_v4_flash_fp8_h200.py +++ b/test/registered/models_e2e/test_deepseek_v4_flash_fp8_h200.py @@ -66,7 +66,7 @@ class TestDSV4FlashFP8H200( "1", "--speculative-num-draft-tokens", "2", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", "--max-running-requests", "128", diff --git a/test/registered/models_e2e/test_gpt_oss_4gpu_bf16.py b/test/registered/models_e2e/test_gpt_oss_4gpu_bf16.py index db0d4ddc0..eeb461b0e 100644 --- a/test/registered/models_e2e/test_gpt_oss_4gpu_bf16.py +++ b/test/registered/models_e2e/test_gpt_oss_4gpu_bf16.py @@ -15,7 +15,7 @@ class TestGptOss4GpuBf16(BaseTestGptOss): expected_score_of_reasoning_effort={ "low": 0.58, }, - other_args=["--tp", "4", "--cuda-graph-max-bs", "200"], + other_args=["--tp", "4", "--cuda-graph-max-bs-decode", "200"], ) diff --git a/test/registered/models_e2e/test_gpt_oss_4gpu_mxfp4.py b/test/registered/models_e2e/test_gpt_oss_4gpu_mxfp4.py index 8475c7eab..af8792c12 100644 --- a/test/registered/models_e2e/test_gpt_oss_4gpu_mxfp4.py +++ b/test/registered/models_e2e/test_gpt_oss_4gpu_mxfp4.py @@ -18,7 +18,7 @@ class TestGptOss4GpuMxfp4(BaseTestGptOss): other_args=[ "--tp", "4", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "200", ], ) diff --git a/test/registered/models_e2e/test_mimo_v2_flash.py b/test/registered/models_e2e/test_mimo_v2_flash.py index 931c19200..a5a7e4ba2 100644 --- a/test/registered/models_e2e/test_mimo_v2_flash.py +++ b/test/registered/models_e2e/test_mimo_v2_flash.py @@ -26,7 +26,7 @@ class TestMiMoV2Flash(GSM8KMixin, SpecDecodingMixin, DefaultServerBase): "fa3", "--max-running-requests", "128", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "64", "--page-size", "64", diff --git a/test/registered/moe/test_hybrid_dp_ep_tp_mtp.py b/test/registered/moe/test_hybrid_dp_ep_tp_mtp.py index 9690fc6ec..49fbaa572 100644 --- a/test/registered/moe/test_hybrid_dp_ep_tp_mtp.py +++ b/test/registered/moe/test_hybrid_dp_ep_tp_mtp.py @@ -414,7 +414,7 @@ class Test10(CustomTestCase): "8", "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", ], ) @@ -455,7 +455,7 @@ class Test11(CustomTestCase): "4", "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", ], ) @@ -496,7 +496,7 @@ class Test12(CustomTestCase): "8", "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", ], ) @@ -536,7 +536,7 @@ class Test13(CustomTestCase): "1", "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", ], ) @@ -579,7 +579,7 @@ class Test14(CustomTestCase): "1", "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", ], ) @@ -622,7 +622,7 @@ class Test15(CustomTestCase): "1", "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", ], ) @@ -664,7 +664,7 @@ class Test16(CustomTestCase): "--enable-dp-lm-head", "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", ], ) @@ -706,7 +706,7 @@ class Test17(CustomTestCase): "--enable-dp-lm-head", "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", ], ) @@ -750,7 +750,7 @@ class Test18(CustomTestCase): "--enable-dp-lm-head", "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", ], ) @@ -794,7 +794,7 @@ class Test19(CustomTestCase): "--enable-dp-lm-head", "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "128", ], ) @@ -1708,7 +1708,7 @@ class Test40(CustomTestCase): "8", "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--max-running-requests", "32", @@ -1761,7 +1761,7 @@ class Test41(CustomTestCase): "4", "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--max-running-requests", "32", @@ -1814,7 +1814,7 @@ class Test42(CustomTestCase): "8", "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--max-running-requests", "32", @@ -1866,7 +1866,7 @@ class Test43(CustomTestCase): "1", "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--max-running-requests", "32", @@ -1921,7 +1921,7 @@ class Test44(CustomTestCase): "1", "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--max-running-requests", "32", @@ -1976,7 +1976,7 @@ class Test45(CustomTestCase): "1", "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--max-running-requests", "32", @@ -2030,7 +2030,7 @@ class Test46(CustomTestCase): "--enable-dp-lm-head", "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--max-running-requests", "32", @@ -2084,7 +2084,7 @@ class Test47(CustomTestCase): "--enable-dp-lm-head", "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--max-running-requests", "32", @@ -2140,7 +2140,7 @@ class Test48(CustomTestCase): "--enable-dp-lm-head", "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--max-running-requests", "32", @@ -2196,7 +2196,7 @@ class Test49(CustomTestCase): "--enable-dp-lm-head", "--moe-a2a-backend", "deepep", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--max-running-requests", "32", diff --git a/test/registered/musa/test_llm_server_smoke_musa.py b/test/registered/musa/test_llm_server_smoke_musa.py index 988571cd9..44f9ca146 100644 --- a/test/registered/musa/test_llm_server_smoke_musa.py +++ b/test/registered/musa/test_llm_server_smoke_musa.py @@ -28,7 +28,7 @@ class TestMusaDeepSeekV2LiteChatServerSmoke(DefaultServerBase): served_model_name, "--attention-backend", "fa3", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--tp-size", "1", diff --git a/test/registered/observability/test_metrics.py b/test/registered/observability/test_metrics.py index 34a765d55..f0287a652 100644 --- a/test/registered/observability/test_metrics.py +++ b/test/registered/observability/test_metrics.py @@ -103,7 +103,12 @@ class TestEnableMetrics(CustomTestCase): envs.SGLANG_ENABLE_METRICS_DEVICE_TIMER.override(True), envs.SGLANG_TEST_RETRACT.override(True), ): - launch_args = ["--enable-metrics", "--cuda-graph-max-bs", 2, *other_args] + launch_args = [ + "--enable-metrics", + "--cuda-graph-max-bs-decode", + 2, + *other_args, + ] if enable_mfu_metrics: launch_args.insert(1, "--enable-mfu-metrics") process = popen_launch_server( diff --git a/test/registered/perf/test_bench_one_batch_2gpu.py b/test/registered/perf/test_bench_one_batch_2gpu.py index ce61984e7..89505b3d4 100644 --- a/test/registered/perf/test_bench_one_batch_2gpu.py +++ b/test/registered/perf/test_bench_one_batch_2gpu.py @@ -19,7 +19,8 @@ class TestBenchOneBatch2GPU(CustomTestCase): def test_moe_tp2_bs1(self): output_throughput = run_bench_offline_throughput( - DEFAULT_MOE_MODEL_NAME_FOR_TEST, ["--tp", "2", "--cuda-graph-max-bs", "2"] + DEFAULT_MOE_MODEL_NAME_FOR_TEST, + ["--tp", "2", "--cuda-graph-max-bs-decode", "2"], ) if is_in_ci(): @@ -35,7 +36,7 @@ class TestBenchOneBatch2GPU(CustomTestCase): def test_torch_compile_tp2_bs1(self): output_throughput = run_bench_offline_throughput( DEFAULT_MODEL_NAME_FOR_TEST, - ["--tp", "2", "--enable-torch-compile", "--cuda-graph-max-bs", "2"], + ["--tp", "2", "--enable-torch-compile", "--cuda-graph-max-bs-decode", "2"], ) if is_in_ci(): diff --git a/test/registered/perf/test_gpt_oss_4gpu_perf.py b/test/registered/perf/test_gpt_oss_4gpu_perf.py index 3cf201f9b..652cd5ccf 100644 --- a/test/registered/perf/test_gpt_oss_4gpu_perf.py +++ b/test/registered/perf/test_gpt_oss_4gpu_perf.py @@ -18,7 +18,7 @@ class TestNightlyGptOss4GpuPerformance(unittest.TestCase): [ "--tp", "4", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "200", "--mem-fraction-static", "0.93", diff --git a/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_hicache_pp_kl.py b/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_hicache_pp_kl.py index 789c3b704..04171f5eb 100644 --- a/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_hicache_pp_kl.py +++ b/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_hicache_pp_kl.py @@ -75,7 +75,7 @@ class TestUnifiedQwen3HiCachePP(UnifiedRadixTreeTestMixin, CustomTestCase): "2", "--mem-fraction-static", "0.8", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--max-running-requests", str(cls.max_running_requests), diff --git a/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_cp.py b/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_cp.py index 5798895a8..66d3ae79b 100644 --- a/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_cp.py +++ b/test/registered/radix_cache/unified_radix_tree/test_unified_radix_cache_kl_cp.py @@ -43,7 +43,7 @@ class TestUnifiedQwen3HiCacheCP(UnifiedRadixTreeTestMixin, CustomTestCase): "--enable-prefill-context-parallel", "--mem-fraction-static", "0.8", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--max-running-requests", str(cls.max_running_requests), diff --git a/test/registered/rl/test_return_routed_experts.py b/test/registered/rl/test_return_routed_experts.py index 90007074d..d8ae2be83 100644 --- a/test/registered/rl/test_return_routed_experts.py +++ b/test/registered/rl/test_return_routed_experts.py @@ -66,7 +66,7 @@ class TestReturnRoutedExperts(CustomTestCase): "deepep", # Force normal-mode dispatch: deepep auto routes decode through # low_latency mode whose buffer (num_max_dispatch_tokens_per_rank) - # is undersized for cuda graph capture at default --cuda-graph-max-bs. + # is undersized for cuda graph capture at default --cuda-graph-max-bs-decode. "--deepep-mode", "normal", ] diff --git a/test/registered/rl/test_update_weights_from_distributed.py b/test/registered/rl/test_update_weights_from_distributed.py index 0aa9f82a6..ddb01f3d0 100644 --- a/test/registered/rl/test_update_weights_from_distributed.py +++ b/test/registered/rl/test_update_weights_from_distributed.py @@ -334,7 +334,7 @@ def init_process_sgl( str(base_gpu_id), "--tp-size", str(tp_size), - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", 2, ), ) diff --git a/test/registered/spec/eagle/test_eagle_dp_attention.py b/test/registered/spec/eagle/test_eagle_dp_attention.py index 352a82244..87ac6dd1b 100644 --- a/test/registered/spec/eagle/test_eagle_dp_attention.py +++ b/test/registered/spec/eagle/test_eagle_dp_attention.py @@ -57,7 +57,7 @@ class TestEAGLE3EngineDPAttention(CustomTestCase): "triton" if is_in_amd_ci() else "fa3", "--mem-fraction-static", "0.75", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "64", ] with envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(1): diff --git a/test/registered/spec/eagle/test_spec_eagle_fa3.py b/test/registered/spec/eagle/test_spec_eagle_fa3.py index db7fff026..979746627 100644 --- a/test/registered/spec/eagle/test_spec_eagle_fa3.py +++ b/test/registered/spec/eagle/test_spec_eagle_fa3.py @@ -44,7 +44,7 @@ class TestEagleLlama2Fa3Page256( attention_backend = "fa3" page_size = 256 chunked_prefill_size = 4096 # must be divisible by page_size (256) - cuda_graph_max_bs = 5 + cuda_graph_max_bs_decode = 5 env_overrides = ((envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY, 1),) diff --git a/test/registered/spec/eagle/test_spec_eagle_stress.py b/test/registered/spec/eagle/test_spec_eagle_stress.py index 353d8a83b..f65832d22 100644 --- a/test/registered/spec/eagle/test_spec_eagle_stress.py +++ b/test/registered/spec/eagle/test_spec_eagle_stress.py @@ -48,7 +48,7 @@ class TestEagle3Topk16V2Retract(Eagle3Base, SpecAccuracyKit, SpecFeatureKit): spec_topk = 16 spec_tokens = 64 disable_overlap = False - cuda_graph_max_bs = 5 + cuda_graph_max_bs_decode = 5 max_running_requests = 64 gsm8k_accept_len_thres = 2.4 extra_args = ("--max-total-tokens", 4500) # small KV to trigger retract diff --git a/test/registered/spec/eagle/test_spec_eagle_topk.py b/test/registered/spec/eagle/test_spec_eagle_topk.py index f260ca37b..8dc700ec1 100644 --- a/test/registered/spec/eagle/test_spec_eagle_topk.py +++ b/test/registered/spec/eagle/test_spec_eagle_topk.py @@ -28,7 +28,7 @@ class TestEagle3Topk16(Eagle3Base, SpecCorrectnessKit, SpecAccuracyKit, SpecLogp spec_topk = 16 spec_tokens = 64 disable_overlap = True # synchronous baseline; SpecV2 subclass flips overlap on - cuda_graph_max_bs = 5 + cuda_graph_max_bs_decode = 5 acc_length_thres = 3.1 batch_accept_len_thres = 1.75 gsm8k_accept_len_thres = 2.4 # EAGLE3 topk16 gsm8k accept ~2.48 @@ -69,7 +69,7 @@ class TestEagleLlama3TokenMap(EagleLlama2Base, SpecAccuracyKit): draft_model = "lmsys/sglang-EAGLE-LLaMA3-Instruct-8B" spec_topk = 4 spec_tokens = 8 - cuda_graph_max_bs = 5 + cuda_graph_max_bs_decode = 5 gsm8k_accept_len_thres = 2.5 # FR-Spec token map lowers accept (~2.57) extra_args = ( "--speculative-token-map", diff --git a/test/registered/spec/eagle/test_spec_eagle_topk_page.py b/test/registered/spec/eagle/test_spec_eagle_topk_page.py index 1db2686c4..3279ff394 100644 --- a/test/registered/spec/eagle/test_spec_eagle_topk_page.py +++ b/test/registered/spec/eagle/test_spec_eagle_topk_page.py @@ -26,7 +26,7 @@ class TestEagle3Page64Topk8(Eagle3Base, SpecAccuracyKit, SpecFeatureKit): spec_topk = 8 spec_tokens = 32 disable_overlap = False - cuda_graph_max_bs = 5 + cuda_graph_max_bs_decode = 5 env_overrides = ((envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY, 1),) diff --git a/test/registered/spec/eagle/test_spec_eagle_triton.py b/test/registered/spec/eagle/test_spec_eagle_triton.py index dde5f04f0..d8f611f73 100644 --- a/test/registered/spec/eagle/test_spec_eagle_triton.py +++ b/test/registered/spec/eagle/test_spec_eagle_triton.py @@ -32,7 +32,7 @@ class TestEagle3Triton( attention_backend = "triton" max_running_requests = 64 - cuda_graph_max_bs = 64 + cuda_graph_max_bs_decode = 64 gsm8k_num_examples = 1000 gsm8k_check_accept_len = False env_overrides = ((envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY, 1),) diff --git a/test/registered/vlm/test_encoder_dp.py b/test/registered/vlm/test_encoder_dp.py index de96ac24b..286ea2d08 100644 --- a/test/registered/vlm/test_encoder_dp.py +++ b/test/registered/vlm/test_encoder_dp.py @@ -19,11 +19,11 @@ MODELS = [ class TestVLMEncoderDP(MMMUMultiModelTestBase): - # --cuda-graph-max-bs 32 last-wins over the kit's default 64. + # --cuda-graph-max-bs-decode 32 last-wins over the kit's default 64. other_args = [ "--mm-enable-dp-encoder", "--tp=4", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", ] diff --git a/test/registered/vlm/test_vision_openai_server_a.py b/test/registered/vlm/test_vision_openai_server_a.py index 85cb642e3..a553368f3 100644 --- a/test/registered/vlm/test_vision_openai_server_a.py +++ b/test/registered/vlm/test_vision_openai_server_a.py @@ -33,13 +33,13 @@ class TestLfm2VlServer(ImageOpenAITestMixin): class TestQwen25VLServer(ImageOpenAITestMixin, VideoOpenAITestMixin): model = "Qwen/Qwen2.5-VL-7B-Instruct" extra_args = [ - "--cuda-graph-max-bs=4", + "--cuda-graph-max-bs-decode=4", ] class TestQwen3VLServer(ImageOpenAITestMixin, VideoOpenAITestMixin): model = "Qwen/Qwen3-VL-30B-A3B-Instruct" - extra_args = ["--cuda-graph-max-bs=4"] + extra_args = ["--cuda-graph-max-bs-decode=4"] class TestQwen3OmniServer(OmniOpenAITestMixin): @@ -66,7 +66,7 @@ class TestQwen2VLContextLengthServer(CustomTestCase): other_args=[ "--context-length", "300", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "4", ], ) @@ -116,7 +116,7 @@ class TestQwen2VLContextLengthServer(CustomTestCase): class TestInternVL25Server(ImageOpenAITestMixin): model = "OpenGVLab/InternVL2_5-2B" extra_args = [ - "--cuda-graph-max-bs=4", + "--cuda-graph-max-bs-decode=4", ] @@ -124,7 +124,7 @@ class TestInternVL25Server(ImageOpenAITestMixin): class TestMiniCPMV4Server(ImageOpenAITestMixin): model = "openbmb/MiniCPM-V-4" extra_args = [ - "--cuda-graph-max-bs=4", + "--cuda-graph-max-bs-decode=4", ] @@ -132,14 +132,14 @@ class TestMiniCPMV4Server(ImageOpenAITestMixin): class TestMiniCPMo26Server(ImageOpenAITestMixin, AudioOpenAITestMixin): model = "openbmb/MiniCPM-o-2_6" extra_args = [ - "--cuda-graph-max-bs=4", + "--cuda-graph-max-bs-decode=4", ] class TestGemma3itServer(ImageOpenAITestMixin): model = "google/gemma-3-4b-it" extra_args = [ - "--cuda-graph-max-bs=4", + "--cuda-graph-max-bs-decode=4", ] @@ -175,7 +175,7 @@ class TestDeepseekOCRServer(TestOpenAIMLLMServerBase): trust_remote_code = False extra_args = [ "--mem-fraction-static=0.70", - "--cuda-graph-max-bs=4", + "--cuda-graph-max-bs-decode=4", ] def verify_single_image_response_for_ocr(self, response): diff --git a/test/registered/vlm/test_vlm_tp4.py b/test/registered/vlm/test_vlm_tp4.py index 7aae595ee..9f4fdc58d 100644 --- a/test/registered/vlm/test_vlm_tp4.py +++ b/test/registered/vlm/test_vlm_tp4.py @@ -34,7 +34,7 @@ class TestVLMTP4(CustomTestCase): other_args=[ "--tp-size", "4", - "--cuda-graph-max-bs", + "--cuda-graph-max-bs-decode", "32", "--mem-fraction-static", "0.8",