diff --git a/.claude/skills/env-var-conventions/SKILL.md b/.claude/skills/env-var-conventions/SKILL.md index fbde953d5..048c6e26b 100644 --- a/.claude/skills/env-var-conventions/SKILL.md +++ b/.claude/skills/env-var-conventions/SKILL.md @@ -137,7 +137,7 @@ The second token signals intent. Pick the right verb up front — renames requir | Verb | Meaning | Example | |---|---|---| -| `ENABLE_FOO` | Knob that turns feature foo on/off. Default in the `EnvBool` encodes prod behavior. | `SGLANG_ENABLE_TORCH_COMPILE`, `SGLANG_ENABLE_SPEC_V2` | +| `ENABLE_FOO` | Knob that turns feature foo on/off. Default in the `EnvBool` encodes prod behavior. | `SGLANG_ENABLE_TORCH_COMPILE`, `SGLANG_ENABLE_OVERLAP_PLAN_STREAM` | | `DISABLE_FOO` | Kill-switch. `DISABLE_FOO=True` turns foo off. | `SGLANG_DISABLE_CONSECUTIVE_PREFILL_OVERLAP` | | `USE_FOO` | Selects which implementation / backend | `SGLANG_USE_AITER`, `SGLANG_USE_DEEPGEMM_BMM` | | `FORCE_FOO` | Overrides autodetection | `SGLANG_FORCE_FP8_MARLIN`, `SGLANG_FORCE_STREAM_INTERVAL` | diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx index 49ca233aa..ace8ac1e6 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3.mdx @@ -396,7 +396,7 @@ For large batch sizes (>48), increase `--max-running-requests` beyond the defaul -The spec-v2 overlap scheduler is enabled by default (`SGLANG_ENABLE_SPEC_V2=True`). It improves performance by overlapping draft and verification stages. Set `SGLANG_ENABLE_SPEC_V2=0` to disable. +The spec-v2 overlap scheduler is enabled by default. It improves performance by overlapping draft and verification stages. Pass `--disable-overlap-schedule` to disable. #### 4.2.4 MLA Optimizations diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx index 6c0d5c4d7..97b5bf04f 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2.mdx @@ -374,7 +374,7 @@ Find optimal values for your workload with [bench_speculative.py](https://github -The spec-v2 overlap scheduler is enabled by default (`SGLANG_ENABLE_SPEC_V2=True`). Set `SGLANG_ENABLE_SPEC_V2=0` to disable. +The spec-v2 overlap scheduler is enabled by default. Pass `--disable-overlap-schedule` to disable. #### 4.2.4 PD Disaggregation diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx index 06cb471cb..e11bd2f9e 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx @@ -156,7 +156,7 @@ The generator currently picks values on the **conservative** side (mirroring an - `low-latency`: steps=3, draft-tokens=4 → largest win at bs=1. - `balanced`: steps=1, draft-tokens=2 → gentler MTP, reduces throughput hit at higher batch. - `high-throughput`: MTP disabled — at saturation the verify step costs more than it saves. -- MTP runs on the v2 speculative path (`SGLANG_ENABLE_SPEC_V2`, enabled by default). +- MTP runs on the v2 speculative path. **EPLB + DeepEP Waterfill (Experimental)** diff --git a/docs_new/cookbook/autoregressive/GLM/GLM-4.5.mdx b/docs_new/cookbook/autoregressive/GLM/GLM-4.5.mdx index 800e740eb..b9d550033 100644 --- a/docs_new/cookbook/autoregressive/GLM/GLM-4.5.mdx +++ b/docs_new/cookbook/autoregressive/GLM/GLM-4.5.mdx @@ -44,7 +44,7 @@ import { GLM45Deployment } from "/src/snippets/autoregressive/glm-45-deployment. ### 3.2 Configuration Tips -- **EAGLE Speculative Decoding:** Supported for GLM-4.5/4.6. Add `--speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4`. The spec-v2 overlap scheduler is enabled by default (`SGLANG_ENABLE_SPEC_V2=True`); set `SGLANG_ENABLE_SPEC_V2=0` to disable. +- **EAGLE Speculative Decoding:** Supported for GLM-4.5/4.6. Add `--speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4`. The spec-v2 overlap scheduler is enabled by default; pass `--disable-overlap-schedule` to disable. - **Thinking Budget:** Use `--enable-custom-logit-processor` flag and pass `Glm4MoeThinkingBudgetLogitProcessor` in requests to cap the model's thinking token count (see section 4.2.3). ## 4. Model Invocation diff --git a/docs_new/cookbook/autoregressive/GLM/GLM-4.6.mdx b/docs_new/cookbook/autoregressive/GLM/GLM-4.6.mdx index d7fd1a8a3..97fa60417 100644 --- a/docs_new/cookbook/autoregressive/GLM/GLM-4.6.mdx +++ b/docs_new/cookbook/autoregressive/GLM/GLM-4.6.mdx @@ -38,7 +38,7 @@ import { GLM46Deployment } from "/src/snippets/autoregressive/glm-46-deployment. ### 3.2 Configuration Tips -- **EAGLE Speculative Decoding:** Supported for GLM-4.5/4.6. Add `--speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4`. The spec-v2 overlap scheduler is enabled by default (`SGLANG_ENABLE_SPEC_V2=True`); set `SGLANG_ENABLE_SPEC_V2=0` to disable. +- **EAGLE Speculative Decoding:** Supported for GLM-4.5/4.6. Add `--speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4`. The spec-v2 overlap scheduler is enabled by default; pass `--disable-overlap-schedule` to disable. - **Thinking Budget:** Use `--enable-custom-logit-processor` flag and pass `Glm4MoeThinkingBudgetLogitProcessor` in requests to cap the model's thinking token count (see section 4.2.3). ## 4. Model Invocation diff --git a/docs_new/cookbook/autoregressive/GLM/GLM-4.7-Flash.mdx b/docs_new/cookbook/autoregressive/GLM/GLM-4.7-Flash.mdx index 2fc3b4e01..1789c0831 100644 --- a/docs_new/cookbook/autoregressive/GLM/GLM-4.7-Flash.mdx +++ b/docs_new/cookbook/autoregressive/GLM/GLM-4.7-Flash.mdx @@ -53,7 +53,7 @@ import { GLM47FlashDeployment } from "/src/snippets/autoregressive/glm-47-flash- ### 3.2 Configuration Tips -- **EAGLE Speculative Decoding:** Supported for GLM-4.7-Flash. Add `--speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4`. The spec-v2 overlap scheduler is enabled by default (`SGLANG_ENABLE_SPEC_V2=True`); set `SGLANG_ENABLE_SPEC_V2=0` to disable. Enable via the interactive command generator above. +- **EAGLE Speculative Decoding:** Supported for GLM-4.7-Flash. Add `--speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4`. The spec-v2 overlap scheduler is enabled by default; pass `--disable-overlap-schedule` to disable. Enable via the interactive command generator above. ## 4. Model Invocation diff --git a/docs_new/cookbook/autoregressive/GLM/GLM-4.7.mdx b/docs_new/cookbook/autoregressive/GLM/GLM-4.7.mdx index 5d568f2aa..ff30024e4 100644 --- a/docs_new/cookbook/autoregressive/GLM/GLM-4.7.mdx +++ b/docs_new/cookbook/autoregressive/GLM/GLM-4.7.mdx @@ -124,7 +124,7 @@ Pick a weight format by hardware: **NVFP4** on NVIDIA Blackwell (B200, GB200), * -- **EAGLE Speculative Decoding:** Supported for GLM-4.7. Add `--speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4`. The spec-v2 overlap scheduler is enabled by default (`SGLANG_ENABLE_SPEC_V2=True`); set `SGLANG_ENABLE_SPEC_V2=0` to disable. Enable via the interactive command generator above. +- **EAGLE Speculative Decoding:** Supported for GLM-4.7. Add `--speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4`. The spec-v2 overlap scheduler is enabled by default; pass `--disable-overlap-schedule` to disable. Enable via the interactive command generator above. - **Thinking Budget:** Use `--enable-custom-logit-processor` flag and pass `Glm4MoeThinkingBudgetLogitProcessor` in requests to cap the model's thinking token count (see section 4.2.3). For general GLM-4.x family launch guidance (AMD ROCm notes and more), see [Launch GLM-4.5 / GLM-4.6 / GLM-4.7 with SGLang](../../../docs/basic_usage/glm45). Per-hardware bench commands and flags are inline in §5.1 below. diff --git a/docs_new/cookbook/autoregressive/OpenAI/GPT-OSS.mdx b/docs_new/cookbook/autoregressive/OpenAI/GPT-OSS.mdx index f5a972a01..646299464 100644 --- a/docs_new/cookbook/autoregressive/OpenAI/GPT-OSS.mdx +++ b/docs_new/cookbook/autoregressive/OpenAI/GPT-OSS.mdx @@ -428,7 +428,7 @@ python3 -m sglang.launch_server \ ``` -The spec-v2 overlap scheduler is enabled by default (`SGLANG_ENABLE_SPEC_V2=True`). It improves performance by overlapping draft and verification stages. Set `SGLANG_ENABLE_SPEC_V2=0` to disable. +The spec-v2 overlap scheduler is enabled by default. It improves performance by overlapping draft and verification stages. Pass `--disable-overlap-schedule` to disable. #### 4.2.4 Responses API and Built-in Tools diff --git a/docs_new/cookbook/autoregressive/Qwen/Qwen3.6.mdx b/docs_new/cookbook/autoregressive/Qwen/Qwen3.6.mdx index 41db3b8e7..38f447fc6 100644 --- a/docs_new/cookbook/autoregressive/Qwen/Qwen3.6.mdx +++ b/docs_new/cookbook/autoregressive/Qwen/Qwen3.6.mdx @@ -150,7 +150,7 @@ All Qwen3.6 variants (MoE 35B-A3B and Dense 27B) fit on a single supported GPU a Deploy Qwen3.6 with the following command (H200, all features enabled). Swap `--model-path` to `Qwen/Qwen3.6-27B-FP8` for the dense 27B variant — all other flags carry over: ```shell Command -SGLANG_ENABLE_SPEC_V2=1 sglang serve \ +sglang serve \ --model-path Qwen/Qwen3.6-35B-A3B-FP8 \ --reasoning-parser qwen3 \ --tool-call-parser qwen3_coder \ diff --git a/docs_new/cookbook/autoregressive/Tencent/Hunyuan3-Preview.mdx b/docs_new/cookbook/autoregressive/Tencent/Hunyuan3-Preview.mdx index 2780630ca..0e730954d 100644 --- a/docs_new/cookbook/autoregressive/Tencent/Hunyuan3-Preview.mdx +++ b/docs_new/cookbook/autoregressive/Tencent/Hunyuan3-Preview.mdx @@ -138,7 +138,7 @@ import { Hunyuan3PreviewDeployment } from '/src/snippets/autoregressive/hunyuan3 `--speculative-algorithm` Speculative decoding via the bundled MTP draft - `EAGLE` + `--speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4` (set env `SGLANG_ENABLE_SPEC_V2=1`) + `EAGLE` + `--speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4` @@ -151,10 +151,10 @@ import { Hunyuan3PreviewDeployment } from '/src/snippets/autoregressive/hunyuan3 **Blackwell (B200 / B300 / GB300):** Auto-selected attention backend can mis-route for HYV3 on Blackwell. Always pass `--attention-backend trtllm_mha` explicitly on Blackwell hardware (the config generator above enforces this). -**Multi-Token Prediction (MTP):** The `Hy3-preview` release bundles an MTP draft module. SGLang runs it via its EAGLE speculative-decoding path — the draft module auto-loads from the same `--model-path`. Enable with the `SGLANG_ENABLE_SPEC_V2=1` env var and the standard MTP flags: +**Multi-Token Prediction (MTP):** The `Hy3-preview` release bundles an MTP draft module. SGLang runs it via its EAGLE speculative-decoding path — the draft module auto-loads from the same `--model-path`. Enable with the standard MTP flags: ```bash Command -SGLANG_ENABLE_SPEC_V2=1 sglang serve \ +sglang serve \ --model-path tencent/Hy3-preview \ --tp 8 \ --speculative-algorithm EAGLE \ diff --git a/docs_new/cookbook/autoregressive/Xiaomi/MiMo-V2.5.mdx b/docs_new/cookbook/autoregressive/Xiaomi/MiMo-V2.5.mdx index efcb37d85..d3b6018de 100644 --- a/docs_new/cookbook/autoregressive/Xiaomi/MiMo-V2.5.mdx +++ b/docs_new/cookbook/autoregressive/Xiaomi/MiMo-V2.5.mdx @@ -83,13 +83,13 @@ import { MiMoV25Deployment } from '/src/snippets/autoregressive/mimo-v25-deploym - **B200**: single node, TP=8 (verified). Uses `--attention-backend fa4` + `--moe-runner-backend flashinfer_trtllm` + `--mem-fraction-static 0.8`. Set `--swa-full-tokens-ratio 0.1` to keep KV-cache footprint within 192 GB HBM. - **GB300**: 2 nodes, TP=8 (verified). Same Blackwell stack as B200; multi-node interconnect requires `NCCL_MNNVL_ENABLE=1 NCCL_CUMEM_ENABLE=1`. Default SWA ratio is fine. - **H100/H200**: 2 nodes × 8 GPUs (TP=16, not yet verified). Uses the Hopper stack (`fa3` + DeepEP + EAGLE multi-layer); fits with `--mem-fraction-static 0.7` and `--swa-full-tokens-ratio 0.3`. DeepEP dispatch tuning: `SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256` avoids memory spikes during prefill. -- EAGLE speculative decoding (3 steps, topk=1) typically yields a 2–3× decode speedup. Requires `SGLANG_ENABLE_SPEC_V2=1` and `--enable-multi-layer-eagle` (both Hopper and Blackwell). See §5.4 for acceptance-rate behavior on natural text vs random prompts. +- EAGLE speculative decoding (3 steps, topk=1) typically yields a 2–3× decode speedup. Requires `--enable-multi-layer-eagle` (both Hopper and Blackwell). See §5.4 for acceptance-rate behavior on natural text vs random prompts. **MiMo-V2.5 (310B):** - The checkpoint has a TP=4-interleaved fused `qkv_proj`; attention-TP per DP group **must** be 4. Use `--dp = TP / 4`; for TP > 4 this also requires DP-attention. Total GPUs must be a multiple of 4. A bare `--tp 8` without `--dp 2` will fail to load with `MiMoV2 fused qkv_proj checkpoint is TP=4-interleaved; got attention tp_size=8`. - Single-node deployments: H100/H200 8× GPUs (`--tp 8 --dp 2`), B200 4× GPUs (`--tp 4`, dp=1, no DP-attn flag needed), GB300 4× GPUs (`--tp 4`, single NVL4 node). FP8 quantization. - `--enable-dp-lm-head` and `--mm-enable-dp-encoder` are required whenever `--enable-dp-attention` is on, to keep LM head and encoder sharding consistent. -- EAGLE MTP uses the checkpoint's MTP weights. Enable with `SGLANG_ENABLE_SPEC_V2=1`, `--speculative-algorithm EAGLE`, and `--enable-multi-layer-eagle` (both Hopper and Blackwell). +- EAGLE MTP uses the checkpoint's MTP weights. Enable with `--speculative-algorithm EAGLE` and `--enable-multi-layer-eagle` (both Hopper and Blackwell). - **Multimodal**: Supports image, video, and audio understanding; see Section 4.3 for invocation examples. **DeepEP (optional toggle, Hopper-only):** diff --git a/docs_new/docs/advanced_features/attention_backend.mdx b/docs_new/docs/advanced_features/attention_backend.mdx index 56f7d5f77..abcb115de 100644 --- a/docs_new/docs/advanced_features/attention_backend.mdx +++ b/docs_new/docs/advanced_features/attention_backend.mdx @@ -305,7 +305,7 @@ Speculative decoding topk: `topk` is the number of draft tokens sampled per step -**Speculative Decoding V2 (Spec V2):** Spec V2 uses overlap scheduling (`SGLANG_ENABLE_SPEC_V2=True`) that benefits various attention backends. Requires `--speculative-eagle-topk 1` and currently applies to EAGLE and EAGLE3. +**Speculative Decoding V2 (Spec V2):** speculative decoding runs the V2 workers with overlap scheduling by default, which benefits various attention backends. Requires `--speculative-eagle-topk 1`. **Verified backends:** TRTLLM MLA, TRTLLM MHA, FA3, Ascend (NPU), Triton. diff --git a/docs_new/docs/advanced_features/speculative_decoding.mdx b/docs_new/docs/advanced_features/speculative_decoding.mdx index b213652b7..5efa8d32f 100644 --- a/docs_new/docs/advanced_features/speculative_decoding.mdx +++ b/docs_new/docs/advanced_features/speculative_decoding.mdx @@ -33,7 +33,6 @@ SGLang provides several speculative decoding options, including EAGLE-2/EAGLE-3, - **You have a DFlash draft checkpoint**: Use **DFLASH** with `--speculative-algorithm DFLASH` and `--speculative-draft-model-path ...`. - **You have a smaller draft LLM**: Use **STANDALONE** (`--speculative-algorithm STANDALONE`). - **No extra model available**: Use **NGRAM** (`--speculative-algorithm NGRAM`, CUDA-only). -- **Want overlap scheduler (experimental)**: Enable **SpecV2** with `SGLANG_ENABLE_SPEC_V2=True` (requires `--speculative-eagle-topk 1`). ### Method comparison (mini table) @@ -101,13 +100,6 @@ SGLang provides several speculative decoding options, including EAGLE-2/EAGLE-3, --speculative-algorithm STANDALONE + --speculative-draft-model-path ... Does not support --enable-dp-attention - - SpecV2 (experimental) - V2 workers + overlap scheduler - N/A - SGLANG_ENABLE_SPEC_V2=True - Only supports --speculative-eagle-topk 1; applies to EAGLE, EAGLE3, STANDALONE - NGRAM Ngram cache from previous tokens @@ -592,19 +584,15 @@ print(response.choices[0].message.content) ## Speculative Decoding V2 (Overlap Scheduler) -SGLang provides an **experimental Speculative Decoding V2** implementation that enables an overlap scheduler and uses V2 speculative workers (e.g. `StandaloneWorkerV2`, `EAGLEWorkerV2`). - -To enable it, set the environment variable: -- `SGLANG_ENABLE_SPEC_V2=True` +Speculative decoding runs the V2 speculative workers (e.g. `StandaloneWorkerV2`, `EAGLEWorkerV2`) with the overlap scheduler enabled by default. Pass `--disable-overlap-schedule` to fall back to the synchronous (non-overlap) path. Notes: -- SpecV2 currently only supports `--speculative-eagle-topk 1`. When SpecV2 is enabled, **set `--speculative-eagle-topk 1` explicitly**. +- The overlap scheduler currently only supports `--speculative-eagle-topk 1`; **set `--speculative-eagle-topk 1` explicitly**. - If you explicitly set `--speculative-eagle-topk > 1`, the server will error. -- If you omit `--speculative-eagle-topk`, auto-tuning may pick `topk > 1` for some models (e.g. Llama). This is incompatible with SpecV2 and may not always trigger an immediate config error, so set `--speculative-eagle-topk 1` explicitly. -- This applies to `EAGLE`, `EAGLE3`, and `STANDALONE`. +- If you omit `--speculative-eagle-topk`, auto-tuning may pick `topk > 1` for some models (e.g. Llama). This is incompatible with the overlap scheduler and may not always trigger an immediate config error, so set `--speculative-eagle-topk 1` explicitly. ```bash Command -SGLANG_ENABLE_SPEC_V2=True python3 -m sglang.launch_server \ +python3 -m sglang.launch_server \ --model Qwen/Qwen2.5-7B-Instruct \ --speculative-algorithm STANDALONE \ --speculative-draft-model-path Qwen/Qwen2.5-1.5B-Instruct \ @@ -926,11 +914,6 @@ Below is a comprehensive list of all speculative decoding parameters available i - - SGLANG_ENABLE_SPEC_V2 - False - Enable Speculative Decoding V2 (overlap scheduler) - SGLANG_NGRAM_FORCE_GREEDY_VERIFY False diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_best_practice.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_best_practice.mdx index 14154242d..d8bee9a44 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_best_practice.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_best_practice.mdx @@ -1108,7 +1108,6 @@ do then echo "${D_IP[$i]}" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export SGLANG_ENABLE_SPEC_V2=1 export HCCL_BUFFSIZE=600 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64 export TASK_QUEUE_ENABLE=1 @@ -1227,7 +1226,6 @@ do then echo "${D_IP[$i]}" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export SGLANG_ENABLE_SPEC_V2=1 export HCCL_BUFFSIZE=800 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=102 export TASK_QUEUE_ENABLE=1 @@ -1351,7 +1349,6 @@ do then echo "${D_IP[$i]}" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export SGLANG_ENABLE_SPEC_V2=1 export HCCL_BUFFSIZE=650 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=16 export TASK_QUEUE_ENABLE=1 @@ -1476,7 +1473,6 @@ do then echo "${D_IP[$i]}" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export SGLANG_ENABLE_SPEC_V2=1 export HCCL_BUFFSIZE=650 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=12 export TASK_QUEUE_ENABLE=1 @@ -1642,7 +1638,6 @@ export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=512 MODEL_PATH=xxx export SGLANG_NPU_USE_MLAPO=1 -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_USE_FIA_NZ=1 @@ -1754,7 +1749,6 @@ do then echo "${D_IP[$i]}" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export SGLANG_ENABLE_SPEC_V2=1 export HCCL_BUFFSIZE=900 export SGLANG_DP_ROUND_ROBIN=1 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=112 @@ -1838,7 +1832,6 @@ export HCCL_BUFFSIZE=1200 export DEEPEP_NORMAL_LONG_SEQ_ROUND=10 export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=512 export SGLANG_NPU_USE_MLAPO=1 -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_USE_FIA_NZ=1 @@ -1952,7 +1945,6 @@ do then echo "${D_IP[$i]}" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export SGLANG_ENABLE_SPEC_V2=1 export HCCL_BUFFSIZE=800 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=78 export TASK_QUEUE_ENABLE=1 @@ -2085,7 +2077,6 @@ do then echo "${D_IP[$i]}" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export SGLANG_ENABLE_SPEC_V2=1 export TASK_QUEUE_ENABLE=0 export SGLANG_SCHEDULER_SKIP_ALL_GATHER=1 @@ -2181,7 +2172,6 @@ export SGLANG_SET_CPU_AFFINITY=1 export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_DP_ROUND_ROBIN=1 export SGLANG_NPU_FUSED_MOE_MODE=2 @@ -2334,7 +2324,6 @@ export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo export HCCL_OP_EXPANSION_MODE="AIV" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100 @@ -2409,7 +2398,6 @@ export HCCL_SOCKET_IFNAME=xxx export GLOO_SOCKET_IFNAME=xxx export HCCL_OP_EXPANSION_MODE="AIV" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=147456 @@ -2490,7 +2478,6 @@ do then echo "${MIX_IP[$i]}" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export SGLANG_ENABLE_SPEC_V2=1 python -m sglang.launch_server --model-path ${MODEL_PATH} \ --host 127.0.0.1 --port 7439 --trust-remote-code \ @@ -2568,7 +2555,6 @@ export HCCL_SOCKET_IFNAME=xxx export GLOO_SOCKET_IFNAME=xxx export HCCL_OP_EXPANSION_MODE="AIV" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 python -m sglang.launch_server --model-path $MODEL_PATH \ --host 127.0.0.1 --port 7439 --trust-remote-code --nnodes 1 --node-rank 0 \ @@ -2639,7 +2625,6 @@ export HCCL_SOCKET_IFNAME=xxx export GLOO_SOCKET_IFNAME=xxx export HCCL_OP_EXPANSION_MODE="AIV" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 python -m sglang.launch_server --model-path $MODEL_PATH \ --host 127.0.0.1 --port 7439 --trust-remote-code --nnodes 1 --node-rank 0 \ @@ -2709,7 +2694,6 @@ export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo export HCCL_OP_EXPANSION_MODE="AIV" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 python -m sglang.launch_server --model-path $MODEL_PATH \ --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ @@ -2779,7 +2763,6 @@ export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo export HCCL_OP_EXPANSION_MODE="AIV" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 python -m sglang.launch_server --model-path $MODEL_PATH \ --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ @@ -2848,7 +2831,6 @@ export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo export HCCL_OP_EXPANSION_MODE="AIV" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 python -m sglang.launch_server --model-path $MODEL_PATH \ --host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \ @@ -2917,7 +2899,6 @@ export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo export HCCL_OP_EXPANSION_MODE="AIV" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 python -m sglang.launch_server --model-path $MODEL_PATH \ --host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \ @@ -2991,7 +2972,6 @@ export HCCL_OP_EXPANSION_MODE="AIV" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200 -export SGLANG_ENABLE_SPEC_V2=1 python -m sglang.launch_server --model-path $MODEL_PATH \ --host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \ @@ -3326,7 +3306,6 @@ export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=330 export ASCEND_USE_FIA=1 export SGLANG_NPU_USE_MULTI_STREAM=0 export SGLANG_WARMUP_TIMEOUT=3600 -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export FORCE_DRAFT_MODEL_NON_QUANT=1 @@ -3417,7 +3396,6 @@ export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo export HCCL_OP_EXPANSION_MODE="AIV" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 python -m sglang.launch_server --model-path $MODEL_PATH \ --host 127.0.0.1 --port 7439 --trust-remote-code --nnodes 1 --node-rank 0 \ @@ -3489,7 +3467,6 @@ export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo export HCCL_OP_EXPANSION_MODE="AIV" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 python -m sglang.launch_server --model-path $MODEL_PATH \ --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ @@ -3561,7 +3538,6 @@ export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo export HCCL_OP_EXPANSION_MODE="AIV" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 python -m sglang.launch_server --model-path $MODEL_PATH \ --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ @@ -3633,7 +3609,6 @@ export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo export HCCL_OP_EXPANSION_MODE="AIV" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 python -m sglang.launch_server --model-path $MODEL_PATH \ --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ @@ -3705,7 +3680,6 @@ export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo export HCCL_OP_EXPANSION_MODE="AIV" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 python -m sglang.launch_server --model-path $MODEL_PATH \ --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ @@ -3777,7 +3751,6 @@ export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo export HCCL_OP_EXPANSION_MODE="AIV" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 python -m sglang.launch_server --model-path $MODEL_PATH \ --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ @@ -3848,7 +3821,6 @@ export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo export HCCL_OP_EXPANSION_MODE="AIV" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 python -m sglang.launch_server --model-path $MODEL_PATH \ --host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \ @@ -3917,7 +3889,6 @@ export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo export HCCL_OP_EXPANSION_MODE="AIV" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 python -m sglang.launch_server --model-path $MODEL_PATH \ --host 127.0.0.1 --port 7239 --trust-remote-code --nnodes 1 --node-rank 0 \ @@ -3980,7 +3951,6 @@ export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo export HCCL_OP_EXPANSION_MODE="AIV" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 python -m sglang.launch_server --model-path $MODEL_PATH \ --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ @@ -4044,7 +4014,6 @@ export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo export HCCL_OP_EXPANSION_MODE="AIV" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 python -m sglang.launch_server --model-path $MODEL_PATH \ --host 127.0.0.1 --port 7339 --trust-remote-code --nnodes 1 --node-rank 0 \ @@ -4111,7 +4080,6 @@ export ASCEND_USE_FIA=1 export SGLANG_NPU_USE_MULTI_STREAM=1 export SGLANG_WARMUP_TIMEOUT=3600 -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export FORCE_DRAFT_MODEL_NON_QUANT=1 @@ -4130,7 +4098,6 @@ export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo export HCCL_OP_EXPANSION_MODE="AIV" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 python3 -m sglang.launch_server --model-path ${MODEL_PATH} \ --page-size 128 \ @@ -4204,7 +4171,6 @@ export ASCEND_USE_FIA=1 export SGLANG_NPU_USE_MULTI_STREAM=1 export SGLANG_WARMUP_TIMEOUT=3600 -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export FORCE_DRAFT_MODEL_NON_QUANT=1 @@ -4223,7 +4189,6 @@ export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo export HCCL_OP_EXPANSION_MODE="AIV" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 python3 -m sglang.launch_server --model-path ${MODEL_PATH} \ --page-size 128 \ @@ -4294,7 +4259,6 @@ export STREAMS_PER_DEVICE=32 export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo export ASCEND_USE_FIA=0 -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` @@ -4369,7 +4333,6 @@ export STREAMS_PER_DEVICE=32 export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo export ASCEND_USE_FIA=0 -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200 @@ -4444,7 +4407,6 @@ export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export HCCL_OP_EXPANSION_MODE="AIV" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=50 export HCCL_SOCKET_IFNAME=lo @@ -4519,7 +4481,6 @@ export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export HCCL_OP_EXPANSION_MODE="AIV" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo @@ -4596,7 +4557,6 @@ export TASK_QUEUE_ENABLE=1 export ASCEND_USE_FIA=1 export SGLANG_NPU_USE_MULTI_STREAM=0 export SGLANG_WARMUP_TIMEOUT=3600 -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export FORCE_DRAFT_MODEL_NON_QUANT=1 export HCCL_BUFFSIZE=2000 @@ -4684,7 +4644,6 @@ export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo export SGLANG_NPU_PROFILING=0 export SGLANG_DISAGGEGATION_WAITING_TIMEOUT=3600 -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 MODEL_PATH=xxx @@ -4762,7 +4721,6 @@ export STREAMS_PER_DEVICE=32 export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100 @@ -4837,7 +4795,6 @@ export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo export SGLANG_NPU_PROFILING=1 -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 # on-demand set device export ASCEND_RT_VISIBLE_DEVICES=4,5 @@ -4908,7 +4865,6 @@ export STREAMS_PER_DEVICE=32 export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100 @@ -4979,7 +4935,6 @@ export STREAMS_PER_DEVICE=32 export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=30 @@ -5052,7 +5007,6 @@ export STREAMS_PER_DEVICE=32 export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=100 @@ -5131,7 +5085,6 @@ export STREAMS_PER_DEVICE=32 export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_NPU_USE_MULTI_STREAM=1 @@ -5218,7 +5171,6 @@ export STREAMS_PER_DEVICE=32 export HCCL_OP_EXPANSION_MODE=AIV export HCCL_SOCKET_IFNAME=lo export GLOO_SOCKET_IFNAME=lo -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_NPU_USE_MULTI_STREAM=1 @@ -5300,7 +5252,6 @@ export TASK_QUEUE_ENABLE=1 export HCCL_BUFFSIZE=1500 export ASCEND_USE_FIA=1 export SGLANG_SET_CPU_AFFINITY=1 -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_NPU_USE_MULTI_STREAM=1 export SGLANG_NPU_FUSED_MOE_MODE=2 @@ -5381,7 +5332,6 @@ export TASK_QUEUE_ENABLE=1 export ASCEND_USE_FIA=1 export HCCL_BUFFSIZE=1600 export SGLANG_SET_CPU_AFFINITY=1 -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=640 export DEEPEP_NORMAL_LONG_SEQ_ROUND=64 @@ -5467,7 +5417,6 @@ export TASK_QUEUE_ENABLE=1 export HCCL_BUFFSIZE=800 export ASCEND_USE_FIA=1 export SGLANG_SET_CPU_AFFINITY=1 -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_NPU_FUSED_MOE_MODE=2 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=204800 @@ -5547,7 +5496,6 @@ export TASK_QUEUE_ENABLE=1 export ASCEND_USE_FIA=1 export HCCL_BUFFSIZE=1600 export SGLANG_SET_CPU_AFFINITY=1 -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=640 export DEEPEP_NORMAL_LONG_SEQ_ROUND=64 @@ -5631,7 +5579,6 @@ export TASK_QUEUE_ENABLE=1 export ASCEND_USE_FIA=1 export HCCL_BUFFSIZE=1600 export SGLANG_SET_CPU_AFFINITY=1 -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=640 export DEEPEP_NORMAL_LONG_SEQ_ROUND=64 @@ -5715,7 +5662,6 @@ export TASK_QUEUE_ENABLE=1 export ASCEND_USE_FIA=0 export HCCL_BUFFSIZE=1600 export SGLANG_SET_CPU_AFFINITY=1 -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=640 export DEEPEP_NORMAL_LONG_SEQ_ROUND=64 @@ -5852,7 +5798,6 @@ do export HCCL_SOCKET_IFNAME=your_nic export GLOO_SOCKET_IFNAME=your_nic export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_NPU_FUSED_MOE_MODE=2 export SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS=96 @@ -5988,7 +5933,6 @@ do export HCCL_SOCKET_IFNAME=your_nic export GLOO_SOCKET_IFNAME=your_nic export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_NPU_FUSED_MOE_MODE=2 export SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS=96 @@ -6074,7 +6018,6 @@ export STREAMS_PER_DEVICE=32 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=48 export HCCL_BUFFSIZE=1200 -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_NPU_USE_MLAPO=1 export SGLANG_NPU_USE_MULTI_STREAM=1 @@ -6150,7 +6093,6 @@ export STREAMS_PER_DEVICE=32 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=96 export HCCL_BUFFSIZE=1200 -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200 @@ -6231,7 +6173,6 @@ P_IP=('your ip1' 'your ip2') P_MASTER="${P_IP[0]}:4567" export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` @@ -6388,7 +6329,6 @@ do echo "${D_IP[$i]}" export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export SGLANG_ENABLE_SPEC_V2=1 export HCCL_BUFFSIZE=650 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=64 export TASK_QUEUE_ENABLE=0 @@ -6528,7 +6468,6 @@ do echo "${D_IP[$i]}" export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export SGLANG_ENABLE_SPEC_V2=1 export HCCL_BUFFSIZE=300 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=40 export TASK_QUEUE_ENABLE=0 @@ -6713,7 +6652,6 @@ do echo "${D_IP[$i]}" export SGLANG_SPEC_ENABLE_OVERLAP_REFLOW=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export SGLANG_ENABLE_SPEC_V2=1 export HCCL_BUFFSIZE=200 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=24 export TASK_QUEUE_ENABLE=0 @@ -6807,7 +6745,6 @@ export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3584 export ASCEND_MF_STORE_URL="tcp://:24669" export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600 export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600 -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export SGLANG_DEEPEP_BF16_DISPATCH=0 @@ -6868,7 +6805,6 @@ export DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=3584 export ASCEND_MF_STORE_URL="tcp://:24669" export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600 export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600 -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export SGLANG_DEEPEP_BF16_DISPATCH=0 diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_deepseek_example.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_deepseek_example.mdx index 78e1cb725..823e2a046 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_deepseek_example.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_deepseek_example.mdx @@ -23,7 +23,6 @@ export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32 export HCCL_BUFFSIZE=1600 #spec overlap -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 #npu acceleration operator @@ -123,7 +122,6 @@ export HCCL_BUFFSIZE=720 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=88 #spec overlap -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 #npu acceleration operator @@ -240,7 +238,6 @@ do then echo "${D_IP[$i]}" export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 - export SGLANG_ENABLE_SPEC_V2=1 export HCCL_BUFFSIZE=650 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=78 export TASK_QUEUE_ENABLE=1 diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_glm5_examples.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_glm5_examples.mdx index 13aa28924..cf22b3748 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_glm5_examples.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_glm5_examples.mdx @@ -133,7 +133,6 @@ source /usr/local/Ascend/nnal/atb/set_env.sh export STREAMS_PER_DEVICE=32 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_NPU_USE_MULTI_STREAM=1 export HCCL_BUFFSIZE=1000 @@ -185,7 +184,6 @@ source /usr/local/Ascend/nnal/atb/set_env.sh export STREAMS_PER_DEVICE=32 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_NPU_USE_MULTI_STREAM=1 export HCCL_BUFFSIZE=1000 @@ -200,7 +198,6 @@ P_IP=('your ip1' 'your ip2') P_MASTER="${P_IP[0]}:your port" export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 LOCAL_HOST1=`hostname -I|awk -F " " '{print$1}'` diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_kimi_k2.5_examples.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_kimi_k2.5_examples.mdx index 445f8fe36..926eab477 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_kimi_k2.5_examples.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_kimi_k2.5_examples.mdx @@ -94,7 +94,6 @@ source /usr/local/Ascend/nnal/atb/set_env.sh export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export STREAMS_PER_DEVICE=32 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=112 @@ -154,7 +153,6 @@ export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 export DEEP_NORMAL_MODE_USE_INT8_QUANT=1 export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=96 export HCCL_BUFFSIZE=2400 -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE=1 export SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES=200 @@ -271,7 +269,6 @@ do export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=32 export HCCL_BUFFSIZE=2400 - export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=60 diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_optimization.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_optimization.mdx index fccd32d9b..3a7fbbc7f 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_optimization.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_optimization.mdx @@ -275,7 +275,7 @@ Speculative decoding reduces per-token latency by generating draft tokens that a `--speculative-eagle-topk` - Limits the number of draft tokens considered per position. Lower values reduce compute on unlikely tokens and are required for the experimental SpecV2 overlap scheduler. Higher values may increase acceptance rates but add overhead. Start with `1` if using SpecV2; otherwise, `4`–`8` is typical. + Limits the number of draft tokens considered per position. Lower values reduce compute on unlikely tokens and are required for the overlap scheduler (enabled by default). Higher values may increase acceptance rates but add overhead. Start with `1` for the overlap scheduler; otherwise, `4`–`8` is typical. `1` @@ -288,11 +288,6 @@ Speculative decoding reduces per-token latency by generating draft tokens that a Enables the overlap plan stream feature for EAGLE v2/v3 speculative decoding workers. This overlaps draft model computation with target model verification, effectively hiding draft latency. Enable when using EAGLE-based speculative decoding; not applicable for NEXTN. `1` - - `SGLANG_ENABLE_SPEC_V2=1` - Enables the experimental SpecV2 overlap scheduler for speculative decoding. Works with `--speculative-eagle-topk 1` to overlap the draft generation and verification stages. Requires `SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1`. - `1` - @@ -766,7 +761,7 @@ Captures the compute graph on NPU and replays it to eliminate kernel launch over ### Speculative Decoding (`--speculative-algorithm`) -Reduces per-token latency by predicting multiple future tokens in a single forward pass, then verifying them against the model. Ascend supports `NEXTN` (for DeepSeek models, using the model's own hidden states) and `EAGLE3` (for Qwen MoE models, using a separate draft model). Controlled by `--speculative-num-steps`, `--speculative-eagle-topk`, and `--speculative-num-draft-tokens`. On Ascend, `SGLANG_ENABLE_SPEC_V2=1` and `SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` enable the optimized overlap scheduler for speculative decoding. +Reduces per-token latency by predicting multiple future tokens in a single forward pass, then verifying them against the model. Ascend supports `NEXTN` (for DeepSeek models, using the model's own hidden states) and `EAGLE3` (for Qwen MoE models, using a separate draft model). Controlled by `--speculative-num-steps`, `--speculative-eagle-topk`, and `--speculative-num-draft-tokens`. On Ascend, `SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1` enables the optimized overlap scheduler for speculative decoding. ### PrefixCache (`--disable-radix-cache`) @@ -774,7 +769,7 @@ Reuses KV cache across requests that share common prompt prefixes (Radix Cache), ### Overlap Schedule -On Ascend, this is primarily controlled via environment variable SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1. For speculative decoding scenarios, combining it with SGLANG_ENABLE_SPEC_V2=1 enables the optimized overlap scheduler. Most effective during decode with speculative decoding and DP attention, where expert dispatch/reduce communication can be overlapped with the next batch's computation. +On Ascend, this is primarily controlled via environment variable SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1, which enables the optimized overlap scheduler for speculative decoding scenarios. Most effective during decode with speculative decoding and DP attention, where expert dispatch/reduce communication can be overlapped with the next batch's computation. ### DP LM Head (`--enable-dp-lm-head`) diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_5_examples.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_5_examples.mdx index ec78a7b32..de4d0d8bd 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_5_examples.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_5_examples.mdx @@ -304,7 +304,6 @@ source /usr/local/Ascend/nnal/atb/set_env.sh export STREAMS_PER_DEVICE=32 export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600 -export SGLANG_ENABLE_SPEC_V2=1 export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 export SGLANG_NPU_USE_MULTI_STREAM=1 export HCCL_BUFFSIZE=1000 diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_examples.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_examples.mdx index f9c06bb20..24f35022e 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_examples.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_npu_qwen3_examples.mdx @@ -43,7 +43,6 @@ export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True export STREAMS_PER_DEVICE=32 export HCCL_OP_EXPANSION_MODE=AIV export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 -export SGLANG_ENABLE_SPEC_V2=1 python -m sglang.launch_server \ --device npu \ diff --git a/python/sglang/srt/arg_groups/deepseek_v4_hook.py b/python/sglang/srt/arg_groups/deepseek_v4_hook.py index 4bdb6e007..f57159a09 100644 --- a/python/sglang/srt/arg_groups/deepseek_v4_hook.py +++ b/python/sglang/srt/arg_groups/deepseek_v4_hook.py @@ -9,7 +9,6 @@ logger = logging.getLogger(__name__) def apply_deepseek_v4_defaults(server_args: "ServerArgs", model_arch: str) -> None: """Apply DeepSeek V4 model-specific server arg defaults and constraints.""" - from sglang.srt.environ import envs from sglang.srt.server_args import ServerArgs server_args.attention_backend = "dsv4" @@ -41,10 +40,6 @@ def apply_deepseek_v4_defaults(server_args: "ServerArgs", model_arch: str) -> No server_args.speculative_eagle_topk == 1 ), f"Only EAGLE speculative algorithm with topk == 1 is supported for {model_arch}" - if not envs.SGLANG_ENABLE_SPEC_V2.get(): - envs.SGLANG_ENABLE_SPEC_V2.set(True) - logger.warning("Spec v2 is enabled for EAGLE speculative decoding.") - if server_args.swa_full_tokens_ratio == ServerArgs.swa_full_tokens_ratio: server_args.swa_full_tokens_ratio = 0.1 logger.info( diff --git a/python/sglang/srt/arg_groups/speculative_hook.py b/python/sglang/srt/arg_groups/speculative_hook.py index f38d953dc..690b684f9 100644 --- a/python/sglang/srt/arg_groups/speculative_hook.py +++ b/python/sglang/srt/arg_groups/speculative_hook.py @@ -1,9 +1,8 @@ import json import logging +import os from typing import TYPE_CHECKING, Optional -from sglang.srt.environ import envs - if TYPE_CHECKING: from sglang.srt.server_args import ServerArgs @@ -63,6 +62,15 @@ def handle_speculative_decoding(server_args: "ServerArgs") -> None: if server_args.speculative_algorithm is not None: server_args.speculative_algorithm = server_args.speculative_algorithm.upper() + # Removal notice for the retired env var; raw os.getenv on purpose -- the + # Envs descriptor is gone. Drop this check after one release. + if os.getenv("SGLANG_ENABLE_SPEC_V2") is not None: + logger.warning( + "SGLANG_ENABLE_SPEC_V2 has been removed: speculative decoding " + "always runs the V2 worker. Use --disable-overlap-schedule to " + "select the non-overlap (synchronous) path." + ) + kwargs = {} override_config_file = server_args.decrypted_draft_config_file @@ -230,14 +238,6 @@ def _handle_dflash(server_args: "ServerArgs") -> None: "Max running requests is reset to 48 for speculative decoding. You can override this by explicitly setting --max-running-requests." ) - # SGLANG_ENABLE_SPEC_V2=False selects the non-overlap (synchronous) spec v2 - # path instead of the overlap-scheduled one; both run the V2 worker. - if ( - not envs.SGLANG_ENABLE_SPEC_V2.get() - and not server_args.disable_overlap_schedule - ): - server_args.disable_overlap_schedule = True - if server_args.enable_mixed_chunk: server_args.enable_mixed_chunk = False logger.warning( @@ -252,14 +252,6 @@ def _handle_frozen_kv_mtp(server_args: "ServerArgs") -> None: "Max running requests is reset to 48 for speculative decoding. You can override this by explicitly setting --max-running-requests." ) - # SGLANG_ENABLE_SPEC_V2=False selects the non-overlap (synchronous) spec v2 - # path instead of the overlap-scheduled one; both run the V2 worker. - if ( - not envs.SGLANG_ENABLE_SPEC_V2.get() - and not server_args.disable_overlap_schedule - ): - server_args.disable_overlap_schedule = True - if server_args.enable_mixed_chunk: server_args.enable_mixed_chunk = False logger.warning( @@ -284,14 +276,6 @@ def _handle_eagle_family(server_args: "ServerArgs") -> None: "Max running requests is reset to 48 for speculative decoding. You can override this by explicitly setting --max-running-requests." ) - # SGLANG_ENABLE_SPEC_V2=False selects the non-overlap (synchronous) spec v2 - # path instead of the overlap-scheduled one; both run the V2 worker. - if ( - not envs.SGLANG_ENABLE_SPEC_V2.get() - and not server_args.disable_overlap_schedule - ): - server_args.disable_overlap_schedule = True - if server_args.disable_overlap_schedule: logger.warning( "Non-overlap (synchronous) spec v2 is used for eagle/eagle3/standalone " diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index e0c9f7e25..a4c8dff0b 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -623,7 +623,6 @@ class Envs: SGLANG_ROPE_CACHE_ALIGN = EnvInt(128) # Overlap Spec V2 - SGLANG_ENABLE_SPEC_V2 = EnvBool(True) SGLANG_ENABLE_OVERLAP_PLAN_STREAM = EnvBool(False) SGLANG_DFLASH_PREFILL_REFILL_TARGET = EnvInt(None) diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index c23957145..fd3e819db 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -2910,7 +2910,7 @@ class ServerArgs: else: raise ValueError( f"Speculative decoding for {model_arch} is not compatible with radix cache when using --mamba-scheduler-strategy no_buffer." - "To use radix cache with speculative decoding, please use --mamba-scheduler-strategy extra_buffer and set SGLANG_ENABLE_SPEC_V2=1." + "To use radix cache with speculative decoding, please use --mamba-scheduler-strategy extra_buffer." ) def _handle_sampling_backend(self): diff --git a/python/sglang/test/server_fixtures/spec_eagle_fixture.py b/python/sglang/test/server_fixtures/spec_eagle_fixture.py index fe3131753..c18538f4e 100644 --- a/python/sglang/test/server_fixtures/spec_eagle_fixture.py +++ b/python/sglang/test/server_fixtures/spec_eagle_fixture.py @@ -6,8 +6,8 @@ backend, env overrides). Pair it with the kits in ``sglang.test.kits.spec_server_kits`` to assemble test classes. The primary axis is ``disable_overlap``: - - ``False`` -> spec v2 (overlap scheduler) - - ``True`` -> spec v1 (overlap disabled) + - ``False`` -> overlap scheduler + - ``True`` -> synchronous (non-overlap) scheduling """ import contextlib @@ -56,7 +56,7 @@ class SpecEagleServerBase(CustomTestCase): # -- runtime config -- page_size = 1 attention_backend = "flashinfer" - # Primary axis: False -> spec v2 (overlap); True -> spec v1 (overlap off). + # Primary axis: False -> overlap scheduler; True -> synchronous (non-overlap). disable_overlap = False mem_fraction_static = 0.75 max_running_requests = 8 @@ -220,7 +220,7 @@ class Eagle3Base(SpecEagleServerBase): class EagleLlama2Base(SpecEagleServerBase): - """EAGLE (Llama-2) config preset. topk=8 tree -> spec v1; gsm8k is low.""" + """EAGLE (Llama-2) config preset. topk=8 tree -> synchronous path; gsm8k is low.""" model = DEFAULT_TARGET_MODEL_EAGLE draft_model = DEFAULT_DRAFT_MODEL_EAGLE @@ -234,5 +234,5 @@ class EagleLlama2Base(SpecEagleServerBase): gsm8k_score_thres = 0.20 acc_length_thres = 3.0 batch_accept_len_thres = 1.8 - # EAGLE topk>1 already routes to v1; force it explicitly to preserve intent. - env_overrides = ((envs.SGLANG_ENABLE_SPEC_V2, False),) + # topk>1 tree verify runs on the synchronous (non-overlap) path. + disable_overlap = True diff --git a/python/sglang/test/server_fixtures/standalone_fixture.py b/python/sglang/test/server_fixtures/standalone_fixture.py index c571ebe7e..1c83005bd 100644 --- a/python/sglang/test/server_fixtures/standalone_fixture.py +++ b/python/sglang/test/server_fixtures/standalone_fixture.py @@ -2,7 +2,7 @@ Variants combine this base with `CustomTestCase` and override class attributes (`attention_backend`, plus optional `speculative_eagle_topk` / -`speculative_num_draft_tokens` / `enable_spec_v2` / +`speculative_num_draft_tokens` / `disable_overlap` / `enable_deterministic_inference`) to select a backend, deterministic mode, and the V1 / V2 spec engine. @@ -37,11 +37,11 @@ class StandaloneServerBase: # Subclasses set these: attention_backend: str = "" - # V2 defaults; V1 subclasses override to (2, 7, False). + # Overlap defaults; synchronous subclasses override to (2, 7, True). speculative_num_steps: int = 4 speculative_eagle_topk: int = 1 speculative_num_draft_tokens: int = 5 - enable_spec_v2: bool = True + disable_overlap: bool = False enable_deterministic_inference: bool = False @classmethod @@ -76,20 +76,19 @@ class StandaloneServerBase: # please don't do this if you want to make your inference workload faster envs.SGLANG_JIT_DEEPGEMM_PRECOMPILE.set(False) envs.SGLANG_ENABLE_JIT_DEEPGEMM.set(False) - if not cls.enable_spec_v2: - envs.SGLANG_ENABLE_SPEC_V2.set(False) + other_args = cls.get_server_args() + if cls.disable_overlap: + other_args = other_args + ["--disable-overlap-schedule"] cls.process = popen_launch_server( cls.model, cls.base_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=cls.get_server_args(), + other_args=other_args, ) @classmethod def tearDownClass(cls): kill_process_tree(cls.process.pid) - if not cls.enable_spec_v2: - envs.SGLANG_ENABLE_SPEC_V2.clear() def test_gsm8k(self): requests.get(self.base_url + "/flush_cache") diff --git a/test/manual/dsv4/test_dsv4_flash_mtp_tp8.py b/test/manual/dsv4/test_dsv4_flash_mtp_tp8.py index 87c63f9c3..0f75b4f27 100644 --- a/test/manual/dsv4/test_dsv4_flash_mtp_tp8.py +++ b/test/manual/dsv4/test_dsv4_flash_mtp_tp8.py @@ -23,7 +23,6 @@ from sglang.test.test_utils import ( DSV4_FLASH_MODEL_PATH = "sgl-project/DeepSeek-V4-Flash-FP8" DSV4_FLASH_BASE_ENV = { - "SGLANG_ENABLE_SPEC_V2": "1", "SGLANG_DSV4_FP4_EXPERTS": "0", } diff --git a/test/manual/dsv4/test_dsv4_pro_mtp.py b/test/manual/dsv4/test_dsv4_pro_mtp.py index 7e5cf62ae..3b066b09c 100644 --- a/test/manual/dsv4/test_dsv4_pro_mtp.py +++ b/test/manual/dsv4/test_dsv4_pro_mtp.py @@ -40,7 +40,6 @@ HONGLOUMENG_PATH = os.environ.get( ) DSV4_PRO_BASE_ENV = { - "SGLANG_ENABLE_SPEC_V2": "1", "SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2": "1", } diff --git a/test/registered/8-gpu-models/test_glm_51_fp8.py b/test/registered/8-gpu-models/test_glm_51_fp8.py index f96cc16b7..8202621a6 100644 --- a/test/registered/8-gpu-models/test_glm_51_fp8.py +++ b/test/registered/8-gpu-models/test_glm_51_fp8.py @@ -51,7 +51,6 @@ class TestGlm51Fp8(unittest.TestCase): tp_size=8, extra_args=COMMON_ARGS + dp_args + MTP_ARGS, variant="TP8+DP8+MTP", - env={"SGLANG_ENABLE_SPEC_V2": "1"}, ), ] diff --git a/test/registered/amd/test_moriep_small.py b/test/registered/amd/test_moriep_small.py index 0e6780c5c..22c7e2e96 100644 --- a/test/registered/amd/test_moriep_small.py +++ b/test/registered/amd/test_moriep_small.py @@ -469,6 +469,7 @@ class TestMTPwithTBOLowLatency(CustomTestCase): "--deepep-mode", "low_latency", "--enable-two-batch-overlap", + "--disable-overlap-schedule", ] ) @@ -476,7 +477,6 @@ class TestMTPwithTBOLowLatency(CustomTestCase): env["SGLANG_USE_AITER"] = "1" env["SGLANG_MORI_DISPATCH_DTYPE"] = "bf16" env["SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK"] = "128" - env["SGLANG_ENABLE_SPEC_V2"] = "false" env["MORI_SHMEM_MODE"] = "ISOLATION" # avoid out of symmetric heap memory # FIXME(billishyahao): enable p2p due to no rdma devices on CI machine # env["MORI_DISABLE_P2P"] = "1" @@ -534,6 +534,7 @@ class TestEPLBMoriStat(CustomTestCase): "normal", "--expert-distribution-recorder-mode", "stat", + "--disable-overlap-schedule", ] ) @@ -541,7 +542,6 @@ class TestEPLBMoriStat(CustomTestCase): env["SGLANG_USE_AITER"] = "1" env["SGLANG_MORI_DISPATCH_DTYPE"] = "bf16" env["SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK"] = "128" - env["SGLANG_ENABLE_SPEC_V2"] = "false" env["SGLANG_EPLB_ROCM_P2P_BATCH_CHUNK_SIZE"] = "32" env["MORI_SHMEM_MODE"] = "ISOLATION" # avoid out of symmetric heap memory # FIXME(billishyahao): enable p2p due to no rdma devices on CI machine diff --git a/test/registered/ep/test_deepep_large.py b/test/registered/ep/test_deepep_large.py index c9c183c3d..ed143cfe3 100644 --- a/test/registered/ep/test_deepep_large.py +++ b/test/registered/ep/test_deepep_large.py @@ -3,7 +3,6 @@ from types import SimpleNamespace import requests -from sglang.srt.environ import envs from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.run_eval import run_eval @@ -87,49 +86,49 @@ class TestDeepseekMTP(CustomTestCase): def setUpClass(cls): cls.model = DEFAULT_DEEPEP_MODEL_NAME_FOR_TEST cls.base_url = DEFAULT_URL_FOR_TEST - with envs.SGLANG_ENABLE_SPEC_V2.override(False): - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--trust-remote-code", - "--tp", - "8", - "--enable-dp-attention", - "--dp", - "8", - "--moe-dense-tp-size", - "1", - "--enable-dp-lm-head", - "--moe-a2a-backend", - "deepep", - "--moe-runner-backend", - "deep_gemm", - "--enable-two-batch-overlap", - "--ep-num-redundant-experts", - "32", - "--ep-dispatch-algorithm", - "dynamic", - "--eplb-algorithm", - "deepseek", - "--cuda-graph-bs", - "64", # TODO: increase it to 128 when TBO is supported in draft_extend - "--max-running-requests", - "512", - "--speculative-algorithm", - "EAGLE", - "--speculative-num-steps", - "1", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "2", - "--disable-radix-cache", - "--model-loader-extra-config", - '{"enable_multithread_load": true,"num_threads": 64}', - ], - ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--disable-overlap-schedule", + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "8", + "--moe-dense-tp-size", + "1", + "--enable-dp-lm-head", + "--moe-a2a-backend", + "deepep", + "--moe-runner-backend", + "deep_gemm", + "--enable-two-batch-overlap", + "--ep-num-redundant-experts", + "32", + "--ep-dispatch-algorithm", + "dynamic", + "--eplb-algorithm", + "deepseek", + "--cuda-graph-bs", + "64", # TODO: increase it to 128 when TBO is supported in draft_extend + "--max-running-requests", + "512", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "1", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "2", + "--disable-radix-cache", + "--model-loader-extra-config", + '{"enable_multithread_load": true,"num_threads": 64}', + ], + ) @classmethod def tearDownClass(cls): diff --git a/test/registered/gb300/test_deepseek_v32.py b/test/registered/gb300/test_deepseek_v32.py index 0f9ff25cd..e370f5e46 100644 --- a/test/registered/gb300/test_deepseek_v32.py +++ b/test/registered/gb300/test_deepseek_v32.py @@ -59,7 +59,6 @@ class TestDeepseekV32(unittest.TestCase): ] + MTP_ARGS, variant="TP4+DP4+DPA+MTP", - env={"SGLANG_ENABLE_SPEC_V2": "1"}, ), ] diff --git a/test/registered/gb300/test_deepseek_v32_nvfp4.py b/test/registered/gb300/test_deepseek_v32_nvfp4.py index f6be6f94a..4640f4fb1 100644 --- a/test/registered/gb300/test_deepseek_v32_nvfp4.py +++ b/test/registered/gb300/test_deepseek_v32_nvfp4.py @@ -62,7 +62,6 @@ class TestDeepseekV32Nvfp4(unittest.TestCase): ] + MTP_ARGS, variant="TP4+DP4+DPA+MTP", - env={"SGLANG_ENABLE_SPEC_V2": "1"}, ), ] diff --git a/test/registered/gb300/test_glm5_fp8.py b/test/registered/gb300/test_glm5_fp8.py index 388f21b63..c9d19bea7 100644 --- a/test/registered/gb300/test_glm5_fp8.py +++ b/test/registered/gb300/test_glm5_fp8.py @@ -50,7 +50,6 @@ class TestGlm5Fp8(unittest.TestCase): + ["--dp-size=4", "--enable-dp-attention"] + MTP_ARGS, variant="TP4+DP4+DPA+MTP", - env={"SGLANG_ENABLE_SPEC_V2": "1"}, ), ] diff --git a/test/registered/gb300/test_glm5_nvfp4.py b/test/registered/gb300/test_glm5_nvfp4.py index 595276c68..816f05881 100644 --- a/test/registered/gb300/test_glm5_nvfp4.py +++ b/test/registered/gb300/test_glm5_nvfp4.py @@ -53,7 +53,6 @@ class TestGlm5Nvfp4(unittest.TestCase): + ["--dp-size=4", "--enable-dp-attention"] + MTP_ARGS, variant="TP4+DP4+DPA+MTP", - env={"SGLANG_ENABLE_SPEC_V2": "1"}, ), ] diff --git a/test/registered/gb300/test_qwen35_fp8.py b/test/registered/gb300/test_qwen35_fp8.py index 1121b1a81..702bebdbc 100644 --- a/test/registered/gb300/test_qwen35_fp8.py +++ b/test/registered/gb300/test_qwen35_fp8.py @@ -55,7 +55,6 @@ class TestQwen35Fp8(unittest.TestCase): + ["--dp-size=4", "--enable-dp-attention"] + MTP_ARGS, variant="TP4+DP4+DPA+MTP", - env={"SGLANG_ENABLE_SPEC_V2": "1"}, ), ] diff --git a/test/registered/gb300/test_qwen35_nvfp4.py b/test/registered/gb300/test_qwen35_nvfp4.py index f48ad701c..7199ff90f 100644 --- a/test/registered/gb300/test_qwen35_nvfp4.py +++ b/test/registered/gb300/test_qwen35_nvfp4.py @@ -59,7 +59,6 @@ class TestQwen35Nvfp4(unittest.TestCase): + ["--dp-size=4", "--enable-dp-attention"] + MTP_ARGS, variant="TP4+DP4+DPA+MTP", - env={"SGLANG_ENABLE_SPEC_V2": "1"}, ), ] diff --git a/test/registered/models_e2e/test_qwen35_fp4_mtp.py b/test/registered/models_e2e/test_qwen35_fp4_mtp.py index 8fefe7d71..a47359548 100644 --- a/test/registered/models_e2e/test_qwen35_fp4_mtp.py +++ b/test/registered/models_e2e/test_qwen35_fp4_mtp.py @@ -3,7 +3,6 @@ from types import SimpleNamespace import requests -from sglang.srt.environ import envs from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.kits.reasoning_kit import ReasoningTokenUsageMixin @@ -92,7 +91,6 @@ class TestQwen35FP4MTP(ReasoningTokenUsageMixin, CustomTestCase): cls.model = QWEN35_FP4_MODEL cls.base_url = DEFAULT_URL_FOR_TEST cls.init_reasoning_token_verifier() - envs.SGLANG_ENABLE_SPEC_V2.set(True) cls.process = popen_launch_server( cls.model, cls.base_url, @@ -102,7 +100,6 @@ class TestQwen35FP4MTP(ReasoningTokenUsageMixin, CustomTestCase): @classmethod def tearDownClass(cls): - envs.SGLANG_ENABLE_SPEC_V2.set(False) kill_process_tree(cls.process.pid) def test_gsm8k(self): @@ -117,7 +114,6 @@ class TestQwen35FP4MTPFlashInfer(ReasoningTokenUsageMixin, CustomTestCase): cls.model = QWEN35_FP4_MODEL cls.base_url = DEFAULT_URL_FOR_TEST cls.init_reasoning_token_verifier() - envs.SGLANG_ENABLE_SPEC_V2.set(True) cls.process = popen_launch_server( cls.model, cls.base_url, @@ -132,7 +128,6 @@ class TestQwen35FP4MTPFlashInfer(ReasoningTokenUsageMixin, CustomTestCase): @classmethod def tearDownClass(cls): - envs.SGLANG_ENABLE_SPEC_V2.set(False) kill_process_tree(cls.process.pid) def test_gsm8k(self): diff --git a/test/registered/models_e2e/test_step3p5_flash_chain_mtp.py b/test/registered/models_e2e/test_step3p5_flash_chain_mtp.py index 223d07fa7..a0c2464d6 100644 --- a/test/registered/models_e2e/test_step3p5_flash_chain_mtp.py +++ b/test/registered/models_e2e/test_step3p5_flash_chain_mtp.py @@ -3,7 +3,6 @@ import unittest import numpy as np import requests -from sglang.srt.environ import envs from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.kits.eval_accuracy_kit import GSM8KMixin from sglang.test.server_fixtures.default_fixture import DefaultServerBase @@ -51,11 +50,6 @@ class TestStep3p5FlashChainMTP(GSM8KMixin, DefaultServerBase): gsm8k_accuracy_thres = 0.83 gsm8k_accept_length_thres = 2.6 - @classmethod - def setUpClass(cls): - with envs.SGLANG_ENABLE_SPEC_V2.override(True): - super().setUpClass() - def test_logprob_spec_v2_match(self): """Verify spec v2 decode logprobs match prefill scoring logprobs. diff --git a/test/registered/perf/test_dpsk_v3_fp4_4gpu_perf.py b/test/registered/perf/test_dpsk_v3_fp4_4gpu_perf.py index 36e5ac3ca..387be756b 100644 --- a/test/registered/perf/test_dpsk_v3_fp4_4gpu_perf.py +++ b/test/registered/perf/test_dpsk_v3_fp4_4gpu_perf.py @@ -55,7 +55,6 @@ class TestDeepseekR1FP4Unified(unittest.TestCase): tp_size=4, extra_args=base_args + mtp_args, variant="TP4+MTP", - env={"SGLANG_ENABLE_SPEC_V2": "1"}, ), ] diff --git a/test/registered/sessions/test_streaming_session.py b/test/registered/sessions/test_streaming_session.py index 119305b5f..8e284a486 100644 --- a/test/registered/sessions/test_streaming_session.py +++ b/test/registered/sessions/test_streaming_session.py @@ -57,7 +57,6 @@ class TestStreamingSessionEagleV2RetractLargePage(TestStreamingSession): "256", ] env_overrides = [ - ("SGLANG_ENABLE_SPEC_V2", True), ("SGLANG_TEST_RETRACT", True), ("SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN", True), ] diff --git a/test/registered/sessions/test_streaming_session_extra.py b/test/registered/sessions/test_streaming_session_extra.py index de296001c..af118a990 100644 --- a/test/registered/sessions/test_streaming_session_extra.py +++ b/test/registered/sessions/test_streaming_session_extra.py @@ -74,7 +74,6 @@ class TestStreamingSessionEagleV2(StreamingSessionServerBase, StreamingSessionKi *_EAGLE3_SPEC_ARGS, ] env_overrides = [ - ("SGLANG_ENABLE_SPEC_V2", True), ("SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN", True), ] diff --git a/test/registered/spec/dflash/test_dflash.py b/test/registered/spec/dflash/test_dflash.py index 6c030c905..ff17ef37f 100644 --- a/test/registered/spec/dflash/test_dflash.py +++ b/test/registered/spec/dflash/test_dflash.py @@ -28,7 +28,8 @@ class TestDFlashServerBase(CustomTestCase, MatchedStopMixin, GSM8KMixin): attention_backend = "flashinfer" page_size = 1 other_launch_args = [] - spec_v2 = False + # Base classes exercise the non-overlap (synchronous) scheduling path. + disable_overlap = True overlap_plan_stream = False model = DEFAULT_TARGET_MODEL_DFLASH draft_model = DEFAULT_DRAFT_MODEL_DFLASH @@ -57,9 +58,10 @@ class TestDFlashServerBase(CustomTestCase, MatchedStopMixin, GSM8KMixin): "--cuda-graph-bs", *[str(i) for i in range(1, cls.max_running_requests + 1)], ] + if cls.disable_overlap: + launch_args.append("--disable-overlap-schedule") launch_args.extend(cls.other_launch_args) with ( - envs.SGLANG_ENABLE_SPEC_V2.override(cls.spec_v2), envs.SGLANG_ENABLE_OVERLAP_PLAN_STREAM.override(cls.overlap_plan_stream), envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(1), envs.SGLANG_ENABLE_ASYNC_ASSERT.override(True), @@ -150,7 +152,7 @@ class TestDFlashServerNoCudaGraph(TestDFlashServerBase): class TestDFlashServerSpecV2(TestDFlashServerBase): - spec_v2 = True + disable_overlap = False def test_radix_attention(self): run_radix_attention_test(self.base_url) diff --git a/test/registered/spec/eagle/test_eagle_constrained_decoding.py b/test/registered/spec/eagle/test_eagle_constrained_decoding.py index f78c30edf..a1fecde38 100644 --- a/test/registered/spec/eagle/test_eagle_constrained_decoding.py +++ b/test/registered/spec/eagle/test_eagle_constrained_decoding.py @@ -1,6 +1,5 @@ import unittest -from sglang.srt.environ import envs from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.kits.json_constrained_kit import JSONConstrainedMixin @@ -31,7 +30,8 @@ class TestEagleConstrainedDecoding( model = DEFAULT_TARGET_MODEL_EAGLE draft_model = DEFAULT_DRAFT_MODEL_EAGLE grammar_backend = "xgrammar" - spec_v2 = False + # Run the synchronous (non-overlap) scheduling path. + disable_overlap = True @classmethod def setUpClass(cls): @@ -59,14 +59,15 @@ class TestEagleConstrainedDecoding( "--grammar-backend", cls.grammar_backend, ] + if cls.disable_overlap: + launch_args.append("--disable-overlap-schedule") launch_args.extend(cls.other_launch_args) - with envs.SGLANG_ENABLE_SPEC_V2.override(cls.spec_v2): - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=launch_args, - ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=launch_args, + ) @classmethod def tearDownClass(cls): @@ -74,7 +75,7 @@ class TestEagleConstrainedDecoding( class TestEagleConstrainedDecodingV2(TestEagleConstrainedDecoding): - spec_v2 = True + disable_overlap = False if __name__ == "__main__": diff --git a/test/registered/spec/test_spec_standalone_extra.py b/test/registered/spec/test_spec_standalone_extra.py index 5903f4a55..ddd37cba3 100644 --- a/test/registered/spec/test_spec_standalone_extra.py +++ b/test/registered/spec/test_spec_standalone_extra.py @@ -13,14 +13,14 @@ class TestStandaloneSpeculativeDecodingBase(StandaloneServerBase, CustomTestCase attention_backend = "fa3" speculative_eagle_topk = 2 speculative_num_draft_tokens = 7 - enable_spec_v2 = False + disable_overlap = True class TestStandaloneSpeculativeDecodingTriton(StandaloneServerBase, CustomTestCase): attention_backend = "triton" speculative_eagle_topk = 2 speculative_num_draft_tokens = 7 - enable_spec_v2 = False + disable_overlap = True enable_deterministic_inference = True @@ -28,7 +28,7 @@ class TestStandaloneSpeculativeDecodingFlashinfer(StandaloneServerBase, CustomTe attention_backend = "flashinfer" speculative_eagle_topk = 2 speculative_num_draft_tokens = 7 - enable_spec_v2 = False + disable_overlap = True if __name__ == "__main__":