minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Use spec v2 by default (#21062)

Browse Source
This commit is contained in:
Qiaolin Yu
2026-04-29 13:40:42 -07:00
committed by GitHub
parent c3ab5bec7d
commit 79dbfe4505
27 changed files with 413 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/environ.py
+1 -1
View File
@@ -458,7 +458,7 @@ class Envs:
SGLANG_ROPE_CACHE_ALIGN = EnvInt(128)
# Overlap Spec V2
SGLANG_ENABLE_SPEC_V2 = EnvBool(False)
SGLANG_ENABLE_SPEC_V2 = EnvBool(True)
SGLANG_ENABLE_OVERLAP_PLAN_STREAM = EnvBool(False)
# Spec Config
python/sglang/srt/server_args.py
+19 -26
View File
@@ -1979,11 +1979,6 @@ class ServerArgs:
logger.info(
"Enable multi-layer EAGLE speculative decoding for MiMoV2 model."
)
if not envs.SGLANG_ENABLE_SPEC_V2.get():
envs.SGLANG_ENABLE_SPEC_V2.set(True)
logger.warning(
"Spec v2 is enabled for multi-layer EAGLE speculative decoding."
)
if self.enable_hierarchical_cache:
self.swa_full_tokens_ratio = 1.0
@@ -2000,11 +1995,6 @@ class ServerArgs:
logger.info(
"Enable multi-layer EAGLE speculative decoding for Step3p5ForCausalLM model."
)
if not envs.SGLANG_ENABLE_SPEC_V2.get():
envs.SGLANG_ENABLE_SPEC_V2.set(True)
logger.warning(
"Spec v2 is enabled for multi-layer EAGLE speculative decoding."
)
if self.enable_hierarchical_cache:
self.swa_full_tokens_ratio = 1.0
logger.warning(
@@ -3411,26 +3401,29 @@ class ServerArgs:
"Max running requests is reset to 48 for speculative decoding. You can override this by explicitly setting --max-running-requests."
)
spec_v1_reason = None
if (
self.speculative_algorithm in ["EAGLE", "EAGLE3", "STANDALONE"]
and envs.SGLANG_ENABLE_SPEC_V2.get()
self.speculative_eagle_topk is not None
and self.speculative_eagle_topk > 1
and not self.disable_overlap_schedule
):
self.disable_overlap_schedule = False
logger.warning(
"Spec v2 is enabled for eagle/eagle3 speculative decoding and overlap schedule is turned on."
)
if (
self.speculative_eagle_topk is not None
and self.speculative_eagle_topk > 1
):
raise ValueError(
"Spec v2 currently only supports topk = 1 for speculative decoding."
)
else:
self.disable_overlap_schedule = True
spec_v1_reason = "spec v2 currently only supports topk = 1"
elif (
not envs.SGLANG_ENABLE_SPEC_V2.get()
and not self.disable_overlap_schedule
):
self.disable_overlap_schedule = True
spec_v1_reason = "SGLANG_ENABLE_SPEC_V2=False"
if self.disable_overlap_schedule:
logger.warning(
"Overlap scheduler is disabled when spec v2 is off or using unsupported speculative algorithm. "
"You can set env SGLANG_ENABLE_SPEC_V2=True to enable the experimental overlap scheduler. "
"Spec v1 is used for eagle/eagle3/standalone speculative decoding because %s.",
spec_v1_reason or "overlap schedule is disabled",
)
else:
logger.warning(
"Spec v2 is enabled by default for eagle/eagle3/standalone speculative decoding."
)
if self.enable_mixed_chunk: