Adjust server args for Mimo-v2-flash model (#15803)

Co-authored-by: Liangsheng Yin <lsyincs@gmail.com>
This commit is contained in:
Ke Bao
2025-12-25 16:04:39 +08:00
committed by GitHub
co-authored by Liangsheng Yin
parent eb3da9c1dd
commit cbf9f13493
+23 -9
View File
@@ -1192,8 +1192,17 @@ class ServerArgs:
self.disable_hybrid_swa_memory = True self.disable_hybrid_swa_memory = True
elif "MiMoV2FlashForCausalLM" in model_arch: elif "MiMoV2FlashForCausalLM" in model_arch:
self.enable_multi_layer_eagle = True if self.speculative_algorithm == "EAGLE":
logger.info("Enable multi-layer eagle for MiMoV2FlashForCausalLM model") self.enable_multi_layer_eagle = True
logger.info(
"Enable multi-layer EAGLE speculative decoding for MiMoV2FlashForCausalLM model."
)
if not envs.SGLANG_ENABLE_SPEC_V2.get():
envs.SGLANG_ENABLE_SPEC_V2.set(True)
logger.warning(
"Spec v2 is enabled for multi-layer EAGLE speculative decoding."
)
self.swa_full_tokens_ratio = 1.0 self.swa_full_tokens_ratio = 1.0
logger.warning( logger.warning(
"Reset swa_full_tokens_ratio to 1.0 for MiMoV2FlashForCausalLM model" "Reset swa_full_tokens_ratio to 1.0 for MiMoV2FlashForCausalLM model"
@@ -1967,19 +1976,24 @@ class ServerArgs:
) )
if ( if (
self.speculative_algorithm == "EAGLE" self.speculative_algorithm in ["EAGLE", "EAGLE3"]
and envs.SGLANG_ENABLE_SPEC_V2.get() and envs.SGLANG_ENABLE_SPEC_V2.get()
): ):
self.disable_overlap_schedule = False self.disable_overlap_schedule = False
logger.warning( logger.warning(
"Beta spec is enabled for eagle speculative decoding and overlap schedule is turned on." "Beta spec is enabled for eagle/eagle3 speculative decoding and overlap schedule is turned on."
) )
if (
if not envs.SGLANG_ENABLE_SPEC_V2.get(): self.speculative_eagle_topk is not None
and self.speculative_eagle_topk > 1
):
raise ValueError(
"Beta spec currently only supports topk = 1 for speculative decoding."
)
else:
self.disable_overlap_schedule = True self.disable_overlap_schedule = True
logger.warning( logger.warning(
"Overlap scheduler is disabled because of using eagle3 or standalone speculative decoding." "Overlap scheduler is disabled when beta spec is off or using unsupported speculative algorithm."
"You can set env SGLANG_ENABLE_SPEC_V2=True to enable the experimental overlap scheduler."
) )
if self.enable_mixed_chunk: if self.enable_mixed_chunk:
@@ -5128,8 +5142,8 @@ def auto_choose_speculative_params(self: ServerArgs):
"BailingMoeV2ForCausalLM", "BailingMoeV2ForCausalLM",
"MistralLarge3ForCausalLM", "MistralLarge3ForCausalLM",
"PixtralForConditionalGeneration", "PixtralForConditionalGeneration",
"MiMoV2FlashForCausalLM",
]: ]:
# The default value for deepseek and gpt-oss
return (3, 1, 4) return (3, 1, 4)
elif arch in ["Grok1ForCausalLM", "Grok1VForCausalLM"]: elif arch in ["Grok1ForCausalLM", "Grok1VForCausalLM"]:
return (5, 4, 8) return (5, 4, 8)