Adjust server args for Mimo-v2-flash model (#15803)
Co-authored-by: Liangsheng Yin <lsyincs@gmail.com>
This commit is contained in:
@@ -1192,8 +1192,17 @@ class ServerArgs:
|
|||||||
self.disable_hybrid_swa_memory = True
|
self.disable_hybrid_swa_memory = True
|
||||||
|
|
||||||
elif "MiMoV2FlashForCausalLM" in model_arch:
|
elif "MiMoV2FlashForCausalLM" in model_arch:
|
||||||
self.enable_multi_layer_eagle = True
|
if self.speculative_algorithm == "EAGLE":
|
||||||
logger.info("Enable multi-layer eagle for MiMoV2FlashForCausalLM model")
|
self.enable_multi_layer_eagle = True
|
||||||
|
logger.info(
|
||||||
|
"Enable multi-layer EAGLE speculative decoding for MiMoV2FlashForCausalLM model."
|
||||||
|
)
|
||||||
|
if not envs.SGLANG_ENABLE_SPEC_V2.get():
|
||||||
|
envs.SGLANG_ENABLE_SPEC_V2.set(True)
|
||||||
|
logger.warning(
|
||||||
|
"Spec v2 is enabled for multi-layer EAGLE speculative decoding."
|
||||||
|
)
|
||||||
|
|
||||||
self.swa_full_tokens_ratio = 1.0
|
self.swa_full_tokens_ratio = 1.0
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Reset swa_full_tokens_ratio to 1.0 for MiMoV2FlashForCausalLM model"
|
"Reset swa_full_tokens_ratio to 1.0 for MiMoV2FlashForCausalLM model"
|
||||||
@@ -1967,19 +1976,24 @@ class ServerArgs:
|
|||||||
)
|
)
|
||||||
|
|
||||||
if (
|
if (
|
||||||
self.speculative_algorithm == "EAGLE"
|
self.speculative_algorithm in ["EAGLE", "EAGLE3"]
|
||||||
and envs.SGLANG_ENABLE_SPEC_V2.get()
|
and envs.SGLANG_ENABLE_SPEC_V2.get()
|
||||||
):
|
):
|
||||||
self.disable_overlap_schedule = False
|
self.disable_overlap_schedule = False
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Beta spec is enabled for eagle speculative decoding and overlap schedule is turned on."
|
"Beta spec is enabled for eagle/eagle3 speculative decoding and overlap schedule is turned on."
|
||||||
)
|
)
|
||||||
|
if (
|
||||||
if not envs.SGLANG_ENABLE_SPEC_V2.get():
|
self.speculative_eagle_topk is not None
|
||||||
|
and self.speculative_eagle_topk > 1
|
||||||
|
):
|
||||||
|
raise ValueError(
|
||||||
|
"Beta spec currently only supports topk = 1 for speculative decoding."
|
||||||
|
)
|
||||||
|
else:
|
||||||
self.disable_overlap_schedule = True
|
self.disable_overlap_schedule = True
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Overlap scheduler is disabled because of using eagle3 or standalone speculative decoding."
|
"Overlap scheduler is disabled when beta spec is off or using unsupported speculative algorithm."
|
||||||
"You can set env SGLANG_ENABLE_SPEC_V2=True to enable the experimental overlap scheduler."
|
|
||||||
)
|
)
|
||||||
|
|
||||||
if self.enable_mixed_chunk:
|
if self.enable_mixed_chunk:
|
||||||
@@ -5128,8 +5142,8 @@ def auto_choose_speculative_params(self: ServerArgs):
|
|||||||
"BailingMoeV2ForCausalLM",
|
"BailingMoeV2ForCausalLM",
|
||||||
"MistralLarge3ForCausalLM",
|
"MistralLarge3ForCausalLM",
|
||||||
"PixtralForConditionalGeneration",
|
"PixtralForConditionalGeneration",
|
||||||
|
"MiMoV2FlashForCausalLM",
|
||||||
]:
|
]:
|
||||||
# The default value for deepseek and gpt-oss
|
|
||||||
return (3, 1, 4)
|
return (3, 1, 4)
|
||||||
elif arch in ["Grok1ForCausalLM", "Grok1VForCausalLM"]:
|
elif arch in ["Grok1ForCausalLM", "Grok1VForCausalLM"]:
|
||||||
return (5, 4, 8)
|
return (5, 4, 8)
|
||||||
|
|||||||
Reference in New Issue
Block a user