[Fix] Fix Qwen3.5 MTP startup with HiCache (#34560)

Co-authored-by: hjzhang <76768149+1e4ves@users.noreply.github.com>
Co-authored-by: YAMY <74099316+YAMY1234@users.noreply.github.com>
This commit is contained in:
DarkraiHL
2026-08-14 10:55:41 -07:00
committed by GitHub
co-authored by hjzhang YAMY
parent d8399af70c
commit 41cd5a7189
5 changed files with 56 additions and 3 deletions
@@ -707,6 +707,7 @@ class ModelConfig:
) )
self.hf_config.architectures[0] = "Qwen3_5ForCausalLMMTP" self.hf_config.architectures[0] = "Qwen3_5ForCausalLMMTP"
self.hf_config.num_nextn_predict_layers = 1 self.hf_config.num_nextn_predict_layers = 1
self.hf_text_config.num_nextn_predict_layers = 1
if is_draft_model and self.hf_config.architectures[0] == "ExaoneMoEForCausalLM": if is_draft_model and self.hf_config.architectures[0] == "ExaoneMoEForCausalLM":
self.hf_config.architectures[0] = "ExaoneMoEForCausalLMMTP" self.hf_config.architectures[0] = "ExaoneMoEForCausalLMMTP"
@@ -930,9 +930,15 @@ def build_full_draft_pools(
server_args: ServerArgs, server_args: ServerArgs,
) -> tuple[list[SidecarPoolSpec], list[PoolEntry]]: ) -> tuple[list[SidecarPoolSpec], list[PoolEntry]]:
"""Build draft KV/DSA sidecars whose indices follow target full KV.""" """Build draft KV/DSA sidecars whose indices follow target full KV."""
from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool from sglang.srt.mem_cache.memory_pool import (
DSATokenToKVPool,
HybridLinearKVPool,
)
pool = draft_kv_pool pool = draft_kv_pool
if isinstance(pool, HybridLinearKVPool):
# Hybrid draft runners keep their sole attention layer in this sub-pool.
pool = pool.full_kv_pool
if pool.layer_num == 0: if pool.layer_num == 0:
return [], [] return [], []
@@ -57,6 +57,14 @@ class TestQwen35WithHiCache(CustomTestCase):
"128", "128",
"--mamba-ssm-dtype", "--mamba-ssm-dtype",
"bfloat16", "bfloat16",
"--speculative-algorithm",
"NEXTN",
"--speculative-num-steps",
"3",
"--speculative-eagle-topk",
"1",
"--speculative-num-draft-tokens",
"4",
"--max-running-requests", "--max-running-requests",
"128", "128",
"--reasoning-parser", "--reasoning-parser",
@@ -4,6 +4,7 @@ import unittest
from types import SimpleNamespace from types import SimpleNamespace
from sglang.srt.configs.model_config import ( from sglang.srt.configs.model_config import (
ModelConfig,
get_hybrid_layer_ids, get_hybrid_layer_ids,
is_embedding_gemma, is_embedding_gemma,
) )
@@ -52,5 +53,22 @@ class TestEmbeddingGemmaConfig(CustomTestCase):
self.assertFalse(is_embedding_gemma(config)) self.assertFalse(is_embedding_gemma(config))
class TestDraftModelConfig(CustomTestCase):
def test_qwen35_mtp_depth_is_synced_to_text_config(self):
config = object.__new__(ModelConfig)
config.is_draft_model = True
config.speculative_algorithm = "EAGLE"
config.hf_config = SimpleNamespace(
architectures=["Qwen3_5MoeForConditionalGeneration"]
)
config.hf_text_config = SimpleNamespace()
config._config_draft_model()
self.assertEqual(config.hf_config.architectures, ["Qwen3_5ForCausalLMMTP"])
self.assertEqual(config.hf_config.num_nextn_predict_layers, 1)
self.assertEqual(config.hf_text_config.num_nextn_predict_layers, 1)
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
@@ -1,8 +1,13 @@
"""Unit test for hybrid HiCache fixed-size budget splitting.""" """Unit tests for hybrid HiCache pool assembly."""
import unittest import unittest
from types import SimpleNamespace
from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import _split_hicache_size from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import (
_split_hicache_size,
build_full_draft_pools,
)
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool
from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase from sglang.test.test_utils import CustomTestCase
@@ -35,5 +40,20 @@ class TestSplitHicacheSize(CustomTestCase):
self.assertEqual(sum(shares), 100) # total budget preserved, not doubled self.assertEqual(sum(shares), 100) # total budget preserved, not doubled
class TestDraftSidecarPoolDispatch(CustomTestCase):
def test_full_builder_unwraps_empty_hybrid_linear_pool(self):
draft_kv_pool = object.__new__(HybridLinearKVPool)
draft_kv_pool.full_kv_pool = SimpleNamespace(layer_num=0)
specs, entries = build_full_draft_pools(
draft_kv_pool=draft_kv_pool,
tree_cache=None,
server_args=None,
)
self.assertEqual(specs, [])
self.assertEqual(entries, [])
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()