[Fix] Fix Qwen3.5 MTP startup with HiCache (#34560)
Co-authored-by: hjzhang <76768149+1e4ves@users.noreply.github.com> Co-authored-by: YAMY <74099316+YAMY1234@users.noreply.github.com>
This commit is contained in:
co-authored by
hjzhang
YAMY
parent
d8399af70c
commit
41cd5a7189
@@ -707,6 +707,7 @@ class ModelConfig:
|
|||||||
)
|
)
|
||||||
self.hf_config.architectures[0] = "Qwen3_5ForCausalLMMTP"
|
self.hf_config.architectures[0] = "Qwen3_5ForCausalLMMTP"
|
||||||
self.hf_config.num_nextn_predict_layers = 1
|
self.hf_config.num_nextn_predict_layers = 1
|
||||||
|
self.hf_text_config.num_nextn_predict_layers = 1
|
||||||
|
|
||||||
if is_draft_model and self.hf_config.architectures[0] == "ExaoneMoEForCausalLM":
|
if is_draft_model and self.hf_config.architectures[0] == "ExaoneMoEForCausalLM":
|
||||||
self.hf_config.architectures[0] = "ExaoneMoEForCausalLMMTP"
|
self.hf_config.architectures[0] = "ExaoneMoEForCausalLMMTP"
|
||||||
|
|||||||
@@ -930,9 +930,15 @@ def build_full_draft_pools(
|
|||||||
server_args: ServerArgs,
|
server_args: ServerArgs,
|
||||||
) -> tuple[list[SidecarPoolSpec], list[PoolEntry]]:
|
) -> tuple[list[SidecarPoolSpec], list[PoolEntry]]:
|
||||||
"""Build draft KV/DSA sidecars whose indices follow target full KV."""
|
"""Build draft KV/DSA sidecars whose indices follow target full KV."""
|
||||||
from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool
|
from sglang.srt.mem_cache.memory_pool import (
|
||||||
|
DSATokenToKVPool,
|
||||||
|
HybridLinearKVPool,
|
||||||
|
)
|
||||||
|
|
||||||
pool = draft_kv_pool
|
pool = draft_kv_pool
|
||||||
|
if isinstance(pool, HybridLinearKVPool):
|
||||||
|
# Hybrid draft runners keep their sole attention layer in this sub-pool.
|
||||||
|
pool = pool.full_kv_pool
|
||||||
if pool.layer_num == 0:
|
if pool.layer_num == 0:
|
||||||
return [], []
|
return [], []
|
||||||
|
|
||||||
|
|||||||
@@ -57,6 +57,14 @@ class TestQwen35WithHiCache(CustomTestCase):
|
|||||||
"128",
|
"128",
|
||||||
"--mamba-ssm-dtype",
|
"--mamba-ssm-dtype",
|
||||||
"bfloat16",
|
"bfloat16",
|
||||||
|
"--speculative-algorithm",
|
||||||
|
"NEXTN",
|
||||||
|
"--speculative-num-steps",
|
||||||
|
"3",
|
||||||
|
"--speculative-eagle-topk",
|
||||||
|
"1",
|
||||||
|
"--speculative-num-draft-tokens",
|
||||||
|
"4",
|
||||||
"--max-running-requests",
|
"--max-running-requests",
|
||||||
"128",
|
"128",
|
||||||
"--reasoning-parser",
|
"--reasoning-parser",
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import unittest
|
|||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
|
|
||||||
from sglang.srt.configs.model_config import (
|
from sglang.srt.configs.model_config import (
|
||||||
|
ModelConfig,
|
||||||
get_hybrid_layer_ids,
|
get_hybrid_layer_ids,
|
||||||
is_embedding_gemma,
|
is_embedding_gemma,
|
||||||
)
|
)
|
||||||
@@ -52,5 +53,22 @@ class TestEmbeddingGemmaConfig(CustomTestCase):
|
|||||||
self.assertFalse(is_embedding_gemma(config))
|
self.assertFalse(is_embedding_gemma(config))
|
||||||
|
|
||||||
|
|
||||||
|
class TestDraftModelConfig(CustomTestCase):
|
||||||
|
def test_qwen35_mtp_depth_is_synced_to_text_config(self):
|
||||||
|
config = object.__new__(ModelConfig)
|
||||||
|
config.is_draft_model = True
|
||||||
|
config.speculative_algorithm = "EAGLE"
|
||||||
|
config.hf_config = SimpleNamespace(
|
||||||
|
architectures=["Qwen3_5MoeForConditionalGeneration"]
|
||||||
|
)
|
||||||
|
config.hf_text_config = SimpleNamespace()
|
||||||
|
|
||||||
|
config._config_draft_model()
|
||||||
|
|
||||||
|
self.assertEqual(config.hf_config.architectures, ["Qwen3_5ForCausalLMMTP"])
|
||||||
|
self.assertEqual(config.hf_config.num_nextn_predict_layers, 1)
|
||||||
|
self.assertEqual(config.hf_text_config.num_nextn_predict_layers, 1)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -1,8 +1,13 @@
|
|||||||
"""Unit test for hybrid HiCache fixed-size budget splitting."""
|
"""Unit tests for hybrid HiCache pool assembly."""
|
||||||
|
|
||||||
import unittest
|
import unittest
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import _split_hicache_size
|
from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import (
|
||||||
|
_split_hicache_size,
|
||||||
|
build_full_draft_pools,
|
||||||
|
)
|
||||||
|
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool
|
||||||
from sglang.test.ci.ci_register import register_cpu_ci
|
from sglang.test.ci.ci_register import register_cpu_ci
|
||||||
from sglang.test.test_utils import CustomTestCase
|
from sglang.test.test_utils import CustomTestCase
|
||||||
|
|
||||||
@@ -35,5 +40,20 @@ class TestSplitHicacheSize(CustomTestCase):
|
|||||||
self.assertEqual(sum(shares), 100) # total budget preserved, not doubled
|
self.assertEqual(sum(shares), 100) # total budget preserved, not doubled
|
||||||
|
|
||||||
|
|
||||||
|
class TestDraftSidecarPoolDispatch(CustomTestCase):
|
||||||
|
def test_full_builder_unwraps_empty_hybrid_linear_pool(self):
|
||||||
|
draft_kv_pool = object.__new__(HybridLinearKVPool)
|
||||||
|
draft_kv_pool.full_kv_pool = SimpleNamespace(layer_num=0)
|
||||||
|
|
||||||
|
specs, entries = build_full_draft_pools(
|
||||||
|
draft_kv_pool=draft_kv_pool,
|
||||||
|
tree_cache=None,
|
||||||
|
server_args=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(specs, [])
|
||||||
|
self.assertEqual(entries, [])
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
Reference in New Issue
Block a user