diff --git a/python/sglang/srt/mem_cache/kv_cache_configurator.py b/python/sglang/srt/mem_cache/kv_cache_configurator.py index a0d9e1e233..3ee589b93c 100644 --- a/python/sglang/srt/mem_cache/kv_cache_configurator.py +++ b/python/sglang/srt/mem_cache/kv_cache_configurator.py @@ -97,6 +97,7 @@ from sglang.srt.platforms import current_platform from sglang.srt.platforms.interface import KVPoolKind, reject_out_of_tree_path from sglang.srt.runtime_context import ( attention_backends, + draft_attn_tp_size, get_context, get_disagg, get_exec, @@ -1059,7 +1060,6 @@ class KVCacheConfigurator: The pool factories decide whether a decline refuses the boot. Asymmetric K/V rows fuse only on backends that carry v_head_dim through to the kernel.""" - from sglang.srt.layers.cp.base import is_cp_enabled from sglang.srt.mem_cache.layout.fused_draft import ( FusedDraftDecision, PlacementContext, @@ -1145,10 +1145,8 @@ class KVCacheConfigurator: aux.draft_model_config, num_layers=int(aux.draft_kv_num_layers), attn_tp_size=( - get_parallel().tp_size - if is_cp_enabled() - and get_parallel().attn_dp_size == 1 - and self.spec_algorithm.is_dflash() + draft_attn_tp_size() + if self.spec_algorithm.is_dflash() else get_parallel().attn_tp_size ), # A DFLASH-family draft config inherits the target's NEXTN depth diff --git a/python/sglang/srt/model_executor/model_runner_components/spec_aux_hidden_state.py b/python/sglang/srt/model_executor/model_runner_components/spec_aux_hidden_state.py index a4cbabc75e..ac2c57ce92 100644 --- a/python/sglang/srt/model_executor/model_runner_components/spec_aux_hidden_state.py +++ b/python/sglang/srt/model_executor/model_runner_components/spec_aux_hidden_state.py @@ -7,6 +7,7 @@ import msgspec from sglang.srt.configs.model_config import ModelConfig, is_deepseek_v4 from sglang.srt.runtime_context import ( + draft_attn_tp_size, get_model, get_parallel, get_spec, @@ -271,12 +272,9 @@ def _resolve_dflash_draft_cell_size( draft_model_config=draft_model_config, draft_num_layers=draft_num_layers, draft_kv_cache_dtype=draft_kv_cache_dtype, - # CP targets run the DFLASH draft on full TP without CP. tp_size=( - get_parallel().tp_size + draft_attn_tp_size() if spec_algorithm.is_dflash() - and get_parallel().attn_cp_size > 1 - and get_parallel().attn_dp_size == 1 else get_parallel().attn_tp_size ), ) diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index 32f0db2552..8df16198c2 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -119,6 +119,8 @@ from sglang.srt.runtime_context import ( get_mm, get_parallel, get_spec, + linear_attn_parallel_group, + linear_attn_tp_size, ) from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.srt.utils.common import ( @@ -397,11 +399,6 @@ class Glm5NextLinearAttention(nn.Module): **kwargs, ) -> None: super().__init__() - from sglang.srt.runtime_context import ( - linear_attn_parallel_group, - linear_attn_tp_size, - ) - head_shard_size = linear_attn_tp_size() head_parallel_group = linear_attn_parallel_group() @@ -842,7 +839,7 @@ class Glm5NextDecoderLayer(nn.Module): update=residual.attn_update, tp_group=SumGroup.TP if config.is_kda_layer(layer_id) - and get_parallel().enable_cp_tp_group_sharing + and linear_attn_parallel_group() == "tp" else SumGroup.ATTN_TP, ), declare_ffn( diff --git a/python/sglang/srt/runtime_context.py b/python/sglang/srt/runtime_context.py index e17d01a4fe..90c16db50b 100644 --- a/python/sglang/srt/runtime_context.py +++ b/python/sglang/srt/runtime_context.py @@ -486,6 +486,23 @@ def linear_attn_tp_size() -> int: return getattr(get_parallel(), f"{linear_attn_parallel_group()}_size") +def draft_attn_parallel_group() -> str: + """The group an attention-owning draft uses, resolved in the target scope. + + With CP and no attention DP, the draft runs on full TP without CP. + Otherwise it stays within attention TP, preserving independent DP drafts. + """ + parallel = get_parallel() + return ( + "tp" if parallel.attn_cp_size > 1 and parallel.attn_dp_size == 1 else "attn_tp" + ) + + +def draft_attn_tp_size() -> int: + """The width of ``draft_attn_parallel_group()`` in the target scope.""" + return getattr(get_parallel(), f"{draft_attn_parallel_group()}_size") + + class _FlagGroupBase(msgspec.Struct): """Shared flag-group behavior: typo-safe writes + transactional ``override()``. diff --git a/python/sglang/srt/speculative/spec_utils.py b/python/sglang/srt/speculative/spec_utils.py index 5d56e1fd25..98459186b5 100644 --- a/python/sglang/srt/speculative/spec_utils.py +++ b/python/sglang/srt/speculative/spec_utils.py @@ -40,6 +40,7 @@ from sglang.srt.mem_cache.allocation import ( assign_req_to_token_pool_func as assign_req_to_token_pool_func, ) from sglang.srt.runtime_context import ( + draft_attn_parallel_group, get_exec, get_parallel, get_spec, @@ -734,19 +735,12 @@ def draft_tp_context(owns_attention: bool): """Enter the TP placement that draft work runs under. Work that owns attention (an attention-owning draft, or DSpark's DP-MoE - sync) runs on the target's attention-TP group. With prefill CP and no - attention DP it uses full TP as ordinary attention TP, matching the borrowed - target LM head. Other draft work keeps the target's TP placement. - Enter it from outside any draft scope. + sync) uses ``draft_attn_parallel_group()``. Other draft work keeps the + target's TP placement. Enter it from outside any draft scope. """ if not owns_attention: return contextlib.nullcontext() - parallel = get_parallel() - group = ( - parallel.tp_group - if parallel.attn_cp_size > 1 and parallel.attn_dp_size == 1 - else parallel.attn_tp_group - ) + group = getattr(get_parallel(), f"{draft_attn_parallel_group()}_group") return patch_tensor_parallel_group(group, owns_attention=True) diff --git a/test/registered/unit/layer_boundary/test_token_order_attention.py b/test/registered/unit/layer_boundary/test_token_order_attention.py index 0d0ea7a471..8ab76ee041 100644 --- a/test/registered/unit/layer_boundary/test_token_order_attention.py +++ b/test/registered/unit/layer_boundary/test_token_order_attention.py @@ -30,6 +30,7 @@ from sglang.srt.layers.layer_boundary.ops import ( ) from sglang.srt.layers.layer_boundary.residual.gated import GatedResidualState from sglang.srt.models import qwen4_exp +from sglang.srt.runtime_context import get_parallel from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -56,14 +57,16 @@ def _step(stage, variant): return stage.plan.paths[variant].entry.prepare.keywords["step"] -def _build_layers(parallel, linear_group): +def _build_layers(parallel): """A linear-attention layer, then a sparse-attention layer.""" config = SimpleNamespace( model_type="qwen4_exp_text", num_hidden_layers=2, ple_layer_ids=[] ) with ( fixture.planning(parallel), - patch.object(qwen4_exp, "linear_attn_parallel_group", lambda: linear_group), + get_parallel().override( + enable_cp_tp_group_sharing=parallel.enable_cp_tp_group_sharing + ), layer_stack(), ): linear = qwen4_exp._build_qwen4_exp_stages( @@ -82,7 +85,7 @@ class TestTokenOrderAttention(CustomTestCase): enable_prefill_cp=True, enable_cp_tp_group_sharing=True, ) - self.linear, self.sparse = _build_layers(parallel, "tp") + self.linear, self.sparse = _build_layers(parallel) def test_linear_attention_reads_the_whole_sequence_in_token_order(self): attn, ffn = self.linear @@ -121,7 +124,7 @@ class TestTokenOrderAttention(CustomTestCase): def test_without_sharing_linear_attention_binds_like_attention(self): parallel = fixture.parallel_of(attn_dp=1, attn_tp=2) - linear, sparse = _build_layers(parallel, "attn_tp") + linear, sparse = _build_layers(parallel) def steps(stage): entry = stage.plan.paths[ORDINARY].entry diff --git a/test/registered/unit/mem_cache/test_fused_draft_placement.py b/test/registered/unit/mem_cache/test_fused_draft_placement.py index aa0735cf31..39fe034262 100644 --- a/test/registered/unit/mem_cache/test_fused_draft_placement.py +++ b/test/registered/unit/mem_cache/test_fused_draft_placement.py @@ -69,8 +69,9 @@ from sglang.srt.mem_cache.layout.fused_draft import ( register_host_kind, ) from sglang.srt.runtime_context import get_parallel, override_platform +from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.test.ci.ci_register import register_cpu_ci -from sglang.test.test_utils import CustomTestCase +from sglang.test.test_utils import CustomTestCase, published_topology register_cpu_ci(est_time=15, suite="base-a-test-cpu") @@ -652,6 +653,9 @@ class TestFusedDraftDecision(CustomTestCase): kv_cache_dtype_flag="auto", draft_backend=None, target_backends=("triton", "triton"), + tp_size=1, + attn_cp_size=1, + attn_dp_size=1, dcp_size=1, hicache=False, retraction_backup=None, @@ -665,10 +669,7 @@ class TestFusedDraftDecision(CustomTestCase): cfg.mambaish_config = None cfg.use_mla_backend = False cfg.is_draft_worker = False - cfg.spec_algorithm = SimpleNamespace( - is_eagle=lambda: algorithm == "EAGLE", - is_dflash_family=lambda: algorithm in ("DFLASH", "DSPARK"), - ) + cfg.spec_algorithm = SpeculativeAlgorithm.from_string(algorithm) cfg.model_config = SimpleNamespace( is_multi_layer_eagle=False, sliding_window_size=None ) @@ -681,7 +682,7 @@ class TestFusedDraftDecision(CustomTestCase): is_hybrid_swa=False, is_deepseek_v4_arch=False, num_nextn_predict_layers=None, - get_num_kv_heads=lambda tp: 4, + get_num_kv_heads=lambda tp: max(1, 4 // tp), head_dim=64, v_head_dim=64, dtype=torch.bfloat16, @@ -697,6 +698,11 @@ class TestFusedDraftDecision(CustomTestCase): speculative_draft_attention_backend=draft_backend, ) with ( + published_topology( + tp_size=tp_size, + attn_cp_size=attn_cp_size, + attn_dp_size=attn_dp_size, + ), patch.object(kvc, "get_memory", return_value=memory), patch.object( kvc, @@ -717,7 +723,7 @@ class TestFusedDraftDecision(CustomTestCase): draft_worker_common, "attention_backends", return_value=target_backends ), patch("sglang.srt.configs.hybrid_arch.mambaish_config", return_value=None), - get_parallel().override(attn_tp_size=1, attn_dcp_size=dcp_size), + get_parallel().override(attn_dcp_size=dcp_size), override_platform(is_xpu=False, is_hip=False), ): return cfg._fused_draft_decision() @@ -725,6 +731,24 @@ class TestFusedDraftDecision(CustomTestCase): def test_a_stateless_draft_is_placed(self): self.assertIsNotNone(self._decide().placement) + def test_cp_draft_placement_keeps_dp_isolated(self): + for algorithm, cp, dp, heads in ( + ("DFLASH", 1, 1, 1), + ("DFLASH", 4, 1, 1), + ("DFLASH", 2, 2, 4), + ("EAGLE", 4, 1, 4), + ("DSPARK", 4, 1, 4), + ): + with self.subTest(algorithm=algorithm, cp=cp, dp=dp): + decision = self._decide( + algorithm=algorithm, + tp_size=4, + attn_cp_size=cp, + attn_dp_size=dp, + ) + self.assertIsNotNone(decision.placement, decision.declined) + self.assertEqual(decision.placement.region("full").head_num, heads) + def test_a_draft_kv_dtype_unlike_the_host_keeps_the_private_pool(self): """A fused draft stores its rows in the host's KV dtype; an explicit draft dtype that differs declines instead of mis-typing those rows, diff --git a/test/registered/unit/spec/test_draft_tp_context.py b/test/registered/unit/spec/test_draft_tp_context.py index 2df9de7ceb..8b8a12365b 100644 --- a/test/registered/unit/spec/test_draft_tp_context.py +++ b/test/registered/unit/spec/test_draft_tp_context.py @@ -1,8 +1,15 @@ import unittest +from types import SimpleNamespace + +import torch from sglang.srt.distributed import parallel_state from sglang.srt.distributed.parallel_state import GroupCoordinator -from sglang.srt.runtime_context import get_parallel +from sglang.srt.model_executor.model_runner_components.spec_aux_hidden_state import ( + _resolve_dflash_draft_cell_size, +) +from sglang.srt.runtime_context import draft_attn_tp_size, get_parallel +from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.srt.speculative.spec_utils import draft_tp_context from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase, published_topology @@ -63,6 +70,54 @@ class TestDraftTpContext(CustomTestCase): self.assertIs(get_parallel().tp_group, attn_tp) self.assertEqual(get_parallel().attn_cp_size, 1) + def test_cp_draft_width_matches_target_side_geometry_and_restores_on_error(self): + with get_parallel().override( + tp_group=self.whole_tp, + attn_tp_group=_group(world_size=1, rank=0), + attn_tp_size=1, + attn_cp_size=4, + attn_dp_size=1, + ): + draft_width = draft_attn_tp_size() + self.assertEqual(draft_width, 4) + with self.assertRaisesRegex(RuntimeError, "draft failed"): + with draft_tp_context(True): + self.assertIs(get_parallel().attn_tp_group, self.whole_tp) + self.assertEqual(get_parallel().attn_tp_size, draft_width) + self.assertEqual(get_parallel().attn_cp_size, 1) + raise RuntimeError("draft failed") + self.assertEqual(get_parallel().attn_tp_size, 1) + self.assertEqual(get_parallel().attn_cp_size, 4) + self.assertFalse(parallel_state._TP_STATE_PATCHED) + + def test_target_kv_budget_matches_draft_execution_geometry(self): + model = SimpleNamespace( + dtype=torch.bfloat16, + head_dim=64, + v_head_dim=32, + get_num_kv_heads=lambda tp: 4 // tp, + ) + for cp, dp, draft_width in ((1, 1, 4), (4, 1, 4), (2, 2, 1)): + with ( + self.subTest(cp=cp, dp=dp), + get_parallel().override( + tp_group=self.whole_tp, + attn_tp_group=_group(world_size=4 // cp // dp, rank=0), + attn_tp_size=4 // cp // dp, + attn_cp_size=cp, + attn_dp_size=dp, + ), + ): + budget = _resolve_dflash_draft_cell_size( + draft_model_config=model, + draft_num_layers=2, + spec_algorithm=SpeculativeAlgorithm.DFLASH, + ) + with draft_tp_context(True): + self.assertEqual(get_parallel().attn_tp_size, draft_width) + heads = model.get_num_kv_heads(get_parallel().attn_tp_size) + self.assertEqual(budget, heads * (64 + 32) * 2 * 2) + if __name__ == "__main__": unittest.main()