[Refactor] attention: centralize linear and draft TP group selection (#45684)

This commit is contained in:
Baizhou Zhang
2026-10-21 18:25:40 -07:00
committed by GitHub
parent 9ec74a53cc
commit 59149553b9
8 changed files with 123 additions and 37 deletions
@@ -97,6 +97,7 @@ from sglang.srt.platforms import current_platform
from sglang.srt.platforms.interface import KVPoolKind, reject_out_of_tree_path
from sglang.srt.runtime_context import (
attention_backends,
draft_attn_tp_size,
get_context,
get_disagg,
get_exec,
@@ -1059,7 +1060,6 @@ class KVCacheConfigurator:
The pool factories decide whether a decline refuses the boot.
Asymmetric K/V rows fuse only on backends that carry v_head_dim through
to the kernel."""
from sglang.srt.layers.cp.base import is_cp_enabled
from sglang.srt.mem_cache.layout.fused_draft import (
FusedDraftDecision,
PlacementContext,
@@ -1145,10 +1145,8 @@ class KVCacheConfigurator:
aux.draft_model_config,
num_layers=int(aux.draft_kv_num_layers),
attn_tp_size=(
get_parallel().tp_size
if is_cp_enabled()
and get_parallel().attn_dp_size == 1
and self.spec_algorithm.is_dflash()
draft_attn_tp_size()
if self.spec_algorithm.is_dflash()
else get_parallel().attn_tp_size
),
# A DFLASH-family draft config inherits the target's NEXTN depth
@@ -7,6 +7,7 @@ import msgspec
from sglang.srt.configs.model_config import ModelConfig, is_deepseek_v4
from sglang.srt.runtime_context import (
draft_attn_tp_size,
get_model,
get_parallel,
get_spec,
@@ -271,12 +272,9 @@ def _resolve_dflash_draft_cell_size(
draft_model_config=draft_model_config,
draft_num_layers=draft_num_layers,
draft_kv_cache_dtype=draft_kv_cache_dtype,
# CP targets run the DFLASH draft on full TP without CP.
tp_size=(
get_parallel().tp_size
draft_attn_tp_size()
if spec_algorithm.is_dflash()
and get_parallel().attn_cp_size > 1
and get_parallel().attn_dp_size == 1
else get_parallel().attn_tp_size
),
)
+3 -6
View File
@@ -119,6 +119,8 @@ from sglang.srt.runtime_context import (
get_mm,
get_parallel,
get_spec,
linear_attn_parallel_group,
linear_attn_tp_size,
)
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.srt.utils.common import (
@@ -397,11 +399,6 @@ class Glm5NextLinearAttention(nn.Module):
**kwargs,
) -> None:
super().__init__()
from sglang.srt.runtime_context import (
linear_attn_parallel_group,
linear_attn_tp_size,
)
head_shard_size = linear_attn_tp_size()
head_parallel_group = linear_attn_parallel_group()
@@ -842,7 +839,7 @@ class Glm5NextDecoderLayer(nn.Module):
update=residual.attn_update,
tp_group=SumGroup.TP
if config.is_kda_layer(layer_id)
and get_parallel().enable_cp_tp_group_sharing
and linear_attn_parallel_group() == "tp"
else SumGroup.ATTN_TP,
),
declare_ffn(
+17
View File
@@ -486,6 +486,23 @@ def linear_attn_tp_size() -> int:
return getattr(get_parallel(), f"{linear_attn_parallel_group()}_size")
def draft_attn_parallel_group() -> str:
"""The group an attention-owning draft uses, resolved in the target scope.
With CP and no attention DP, the draft runs on full TP without CP.
Otherwise it stays within attention TP, preserving independent DP drafts.
"""
parallel = get_parallel()
return (
"tp" if parallel.attn_cp_size > 1 and parallel.attn_dp_size == 1 else "attn_tp"
)
def draft_attn_tp_size() -> int:
"""The width of ``draft_attn_parallel_group()`` in the target scope."""
return getattr(get_parallel(), f"{draft_attn_parallel_group()}_size")
class _FlagGroupBase(msgspec.Struct):
"""Shared flag-group behavior: typo-safe writes + transactional ``override()``.
+4 -10
View File
@@ -40,6 +40,7 @@ from sglang.srt.mem_cache.allocation import (
assign_req_to_token_pool_func as assign_req_to_token_pool_func,
)
from sglang.srt.runtime_context import (
draft_attn_parallel_group,
get_exec,
get_parallel,
get_spec,
@@ -734,19 +735,12 @@ def draft_tp_context(owns_attention: bool):
"""Enter the TP placement that draft work runs under.
Work that owns attention (an attention-owning draft, or DSpark's DP-MoE
sync) runs on the target's attention-TP group. With prefill CP and no
attention DP it uses full TP as ordinary attention TP, matching the borrowed
target LM head. Other draft work keeps the target's TP placement.
Enter it from outside any draft scope.
sync) uses ``draft_attn_parallel_group()``. Other draft work keeps the
target's TP placement. Enter it from outside any draft scope.
"""
if not owns_attention:
return contextlib.nullcontext()
parallel = get_parallel()
group = (
parallel.tp_group
if parallel.attn_cp_size > 1 and parallel.attn_dp_size == 1
else parallel.attn_tp_group
)
group = getattr(get_parallel(), f"{draft_attn_parallel_group()}_group")
return patch_tensor_parallel_group(group, owns_attention=True)