[Refactor] attention: centralize linear and draft TP group selection (#45684)
This commit is contained in:
@@ -97,6 +97,7 @@ from sglang.srt.platforms import current_platform
|
||||
from sglang.srt.platforms.interface import KVPoolKind, reject_out_of_tree_path
|
||||
from sglang.srt.runtime_context import (
|
||||
attention_backends,
|
||||
draft_attn_tp_size,
|
||||
get_context,
|
||||
get_disagg,
|
||||
get_exec,
|
||||
@@ -1059,7 +1060,6 @@ class KVCacheConfigurator:
|
||||
The pool factories decide whether a decline refuses the boot.
|
||||
Asymmetric K/V rows fuse only on backends that carry v_head_dim through
|
||||
to the kernel."""
|
||||
from sglang.srt.layers.cp.base import is_cp_enabled
|
||||
from sglang.srt.mem_cache.layout.fused_draft import (
|
||||
FusedDraftDecision,
|
||||
PlacementContext,
|
||||
@@ -1145,10 +1145,8 @@ class KVCacheConfigurator:
|
||||
aux.draft_model_config,
|
||||
num_layers=int(aux.draft_kv_num_layers),
|
||||
attn_tp_size=(
|
||||
get_parallel().tp_size
|
||||
if is_cp_enabled()
|
||||
and get_parallel().attn_dp_size == 1
|
||||
and self.spec_algorithm.is_dflash()
|
||||
draft_attn_tp_size()
|
||||
if self.spec_algorithm.is_dflash()
|
||||
else get_parallel().attn_tp_size
|
||||
),
|
||||
# A DFLASH-family draft config inherits the target's NEXTN depth
|
||||
|
||||
@@ -7,6 +7,7 @@ import msgspec
|
||||
|
||||
from sglang.srt.configs.model_config import ModelConfig, is_deepseek_v4
|
||||
from sglang.srt.runtime_context import (
|
||||
draft_attn_tp_size,
|
||||
get_model,
|
||||
get_parallel,
|
||||
get_spec,
|
||||
@@ -271,12 +272,9 @@ def _resolve_dflash_draft_cell_size(
|
||||
draft_model_config=draft_model_config,
|
||||
draft_num_layers=draft_num_layers,
|
||||
draft_kv_cache_dtype=draft_kv_cache_dtype,
|
||||
# CP targets run the DFLASH draft on full TP without CP.
|
||||
tp_size=(
|
||||
get_parallel().tp_size
|
||||
draft_attn_tp_size()
|
||||
if spec_algorithm.is_dflash()
|
||||
and get_parallel().attn_cp_size > 1
|
||||
and get_parallel().attn_dp_size == 1
|
||||
else get_parallel().attn_tp_size
|
||||
),
|
||||
)
|
||||
|
||||
@@ -119,6 +119,8 @@ from sglang.srt.runtime_context import (
|
||||
get_mm,
|
||||
get_parallel,
|
||||
get_spec,
|
||||
linear_attn_parallel_group,
|
||||
linear_attn_tp_size,
|
||||
)
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
from sglang.srt.utils.common import (
|
||||
@@ -397,11 +399,6 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
from sglang.srt.runtime_context import (
|
||||
linear_attn_parallel_group,
|
||||
linear_attn_tp_size,
|
||||
)
|
||||
|
||||
head_shard_size = linear_attn_tp_size()
|
||||
head_parallel_group = linear_attn_parallel_group()
|
||||
|
||||
@@ -842,7 +839,7 @@ class Glm5NextDecoderLayer(nn.Module):
|
||||
update=residual.attn_update,
|
||||
tp_group=SumGroup.TP
|
||||
if config.is_kda_layer(layer_id)
|
||||
and get_parallel().enable_cp_tp_group_sharing
|
||||
and linear_attn_parallel_group() == "tp"
|
||||
else SumGroup.ATTN_TP,
|
||||
),
|
||||
declare_ffn(
|
||||
|
||||
@@ -486,6 +486,23 @@ def linear_attn_tp_size() -> int:
|
||||
return getattr(get_parallel(), f"{linear_attn_parallel_group()}_size")
|
||||
|
||||
|
||||
def draft_attn_parallel_group() -> str:
|
||||
"""The group an attention-owning draft uses, resolved in the target scope.
|
||||
|
||||
With CP and no attention DP, the draft runs on full TP without CP.
|
||||
Otherwise it stays within attention TP, preserving independent DP drafts.
|
||||
"""
|
||||
parallel = get_parallel()
|
||||
return (
|
||||
"tp" if parallel.attn_cp_size > 1 and parallel.attn_dp_size == 1 else "attn_tp"
|
||||
)
|
||||
|
||||
|
||||
def draft_attn_tp_size() -> int:
|
||||
"""The width of ``draft_attn_parallel_group()`` in the target scope."""
|
||||
return getattr(get_parallel(), f"{draft_attn_parallel_group()}_size")
|
||||
|
||||
|
||||
class _FlagGroupBase(msgspec.Struct):
|
||||
"""Shared flag-group behavior: typo-safe writes + transactional ``override()``.
|
||||
|
||||
|
||||
@@ -40,6 +40,7 @@ from sglang.srt.mem_cache.allocation import (
|
||||
assign_req_to_token_pool_func as assign_req_to_token_pool_func,
|
||||
)
|
||||
from sglang.srt.runtime_context import (
|
||||
draft_attn_parallel_group,
|
||||
get_exec,
|
||||
get_parallel,
|
||||
get_spec,
|
||||
@@ -734,19 +735,12 @@ def draft_tp_context(owns_attention: bool):
|
||||
"""Enter the TP placement that draft work runs under.
|
||||
|
||||
Work that owns attention (an attention-owning draft, or DSpark's DP-MoE
|
||||
sync) runs on the target's attention-TP group. With prefill CP and no
|
||||
attention DP it uses full TP as ordinary attention TP, matching the borrowed
|
||||
target LM head. Other draft work keeps the target's TP placement.
|
||||
Enter it from outside any draft scope.
|
||||
sync) uses ``draft_attn_parallel_group()``. Other draft work keeps the
|
||||
target's TP placement. Enter it from outside any draft scope.
|
||||
"""
|
||||
if not owns_attention:
|
||||
return contextlib.nullcontext()
|
||||
parallel = get_parallel()
|
||||
group = (
|
||||
parallel.tp_group
|
||||
if parallel.attn_cp_size > 1 and parallel.attn_dp_size == 1
|
||||
else parallel.attn_tp_group
|
||||
)
|
||||
group = getattr(get_parallel(), f"{draft_attn_parallel_group()}_group")
|
||||
return patch_tensor_parallel_group(group, owns_attention=True)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user