[Feat] cp: Enable GLM-5.3 Flash interleave BCG with batched prefill and DFlash2 (#43182)

This commit is contained in:
Baizhou Zhang
2026-10-21 15:51:28 -07:00
committed by GitHub
parent 42db2db4a8
commit f4035cab0b
18 changed files with 360 additions and 143 deletions
@@ -17,7 +17,8 @@ def _dsa_cp_interleave_q_seqs_kernel(
cp_size: tl.constexpr,
cp_rank: tl.constexpr,
):
extra_seq = 0
# Eager metadata uses int32, while prefill graph buffers may use int64.
extra_seq = tl.full((), 0, in_seqs_ptr.dtype.element_ty)
bs_idx = 0
for bs in range(tokens):
cur_len = tl.load(in_seqs_ptr + bs)
@@ -47,21 +47,21 @@ def resolve_cp_tp_group_sharing(server_args: Any, model: Any) -> None:
raise ValueError(f"CP-TP group sharing requires --cp-strategy {strategy}.")
# This hook runs before speculative defaults and NEXTN alias resolution.
# Native MTP keeps the target's CP/TP placement; other draft architectures
# have their own topology and model-boundary contracts. Only GLM-5.3-Flash
# has its MTP ported.
# Native MTP keeps the target's CP/TP placement. DFLASH consumes the
# target's gathered hidden states and runs its draft with ordinary TP.
# Only GLM-5.3-Flash has these speculative paths ported.
if cfg.speculative_algorithm is not None:
if model.hf_config.architectures[0] != "Glm5NextForConditionalGeneration":
raise ValueError(
"--speculative-algorithm is not supported with CP-TP group sharing."
)
if (
if cfg.speculative_algorithm != "DFLASH" and (
cfg.speculative_algorithm.upper() not in ("EAGLE", "NEXTN")
or cfg.speculative_draft_model_path not in (None, cfg.model_path)
or cfg.speculative_eagle_topk not in (None, 1)
):
raise ValueError(
"CP/TP group sharing only supports native MTP with top-k 1."
"CP/TP group sharing only supports DFLASH or native MTP with top-k 1."
)
unsupported = {
@@ -108,12 +108,10 @@ def resolve_cp_tp_group_sharing(server_args: Any, model: Any) -> None:
f"{name}={count} must be divisible by --tp-size={cfg.tp_size}."
)
declare_resolution(
server_args,
"resolve_cp_tp_group_sharing",
enable_cp_tp_group_sharing=True,
# The sequence collectives have batch-dependent sizes.
cuda_graph_config=with_phase(
fields = {"enable_cp_tp_group_sharing": True}
if model.hf_config.architectures[0] == "Qwen4ExpForConditionalGeneration":
# Qwen's sequence collectives still have batch-dependent sizes.
fields["cuda_graph_config"] = with_phase(
cfg.cuda_graph_config, Phase.PREFILL, backend=Backend.DISABLED
),
)
)
declare_resolution(server_args, "resolve_cp_tp_group_sharing", **fields)
+15 -9
View File
@@ -305,15 +305,21 @@ def handle_model_specific_adjustments(server_args: Any):
# The DSA CP field declarations moved to the override
# registry (arg_groups/overrides.py:
# _deepseek_family_overrides).
declare_resolution(
server_args,
"_handle_model_specific_adjustments",
cuda_graph_config=with_phase(
cfg.cuda_graph_config,
Phase.PREFILL,
backend=Backend.DISABLED,
),
)
from sglang.srt.layers.cp.bcg import supports_prefill_cp_bcg
if not (
cfg.cuda_graph_config.prefill.backend == Backend.BREAKABLE
and supports_prefill_cp_bcg(server_args)
):
declare_resolution(
server_args,
"_handle_model_specific_adjustments",
cuda_graph_config=with_phase(
cfg.cuda_graph_config,
Phase.PREFILL,
backend=Backend.DISABLED,
),
)
else:
# Pure TP and partial DP Attention mode is active for DSA, logging a warning
attn_tp_size = derive_attn_tp_size(
@@ -139,8 +139,7 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
f"ep_size={overrides.get('ep_size', cfg.ep_size)}, tp_size={cfg.tp_size}, "
f"attn_cp_size={attn_cp_size}, "
f"kv_cache_dtype={cfg.kv_cache_dtype}, "
f"moe_a2a_backend={overrides.get('moe_a2a_backend', cfg.moe_a2a_backend)}, "
f"cuda_graph_config[prefill].backend=disabled"
f"moe_a2a_backend={overrides.get('moe_a2a_backend', cfg.moe_a2a_backend)}"
)
# Deferred import to avoid a circular import at module-load
@@ -2,6 +2,7 @@
import torch
from sglang.srt.layers.cp.interleave import is_interleave_extend
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
eager_on_graph,
)
@@ -22,6 +23,11 @@ def _kpool_indexer_prefill_with_output(
# Resolve the live batch inside the eager break, never from capture args.
forward_batch = get_tc_piecewise_forward_context().forward_batch
n = forward_batch.extend_num_tokens
num_logical = sum(forward_batch.extend_seq_lens_cpu)
if is_interleave_extend(forward_batch):
from sglang.srt.layers.cp.bcg.interleave_bcg import cp_interleave_indexer_rows
n, num_logical = cp_interleave_indexer_rows(forward_batch)
if n is None or not 0 <= n <= x.shape[0]:
raise ValueError(f"Invalid pooled-indexer prefill token count: {n}")
if n > q_lora.shape[0] or n > positions.shape[0]:
@@ -37,7 +43,6 @@ def _kpool_indexer_prefill_with_output(
)
if not return_indices:
return
num_logical = sum(forward_batch.extend_seq_lens_cpu)
if (
result is None
or result.ndim != 2
@@ -0,0 +1,64 @@
"""Strategy dispatch for prefill context parallel CUDA graphs."""
from __future__ import annotations
from typing import TYPE_CHECKING
from sglang.srt.arg_groups.overrides import (
attention_backends_of,
resolved_view,
resolving_view,
)
from sglang.srt.layers.cp.base import (
ContextParallelStrategyKind,
CPAttentionBackendKind,
)
if TYPE_CHECKING:
from sglang.srt.server_args import ServerArgs
PREFILL_CP_BCG_SUPPORTED_COMBINATIONS: set[
tuple[ContextParallelStrategyKind, CPAttentionBackendKind]
] = {
(ContextParallelStrategyKind.ZIGZAG, CPAttentionBackendKind.TRTLLM_MHA),
(ContextParallelStrategyKind.INTERLEAVE, CPAttentionBackendKind.DSA),
}
def get_cp_bcg_input_type(strategy: ContextParallelStrategyKind):
# Keep runtime adapters lazy: config resolution also queries CP BCG support.
if strategy == ContextParallelStrategyKind.INTERLEAVE:
from sglang.srt.layers.cp.bcg.interleave_bcg import InterleaveCPBCGInput
return InterleaveCPBCGInput
if strategy == ContextParallelStrategyKind.ZIGZAG:
from sglang.srt.layers.cp.bcg.zigzag_bcg import ZigzagCPBCGInput
return ZigzagCPBCGInput
raise ValueError(f"Unsupported CP BCG strategy: {strategy}")
def supports_prefill_cp_bcg(server_args: ServerArgs) -> bool:
"""Return whether the selected prefill-CP configuration supports BCG."""
cfg = resolving_view(server_args)
resolved = resolved_view(server_args)
if not (
cfg.enable_prefill_cp
and cfg.pp_size == 1
and resolved.attn_cp_size == cfg.tp_size
):
return False
prefill_attention_backend, _ = attention_backends_of(resolved)
try:
combination = (
ContextParallelStrategyKind.from_string(cfg.cp_strategy),
CPAttentionBackendKind.from_string(prefill_attention_backend),
)
except ValueError:
# Other backends (or an unresolved backend/strategy) cannot use CP BCG.
return False
if combination not in PREFILL_CP_BCG_SUPPORTED_COMBINATIONS:
return False
return get_cp_bcg_input_type(combination[0]).supports(server_args)
@@ -22,23 +22,19 @@ from typing import TYPE_CHECKING, Any, Dict, Optional
import torch
from sglang.srt.arg_groups.overrides import (
attention_backends_of,
resolved_view,
resolving_view,
)
from sglang.srt.layers.cp.base import (
ContextParallelStrategyKind,
CPAttentionBackendKind,
get_cp_strategy,
get_cp_strategy_kind,
)
from sglang.srt.layers.cp.interleave import InterleaveCPStrategy
from sglang.srt.layers.cp.padding import get_cp_padding_align_size
from sglang.srt.layers.cp.utils import (
cp_gather_after_forward,
cp_split_before_forward,
prepare_cp_forward,
)
from sglang.srt.layers.cp.zigzag import ZigzagCPStrategy
from sglang.srt.model_executor.forward_batch_info import PPProxyTensors
if TYPE_CHECKING:
@@ -49,48 +45,18 @@ if TYPE_CHECKING:
from sglang.srt.server_args import ServerArgs
PREFILL_CP_BCG_SUPPORTED_COMBINATIONS: set[
tuple[ContextParallelStrategyKind, CPAttentionBackendKind]
] = {
(ContextParallelStrategyKind.ZIGZAG, CPAttentionBackendKind.TRTLLM_MHA),
}
def supports_prefill_cp_bcg(server_args: ServerArgs) -> bool:
"""Return whether the selected prefill-CP configuration supports BCG."""
cfg = resolving_view(server_args)
resolved = resolved_view(server_args)
if not (
cfg.enable_prefill_cp
and cfg.pp_size == 1
and resolved.attn_cp_size == cfg.tp_size
):
return False
prefill_attention_backend, _ = attention_backends_of(resolved)
try:
combination = (
ContextParallelStrategyKind.from_string(cfg.cp_strategy),
CPAttentionBackendKind.from_string(prefill_attention_backend),
)
except ValueError:
# Other backends (or an unresolved backend/strategy) cannot use CP BCG.
return False
return combination in PREFILL_CP_BCG_SUPPORTED_COMBINATIONS
def enable_cp_bcg_capture(server_args: ServerArgs) -> bool:
"""Return whether CP breakable prefill capture is enabled."""
return supports_prefill_cp_bcg(server_args)
def filter_prefill_cp_bcg_capture_num_tokens(
capture_num_tokens: list[int], server_args: ServerArgs
) -> list[int]:
"""Keep only token buckets where the selected CP strategy can run."""
min_num_tokens = resolved_view(server_args).attn_cp_size
if resolving_view(server_args).cp_strategy == "zigzag":
min_num_tokens *= 2
from sglang.srt.layers.cp.bcg import get_cp_bcg_input_type
input_type = get_cp_bcg_input_type(
ContextParallelStrategyKind.from_string(resolving_view(server_args).cp_strategy)
)
min_num_tokens = input_type.min_capture_tokens(
resolved_view(server_args).attn_cp_size
)
filtered = [size for size in capture_num_tokens if size >= min_num_tokens]
if not filtered:
raise ValueError(
@@ -123,6 +89,10 @@ class PrefillCPBCGInput:
@classmethod
def create(cls, runner: PrefillCudaGraphRunner) -> PrefillCPBCGInput:
if cls is PrefillCPBCGInput:
from sglang.srt.layers.cp.bcg import get_cp_bcg_input_type
return get_cp_bcg_input_type(get_cp_strategy_kind()).create(runner)
with torch.device(runner.device):
return cls(
input_embeds=torch.zeros(
@@ -138,29 +108,29 @@ class PrefillCPBCGInput:
),
)
def allows_replay(self, batch_size, num_tokens, prefix_lens, contains_mm_inputs):
return True
def model_positions(self, forward_batch):
# No override: retain the runner's existing model/mRoPE selection.
return None
@staticmethod
def supports(server_args: ServerArgs) -> bool:
return True
@staticmethod
def min_capture_tokens(cp_size: int) -> int:
return cp_size
def _max_local_tokens(self, extend_seq_lens: Any) -> int:
raise NotImplementedError
def required_local_tokens(self, extend_seq_lens: Any) -> Optional[int]:
"""Return the largest aligned local shard, uniformly across CP ranks."""
strategy = get_cp_strategy()
if extend_seq_lens is None:
return None
if isinstance(strategy, InterleaveCPStrategy):
num_tokens = sum(int(length) for length in extend_seq_lens)
max_local_tokens = (num_tokens + strategy.cp_size - 1) // strategy.cp_size
elif isinstance(strategy, ZigzagCPStrategy):
cp_segment_num = strategy.cp_size * 2
per_rank_logical_tokens = [0] * strategy.cp_size
for raw_length in extend_seq_lens:
base, remainder = divmod(int(raw_length), cp_segment_num)
for rank in range(strategy.cp_size):
opposite_rank = cp_segment_num - 1 - rank
per_rank_logical_tokens[rank] += (
base * 2
+ int(rank < remainder)
+ int(opposite_rank < remainder)
)
max_local_tokens = max(per_rank_logical_tokens)
else:
if extend_seq_lens is None or len(extend_seq_lens) == 0:
return None
max_local_tokens = self._max_local_tokens(extend_seq_lens)
align_size = get_cp_padding_align_size()
return (max_local_tokens + align_size - 1) // align_size * align_size
@@ -217,7 +187,6 @@ class PrefillCPBCGInput:
# Replay batches may reuse a ForwardBatch object whose metadata was
# built for a different request layout. Always rebuild before sharding.
forward_batch.attn_cp_metadata = None
out_cache_loc = forward_batch.out_cache_loc
prepare_cp_forward(forward_batch)
captured_local_tokens = None
@@ -286,16 +255,6 @@ class PrefillCPBCGInput:
forward_batch.positions = positions
self.live_local_tokens = live_local_tokens
if isinstance(get_cp_strategy(), InterleaveCPStrategy):
metadata = forward_batch.attn_cp_metadata
cp_size = len(metadata.per_rank_actual_token)
base, remainder = divmod(static_num_tokens, cp_size)
metadata.total_seq_lens = static_num_tokens
metadata.per_rank_logical_token = [
base + int(rank < remainder) for rank in range(cp_size)
]
forward_batch.out_cache_loc = out_cache_loc
def execute_prefill_cp_bcg(
runner: PrefillCudaGraphRunner,
@@ -0,0 +1,85 @@
"""Experimental, exact-shape interleave BCG adapter for GLM-5.3 Flash.
Keep full-sequence KPool positions separate from the local DSA positions.
Prefix-free text batches replay when their total token count matches a capture,
independently of request boundaries. Other batches retain eager CP, including MM.
"""
import torch
from sglang.srt.arg_groups.overrides import (
model_config_of,
resolved_view,
resolving_view,
)
from sglang.srt.layers.cp.bcg.bcg import PrefillCPBCGInput
from sglang.srt.layers.cp.utils import cp_interleave_input_ids
from sglang.srt.runtime_context import get_parallel
class InterleaveCPBCGInput(PrefillCPBCGInput):
@classmethod
def create(cls, runner):
result = super().create(runner)
# Each local shard is CP-aligned before rank-major MoE all-gather.
alignment = get_parallel().attn_cp_size ** 2
capacity = (runner.max_num_tokens + alignment - 1) // alignment * alignment
result.input_ids_global = torch.zeros(
capacity, dtype=torch.int64, device=runner.device
)
return result
def allows_replay(self, batch_size, num_tokens, prefix_lens, contains_mm_inputs):
return (
batch_size > 0
and num_tokens in self.bucket_local_tokens
and not contains_mm_inputs
and prefix_lens is not None
and not any(prefix_lens)
)
@staticmethod
def supports(server_args):
cfg = resolving_view(server_args)
return (
cfg.dp_size == 1
and cfg.ep_size == 1
and resolved_view(server_args).attn_cp_size > 1
and "Glm5NextForConditionalGeneration"
in model_config_of(server_args).hf_config.architectures
)
def _max_local_tokens(self, extend_seq_lens):
size = get_parallel().attn_cp_size
return (sum(int(length) for length in extend_seq_lens) + size - 1) // size
def model_positions(self, forward_batch):
return self.positions[: self.live_local_tokens]
def prepare(self, runner, forward_batch, *, static_num_tokens, capture):
global_positions = forward_batch.positions
out_cache_loc = forward_batch.out_cache_loc
super().prepare(
runner, forward_batch, static_num_tokens=static_num_tokens, capture=capture
)
metadata = forward_batch.attn_cp_metadata
cp_size = len(metadata.per_rank_actual_token)
base, remainder = divmod(static_num_tokens, cp_size)
metadata.total_seq_lens = static_num_tokens
metadata.per_rank_logical_token = [
base + int(rank < remainder) for rank in range(cp_size)
]
forward_batch.out_cache_loc = out_cache_loc
# KPool compresses consecutive global tokens; only the model argument
# (DSA RoPE) is local. Do not replace ForwardBatch's global positions.
forward_batch.positions = global_positions
ids = cp_interleave_input_ids(forward_batch.input_ids, forward_batch)
self.input_ids_global[: ids.shape[0]].copy_(ids)
forward_batch.input_ids_global = self.input_ids_global[: ids.shape[0]]
def cp_interleave_indexer_rows(forward_batch):
metadata = forward_batch.attn_cp_metadata
rank = get_parallel().attn_cp_rank
logical = metadata.per_rank_logical_token or metadata.per_rank_actual_token
return metadata.per_rank_actual_token[rank], logical[rank]
@@ -0,0 +1,23 @@
"""Zigzag-specific capture geometry for prefill context parallel CUDA graphs."""
from sglang.srt.layers.cp.base import get_cp_strategy
from sglang.srt.layers.cp.bcg.bcg import PrefillCPBCGInput
class ZigzagCPBCGInput(PrefillCPBCGInput):
@staticmethod
def min_capture_tokens(cp_size: int) -> int:
return cp_size * 2
def _max_local_tokens(self, extend_seq_lens) -> int:
cp_size = get_cp_strategy().cp_size
cp_segment_num = cp_size * 2
per_rank_logical_tokens = [0] * cp_size
for raw_length in extend_seq_lens:
base, remainder = divmod(int(raw_length), cp_segment_num)
for rank in range(cp_size):
opposite_rank = cp_segment_num - 1 - rank
per_rank_logical_tokens[rank] += (
base * 2 + int(rank < remainder) + int(opposite_rank < remainder)
)
return max(per_rank_logical_tokens)
@@ -1059,6 +1059,7 @@ class KVCacheConfigurator:
The pool factories decide whether a decline refuses the boot.
Asymmetric K/V rows fuse only on backends that carry v_head_dim through
to the kernel."""
from sglang.srt.layers.cp.base import is_cp_enabled
from sglang.srt.mem_cache.layout.fused_draft import (
FusedDraftDecision,
PlacementContext,
@@ -1143,7 +1144,13 @@ class KVCacheConfigurator:
profile = draft_kv_profile(
aux.draft_model_config,
num_layers=int(aux.draft_kv_num_layers),
attn_tp_size=get_parallel().attn_tp_size,
attn_tp_size=(
get_parallel().tp_size
if is_cp_enabled()
and get_parallel().attn_dp_size == 1
and self.spec_algorithm.is_dflash()
else get_parallel().attn_tp_size
),
# A DFLASH-family draft config inherits the target's NEXTN depth
# count; its draft is one block, replicated per runner.
num_depths=None if self.spec_algorithm.is_eagle() else 1,
@@ -511,6 +511,15 @@ def capture_prefill_graph(
if model_runner.is_draft_worker and not force_for_draft_worker:
return result(None)
if model_runner.is_draft_worker and get_parallel().enable_prefill_cp:
from sglang.srt.layers.cp.base import is_interleave
if is_interleave():
# EagleWorkerV2 requests capture again with force_for_draft_worker=True
# after init_lm_head, bypassing the guard above. The interleave adapter
# supports only target prefill; draft CP must stay eager on that call too.
return result(None)
# Skip prefill CG for EAGLE target on tc_piecewise when the fixed server
# capture ceiling is below FULL. EAGLE target prefill requests FULL, so a
# NULL or LAST graph is dead; capturing it can perturb FP4/TRTLLM-MoE
@@ -234,6 +234,7 @@ def _resolve_dflash_aux_hidden_state(
config.dflash_draft_cell_size_per_token = _resolve_dflash_draft_cell_size(
draft_model_config=draft_model_config,
draft_num_layers=int(draft_num_layers),
spec_algorithm=spec_algorithm,
)
@@ -241,6 +242,7 @@ def _resolve_dflash_draft_cell_size(
*,
draft_model_config: ModelConfig,
draft_num_layers: int,
spec_algorithm: SpeculativeAlgorithm,
) -> int | None:
"""Bytes/token the DFLASH draft KV pool will cost the target's pool budget.
@@ -269,8 +271,14 @@ def _resolve_dflash_draft_cell_size(
draft_model_config=draft_model_config,
draft_num_layers=draft_num_layers,
draft_kv_cache_dtype=draft_kv_cache_dtype,
# The draft pool shards KV heads like the target pool (by attention TP).
tp_size=get_parallel().attn_tp_size,
# CP targets run the DFLASH draft on full TP without CP.
tp_size=(
get_parallel().tp_size
if spec_algorithm.is_dflash()
and get_parallel().attn_cp_size > 1
and get_parallel().attn_dp_size == 1
else get_parallel().attn_tp_size
),
)
except Exception as e: # noqa: BLE001
logger.warning(
@@ -56,13 +56,9 @@ from sglang.srt.configs.model_config import is_qwen4_exp
from sglang.srt.distributed.parallel_state import graph_capture
from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
from sglang.srt.layers.cp.bcg import (
from sglang.srt.layers.cp.bcg import supports_prefill_cp_bcg
from sglang.srt.layers.cp.bcg.bcg import (
PrefillCPBCGInput,
)
from sglang.srt.layers.cp.bcg import (
enable_cp_bcg_capture as should_enable_cp_bcg_capture,
)
from sglang.srt.layers.cp.bcg import (
execute_prefill_cp_bcg,
filter_prefill_cp_bcg_capture_num_tokens,
)
@@ -612,7 +608,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
server_args = model_runner.server_args
self.enable_cp_bcg_capture = isinstance(
self.backend, BreakableCudaGraphBackend
) and should_enable_cp_bcg_capture(server_args)
) and supports_prefill_cp_bcg(server_args)
if self.enable_cp_bcg_capture:
if self.max_context_size is not None:
# TODO(SYChen123): Preserve max_seq_len_override through CP's padded
@@ -784,6 +780,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
def _get_layer_model_positions(self, forward_batch: ForwardBatch) -> torch.Tensor:
"""Mirror outer multimodal wrappers when BCG captures layer_model directly."""
if self.enable_cp_bcg_capture:
cp_positions = self.prefill_cp_bcg_input.model_positions(forward_batch)
if cp_positions is not None:
return cp_positions
if forward_batch.mrope_positions is None:
return forward_batch.positions
@@ -1336,6 +1336,12 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
``capture_hidden_mode=None`` when unknown at the call site (it is
rank-uniform; forward-time-only checking cannot split the group).
"""
if getattr(
self, "enable_cp_bcg_capture", False
) and not self.prefill_cp_bcg_input.allows_replay(
batch_size, num_tokens, prefix_lens, contains_mm_inputs
):
return False
if contains_mm_inputs and (
self._qwen_bcg_hc_sidechannel or self._qwen_bcg_mtp_draft
):
@@ -411,10 +411,11 @@ class DFlashWorkerV2(BaseSpecWorker):
self._tp_sync = SpecTpSync(get_dp_tp_group())
check_simulate_acc_token_mode()
# Under dp attention, the draft worker runs on the per-DP attn-TP
# group, independent of idle peer DP ranks; it is built and run under
# the same placement.
self.draft_owns_attention = get_parallel().attn_dp_enabled
# DP drafts use per-DP attention TP, independent of idle peers. CP
# drafts use full TP without CP, including their backend and KV pool.
self.draft_owns_attention = (
get_parallel().attn_dp_enabled or get_parallel().attn_cp_size > 1
)
# Inside the draft scope the context answers the draft's narrowed rank.
self._target_tp_rank = get_parallel().tp_rank
with draft_pp_context(), draft_tp_context(self.draft_owns_attention):
@@ -657,13 +658,14 @@ class DFlashWorkerV2(BaseSpecWorker):
# enabled, the draft worker keeps a private compact req->token table
# over the same global KV index space, so radix-cache/prefix-hit KV
# remains reusable while draft attention sees only the recent window.
self._draft_worker.alloc_memory_pool(
memory_pool_config=memory_pool_config,
req_to_token_pool=(
None if self.use_compact_draft_cache else req_to_token_pool
),
token_to_kv_pool_allocator=token_to_kv_pool_allocator,
)
with draft_tp_context(self.draft_owns_attention):
self._draft_worker.alloc_memory_pool(
memory_pool_config=memory_pool_config,
req_to_token_pool=(
None if self.use_compact_draft_cache else req_to_token_pool
),
token_to_kv_pool_allocator=token_to_kv_pool_allocator,
)
def init_attention_backends(self):
with (
+10 -5
View File
@@ -734,15 +734,20 @@ def draft_tp_context(owns_attention: bool):
"""Enter the TP placement that draft work runs under.
Work that owns attention (an attention-owning draft, or DSpark's DP-MoE
sync) runs on the target's attention-TP group; other draft work keeps the
target's TP placement. Enter it from outside any draft scope, where
``attn_tp_group`` is still the target's.
sync) runs on the target's attention-TP group. With prefill CP and no
attention DP it uses full TP as ordinary attention TP, matching the borrowed
target LM head. Other draft work keeps the target's TP placement.
Enter it from outside any draft scope.
"""
if not owns_attention:
return contextlib.nullcontext()
return patch_tensor_parallel_group(
get_parallel().attn_tp_group, owns_attention=True
parallel = get_parallel()
group = (
parallel.tp_group
if parallel.attn_cp_size > 1 and parallel.attn_dp_size == 1
else parallel.attn_tp_group
)
return patch_tensor_parallel_group(group, owns_attention=True)
def spec_stage_span(name: str):