[Feat] cp: Enable GLM-5.3 Flash interleave BCG with batched prefill and DFlash2 (#43182)
This commit is contained in:
@@ -17,7 +17,8 @@ def _dsa_cp_interleave_q_seqs_kernel(
|
||||
cp_size: tl.constexpr,
|
||||
cp_rank: tl.constexpr,
|
||||
):
|
||||
extra_seq = 0
|
||||
# Eager metadata uses int32, while prefill graph buffers may use int64.
|
||||
extra_seq = tl.full((), 0, in_seqs_ptr.dtype.element_ty)
|
||||
bs_idx = 0
|
||||
for bs in range(tokens):
|
||||
cur_len = tl.load(in_seqs_ptr + bs)
|
||||
|
||||
@@ -47,21 +47,21 @@ def resolve_cp_tp_group_sharing(server_args: Any, model: Any) -> None:
|
||||
raise ValueError(f"CP-TP group sharing requires --cp-strategy {strategy}.")
|
||||
|
||||
# This hook runs before speculative defaults and NEXTN alias resolution.
|
||||
# Native MTP keeps the target's CP/TP placement; other draft architectures
|
||||
# have their own topology and model-boundary contracts. Only GLM-5.3-Flash
|
||||
# has its MTP ported.
|
||||
# Native MTP keeps the target's CP/TP placement. DFLASH consumes the
|
||||
# target's gathered hidden states and runs its draft with ordinary TP.
|
||||
# Only GLM-5.3-Flash has these speculative paths ported.
|
||||
if cfg.speculative_algorithm is not None:
|
||||
if model.hf_config.architectures[0] != "Glm5NextForConditionalGeneration":
|
||||
raise ValueError(
|
||||
"--speculative-algorithm is not supported with CP-TP group sharing."
|
||||
)
|
||||
if (
|
||||
if cfg.speculative_algorithm != "DFLASH" and (
|
||||
cfg.speculative_algorithm.upper() not in ("EAGLE", "NEXTN")
|
||||
or cfg.speculative_draft_model_path not in (None, cfg.model_path)
|
||||
or cfg.speculative_eagle_topk not in (None, 1)
|
||||
):
|
||||
raise ValueError(
|
||||
"CP/TP group sharing only supports native MTP with top-k 1."
|
||||
"CP/TP group sharing only supports DFLASH or native MTP with top-k 1."
|
||||
)
|
||||
|
||||
unsupported = {
|
||||
@@ -108,12 +108,10 @@ def resolve_cp_tp_group_sharing(server_args: Any, model: Any) -> None:
|
||||
f"{name}={count} must be divisible by --tp-size={cfg.tp_size}."
|
||||
)
|
||||
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"resolve_cp_tp_group_sharing",
|
||||
enable_cp_tp_group_sharing=True,
|
||||
# The sequence collectives have batch-dependent sizes.
|
||||
cuda_graph_config=with_phase(
|
||||
fields = {"enable_cp_tp_group_sharing": True}
|
||||
if model.hf_config.architectures[0] == "Qwen4ExpForConditionalGeneration":
|
||||
# Qwen's sequence collectives still have batch-dependent sizes.
|
||||
fields["cuda_graph_config"] = with_phase(
|
||||
cfg.cuda_graph_config, Phase.PREFILL, backend=Backend.DISABLED
|
||||
),
|
||||
)
|
||||
)
|
||||
declare_resolution(server_args, "resolve_cp_tp_group_sharing", **fields)
|
||||
|
||||
@@ -305,15 +305,21 @@ def handle_model_specific_adjustments(server_args: Any):
|
||||
# The DSA CP field declarations moved to the override
|
||||
# registry (arg_groups/overrides.py:
|
||||
# _deepseek_family_overrides).
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"_handle_model_specific_adjustments",
|
||||
cuda_graph_config=with_phase(
|
||||
cfg.cuda_graph_config,
|
||||
Phase.PREFILL,
|
||||
backend=Backend.DISABLED,
|
||||
),
|
||||
)
|
||||
from sglang.srt.layers.cp.bcg import supports_prefill_cp_bcg
|
||||
|
||||
if not (
|
||||
cfg.cuda_graph_config.prefill.backend == Backend.BREAKABLE
|
||||
and supports_prefill_cp_bcg(server_args)
|
||||
):
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"_handle_model_specific_adjustments",
|
||||
cuda_graph_config=with_phase(
|
||||
cfg.cuda_graph_config,
|
||||
Phase.PREFILL,
|
||||
backend=Backend.DISABLED,
|
||||
),
|
||||
)
|
||||
else:
|
||||
# Pure TP and partial DP Attention mode is active for DSA, logging a warning
|
||||
attn_tp_size = derive_attn_tp_size(
|
||||
|
||||
@@ -139,8 +139,7 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
f"ep_size={overrides.get('ep_size', cfg.ep_size)}, tp_size={cfg.tp_size}, "
|
||||
f"attn_cp_size={attn_cp_size}, "
|
||||
f"kv_cache_dtype={cfg.kv_cache_dtype}, "
|
||||
f"moe_a2a_backend={overrides.get('moe_a2a_backend', cfg.moe_a2a_backend)}, "
|
||||
f"cuda_graph_config[prefill].backend=disabled"
|
||||
f"moe_a2a_backend={overrides.get('moe_a2a_backend', cfg.moe_a2a_backend)}"
|
||||
)
|
||||
|
||||
# Deferred import to avoid a circular import at module-load
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.cp.interleave import is_interleave_extend
|
||||
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
|
||||
eager_on_graph,
|
||||
)
|
||||
@@ -22,6 +23,11 @@ def _kpool_indexer_prefill_with_output(
|
||||
# Resolve the live batch inside the eager break, never from capture args.
|
||||
forward_batch = get_tc_piecewise_forward_context().forward_batch
|
||||
n = forward_batch.extend_num_tokens
|
||||
num_logical = sum(forward_batch.extend_seq_lens_cpu)
|
||||
if is_interleave_extend(forward_batch):
|
||||
from sglang.srt.layers.cp.bcg.interleave_bcg import cp_interleave_indexer_rows
|
||||
|
||||
n, num_logical = cp_interleave_indexer_rows(forward_batch)
|
||||
if n is None or not 0 <= n <= x.shape[0]:
|
||||
raise ValueError(f"Invalid pooled-indexer prefill token count: {n}")
|
||||
if n > q_lora.shape[0] or n > positions.shape[0]:
|
||||
@@ -37,7 +43,6 @@ def _kpool_indexer_prefill_with_output(
|
||||
)
|
||||
if not return_indices:
|
||||
return
|
||||
num_logical = sum(forward_batch.extend_seq_lens_cpu)
|
||||
if (
|
||||
result is None
|
||||
or result.ndim != 2
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Strategy dispatch for prefill context parallel CUDA graphs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sglang.srt.arg_groups.overrides import (
|
||||
attention_backends_of,
|
||||
resolved_view,
|
||||
resolving_view,
|
||||
)
|
||||
from sglang.srt.layers.cp.base import (
|
||||
ContextParallelStrategyKind,
|
||||
CPAttentionBackendKind,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
|
||||
|
||||
PREFILL_CP_BCG_SUPPORTED_COMBINATIONS: set[
|
||||
tuple[ContextParallelStrategyKind, CPAttentionBackendKind]
|
||||
] = {
|
||||
(ContextParallelStrategyKind.ZIGZAG, CPAttentionBackendKind.TRTLLM_MHA),
|
||||
(ContextParallelStrategyKind.INTERLEAVE, CPAttentionBackendKind.DSA),
|
||||
}
|
||||
|
||||
|
||||
def get_cp_bcg_input_type(strategy: ContextParallelStrategyKind):
|
||||
# Keep runtime adapters lazy: config resolution also queries CP BCG support.
|
||||
if strategy == ContextParallelStrategyKind.INTERLEAVE:
|
||||
from sglang.srt.layers.cp.bcg.interleave_bcg import InterleaveCPBCGInput
|
||||
|
||||
return InterleaveCPBCGInput
|
||||
if strategy == ContextParallelStrategyKind.ZIGZAG:
|
||||
from sglang.srt.layers.cp.bcg.zigzag_bcg import ZigzagCPBCGInput
|
||||
|
||||
return ZigzagCPBCGInput
|
||||
raise ValueError(f"Unsupported CP BCG strategy: {strategy}")
|
||||
|
||||
|
||||
def supports_prefill_cp_bcg(server_args: ServerArgs) -> bool:
|
||||
"""Return whether the selected prefill-CP configuration supports BCG."""
|
||||
|
||||
cfg = resolving_view(server_args)
|
||||
resolved = resolved_view(server_args)
|
||||
if not (
|
||||
cfg.enable_prefill_cp
|
||||
and cfg.pp_size == 1
|
||||
and resolved.attn_cp_size == cfg.tp_size
|
||||
):
|
||||
return False
|
||||
prefill_attention_backend, _ = attention_backends_of(resolved)
|
||||
try:
|
||||
combination = (
|
||||
ContextParallelStrategyKind.from_string(cfg.cp_strategy),
|
||||
CPAttentionBackendKind.from_string(prefill_attention_backend),
|
||||
)
|
||||
except ValueError:
|
||||
# Other backends (or an unresolved backend/strategy) cannot use CP BCG.
|
||||
return False
|
||||
if combination not in PREFILL_CP_BCG_SUPPORTED_COMBINATIONS:
|
||||
return False
|
||||
return get_cp_bcg_input_type(combination[0]).supports(server_args)
|
||||
@@ -22,23 +22,19 @@ from typing import TYPE_CHECKING, Any, Dict, Optional
|
||||
import torch
|
||||
|
||||
from sglang.srt.arg_groups.overrides import (
|
||||
attention_backends_of,
|
||||
resolved_view,
|
||||
resolving_view,
|
||||
)
|
||||
from sglang.srt.layers.cp.base import (
|
||||
ContextParallelStrategyKind,
|
||||
CPAttentionBackendKind,
|
||||
get_cp_strategy,
|
||||
get_cp_strategy_kind,
|
||||
)
|
||||
from sglang.srt.layers.cp.interleave import InterleaveCPStrategy
|
||||
from sglang.srt.layers.cp.padding import get_cp_padding_align_size
|
||||
from sglang.srt.layers.cp.utils import (
|
||||
cp_gather_after_forward,
|
||||
cp_split_before_forward,
|
||||
prepare_cp_forward,
|
||||
)
|
||||
from sglang.srt.layers.cp.zigzag import ZigzagCPStrategy
|
||||
from sglang.srt.model_executor.forward_batch_info import PPProxyTensors
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -49,48 +45,18 @@ if TYPE_CHECKING:
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
|
||||
|
||||
PREFILL_CP_BCG_SUPPORTED_COMBINATIONS: set[
|
||||
tuple[ContextParallelStrategyKind, CPAttentionBackendKind]
|
||||
] = {
|
||||
(ContextParallelStrategyKind.ZIGZAG, CPAttentionBackendKind.TRTLLM_MHA),
|
||||
}
|
||||
|
||||
|
||||
def supports_prefill_cp_bcg(server_args: ServerArgs) -> bool:
|
||||
"""Return whether the selected prefill-CP configuration supports BCG."""
|
||||
|
||||
cfg = resolving_view(server_args)
|
||||
resolved = resolved_view(server_args)
|
||||
if not (
|
||||
cfg.enable_prefill_cp
|
||||
and cfg.pp_size == 1
|
||||
and resolved.attn_cp_size == cfg.tp_size
|
||||
):
|
||||
return False
|
||||
prefill_attention_backend, _ = attention_backends_of(resolved)
|
||||
try:
|
||||
combination = (
|
||||
ContextParallelStrategyKind.from_string(cfg.cp_strategy),
|
||||
CPAttentionBackendKind.from_string(prefill_attention_backend),
|
||||
)
|
||||
except ValueError:
|
||||
# Other backends (or an unresolved backend/strategy) cannot use CP BCG.
|
||||
return False
|
||||
return combination in PREFILL_CP_BCG_SUPPORTED_COMBINATIONS
|
||||
|
||||
|
||||
def enable_cp_bcg_capture(server_args: ServerArgs) -> bool:
|
||||
"""Return whether CP breakable prefill capture is enabled."""
|
||||
return supports_prefill_cp_bcg(server_args)
|
||||
|
||||
|
||||
def filter_prefill_cp_bcg_capture_num_tokens(
|
||||
capture_num_tokens: list[int], server_args: ServerArgs
|
||||
) -> list[int]:
|
||||
"""Keep only token buckets where the selected CP strategy can run."""
|
||||
min_num_tokens = resolved_view(server_args).attn_cp_size
|
||||
if resolving_view(server_args).cp_strategy == "zigzag":
|
||||
min_num_tokens *= 2
|
||||
from sglang.srt.layers.cp.bcg import get_cp_bcg_input_type
|
||||
|
||||
input_type = get_cp_bcg_input_type(
|
||||
ContextParallelStrategyKind.from_string(resolving_view(server_args).cp_strategy)
|
||||
)
|
||||
min_num_tokens = input_type.min_capture_tokens(
|
||||
resolved_view(server_args).attn_cp_size
|
||||
)
|
||||
filtered = [size for size in capture_num_tokens if size >= min_num_tokens]
|
||||
if not filtered:
|
||||
raise ValueError(
|
||||
@@ -123,6 +89,10 @@ class PrefillCPBCGInput:
|
||||
|
||||
@classmethod
|
||||
def create(cls, runner: PrefillCudaGraphRunner) -> PrefillCPBCGInput:
|
||||
if cls is PrefillCPBCGInput:
|
||||
from sglang.srt.layers.cp.bcg import get_cp_bcg_input_type
|
||||
|
||||
return get_cp_bcg_input_type(get_cp_strategy_kind()).create(runner)
|
||||
with torch.device(runner.device):
|
||||
return cls(
|
||||
input_embeds=torch.zeros(
|
||||
@@ -138,29 +108,29 @@ class PrefillCPBCGInput:
|
||||
),
|
||||
)
|
||||
|
||||
def allows_replay(self, batch_size, num_tokens, prefix_lens, contains_mm_inputs):
|
||||
return True
|
||||
|
||||
def model_positions(self, forward_batch):
|
||||
# No override: retain the runner's existing model/mRoPE selection.
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def supports(server_args: ServerArgs) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def min_capture_tokens(cp_size: int) -> int:
|
||||
return cp_size
|
||||
|
||||
def _max_local_tokens(self, extend_seq_lens: Any) -> int:
|
||||
raise NotImplementedError
|
||||
|
||||
def required_local_tokens(self, extend_seq_lens: Any) -> Optional[int]:
|
||||
"""Return the largest aligned local shard, uniformly across CP ranks."""
|
||||
strategy = get_cp_strategy()
|
||||
if extend_seq_lens is None:
|
||||
return None
|
||||
if isinstance(strategy, InterleaveCPStrategy):
|
||||
num_tokens = sum(int(length) for length in extend_seq_lens)
|
||||
max_local_tokens = (num_tokens + strategy.cp_size - 1) // strategy.cp_size
|
||||
elif isinstance(strategy, ZigzagCPStrategy):
|
||||
cp_segment_num = strategy.cp_size * 2
|
||||
per_rank_logical_tokens = [0] * strategy.cp_size
|
||||
for raw_length in extend_seq_lens:
|
||||
base, remainder = divmod(int(raw_length), cp_segment_num)
|
||||
for rank in range(strategy.cp_size):
|
||||
opposite_rank = cp_segment_num - 1 - rank
|
||||
per_rank_logical_tokens[rank] += (
|
||||
base * 2
|
||||
+ int(rank < remainder)
|
||||
+ int(opposite_rank < remainder)
|
||||
)
|
||||
max_local_tokens = max(per_rank_logical_tokens)
|
||||
else:
|
||||
if extend_seq_lens is None or len(extend_seq_lens) == 0:
|
||||
return None
|
||||
max_local_tokens = self._max_local_tokens(extend_seq_lens)
|
||||
align_size = get_cp_padding_align_size()
|
||||
return (max_local_tokens + align_size - 1) // align_size * align_size
|
||||
|
||||
@@ -217,7 +187,6 @@ class PrefillCPBCGInput:
|
||||
# Replay batches may reuse a ForwardBatch object whose metadata was
|
||||
# built for a different request layout. Always rebuild before sharding.
|
||||
forward_batch.attn_cp_metadata = None
|
||||
out_cache_loc = forward_batch.out_cache_loc
|
||||
prepare_cp_forward(forward_batch)
|
||||
|
||||
captured_local_tokens = None
|
||||
@@ -286,16 +255,6 @@ class PrefillCPBCGInput:
|
||||
forward_batch.positions = positions
|
||||
self.live_local_tokens = live_local_tokens
|
||||
|
||||
if isinstance(get_cp_strategy(), InterleaveCPStrategy):
|
||||
metadata = forward_batch.attn_cp_metadata
|
||||
cp_size = len(metadata.per_rank_actual_token)
|
||||
base, remainder = divmod(static_num_tokens, cp_size)
|
||||
metadata.total_seq_lens = static_num_tokens
|
||||
metadata.per_rank_logical_token = [
|
||||
base + int(rank < remainder) for rank in range(cp_size)
|
||||
]
|
||||
forward_batch.out_cache_loc = out_cache_loc
|
||||
|
||||
|
||||
def execute_prefill_cp_bcg(
|
||||
runner: PrefillCudaGraphRunner,
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Experimental, exact-shape interleave BCG adapter for GLM-5.3 Flash.
|
||||
|
||||
Keep full-sequence KPool positions separate from the local DSA positions.
|
||||
Prefix-free text batches replay when their total token count matches a capture,
|
||||
independently of request boundaries. Other batches retain eager CP, including MM.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.arg_groups.overrides import (
|
||||
model_config_of,
|
||||
resolved_view,
|
||||
resolving_view,
|
||||
)
|
||||
from sglang.srt.layers.cp.bcg.bcg import PrefillCPBCGInput
|
||||
from sglang.srt.layers.cp.utils import cp_interleave_input_ids
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
|
||||
class InterleaveCPBCGInput(PrefillCPBCGInput):
|
||||
@classmethod
|
||||
def create(cls, runner):
|
||||
result = super().create(runner)
|
||||
# Each local shard is CP-aligned before rank-major MoE all-gather.
|
||||
alignment = get_parallel().attn_cp_size ** 2
|
||||
capacity = (runner.max_num_tokens + alignment - 1) // alignment * alignment
|
||||
result.input_ids_global = torch.zeros(
|
||||
capacity, dtype=torch.int64, device=runner.device
|
||||
)
|
||||
return result
|
||||
|
||||
def allows_replay(self, batch_size, num_tokens, prefix_lens, contains_mm_inputs):
|
||||
return (
|
||||
batch_size > 0
|
||||
and num_tokens in self.bucket_local_tokens
|
||||
and not contains_mm_inputs
|
||||
and prefix_lens is not None
|
||||
and not any(prefix_lens)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def supports(server_args):
|
||||
cfg = resolving_view(server_args)
|
||||
return (
|
||||
cfg.dp_size == 1
|
||||
and cfg.ep_size == 1
|
||||
and resolved_view(server_args).attn_cp_size > 1
|
||||
and "Glm5NextForConditionalGeneration"
|
||||
in model_config_of(server_args).hf_config.architectures
|
||||
)
|
||||
|
||||
def _max_local_tokens(self, extend_seq_lens):
|
||||
size = get_parallel().attn_cp_size
|
||||
return (sum(int(length) for length in extend_seq_lens) + size - 1) // size
|
||||
|
||||
def model_positions(self, forward_batch):
|
||||
return self.positions[: self.live_local_tokens]
|
||||
|
||||
def prepare(self, runner, forward_batch, *, static_num_tokens, capture):
|
||||
global_positions = forward_batch.positions
|
||||
out_cache_loc = forward_batch.out_cache_loc
|
||||
super().prepare(
|
||||
runner, forward_batch, static_num_tokens=static_num_tokens, capture=capture
|
||||
)
|
||||
metadata = forward_batch.attn_cp_metadata
|
||||
cp_size = len(metadata.per_rank_actual_token)
|
||||
base, remainder = divmod(static_num_tokens, cp_size)
|
||||
metadata.total_seq_lens = static_num_tokens
|
||||
metadata.per_rank_logical_token = [
|
||||
base + int(rank < remainder) for rank in range(cp_size)
|
||||
]
|
||||
forward_batch.out_cache_loc = out_cache_loc
|
||||
# KPool compresses consecutive global tokens; only the model argument
|
||||
# (DSA RoPE) is local. Do not replace ForwardBatch's global positions.
|
||||
forward_batch.positions = global_positions
|
||||
ids = cp_interleave_input_ids(forward_batch.input_ids, forward_batch)
|
||||
self.input_ids_global[: ids.shape[0]].copy_(ids)
|
||||
forward_batch.input_ids_global = self.input_ids_global[: ids.shape[0]]
|
||||
|
||||
|
||||
def cp_interleave_indexer_rows(forward_batch):
|
||||
metadata = forward_batch.attn_cp_metadata
|
||||
rank = get_parallel().attn_cp_rank
|
||||
logical = metadata.per_rank_logical_token or metadata.per_rank_actual_token
|
||||
return metadata.per_rank_actual_token[rank], logical[rank]
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Zigzag-specific capture geometry for prefill context parallel CUDA graphs."""
|
||||
|
||||
from sglang.srt.layers.cp.base import get_cp_strategy
|
||||
from sglang.srt.layers.cp.bcg.bcg import PrefillCPBCGInput
|
||||
|
||||
|
||||
class ZigzagCPBCGInput(PrefillCPBCGInput):
|
||||
@staticmethod
|
||||
def min_capture_tokens(cp_size: int) -> int:
|
||||
return cp_size * 2
|
||||
|
||||
def _max_local_tokens(self, extend_seq_lens) -> int:
|
||||
cp_size = get_cp_strategy().cp_size
|
||||
cp_segment_num = cp_size * 2
|
||||
per_rank_logical_tokens = [0] * cp_size
|
||||
for raw_length in extend_seq_lens:
|
||||
base, remainder = divmod(int(raw_length), cp_segment_num)
|
||||
for rank in range(cp_size):
|
||||
opposite_rank = cp_segment_num - 1 - rank
|
||||
per_rank_logical_tokens[rank] += (
|
||||
base * 2 + int(rank < remainder) + int(opposite_rank < remainder)
|
||||
)
|
||||
return max(per_rank_logical_tokens)
|
||||
@@ -1059,6 +1059,7 @@ class KVCacheConfigurator:
|
||||
The pool factories decide whether a decline refuses the boot.
|
||||
Asymmetric K/V rows fuse only on backends that carry v_head_dim through
|
||||
to the kernel."""
|
||||
from sglang.srt.layers.cp.base import is_cp_enabled
|
||||
from sglang.srt.mem_cache.layout.fused_draft import (
|
||||
FusedDraftDecision,
|
||||
PlacementContext,
|
||||
@@ -1143,7 +1144,13 @@ class KVCacheConfigurator:
|
||||
profile = draft_kv_profile(
|
||||
aux.draft_model_config,
|
||||
num_layers=int(aux.draft_kv_num_layers),
|
||||
attn_tp_size=get_parallel().attn_tp_size,
|
||||
attn_tp_size=(
|
||||
get_parallel().tp_size
|
||||
if is_cp_enabled()
|
||||
and get_parallel().attn_dp_size == 1
|
||||
and self.spec_algorithm.is_dflash()
|
||||
else get_parallel().attn_tp_size
|
||||
),
|
||||
# A DFLASH-family draft config inherits the target's NEXTN depth
|
||||
# count; its draft is one block, replicated per runner.
|
||||
num_depths=None if self.spec_algorithm.is_eagle() else 1,
|
||||
|
||||
@@ -511,6 +511,15 @@ def capture_prefill_graph(
|
||||
if model_runner.is_draft_worker and not force_for_draft_worker:
|
||||
return result(None)
|
||||
|
||||
if model_runner.is_draft_worker and get_parallel().enable_prefill_cp:
|
||||
from sglang.srt.layers.cp.base import is_interleave
|
||||
|
||||
if is_interleave():
|
||||
# EagleWorkerV2 requests capture again with force_for_draft_worker=True
|
||||
# after init_lm_head, bypassing the guard above. The interleave adapter
|
||||
# supports only target prefill; draft CP must stay eager on that call too.
|
||||
return result(None)
|
||||
|
||||
# Skip prefill CG for EAGLE target on tc_piecewise when the fixed server
|
||||
# capture ceiling is below FULL. EAGLE target prefill requests FULL, so a
|
||||
# NULL or LAST graph is dead; capturing it can perturb FP4/TRTLLM-MoE
|
||||
|
||||
@@ -234,6 +234,7 @@ def _resolve_dflash_aux_hidden_state(
|
||||
config.dflash_draft_cell_size_per_token = _resolve_dflash_draft_cell_size(
|
||||
draft_model_config=draft_model_config,
|
||||
draft_num_layers=int(draft_num_layers),
|
||||
spec_algorithm=spec_algorithm,
|
||||
)
|
||||
|
||||
|
||||
@@ -241,6 +242,7 @@ def _resolve_dflash_draft_cell_size(
|
||||
*,
|
||||
draft_model_config: ModelConfig,
|
||||
draft_num_layers: int,
|
||||
spec_algorithm: SpeculativeAlgorithm,
|
||||
) -> int | None:
|
||||
"""Bytes/token the DFLASH draft KV pool will cost the target's pool budget.
|
||||
|
||||
@@ -269,8 +271,14 @@ def _resolve_dflash_draft_cell_size(
|
||||
draft_model_config=draft_model_config,
|
||||
draft_num_layers=draft_num_layers,
|
||||
draft_kv_cache_dtype=draft_kv_cache_dtype,
|
||||
# The draft pool shards KV heads like the target pool (by attention TP).
|
||||
tp_size=get_parallel().attn_tp_size,
|
||||
# CP targets run the DFLASH draft on full TP without CP.
|
||||
tp_size=(
|
||||
get_parallel().tp_size
|
||||
if spec_algorithm.is_dflash()
|
||||
and get_parallel().attn_cp_size > 1
|
||||
and get_parallel().attn_dp_size == 1
|
||||
else get_parallel().attn_tp_size
|
||||
),
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning(
|
||||
|
||||
@@ -56,13 +56,9 @@ from sglang.srt.configs.model_config import is_qwen4_exp
|
||||
from sglang.srt.distributed.parallel_state import graph_capture
|
||||
from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds
|
||||
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
|
||||
from sglang.srt.layers.cp.bcg import (
|
||||
from sglang.srt.layers.cp.bcg import supports_prefill_cp_bcg
|
||||
from sglang.srt.layers.cp.bcg.bcg import (
|
||||
PrefillCPBCGInput,
|
||||
)
|
||||
from sglang.srt.layers.cp.bcg import (
|
||||
enable_cp_bcg_capture as should_enable_cp_bcg_capture,
|
||||
)
|
||||
from sglang.srt.layers.cp.bcg import (
|
||||
execute_prefill_cp_bcg,
|
||||
filter_prefill_cp_bcg_capture_num_tokens,
|
||||
)
|
||||
@@ -612,7 +608,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
server_args = model_runner.server_args
|
||||
self.enable_cp_bcg_capture = isinstance(
|
||||
self.backend, BreakableCudaGraphBackend
|
||||
) and should_enable_cp_bcg_capture(server_args)
|
||||
) and supports_prefill_cp_bcg(server_args)
|
||||
if self.enable_cp_bcg_capture:
|
||||
if self.max_context_size is not None:
|
||||
# TODO(SYChen123): Preserve max_seq_len_override through CP's padded
|
||||
@@ -784,6 +780,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
|
||||
def _get_layer_model_positions(self, forward_batch: ForwardBatch) -> torch.Tensor:
|
||||
"""Mirror outer multimodal wrappers when BCG captures layer_model directly."""
|
||||
if self.enable_cp_bcg_capture:
|
||||
cp_positions = self.prefill_cp_bcg_input.model_positions(forward_batch)
|
||||
if cp_positions is not None:
|
||||
return cp_positions
|
||||
if forward_batch.mrope_positions is None:
|
||||
return forward_batch.positions
|
||||
|
||||
@@ -1336,6 +1336,12 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
``capture_hidden_mode=None`` when unknown at the call site (it is
|
||||
rank-uniform; forward-time-only checking cannot split the group).
|
||||
"""
|
||||
if getattr(
|
||||
self, "enable_cp_bcg_capture", False
|
||||
) and not self.prefill_cp_bcg_input.allows_replay(
|
||||
batch_size, num_tokens, prefix_lens, contains_mm_inputs
|
||||
):
|
||||
return False
|
||||
if contains_mm_inputs and (
|
||||
self._qwen_bcg_hc_sidechannel or self._qwen_bcg_mtp_draft
|
||||
):
|
||||
|
||||
@@ -411,10 +411,11 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
self._tp_sync = SpecTpSync(get_dp_tp_group())
|
||||
check_simulate_acc_token_mode()
|
||||
|
||||
# Under dp attention, the draft worker runs on the per-DP attn-TP
|
||||
# group, independent of idle peer DP ranks; it is built and run under
|
||||
# the same placement.
|
||||
self.draft_owns_attention = get_parallel().attn_dp_enabled
|
||||
# DP drafts use per-DP attention TP, independent of idle peers. CP
|
||||
# drafts use full TP without CP, including their backend and KV pool.
|
||||
self.draft_owns_attention = (
|
||||
get_parallel().attn_dp_enabled or get_parallel().attn_cp_size > 1
|
||||
)
|
||||
# Inside the draft scope the context answers the draft's narrowed rank.
|
||||
self._target_tp_rank = get_parallel().tp_rank
|
||||
with draft_pp_context(), draft_tp_context(self.draft_owns_attention):
|
||||
@@ -657,13 +658,14 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
# enabled, the draft worker keeps a private compact req->token table
|
||||
# over the same global KV index space, so radix-cache/prefix-hit KV
|
||||
# remains reusable while draft attention sees only the recent window.
|
||||
self._draft_worker.alloc_memory_pool(
|
||||
memory_pool_config=memory_pool_config,
|
||||
req_to_token_pool=(
|
||||
None if self.use_compact_draft_cache else req_to_token_pool
|
||||
),
|
||||
token_to_kv_pool_allocator=token_to_kv_pool_allocator,
|
||||
)
|
||||
with draft_tp_context(self.draft_owns_attention):
|
||||
self._draft_worker.alloc_memory_pool(
|
||||
memory_pool_config=memory_pool_config,
|
||||
req_to_token_pool=(
|
||||
None if self.use_compact_draft_cache else req_to_token_pool
|
||||
),
|
||||
token_to_kv_pool_allocator=token_to_kv_pool_allocator,
|
||||
)
|
||||
|
||||
def init_attention_backends(self):
|
||||
with (
|
||||
|
||||
@@ -734,15 +734,20 @@ def draft_tp_context(owns_attention: bool):
|
||||
"""Enter the TP placement that draft work runs under.
|
||||
|
||||
Work that owns attention (an attention-owning draft, or DSpark's DP-MoE
|
||||
sync) runs on the target's attention-TP group; other draft work keeps the
|
||||
target's TP placement. Enter it from outside any draft scope, where
|
||||
``attn_tp_group`` is still the target's.
|
||||
sync) runs on the target's attention-TP group. With prefill CP and no
|
||||
attention DP it uses full TP as ordinary attention TP, matching the borrowed
|
||||
target LM head. Other draft work keeps the target's TP placement.
|
||||
Enter it from outside any draft scope.
|
||||
"""
|
||||
if not owns_attention:
|
||||
return contextlib.nullcontext()
|
||||
return patch_tensor_parallel_group(
|
||||
get_parallel().attn_tp_group, owns_attention=True
|
||||
parallel = get_parallel()
|
||||
group = (
|
||||
parallel.tp_group
|
||||
if parallel.attn_cp_size > 1 and parallel.attn_dp_size == 1
|
||||
else parallel.attn_tp_group
|
||||
)
|
||||
return patch_tensor_parallel_group(group, owns_attention=True)
|
||||
|
||||
|
||||
def spec_stage_span(name: str):
|
||||
|
||||
Reference in New Issue
Block a user