From f4035cab0be4dd54099eeb044a701ad9aeecb5b4 Mon Sep 17 00:00:00 2001 From: Baizhou Zhang Date: Wed, 21 Oct 2026 15:51:28 -0700 Subject: [PATCH] [Feat] cp: Enable GLM-5.3 Flash interleave BCG with batched prefill and DFlash2 (#43182) --- .../kernels/ops/attention/dsa/cp_split.py | 3 +- .../arg_groups/cp_tp_group_sharing_hook.py | 24 ++-- python/sglang/srt/arg_groups/model_hook.py | 24 ++-- .../arg_groups/model_overrides/deepseek_v2.py | 3 +- .../attention/dsa/kpool_prefill_cuda_graph.py | 7 +- python/sglang/srt/layers/cp/bcg/__init__.py | 64 +++++++++++ python/sglang/srt/layers/cp/{ => bcg}/bcg.py | 107 ++++++------------ .../srt/layers/cp/bcg/interleave_bcg.py | 85 ++++++++++++++ python/sglang/srt/layers/cp/bcg/zigzag_bcg.py | 23 ++++ .../srt/mem_cache/kv_cache_configurator.py | 9 +- .../cuda_graph_setup.py | 9 ++ .../spec_aux_hidden_state.py | 12 +- .../runner/prefill_cuda_graph_runner.py | 20 ++-- .../srt/speculative/dflash_worker_v2.py | 24 ++-- python/sglang/srt/speculative/spec_utils.py | 15 ++- test/registered/cp/test_cp_strategy_unit.py | 44 +++++-- test/registered/cp/test_glm53_flash_cp.py | 18 +-- .../unit/spec/test_draft_tp_context.py | 12 ++ 18 files changed, 360 insertions(+), 143 deletions(-) create mode 100644 python/sglang/srt/layers/cp/bcg/__init__.py rename python/sglang/srt/layers/cp/{ => bcg}/bcg.py (79%) create mode 100644 python/sglang/srt/layers/cp/bcg/interleave_bcg.py create mode 100644 python/sglang/srt/layers/cp/bcg/zigzag_bcg.py diff --git a/python/sglang/kernels/ops/attention/dsa/cp_split.py b/python/sglang/kernels/ops/attention/dsa/cp_split.py index d346facfc1..a18dede618 100644 --- a/python/sglang/kernels/ops/attention/dsa/cp_split.py +++ b/python/sglang/kernels/ops/attention/dsa/cp_split.py @@ -17,7 +17,8 @@ def _dsa_cp_interleave_q_seqs_kernel( cp_size: tl.constexpr, cp_rank: tl.constexpr, ): - extra_seq = 0 + # Eager metadata uses int32, while prefill graph buffers may use int64. + extra_seq = tl.full((), 0, in_seqs_ptr.dtype.element_ty) bs_idx = 0 for bs in range(tokens): cur_len = tl.load(in_seqs_ptr + bs) diff --git a/python/sglang/srt/arg_groups/cp_tp_group_sharing_hook.py b/python/sglang/srt/arg_groups/cp_tp_group_sharing_hook.py index 8a1b51e0ad..d5bdc80dae 100644 --- a/python/sglang/srt/arg_groups/cp_tp_group_sharing_hook.py +++ b/python/sglang/srt/arg_groups/cp_tp_group_sharing_hook.py @@ -47,21 +47,21 @@ def resolve_cp_tp_group_sharing(server_args: Any, model: Any) -> None: raise ValueError(f"CP-TP group sharing requires --cp-strategy {strategy}.") # This hook runs before speculative defaults and NEXTN alias resolution. - # Native MTP keeps the target's CP/TP placement; other draft architectures - # have their own topology and model-boundary contracts. Only GLM-5.3-Flash - # has its MTP ported. + # Native MTP keeps the target's CP/TP placement. DFLASH consumes the + # target's gathered hidden states and runs its draft with ordinary TP. + # Only GLM-5.3-Flash has these speculative paths ported. if cfg.speculative_algorithm is not None: if model.hf_config.architectures[0] != "Glm5NextForConditionalGeneration": raise ValueError( "--speculative-algorithm is not supported with CP-TP group sharing." ) - if ( + if cfg.speculative_algorithm != "DFLASH" and ( cfg.speculative_algorithm.upper() not in ("EAGLE", "NEXTN") or cfg.speculative_draft_model_path not in (None, cfg.model_path) or cfg.speculative_eagle_topk not in (None, 1) ): raise ValueError( - "CP/TP group sharing only supports native MTP with top-k 1." + "CP/TP group sharing only supports DFLASH or native MTP with top-k 1." ) unsupported = { @@ -108,12 +108,10 @@ def resolve_cp_tp_group_sharing(server_args: Any, model: Any) -> None: f"{name}={count} must be divisible by --tp-size={cfg.tp_size}." ) - declare_resolution( - server_args, - "resolve_cp_tp_group_sharing", - enable_cp_tp_group_sharing=True, - # The sequence collectives have batch-dependent sizes. - cuda_graph_config=with_phase( + fields = {"enable_cp_tp_group_sharing": True} + if model.hf_config.architectures[0] == "Qwen4ExpForConditionalGeneration": + # Qwen's sequence collectives still have batch-dependent sizes. + fields["cuda_graph_config"] = with_phase( cfg.cuda_graph_config, Phase.PREFILL, backend=Backend.DISABLED - ), - ) + ) + declare_resolution(server_args, "resolve_cp_tp_group_sharing", **fields) diff --git a/python/sglang/srt/arg_groups/model_hook.py b/python/sglang/srt/arg_groups/model_hook.py index eeb90fceed..67407f3992 100644 --- a/python/sglang/srt/arg_groups/model_hook.py +++ b/python/sglang/srt/arg_groups/model_hook.py @@ -305,15 +305,21 @@ def handle_model_specific_adjustments(server_args: Any): # The DSA CP field declarations moved to the override # registry (arg_groups/overrides.py: # _deepseek_family_overrides). - declare_resolution( - server_args, - "_handle_model_specific_adjustments", - cuda_graph_config=with_phase( - cfg.cuda_graph_config, - Phase.PREFILL, - backend=Backend.DISABLED, - ), - ) + from sglang.srt.layers.cp.bcg import supports_prefill_cp_bcg + + if not ( + cfg.cuda_graph_config.prefill.backend == Backend.BREAKABLE + and supports_prefill_cp_bcg(server_args) + ): + declare_resolution( + server_args, + "_handle_model_specific_adjustments", + cuda_graph_config=with_phase( + cfg.cuda_graph_config, + Phase.PREFILL, + backend=Backend.DISABLED, + ), + ) else: # Pure TP and partial DP Attention mode is active for DSA, logging a warning attn_tp_size = derive_attn_tp_size( diff --git a/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py b/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py index a620c1eeba..f0e02c2b9a 100644 --- a/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py +++ b/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py @@ -139,8 +139,7 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict: f"ep_size={overrides.get('ep_size', cfg.ep_size)}, tp_size={cfg.tp_size}, " f"attn_cp_size={attn_cp_size}, " f"kv_cache_dtype={cfg.kv_cache_dtype}, " - f"moe_a2a_backend={overrides.get('moe_a2a_backend', cfg.moe_a2a_backend)}, " - f"cuda_graph_config[prefill].backend=disabled" + f"moe_a2a_backend={overrides.get('moe_a2a_backend', cfg.moe_a2a_backend)}" ) # Deferred import to avoid a circular import at module-load diff --git a/python/sglang/srt/layers/attention/dsa/kpool_prefill_cuda_graph.py b/python/sglang/srt/layers/attention/dsa/kpool_prefill_cuda_graph.py index 385607af6f..6fc4f8f827 100644 --- a/python/sglang/srt/layers/attention/dsa/kpool_prefill_cuda_graph.py +++ b/python/sglang/srt/layers/attention/dsa/kpool_prefill_cuda_graph.py @@ -2,6 +2,7 @@ import torch +from sglang.srt.layers.cp.interleave import is_interleave_extend from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import ( eager_on_graph, ) @@ -22,6 +23,11 @@ def _kpool_indexer_prefill_with_output( # Resolve the live batch inside the eager break, never from capture args. forward_batch = get_tc_piecewise_forward_context().forward_batch n = forward_batch.extend_num_tokens + num_logical = sum(forward_batch.extend_seq_lens_cpu) + if is_interleave_extend(forward_batch): + from sglang.srt.layers.cp.bcg.interleave_bcg import cp_interleave_indexer_rows + + n, num_logical = cp_interleave_indexer_rows(forward_batch) if n is None or not 0 <= n <= x.shape[0]: raise ValueError(f"Invalid pooled-indexer prefill token count: {n}") if n > q_lora.shape[0] or n > positions.shape[0]: @@ -37,7 +43,6 @@ def _kpool_indexer_prefill_with_output( ) if not return_indices: return - num_logical = sum(forward_batch.extend_seq_lens_cpu) if ( result is None or result.ndim != 2 diff --git a/python/sglang/srt/layers/cp/bcg/__init__.py b/python/sglang/srt/layers/cp/bcg/__init__.py new file mode 100644 index 0000000000..880dd3f9b9 --- /dev/null +++ b/python/sglang/srt/layers/cp/bcg/__init__.py @@ -0,0 +1,64 @@ +"""Strategy dispatch for prefill context parallel CUDA graphs.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sglang.srt.arg_groups.overrides import ( + attention_backends_of, + resolved_view, + resolving_view, +) +from sglang.srt.layers.cp.base import ( + ContextParallelStrategyKind, + CPAttentionBackendKind, +) + +if TYPE_CHECKING: + from sglang.srt.server_args import ServerArgs + + +PREFILL_CP_BCG_SUPPORTED_COMBINATIONS: set[ + tuple[ContextParallelStrategyKind, CPAttentionBackendKind] +] = { + (ContextParallelStrategyKind.ZIGZAG, CPAttentionBackendKind.TRTLLM_MHA), + (ContextParallelStrategyKind.INTERLEAVE, CPAttentionBackendKind.DSA), +} + + +def get_cp_bcg_input_type(strategy: ContextParallelStrategyKind): + # Keep runtime adapters lazy: config resolution also queries CP BCG support. + if strategy == ContextParallelStrategyKind.INTERLEAVE: + from sglang.srt.layers.cp.bcg.interleave_bcg import InterleaveCPBCGInput + + return InterleaveCPBCGInput + if strategy == ContextParallelStrategyKind.ZIGZAG: + from sglang.srt.layers.cp.bcg.zigzag_bcg import ZigzagCPBCGInput + + return ZigzagCPBCGInput + raise ValueError(f"Unsupported CP BCG strategy: {strategy}") + + +def supports_prefill_cp_bcg(server_args: ServerArgs) -> bool: + """Return whether the selected prefill-CP configuration supports BCG.""" + + cfg = resolving_view(server_args) + resolved = resolved_view(server_args) + if not ( + cfg.enable_prefill_cp + and cfg.pp_size == 1 + and resolved.attn_cp_size == cfg.tp_size + ): + return False + prefill_attention_backend, _ = attention_backends_of(resolved) + try: + combination = ( + ContextParallelStrategyKind.from_string(cfg.cp_strategy), + CPAttentionBackendKind.from_string(prefill_attention_backend), + ) + except ValueError: + # Other backends (or an unresolved backend/strategy) cannot use CP BCG. + return False + if combination not in PREFILL_CP_BCG_SUPPORTED_COMBINATIONS: + return False + return get_cp_bcg_input_type(combination[0]).supports(server_args) diff --git a/python/sglang/srt/layers/cp/bcg.py b/python/sglang/srt/layers/cp/bcg/bcg.py similarity index 79% rename from python/sglang/srt/layers/cp/bcg.py rename to python/sglang/srt/layers/cp/bcg/bcg.py index c6b7798e10..5cfc835e90 100644 --- a/python/sglang/srt/layers/cp/bcg.py +++ b/python/sglang/srt/layers/cp/bcg/bcg.py @@ -22,23 +22,19 @@ from typing import TYPE_CHECKING, Any, Dict, Optional import torch from sglang.srt.arg_groups.overrides import ( - attention_backends_of, resolved_view, resolving_view, ) from sglang.srt.layers.cp.base import ( ContextParallelStrategyKind, - CPAttentionBackendKind, - get_cp_strategy, + get_cp_strategy_kind, ) -from sglang.srt.layers.cp.interleave import InterleaveCPStrategy from sglang.srt.layers.cp.padding import get_cp_padding_align_size from sglang.srt.layers.cp.utils import ( cp_gather_after_forward, cp_split_before_forward, prepare_cp_forward, ) -from sglang.srt.layers.cp.zigzag import ZigzagCPStrategy from sglang.srt.model_executor.forward_batch_info import PPProxyTensors if TYPE_CHECKING: @@ -49,48 +45,18 @@ if TYPE_CHECKING: from sglang.srt.server_args import ServerArgs -PREFILL_CP_BCG_SUPPORTED_COMBINATIONS: set[ - tuple[ContextParallelStrategyKind, CPAttentionBackendKind] -] = { - (ContextParallelStrategyKind.ZIGZAG, CPAttentionBackendKind.TRTLLM_MHA), -} - - -def supports_prefill_cp_bcg(server_args: ServerArgs) -> bool: - """Return whether the selected prefill-CP configuration supports BCG.""" - - cfg = resolving_view(server_args) - resolved = resolved_view(server_args) - if not ( - cfg.enable_prefill_cp - and cfg.pp_size == 1 - and resolved.attn_cp_size == cfg.tp_size - ): - return False - prefill_attention_backend, _ = attention_backends_of(resolved) - try: - combination = ( - ContextParallelStrategyKind.from_string(cfg.cp_strategy), - CPAttentionBackendKind.from_string(prefill_attention_backend), - ) - except ValueError: - # Other backends (or an unresolved backend/strategy) cannot use CP BCG. - return False - return combination in PREFILL_CP_BCG_SUPPORTED_COMBINATIONS - - -def enable_cp_bcg_capture(server_args: ServerArgs) -> bool: - """Return whether CP breakable prefill capture is enabled.""" - return supports_prefill_cp_bcg(server_args) - - def filter_prefill_cp_bcg_capture_num_tokens( capture_num_tokens: list[int], server_args: ServerArgs ) -> list[int]: """Keep only token buckets where the selected CP strategy can run.""" - min_num_tokens = resolved_view(server_args).attn_cp_size - if resolving_view(server_args).cp_strategy == "zigzag": - min_num_tokens *= 2 + from sglang.srt.layers.cp.bcg import get_cp_bcg_input_type + + input_type = get_cp_bcg_input_type( + ContextParallelStrategyKind.from_string(resolving_view(server_args).cp_strategy) + ) + min_num_tokens = input_type.min_capture_tokens( + resolved_view(server_args).attn_cp_size + ) filtered = [size for size in capture_num_tokens if size >= min_num_tokens] if not filtered: raise ValueError( @@ -123,6 +89,10 @@ class PrefillCPBCGInput: @classmethod def create(cls, runner: PrefillCudaGraphRunner) -> PrefillCPBCGInput: + if cls is PrefillCPBCGInput: + from sglang.srt.layers.cp.bcg import get_cp_bcg_input_type + + return get_cp_bcg_input_type(get_cp_strategy_kind()).create(runner) with torch.device(runner.device): return cls( input_embeds=torch.zeros( @@ -138,29 +108,29 @@ class PrefillCPBCGInput: ), ) + def allows_replay(self, batch_size, num_tokens, prefix_lens, contains_mm_inputs): + return True + + def model_positions(self, forward_batch): + # No override: retain the runner's existing model/mRoPE selection. + return None + + @staticmethod + def supports(server_args: ServerArgs) -> bool: + return True + + @staticmethod + def min_capture_tokens(cp_size: int) -> int: + return cp_size + + def _max_local_tokens(self, extend_seq_lens: Any) -> int: + raise NotImplementedError + def required_local_tokens(self, extend_seq_lens: Any) -> Optional[int]: """Return the largest aligned local shard, uniformly across CP ranks.""" - strategy = get_cp_strategy() - if extend_seq_lens is None: - return None - if isinstance(strategy, InterleaveCPStrategy): - num_tokens = sum(int(length) for length in extend_seq_lens) - max_local_tokens = (num_tokens + strategy.cp_size - 1) // strategy.cp_size - elif isinstance(strategy, ZigzagCPStrategy): - cp_segment_num = strategy.cp_size * 2 - per_rank_logical_tokens = [0] * strategy.cp_size - for raw_length in extend_seq_lens: - base, remainder = divmod(int(raw_length), cp_segment_num) - for rank in range(strategy.cp_size): - opposite_rank = cp_segment_num - 1 - rank - per_rank_logical_tokens[rank] += ( - base * 2 - + int(rank < remainder) - + int(opposite_rank < remainder) - ) - max_local_tokens = max(per_rank_logical_tokens) - else: + if extend_seq_lens is None or len(extend_seq_lens) == 0: return None + max_local_tokens = self._max_local_tokens(extend_seq_lens) align_size = get_cp_padding_align_size() return (max_local_tokens + align_size - 1) // align_size * align_size @@ -217,7 +187,6 @@ class PrefillCPBCGInput: # Replay batches may reuse a ForwardBatch object whose metadata was # built for a different request layout. Always rebuild before sharding. forward_batch.attn_cp_metadata = None - out_cache_loc = forward_batch.out_cache_loc prepare_cp_forward(forward_batch) captured_local_tokens = None @@ -286,16 +255,6 @@ class PrefillCPBCGInput: forward_batch.positions = positions self.live_local_tokens = live_local_tokens - if isinstance(get_cp_strategy(), InterleaveCPStrategy): - metadata = forward_batch.attn_cp_metadata - cp_size = len(metadata.per_rank_actual_token) - base, remainder = divmod(static_num_tokens, cp_size) - metadata.total_seq_lens = static_num_tokens - metadata.per_rank_logical_token = [ - base + int(rank < remainder) for rank in range(cp_size) - ] - forward_batch.out_cache_loc = out_cache_loc - def execute_prefill_cp_bcg( runner: PrefillCudaGraphRunner, diff --git a/python/sglang/srt/layers/cp/bcg/interleave_bcg.py b/python/sglang/srt/layers/cp/bcg/interleave_bcg.py new file mode 100644 index 0000000000..d90299cfcd --- /dev/null +++ b/python/sglang/srt/layers/cp/bcg/interleave_bcg.py @@ -0,0 +1,85 @@ +"""Experimental, exact-shape interleave BCG adapter for GLM-5.3 Flash. + +Keep full-sequence KPool positions separate from the local DSA positions. +Prefix-free text batches replay when their total token count matches a capture, +independently of request boundaries. Other batches retain eager CP, including MM. +""" + +import torch + +from sglang.srt.arg_groups.overrides import ( + model_config_of, + resolved_view, + resolving_view, +) +from sglang.srt.layers.cp.bcg.bcg import PrefillCPBCGInput +from sglang.srt.layers.cp.utils import cp_interleave_input_ids +from sglang.srt.runtime_context import get_parallel + + +class InterleaveCPBCGInput(PrefillCPBCGInput): + @classmethod + def create(cls, runner): + result = super().create(runner) + # Each local shard is CP-aligned before rank-major MoE all-gather. + alignment = get_parallel().attn_cp_size ** 2 + capacity = (runner.max_num_tokens + alignment - 1) // alignment * alignment + result.input_ids_global = torch.zeros( + capacity, dtype=torch.int64, device=runner.device + ) + return result + + def allows_replay(self, batch_size, num_tokens, prefix_lens, contains_mm_inputs): + return ( + batch_size > 0 + and num_tokens in self.bucket_local_tokens + and not contains_mm_inputs + and prefix_lens is not None + and not any(prefix_lens) + ) + + @staticmethod + def supports(server_args): + cfg = resolving_view(server_args) + return ( + cfg.dp_size == 1 + and cfg.ep_size == 1 + and resolved_view(server_args).attn_cp_size > 1 + and "Glm5NextForConditionalGeneration" + in model_config_of(server_args).hf_config.architectures + ) + + def _max_local_tokens(self, extend_seq_lens): + size = get_parallel().attn_cp_size + return (sum(int(length) for length in extend_seq_lens) + size - 1) // size + + def model_positions(self, forward_batch): + return self.positions[: self.live_local_tokens] + + def prepare(self, runner, forward_batch, *, static_num_tokens, capture): + global_positions = forward_batch.positions + out_cache_loc = forward_batch.out_cache_loc + super().prepare( + runner, forward_batch, static_num_tokens=static_num_tokens, capture=capture + ) + metadata = forward_batch.attn_cp_metadata + cp_size = len(metadata.per_rank_actual_token) + base, remainder = divmod(static_num_tokens, cp_size) + metadata.total_seq_lens = static_num_tokens + metadata.per_rank_logical_token = [ + base + int(rank < remainder) for rank in range(cp_size) + ] + forward_batch.out_cache_loc = out_cache_loc + # KPool compresses consecutive global tokens; only the model argument + # (DSA RoPE) is local. Do not replace ForwardBatch's global positions. + forward_batch.positions = global_positions + ids = cp_interleave_input_ids(forward_batch.input_ids, forward_batch) + self.input_ids_global[: ids.shape[0]].copy_(ids) + forward_batch.input_ids_global = self.input_ids_global[: ids.shape[0]] + + +def cp_interleave_indexer_rows(forward_batch): + metadata = forward_batch.attn_cp_metadata + rank = get_parallel().attn_cp_rank + logical = metadata.per_rank_logical_token or metadata.per_rank_actual_token + return metadata.per_rank_actual_token[rank], logical[rank] diff --git a/python/sglang/srt/layers/cp/bcg/zigzag_bcg.py b/python/sglang/srt/layers/cp/bcg/zigzag_bcg.py new file mode 100644 index 0000000000..813c99ef35 --- /dev/null +++ b/python/sglang/srt/layers/cp/bcg/zigzag_bcg.py @@ -0,0 +1,23 @@ +"""Zigzag-specific capture geometry for prefill context parallel CUDA graphs.""" + +from sglang.srt.layers.cp.base import get_cp_strategy +from sglang.srt.layers.cp.bcg.bcg import PrefillCPBCGInput + + +class ZigzagCPBCGInput(PrefillCPBCGInput): + @staticmethod + def min_capture_tokens(cp_size: int) -> int: + return cp_size * 2 + + def _max_local_tokens(self, extend_seq_lens) -> int: + cp_size = get_cp_strategy().cp_size + cp_segment_num = cp_size * 2 + per_rank_logical_tokens = [0] * cp_size + for raw_length in extend_seq_lens: + base, remainder = divmod(int(raw_length), cp_segment_num) + for rank in range(cp_size): + opposite_rank = cp_segment_num - 1 - rank + per_rank_logical_tokens[rank] += ( + base * 2 + int(rank < remainder) + int(opposite_rank < remainder) + ) + return max(per_rank_logical_tokens) diff --git a/python/sglang/srt/mem_cache/kv_cache_configurator.py b/python/sglang/srt/mem_cache/kv_cache_configurator.py index 660941cbc6..a0d9e1e233 100644 --- a/python/sglang/srt/mem_cache/kv_cache_configurator.py +++ b/python/sglang/srt/mem_cache/kv_cache_configurator.py @@ -1059,6 +1059,7 @@ class KVCacheConfigurator: The pool factories decide whether a decline refuses the boot. Asymmetric K/V rows fuse only on backends that carry v_head_dim through to the kernel.""" + from sglang.srt.layers.cp.base import is_cp_enabled from sglang.srt.mem_cache.layout.fused_draft import ( FusedDraftDecision, PlacementContext, @@ -1143,7 +1144,13 @@ class KVCacheConfigurator: profile = draft_kv_profile( aux.draft_model_config, num_layers=int(aux.draft_kv_num_layers), - attn_tp_size=get_parallel().attn_tp_size, + attn_tp_size=( + get_parallel().tp_size + if is_cp_enabled() + and get_parallel().attn_dp_size == 1 + and self.spec_algorithm.is_dflash() + else get_parallel().attn_tp_size + ), # A DFLASH-family draft config inherits the target's NEXTN depth # count; its draft is one block, replicated per runner. num_depths=None if self.spec_algorithm.is_eagle() else 1, diff --git a/python/sglang/srt/model_executor/model_runner_components/cuda_graph_setup.py b/python/sglang/srt/model_executor/model_runner_components/cuda_graph_setup.py index 49c8191fc3..6e4ad28ca7 100644 --- a/python/sglang/srt/model_executor/model_runner_components/cuda_graph_setup.py +++ b/python/sglang/srt/model_executor/model_runner_components/cuda_graph_setup.py @@ -511,6 +511,15 @@ def capture_prefill_graph( if model_runner.is_draft_worker and not force_for_draft_worker: return result(None) + if model_runner.is_draft_worker and get_parallel().enable_prefill_cp: + from sglang.srt.layers.cp.base import is_interleave + + if is_interleave(): + # EagleWorkerV2 requests capture again with force_for_draft_worker=True + # after init_lm_head, bypassing the guard above. The interleave adapter + # supports only target prefill; draft CP must stay eager on that call too. + return result(None) + # Skip prefill CG for EAGLE target on tc_piecewise when the fixed server # capture ceiling is below FULL. EAGLE target prefill requests FULL, so a # NULL or LAST graph is dead; capturing it can perturb FP4/TRTLLM-MoE diff --git a/python/sglang/srt/model_executor/model_runner_components/spec_aux_hidden_state.py b/python/sglang/srt/model_executor/model_runner_components/spec_aux_hidden_state.py index d8a70ce57e..a4cbabc75e 100644 --- a/python/sglang/srt/model_executor/model_runner_components/spec_aux_hidden_state.py +++ b/python/sglang/srt/model_executor/model_runner_components/spec_aux_hidden_state.py @@ -234,6 +234,7 @@ def _resolve_dflash_aux_hidden_state( config.dflash_draft_cell_size_per_token = _resolve_dflash_draft_cell_size( draft_model_config=draft_model_config, draft_num_layers=int(draft_num_layers), + spec_algorithm=spec_algorithm, ) @@ -241,6 +242,7 @@ def _resolve_dflash_draft_cell_size( *, draft_model_config: ModelConfig, draft_num_layers: int, + spec_algorithm: SpeculativeAlgorithm, ) -> int | None: """Bytes/token the DFLASH draft KV pool will cost the target's pool budget. @@ -269,8 +271,14 @@ def _resolve_dflash_draft_cell_size( draft_model_config=draft_model_config, draft_num_layers=draft_num_layers, draft_kv_cache_dtype=draft_kv_cache_dtype, - # The draft pool shards KV heads like the target pool (by attention TP). - tp_size=get_parallel().attn_tp_size, + # CP targets run the DFLASH draft on full TP without CP. + tp_size=( + get_parallel().tp_size + if spec_algorithm.is_dflash() + and get_parallel().attn_cp_size > 1 + and get_parallel().attn_dp_size == 1 + else get_parallel().attn_tp_size + ), ) except Exception as e: # noqa: BLE001 logger.warning( diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index fc7e93b9f2..ead3da2d6e 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -56,13 +56,9 @@ from sglang.srt.configs.model_config import is_qwen4_exp from sglang.srt.distributed.parallel_state import graph_capture from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp -from sglang.srt.layers.cp.bcg import ( +from sglang.srt.layers.cp.bcg import supports_prefill_cp_bcg +from sglang.srt.layers.cp.bcg.bcg import ( PrefillCPBCGInput, -) -from sglang.srt.layers.cp.bcg import ( - enable_cp_bcg_capture as should_enable_cp_bcg_capture, -) -from sglang.srt.layers.cp.bcg import ( execute_prefill_cp_bcg, filter_prefill_cp_bcg_capture_num_tokens, ) @@ -612,7 +608,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): server_args = model_runner.server_args self.enable_cp_bcg_capture = isinstance( self.backend, BreakableCudaGraphBackend - ) and should_enable_cp_bcg_capture(server_args) + ) and supports_prefill_cp_bcg(server_args) if self.enable_cp_bcg_capture: if self.max_context_size is not None: # TODO(SYChen123): Preserve max_seq_len_override through CP's padded @@ -784,6 +780,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): def _get_layer_model_positions(self, forward_batch: ForwardBatch) -> torch.Tensor: """Mirror outer multimodal wrappers when BCG captures layer_model directly.""" + if self.enable_cp_bcg_capture: + cp_positions = self.prefill_cp_bcg_input.model_positions(forward_batch) + if cp_positions is not None: + return cp_positions if forward_batch.mrope_positions is None: return forward_batch.positions @@ -1336,6 +1336,12 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): ``capture_hidden_mode=None`` when unknown at the call site (it is rank-uniform; forward-time-only checking cannot split the group). """ + if getattr( + self, "enable_cp_bcg_capture", False + ) and not self.prefill_cp_bcg_input.allows_replay( + batch_size, num_tokens, prefix_lens, contains_mm_inputs + ): + return False if contains_mm_inputs and ( self._qwen_bcg_hc_sidechannel or self._qwen_bcg_mtp_draft ): diff --git a/python/sglang/srt/speculative/dflash_worker_v2.py b/python/sglang/srt/speculative/dflash_worker_v2.py index 36596e2026..404620e2b7 100644 --- a/python/sglang/srt/speculative/dflash_worker_v2.py +++ b/python/sglang/srt/speculative/dflash_worker_v2.py @@ -411,10 +411,11 @@ class DFlashWorkerV2(BaseSpecWorker): self._tp_sync = SpecTpSync(get_dp_tp_group()) check_simulate_acc_token_mode() - # Under dp attention, the draft worker runs on the per-DP attn-TP - # group, independent of idle peer DP ranks; it is built and run under - # the same placement. - self.draft_owns_attention = get_parallel().attn_dp_enabled + # DP drafts use per-DP attention TP, independent of idle peers. CP + # drafts use full TP without CP, including their backend and KV pool. + self.draft_owns_attention = ( + get_parallel().attn_dp_enabled or get_parallel().attn_cp_size > 1 + ) # Inside the draft scope the context answers the draft's narrowed rank. self._target_tp_rank = get_parallel().tp_rank with draft_pp_context(), draft_tp_context(self.draft_owns_attention): @@ -657,13 +658,14 @@ class DFlashWorkerV2(BaseSpecWorker): # enabled, the draft worker keeps a private compact req->token table # over the same global KV index space, so radix-cache/prefix-hit KV # remains reusable while draft attention sees only the recent window. - self._draft_worker.alloc_memory_pool( - memory_pool_config=memory_pool_config, - req_to_token_pool=( - None if self.use_compact_draft_cache else req_to_token_pool - ), - token_to_kv_pool_allocator=token_to_kv_pool_allocator, - ) + with draft_tp_context(self.draft_owns_attention): + self._draft_worker.alloc_memory_pool( + memory_pool_config=memory_pool_config, + req_to_token_pool=( + None if self.use_compact_draft_cache else req_to_token_pool + ), + token_to_kv_pool_allocator=token_to_kv_pool_allocator, + ) def init_attention_backends(self): with ( diff --git a/python/sglang/srt/speculative/spec_utils.py b/python/sglang/srt/speculative/spec_utils.py index 40e595591d..5d56e1fd25 100644 --- a/python/sglang/srt/speculative/spec_utils.py +++ b/python/sglang/srt/speculative/spec_utils.py @@ -734,15 +734,20 @@ def draft_tp_context(owns_attention: bool): """Enter the TP placement that draft work runs under. Work that owns attention (an attention-owning draft, or DSpark's DP-MoE - sync) runs on the target's attention-TP group; other draft work keeps the - target's TP placement. Enter it from outside any draft scope, where - ``attn_tp_group`` is still the target's. + sync) runs on the target's attention-TP group. With prefill CP and no + attention DP it uses full TP as ordinary attention TP, matching the borrowed + target LM head. Other draft work keeps the target's TP placement. + Enter it from outside any draft scope. """ if not owns_attention: return contextlib.nullcontext() - return patch_tensor_parallel_group( - get_parallel().attn_tp_group, owns_attention=True + parallel = get_parallel() + group = ( + parallel.tp_group + if parallel.attn_cp_size > 1 and parallel.attn_dp_size == 1 + else parallel.attn_tp_group ) + return patch_tensor_parallel_group(group, owns_attention=True) def spec_stage_span(name: str): diff --git a/test/registered/cp/test_cp_strategy_unit.py b/test/registered/cp/test_cp_strategy_unit.py index 13250553df..02f268fe0d 100644 --- a/test/registered/cp/test_cp_strategy_unit.py +++ b/test/registered/cp/test_cp_strategy_unit.py @@ -15,7 +15,8 @@ from sglang.srt.layers.cp.base import ( is_interleave, is_zigzag, ) -from sglang.srt.layers.cp.bcg import PrefillCPBCGInput +from sglang.srt.layers.cp.bcg.interleave_bcg import InterleaveCPBCGInput +from sglang.srt.layers.cp.bcg.zigzag_bcg import ZigzagCPBCGInput from sglang.srt.layers.cp.interleave import InterleaveCPStrategy from sglang.srt.layers.cp.padding import ( get_cp_padding_align_size, @@ -158,11 +159,38 @@ class TestPrefillCPBCGReplay(CustomTestCase): cp_strategy="zigzag", ) + def test_interleave_replay_uses_total_tokens_across_requests(self): + """A captured token shape remains reusable when request boundaries change.""" + runner = self._make_runner() + runner.prefill_cp_bcg_input = InterleaveCPBCGInput( + input_embeds=torch.empty(0), + positions=torch.empty(0), + bucket_local_tokens={2048: 512, 2304: 576}, + ) + batch = self._make_forward_batch() + init_cp_strategy(enable_prefill_cp=True, cp_size=4, cp_strategy="interleave") + for rank in range(4): + with ( + self.subTest(rank=rank), + get_parallel().override(attn_cp_rank=rank, attn_cp_size=4), + ): + # Uneven request boundaries must not round each request up + # independently: interleave shards their concatenation. + self.assertEqual( + runner.prefill_cp_bcg_input.required_local_tokens( + batch.extend_seq_lens_cpu + ), + 512, + ) + self.assertTrue(runner.can_run_graph(batch)) + batch.extend_prefix_lens_cpu = [0, 1, 0] + self.assertFalse(runner.can_run_graph(batch)) + def test_local_capacity_overflow_uses_next_capture_bucket(self): runner = self._make_runner() runner.capture_num_tokens.append(2560) runner.max_num_tokens = 2560 - runner.prefill_cp_bcg_input = PrefillCPBCGInput( + runner.prefill_cp_bcg_input = ZigzagCPBCGInput( input_embeds=torch.empty(0), positions=torch.empty(0), bucket_local_tokens={2048: 512, 2304: 576, 2560: 640}, @@ -172,7 +200,7 @@ class TestPrefillCPBCGReplay(CustomTestCase): with ( patch( - "sglang.srt.layers.cp.bcg.get_cp_padding_align_size", + "sglang.srt.layers.cp.bcg.bcg.get_cp_padding_align_size", return_value=8, ), ): @@ -194,7 +222,7 @@ class TestPrefillCPBCGReplay(CustomTestCase): def test_bucket_search_preserves_two_x_padding_limit(self): runner = self._make_runner() - runner.prefill_cp_bcg_input = PrefillCPBCGInput( + runner.prefill_cp_bcg_input = ZigzagCPBCGInput( input_embeds=torch.empty(0), positions=torch.empty(0), bucket_local_tokens={2048: 512, 2304: 576}, @@ -211,7 +239,7 @@ class TestPrefillCPBCGReplay(CustomTestCase): def test_bucket_search_falls_back_when_no_capture_has_capacity(self): runner = self._make_runner() - runner.prefill_cp_bcg_input = PrefillCPBCGInput( + runner.prefill_cp_bcg_input = ZigzagCPBCGInput( input_embeds=torch.empty(0), positions=torch.empty(0), bucket_local_tokens={2048: 512, 2304: 516}, @@ -222,7 +250,7 @@ class TestPrefillCPBCGReplay(CustomTestCase): with ( get_parallel().override(attn_cp_rank=0, attn_cp_size=4), patch( - "sglang.srt.layers.cp.bcg.get_cp_padding_align_size", + "sglang.srt.layers.cp.bcg.bcg.get_cp_padding_align_size", return_value=8, ), ): @@ -240,7 +268,7 @@ class TestPrefillCPBCGReplay(CustomTestCase): raise StopAfterRecordingFill runner = self._make_runner() - runner.prefill_cp_bcg_input = PrefillCPBCGInput( + runner.prefill_cp_bcg_input = ZigzagCPBCGInput( input_embeds=torch.empty(0), positions=torch.empty(0), bucket_local_tokens={2048: 512, 2304: 576}, @@ -252,7 +280,7 @@ class TestPrefillCPBCGReplay(CustomTestCase): with ( get_parallel().override(attn_cp_rank=0, attn_cp_size=4), patch( - "sglang.srt.layers.cp.bcg.get_cp_padding_align_size", + "sglang.srt.layers.cp.bcg.bcg.get_cp_padding_align_size", return_value=8, ), self.assertRaises(StopAfterRecordingFill), diff --git a/test/registered/cp/test_glm53_flash_cp.py b/test/registered/cp/test_glm53_flash_cp.py index 1f232facda..5016ef9804 100644 --- a/test/registered/cp/test_glm53_flash_cp.py +++ b/test/registered/cp/test_glm53_flash_cp.py @@ -1,4 +1,4 @@ -"""GLM-5.3-Flash interleave CP4 + native MTP on four Blackwell GPUs.""" +"""GLM-5.3-Flash interleave CP4 + DFlash2 on four Blackwell GPUs.""" import base64 import io @@ -44,13 +44,13 @@ class TestGLM53FlashB200ContextParallel( "--tool-call-parser", "auto", "--speculative-algorithm", - "EAGLE", - "--speculative-num-steps", - "5", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "6", + "DFLASH", + "--speculative-draft-model-path", + "incoai/GLM-5.3-Flash-DFlash2", + "--speculative-draft-attention-backend", + "fa4", + "--mem-fraction-static", + "0.85", "--enable-prefill-cp", "--cp-strategy", "interleave", @@ -85,7 +85,7 @@ class TestGLM53FlashB200ContextParallel( def test_gsm8k(self): super().test_gsm8k() # Require the metric as well as the floor: an accuracy pass alone must - # not hide disabled MTP or a draft whose tokens are mostly rejected. + # not hide disabled speculation or a draft whose tokens are mostly rejected. response = requests.get(self.base_url + "/server_info", timeout=30) response.raise_for_status() self.assertGreater( diff --git a/test/registered/unit/spec/test_draft_tp_context.py b/test/registered/unit/spec/test_draft_tp_context.py index 1e1b6795c6..2df9de7ceb 100644 --- a/test/registered/unit/spec/test_draft_tp_context.py +++ b/test/registered/unit/spec/test_draft_tp_context.py @@ -51,6 +51,18 @@ class TestDraftTpContext(CustomTestCase): self.assertEqual(parallel.attn_dp_size, 2) self.assertFalse(parallel_state._TP_STATE_PATCHED) + def test_cp_does_not_join_independent_dp_drafts(self): + attn_tp = _group(world_size=1, rank=0) + with get_parallel().override( + tp_group=self.whole_tp, + attn_tp_group=attn_tp, + attn_tp_size=1, + attn_cp_size=2, + ): + with draft_tp_context(True): + self.assertIs(get_parallel().tp_group, attn_tp) + self.assertEqual(get_parallel().attn_cp_size, 1) + if __name__ == "__main__": unittest.main()