From 54409c2fa0812b5801197db658f695335f484c48 Mon Sep 17 00:00:00 2001 From: BingjiaWang Date: Mon, 12 Oct 2026 16:03:10 +0800 Subject: [PATCH] [BCG][DSV4-Flash][CP] Support DeepSeek V4 prefill with interleave CP and breakable CUDA graphs (#38603) --- .../layers/attention/deepseek_v4_backend.py | 66 +-- python/sglang/srt/layers/cp/bcg.py | 84 +++- .../runner/prefill_cuda_graph_runner.py | 13 + .../layers/cp/test_dsv4_prefill_cp_bcg.py | 406 ++++++++++++++++++ 4 files changed, 520 insertions(+), 49 deletions(-) create mode 100644 test/registered/unit/layers/cp/test_dsv4_prefill_cp_bcg.py diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index 4420fdecd3..ffcbedaec9 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -1394,6 +1394,10 @@ class DeepseekV4AttnBackend( (0, padded_num_tokens - swa_replay_start.shape[0]), ) + num_write_tokens = ( + out_cache_loc.shape[0] if use_prefill_cuda_graph else num_tokens + ) + seq_lens_casual, req_pool_indices_repeated = self.expand_prefill_casually( num_tokens=num_tokens, seq_lens=seq_lens_cpu, @@ -1414,13 +1418,13 @@ class DeepseekV4AttnBackend( is_prefill=True, dspark_block_size=dspark_block_size, dspark_swa_buffers=dspark_swa_buffers, - num_tokens=num_tokens if cp_active else None, + num_tokens=num_write_tokens if cp_active else None, swa_replay_start=swa_replay_start, num_groups=len(extend_seq_lens_cpu), ) if cp_active: core_attn_metadata.apply_cp_reindex( - num_tokens=num_tokens, local_index=cp_metadata.local_index + num_tokens=num_write_tokens, local_index=cp_metadata.local_index ) core_attn_metadata.init_flashmla_related(is_prefill=True) indexer_metadata = ( @@ -2364,6 +2368,19 @@ class DeepseekV4AttnBackend( self.forward_metadata.core_attn_metadata.request_window_layout ) + def _use_sparse_prefill( + self, forward_batch: ForwardBatch, *, num_qo_tokens: int + ) -> bool: + return ( + forward_batch.forward_mode.is_extend_without_speculative() + and not is_cp_active(forward_batch) + and not get_platform().is_sm120 + and ( + num_qo_tokens > _LARGE_INDEXER_QUERY_THRESHOLD + or envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.get() + ) + ) + def prepare_prefill_shared_read_snapshot( self, forward_batch: ForwardBatch, *, num_qo_tokens: int ) -> None: @@ -2386,15 +2403,9 @@ class DeepseekV4AttnBackend( assert isinstance(metadata, DSV4Metadata) # The tail never takes the sparse path, so it carries no chunk cache. - use_sparse_prefill = ( - not get_platform().is_sm120 - and metadata.late_layer_tail is None - and ( - num_qo_tokens > _LARGE_INDEXER_QUERY_THRESHOLD - or envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.get() - ) - ) - if use_sparse_prefill: + if metadata.late_layer_tail is None and self._use_sparse_prefill( + forward_batch, num_qo_tokens=num_qo_tokens + ): metadata.sparse_prefill_cache = self._build_sparse_prefill_chunk_cache( forward_batch, metadata.core_attn_metadata, num_qo_tokens=num_qo_tokens ) @@ -2413,14 +2424,21 @@ class DeepseekV4AttnBackend( assert seq_lens_cpu is not None # Decoder tails require explicit SWA indices to exclude unwritten history. tail = self.forward_metadata.late_layer_tail - request_layout = core_attn_metadata.request_window_layout - assert tail is None or request_layout is not None extend_seq_lens = forward_batch.extend_seq_lens extend_seq_lens_cpu = forward_batch.extend_seq_lens_cpu if tail is not None: extend_seq_lens = tail.extend_seq_lens extend_seq_lens_cpu = tail.extend_seq_lens_cpu assert extend_seq_lens_cpu is not None + num_global_queries = sum(extend_seq_lens_cpu) + assert num_qo_tokens >= num_global_queries, ( + "DSV4 sparse prefill requires global query rows: " + f"allocated {num_qo_tokens}, but request offsets cover {num_global_queries}. " + "CP-local queries must use paged attention." + ) + # Reject undersized query buffers before accessing attention metadata. + request_layout = core_attn_metadata.request_window_layout + assert tail is None or request_layout is not None seq_lens_cpu_list = seq_lens_cpu.tolist() total_swa = sum( min(int(seq_len), int(extend_len) + SWA_WINDOW - 1) @@ -3480,24 +3498,12 @@ class DeepseekV4AttnBackend( f"{extra_indices.shape=}'s last dimension is not aligned to 64" ) - # RequestWindow sparse gathering does not support CP yet. + # The tail stays dense: its window floor lives in swa_page_indices, + # which the sparse chunk cache ignores. if ( - forward_batch.forward_mode.is_extend_without_speculative() - and not get_platform().is_sm120 - and ( - ( - token_to_kv_pool.request_window is None - and self.forward_metadata.late_layer_tail is None - ) - or ( - token_to_kv_pool.request_window is not None - and not is_cp_active(forward_batch) - ) - ) - and ( - q.shape[0] > _LARGE_INDEXER_QUERY_THRESHOLD - or envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.get() - ) + self._use_sparse_prefill(forward_batch, num_qo_tokens=q.shape[0]) + and self.forward_metadata.late_layer_tail is None + and token_to_kv_pool.request_window is None ): if use_dsv4_q8kv8_sparse_prefill(self.dsv4_prefill_backend): return self._forward_prefill_sparse_q8kv8( diff --git a/python/sglang/srt/layers/cp/bcg.py b/python/sglang/srt/layers/cp/bcg.py index 6631c9d48a..4d1f1a5d77 100644 --- a/python/sglang/srt/layers/cp/bcg.py +++ b/python/sglang/srt/layers/cp/bcg.py @@ -27,6 +27,7 @@ from sglang.srt.arg_groups.overrides import ( resolving_view, ) from sglang.srt.layers.cp.base import get_cp_strategy +from sglang.srt.layers.cp.interleave import InterleaveCPStrategy from sglang.srt.layers.cp.padding import get_cp_padding_align_size from sglang.srt.layers.cp.utils import ( cp_gather_after_forward, @@ -54,8 +55,10 @@ def supports_prefill_cp_bcg(server_args: ServerArgs) -> bool: cfg.enable_prefill_cp and cfg.pp_size == 1 and resolved.attn_cp_size == cfg.tp_size - and cfg.cp_strategy == "zigzag" - and prefill_attention_backend == "trtllm_mha" + and ( + (cfg.cp_strategy == "zigzag" and prefill_attention_backend == "trtllm_mha") + or (cfg.cp_strategy == "interleave" and prefill_attention_backend == "dsv4") + ) ) @@ -67,8 +70,10 @@ def enable_cp_bcg_capture(server_args: ServerArgs) -> bool: def filter_prefill_cp_bcg_capture_num_tokens( capture_num_tokens: list[int], server_args: ServerArgs ) -> list[int]: - """Keep only token buckets where the zigzag CP strategy can run.""" - min_num_tokens = resolved_view(server_args).attn_cp_size * 2 + """Keep only token buckets where the selected CP strategy can run.""" + min_num_tokens = resolved_view(server_args).attn_cp_size + if resolving_view(server_args).cp_strategy == "zigzag": + min_num_tokens *= 2 filtered = [size for size in capture_num_tokens if size >= min_num_tokens] if not filtered: raise ValueError( @@ -117,24 +122,30 @@ class PrefillCPBCGInput: ) def required_local_tokens(self, extend_seq_lens: Any) -> Optional[int]: - """Return the aligned CP-local rows required by a live zigzag layout.""" + """Return the largest aligned local shard, uniformly across CP ranks.""" strategy = get_cp_strategy() - if not isinstance(strategy, ZigzagCPStrategy) or extend_seq_lens is None: + if extend_seq_lens is None: + return None + if isinstance(strategy, InterleaveCPStrategy): + num_tokens = sum(int(length) for length in extend_seq_lens) + max_local_tokens = (num_tokens + strategy.cp_size - 1) // strategy.cp_size + elif isinstance(strategy, ZigzagCPStrategy): + cp_segment_num = strategy.cp_size * 2 + per_rank_logical_tokens = [0] * strategy.cp_size + for raw_length in extend_seq_lens: + base, remainder = divmod(int(raw_length), cp_segment_num) + for rank in range(strategy.cp_size): + opposite_rank = cp_segment_num - 1 - rank + per_rank_logical_tokens[rank] += ( + base * 2 + + int(rank < remainder) + + int(opposite_rank < remainder) + ) + max_local_tokens = max(per_rank_logical_tokens) + else: return None - - cp_segment_num = strategy.cp_size * 2 - per_rank_logical_tokens = [0] * strategy.cp_size - for raw_length in extend_seq_lens: - base, remainder = divmod(int(raw_length), cp_segment_num) - for rank in range(strategy.cp_size): - opposite_rank = cp_segment_num - 1 - rank - per_rank_logical_tokens[rank] += ( - base * 2 + int(rank < remainder) + int(opposite_rank < remainder) - ) align_size = get_cp_padding_align_size() - return ( - (max(per_rank_logical_tokens) + align_size - 1) // align_size * align_size - ) + return (max_local_tokens + align_size - 1) // align_size * align_size def select_replay_bucket( self, @@ -189,6 +200,7 @@ class PrefillCPBCGInput: # Replay batches may reuse a ForwardBatch object whose metadata was # built for a different request layout. Always rebuild before sharding. forward_batch.attn_cp_metadata = None + out_cache_loc = forward_batch.out_cache_loc prepare_cp_forward(forward_batch) captured_local_tokens = None @@ -257,6 +269,16 @@ class PrefillCPBCGInput: forward_batch.positions = positions self.live_local_tokens = live_local_tokens + if isinstance(get_cp_strategy(), InterleaveCPStrategy): + metadata = forward_batch.attn_cp_metadata + cp_size = len(metadata.per_rank_actual_token) + base, remainder = divmod(static_num_tokens, cp_size) + metadata.total_seq_lens = static_num_tokens + metadata.per_rank_logical_token = [ + base + int(rank < remainder) for rank in range(cp_size) + ] + forward_batch.out_cache_loc = out_cache_loc + def execute_prefill_cp_bcg( runner: PrefillCudaGraphRunner, @@ -307,10 +329,34 @@ def execute_prefill_cp_bcg( static_forward_batch, torch.cuda.current_stream(), ) + hidden_states = _slice_output_rows(hidden_states, raw_num_tokens) + if aux_hidden_states is not None: + + def gather_aux(aux): + return _slice_output_rows( + cp_gather_after_forward( + aux, static_forward_batch, torch.cuda.current_stream() + ), + raw_num_tokens, + ) + + if torch.is_tensor(aux_hidden_states): + aux_hidden_states = gather_aux(aux_hidden_states) + else: + aux_hidden_states = [gather_aux(aux) for aux in aux_hidden_states] + + # Mirror the DSV4 outer forward and the eager CP runner: the body + # returns (normalized hidden states, pre-mHC-head hidden states). + logits_kwargs = {} + if isinstance(hidden_states, tuple): + hidden_states, hidden_states_before_norm = hidden_states + if aux_hidden_states is None: + logits_kwargs["hidden_states_before_norm"] = hidden_states_before_norm return model.logits_processor( forward_batch.input_ids, hidden_states, model.lm_head, forward_batch, aux_hidden_states, + **logits_kwargs, ) diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index 7a11965ca4..e8e91c21a6 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -593,6 +593,12 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): self.capture_num_tokens, server_args ) self.prefill_cp_bcg_input = PrefillCPBCGInput.create(self) + logger.info( + "Prefill CP breakable CUDA graph enabled: strategy=%s, cp_size=%d", + get_parallel().cp_strategy, + get_parallel().attn_cp_size, + ) + if self.max_context_size is not None and not ( model_runner.attn_backend.supports_prefill_cuda_graph_max_context_size ): @@ -1370,6 +1376,13 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): if self._has_inactive_dp_rank(forward_batch): return False + # A batch can be too small to activate CP. Its unsharded inputs must + # not replay a model body captured with CP-local rows. + if getattr(self, "enable_cp_bcg_capture", False) and not is_cp_active( + forward_batch + ): + return False + # Non-DP local check (sole decision for tp-only). batch_max_context_len = ( self._batch_max_context_len(forward_batch) diff --git a/test/registered/unit/layers/cp/test_dsv4_prefill_cp_bcg.py b/test/registered/unit/layers/cp/test_dsv4_prefill_cp_bcg.py new file mode 100644 index 0000000000..ab32ef791b --- /dev/null +++ b/test/registered/unit/layers/cp/test_dsv4_prefill_cp_bcg.py @@ -0,0 +1,406 @@ +"""CPU checks for DSV4 interleave BCG padding and the eager logits tail.""" + +import unittest +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import torch + +from sglang.srt.layers.cp.base import get_cp_strategy, init_cp_strategy +from sglang.srt.layers.cp.bcg import ( + PrefillCPBCGInput, + execute_prefill_cp_bcg, + filter_prefill_cp_bcg_capture_num_tokens, + supports_prefill_cp_bcg, +) +from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.srt.runtime_context import get_parallel +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class TestDSV4PrefillCPBCG(unittest.TestCase): + def tearDown(self): + init_cp_strategy(enable_prefill_cp=False, cp_size=1, cp_strategy="zigzag") + + def test_sparse_prefill_never_uses_cp_local_queries(self): + from sglang.srt.layers.attention.deepseek_v4_backend import ( + DeepseekV4AttnBackend, + ) + + batch = SimpleNamespace(forward_mode=ForwardMode.EXTEND) + for cp_active in (False, True): + for forced in (False, True): + for num_tokens in (256, 11673, 11674, 16384, 32768): + with ( + patch( + "sglang.srt.layers.attention.deepseek_v4_backend.is_cp_active", + return_value=cp_active, + ), + patch( + "sglang.srt.layers.attention.deepseek_v4_backend.get_platform", + return_value=SimpleNamespace(is_sm120=False), + ), + patch( + "sglang.srt.layers.attention.deepseek_v4_backend.envs" + ) as envs, + ): + envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.get.return_value = ( + forced + ) + selected = DeepseekV4AttnBackend._use_sparse_prefill( + None, batch, num_qo_tokens=num_tokens + ) + self.assertEqual( + selected, not cp_active and (forced or num_tokens > 11673) + ) + + def test_sparse_metadata_rejects_short_query_buffer_before_launch(self): + from sglang.srt.layers.attention.deepseek_v4_backend import ( + DeepseekV4AttnBackend, + ) + + backend = SimpleNamespace( + forward_metadata=SimpleNamespace(late_layer_tail=None) + ) + for lengths in ([32768], [8192, 24576]): + batch = SimpleNamespace( + seq_lens_cpu=torch.tensor(lengths), + extend_seq_lens=torch.tensor(lengths, dtype=torch.int32), + extend_seq_lens_cpu=lengths, + ) + # No core attention metadata or pool is present: the invariant + # must fail before accessing either or launching the index combiner. + with self.assertRaisesRegex(AssertionError, "allocated 16384.*32768"): + DeepseekV4AttnBackend._build_sparse_prefill_chunk_cache( + backend, batch, core_attn_metadata=None, num_qo_tokens=16384 + ) + + def test_inactive_cp_batch_cannot_replay_cp_body(self): + from sglang.srt.model_executor.runner.prefill_cuda_graph_runner import ( + PrefillCudaGraphRunner, + ) + + init_cp_strategy(enable_prefill_cp=True, cp_size=2, cp_strategy="interleave") + runner = SimpleNamespace( + can_replay_locally=lambda **kwargs: True, + _has_inactive_dp_rank=lambda batch: False, + enable_cp_bcg_capture=True, + enable_lora=False, + ) + batch = SimpleNamespace( + global_num_tokens_cpu=None, + batch_size=1, + input_ids=[1], + input_embeds=None, + replace_embeds=None, + extend_prefix_lens_cpu=[0], + forward_mode=ForwardMode.EXTEND, + capture_hidden_mode=None, + return_logprob=False, + extend_seq_lens_cpu=[1], + ) + self.assertFalse(PrefillCudaGraphRunner.can_run_graph(runner, batch)) + + def test_attention_keeps_global_write_bucket_and_live_sequence_lengths(self): + from sglang.srt.layers.attention.deepseek_v4_backend import ( + DeepseekV4AttnBackend, + ) + + for use_graph, write_tokens in ((False, 11), (True, 16)): + core = Mock() + backend = SimpleNamespace( + has_c4=True, + has_c128=True, + low_ratio_prefill_graph=False, + expand_prefill_casually=Mock( + return_value=(torch.ones(16), torch.zeros(16)) + ), + make_core_attn_metadata=Mock(return_value=core), + init_forward_metadata_indexer=Mock(return_value=None), + req_to_token=object(), + token_to_kv_pool=object(), + ) + out_loc = torch.arange(write_tokens) + batch = SimpleNamespace( + attn_cp_metadata=SimpleNamespace( + per_rank_actual_token=[8, 8], local_index=None + ) + ) + with ( + patch( + "sglang.srt.layers.attention.deepseek_v4_backend.is_cp_active", + return_value=True, + ), + patch( + "sglang.srt.layers.attention.deepseek_v4_backend.create_paged_compressor_data", + return_value=None, + ) as planner, + patch("sglang.srt.layers.attention.deepseek_v4_backend.envs") as envs, + ): + envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get.return_value = False + DeepseekV4AttnBackend.init_forward_metadata_prefill( + backend, + max_seq_len=128, + req_pool_indices=torch.tensor([2, 3]), + seq_lens=torch.tensor([22, 29]), + seq_lens_cpu=[22, 29], + out_cache_loc=out_loc, + num_tokens=11, + extend_seq_lens=torch.tensor([2, 9]), + extend_seq_lens_cpu=[2, 9], + use_prefill_cuda_graph=use_graph, + forward_batch=batch, + ) + self.assertEqual( + backend.expand_prefill_casually.call_args.kwargs["num_tokens"], 11 + ) + self.assertEqual( + backend.expand_prefill_casually.call_args.kwargs["padded_num_tokens"], + 16, + ) + self.assertEqual( + backend.make_core_attn_metadata.call_args.kwargs["num_tokens"], + write_tokens, + ) + core.apply_cp_reindex.assert_called_once_with( + num_tokens=write_tokens, local_index=batch.attn_cp_metadata.local_index + ) + if use_graph: + for call in planner.call_args_list: + self.assertEqual(call.kwargs["num_q_tokens"], 16) + self.assertIsNone(call.kwargs["seq_lens_cpu"]) + + def test_supported_configurations_and_capture_buckets(self): + cfg = SimpleNamespace( + enable_prefill_cp=True, + pp_size=1, + attn_cp_size=2, + tp_size=2, + cp_strategy="interleave", + ) + with ( + patch("sglang.srt.layers.cp.bcg.resolving_view", return_value=cfg), + patch("sglang.srt.layers.cp.bcg.resolved_view", return_value=cfg), + patch( + "sglang.srt.layers.cp.bcg.attention_backends_of", + return_value=("dsv4", "dsv4"), + ), + ): + self.assertTrue(supports_prefill_cp_bcg(cfg)) + self.assertEqual( + filter_prefill_cp_bcg_capture_num_tokens([1, 2, 4, 8], cfg), [2, 4, 8] + ) + cfg.pp_size = 2 + self.assertFalse(supports_prefill_cp_bcg(cfg)) + cfg.pp_size = 1 + cfg.tp_size = 4 + self.assertFalse(supports_prefill_cp_bcg(cfg)) + cfg.tp_size = 2 + cfg.cp_strategy = "zigzag" + self.assertFalse(supports_prefill_cp_bcg(cfg)) + self.assertEqual( + filter_prefill_cp_bcg_capture_num_tokens([1, 2, 4, 8], cfg), [4, 8] + ) + + def test_interleave_capacity_is_independent_of_request_boundaries(self): + init_cp_strategy(enable_prefill_cp=True, cp_size=4, cp_strategy="interleave") + inputs = PrefillCPBCGInput(torch.empty(0), torch.empty(0), {16: 4, 32: 8}) + with patch( + "sglang.srt.layers.cp.bcg.get_cp_padding_align_size", return_value=4 + ): + for lengths in ([13], [1, 3, 9], [4, 4, 4, 1]): + self.assertEqual(inputs.required_local_tokens(lengths), 4) + self.assertEqual( + inputs.select_replay_bucket_for_batch( + num_tokens=13, + extend_seq_lens=lengths, + capture_num_tokens=[16, 32], + max_padding_factor=2, + ), + 16, + ) + self.assertEqual(inputs.required_local_tokens([1, 16]), 8) + + @staticmethod + def _batch(lengths, bucket): + n = sum(lengths) + ids = torch.zeros(bucket, dtype=torch.int64) + ids[:n] = torch.arange(1, n + 1) + positions = torch.zeros(bucket, dtype=torch.int64) + positions[:n] = torch.cat([torch.arange(length) + 20 for length in lengths]) + out_cache_loc = torch.zeros(bucket, dtype=torch.int64) + out_cache_loc[:n] = torch.arange(256, 256 + n) + return SimpleNamespace( + input_ids=ids, + positions=positions, + out_cache_loc=out_cache_loc, + forward_mode=ForwardMode.EXTEND, + extend_num_tokens=n, + extend_seq_lens_cpu=list(lengths), + seq_lens_cpu=[length + 20 for length in lengths], + global_num_tokens_cpu=None, + attn_cp_metadata=None, + ) + + def test_repeated_padded_replay_preserves_addresses_and_global_order(self): + for cp_size, bucket in ((2, 16), (4, 16), (4, 18)): + init_cp_strategy( + enable_prefill_cp=True, cp_size=cp_size, cp_strategy="interleave" + ) + inputs = [ + PrefillCPBCGInput( + torch.zeros(bucket, 1), torch.zeros(bucket, dtype=torch.int64) + ) + for _ in range(cp_size) + ] + model = SimpleNamespace( + get_input_embeddings=lambda: lambda ids: ids[:, None].float() + ) + runner = SimpleNamespace(model_runner=SimpleNamespace(model=model)) + for lengths, capture in ( + ([bucket], True), + ([2, 9], False), + ([1, 3, 5], False), + ([bucket], False), + ): + batches, shards = [], [] + for rank in range(cp_size): + batch = self._batch(lengths, bucket) + cache_loc = batch.out_cache_loc + global_positions = batch.positions.clone() + with ( + get_parallel().override( + attn_cp_rank=rank, + attn_cp_size=cp_size, + enable_cp_tp_group_sharing=False, + ), + patch( + "sglang.srt.layers.cp.padding.get_cp_padding_align_size", + return_value=cp_size, + ), + ): + inputs[rank].prepare( + runner, batch, static_num_tokens=bucket, capture=capture + ) + n = sum(lengths) + local_ids = torch.arange(1, n + 1)[rank::cp_size].float() + torch.testing.assert_close( + batch.input_embeds[: len(local_ids), 0], local_ids + ) + torch.testing.assert_close( + batch.positions[: len(local_ids)], + global_positions[:n][rank::cp_size], + ) + self.assertEqual( + torch.count_nonzero( + batch.input_embeds[len(local_ids) :] + ).item(), + 0, + ) + self.assertEqual( + torch.count_nonzero(batch.positions[len(local_ids) :]).item(), 0 + ) + self.assertEqual( + batch.input_embeds.data_ptr(), + inputs[rank].input_embeds.data_ptr(), + ) + self.assertEqual( + batch.positions.data_ptr(), inputs[rank].positions.data_ptr() + ) + self.assertIs(batch.out_cache_loc, cache_loc) + self.assertEqual(batch.out_cache_loc.shape[0], bucket) + self.assertEqual(batch.attn_cp_metadata.total_seq_lens, bucket) + self.assertEqual(batch.extend_seq_lens_cpu, list(lengths)) + self.assertEqual( + batch.input_embeds.shape[0], + inputs[rank].bucket_local_tokens[bucket], + ) + batches.append(batch) + shards.append(batch.input_embeds.clone()) + + def gather(output, local): + output.copy_(torch.cat(shards)) + + with ( + get_parallel().override( + attn_cp_rank=0, attn_cp_size=cp_size, attn_cp_group=object() + ), + patch( + "sglang.srt.layers.cp.interleave.attn_cp_all_gather_into_tensor", + side_effect=gather, + ), + patch( + "sglang.srt.layers.cp.interleave.is_allocation_symmetric", + return_value=False, + ), + patch( + "sglang.srt.layers.cp.interleave.use_symmetric_memory", + return_value=nullcontext(), + ), + ): + gathered = get_cp_strategy().gather_kv_cache(shards[0], batches[0]) + self.assertEqual(gathered.shape, (bucket, 1)) + torch.testing.assert_close( + gathered[:n, 0], torch.arange(1, n + 1).float() + ) + self.assertEqual(torch.count_nonzero(gathered[n:]).item(), 0) + + def test_logits_tail_gathers_dspark_aux_and_trims_padding(self): + for capture_aux in (False, True): + hidden = torch.arange(8).view(8, 1).float() + pre_head = hidden + 100 + aux = [hidden + 200, hidden + 300] + body_output = ( + ((hidden, pre_head), aux) if capture_aux else (hidden, pre_head) + ) + logits = Mock(return_value="logits") + model = SimpleNamespace( + capture_aux_hidden_states=capture_aux, + pp_group=SimpleNamespace(is_last_rank=True), + logits_processor=logits, + lm_head=object(), + ) + runner = SimpleNamespace( + prefill_cp_bcg_input=SimpleNamespace(live_local_tokens=8), + model_runner=SimpleNamespace(model=model), + _prefill_forward_context=lambda *args, **kwargs: nullcontext(), + backend=SimpleNamespace(replay=lambda *args, **kwargs: body_output), + ) + batch = SimpleNamespace(input_ids=torch.arange(11)) + + def gather(value, *args): + if isinstance(value, tuple): + return tuple(gather(item) for item in value) + return value.repeat_interleave(2, dim=0) + + with ( + patch( + "sglang.srt.layers.cp.bcg.cp_gather_after_forward", + side_effect=gather, + ), + patch( + "sglang.srt.layers.cp.bcg.torch.cuda.current_stream", + return_value=None, + ), + ): + self.assertEqual( + execute_prefill_cp_bcg(runner, batch, batch, 16, 11), "logits" + ) + args, kwargs = logits.call_args + torch.testing.assert_close(args[1], gather(hidden)[:11]) + if capture_aux: + self.assertNotIn("hidden_states_before_norm", kwargs) + for actual, expected in zip(args[4], aux): + torch.testing.assert_close(actual, gather(expected)[:11]) + else: + self.assertIsNone(args[4]) + torch.testing.assert_close( + kwargs["hidden_states_before_norm"], gather(pre_head)[:11] + ) + + +if __name__ == "__main__": + unittest.main()