diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index 0ee3eb5ab9..e7f4c72c08 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -1950,15 +1950,16 @@ class DeepseekV4AttnBackend( ) -> None: """Write every row's compressed KV for the source layer the tail starts at, before enter_late_layer_tail; the layer's own pass inside the tail then - skips its compressor.""" - self.forward_low_ratio_sources( - layer=layer, - x=x, - q_lora=None, - positions=positions, - forward_batch=forward_batch, - run_indexer=False, - ) + skips its compressor, also on a rank with no rows to write.""" + if x.shape[0] > 0: + self.forward_low_ratio_sources( + layer=layer, + x=x, + q_lora=None, + positions=positions, + forward_batch=forward_batch, + run_indexer=False, + ) self.tail_source_kv_layer_id = layer.layer_id def enter_late_layer_tail(self, forward_batch: ForwardBatch) -> tuple: @@ -3211,23 +3212,18 @@ class DeepseekV4AttnBackend( return hoisted_req = getattr(meta, "low_ratio_req_indices", None) hoisted_pos = getattr(meta, "low_ratio_pos_i64", None) - if hoisted_pos is not None and hoisted_pos.shape[0] < positions.shape[0]: - # DP attention pads an eager extend past the rows extend_seq_lens cover. - num_real = hoisted_pos.shape[0] - x, positions = x[:num_real], positions[:num_real] - if q_lora is not None: - q_lora = q_lora[:num_real] - if ( - hoisted_req is not None - and hoisted_pos is not None - and hoisted_pos.shape[0] == positions.shape[0] - ): - # Bucket-sized under the prefill graph; an eager break sees the - # live rows only and falls through. - req, pos = hoisted_req, hoisted_pos - else: + if hoisted_pos is None: req = token_req_indices(forward_batch, num_tokens=positions.shape[0]) pos = positions + else: + # The hoisted rows list the live rows first. DP attention pads an eager + # extend past them; an eager break in the late-layer tail sees fewer, and + # there extend_seq_lens still spans the whole extend. + n = min(hoisted_pos.shape[0], positions.shape[0]) + x, positions = x[:n], positions[:n] + if q_lora is not None: + q_lora = q_lora[:n] + req, pos = hoisted_req[:n], hoisted_pos[:n] if ( forward_batch.forward_mode.is_extend() and self._low_ratio_in_prefill_graph() diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index f8e52b948d..2142b5e2f5 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -849,6 +849,34 @@ bcg_deepseek_v4_low_ratio_sources = eager_on_graph( ) +def _low_ratio_sources_break(attn_backend, forward_batch) -> bool: + """Whether the ratio-1/2 sources run as an eager break: inside a breakable graph + without the in-graph low-ratio path, or in a decoder tail graph.""" + return ( + forward_batch.forward_mode.is_extend() + and is_in_breakable_cuda_graph() + and ( + not getattr(attn_backend, "low_ratio_prefill_graph", False) + or in_replay_tail_graph() + ) + ) + + +def deepseek_v4_write_tail_source_kv(layer, x, positions) -> None: + # Like the sources above: the compressor syncs with the host. + forward_batch = get_tc_piecewise_forward_context().forward_batch + real_num_tokens = forward_batch.global_num_token_non_padded_cpu + get_attn_backend().write_tail_source_kv( + layer=layer, + x=x[:real_num_tokens], + positions=positions[:real_num_tokens], + forward_batch=forward_batch, + ) + + +bcg_deepseek_v4_write_tail_source_kv = eager_on_graph(deepseek_v4_write_tail_source_kv) + + def deepseek_v4_engram_hash_ids(hasher, input_ids: torch.Tensor) -> torch.Tensor: # The hasher reads per-request rows, so it cannot run inside the CUDA graph. forward_batch = get_tc_piecewise_forward_context().forward_batch @@ -2425,14 +2453,7 @@ class MQALayer(MqaAttentionBase): if self.compress_ratio in (1, 2) and ( self.compressor is not None or self.indexer is not None ): - if ( - forward_batch.forward_mode.is_extend() - and is_in_breakable_cuda_graph() - and ( - not getattr(attn_backend, "low_ratio_prefill_graph", False) - or in_replay_tail_graph() - ) - ): + if _low_ratio_sources_break(attn_backend, forward_batch): bcg_deepseek_v4_low_ratio_sources(self, x, q_lora, positions) else: attn_backend.forward_low_ratio_sources( @@ -3975,12 +3996,17 @@ class DeepseekV4Model(nn.Module): source_x = fused_boundary_attn_input(source, state) else: source_x = mhc.combine(source.attn_hc, state) - attn_backend.write_tail_source_kv( - layer=source.self_attn, - x=source_x, - positions=positions, - forward_batch=forward_batch, - ) + if _low_ratio_sources_break(attn_backend, forward_batch): + bcg_deepseek_v4_write_tail_source_kv( + source.self_attn, source_x, positions + ) + else: + attn_backend.write_tail_source_kv( + layer=source.self_attn, + x=source_x, + positions=positions, + forward_batch=forward_batch, + ) # Decode reaches back at most SWA_WINDOW positions. if tail_in_graph: # Eager break: re-run at replay so the backend swaps onto the diff --git a/test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py b/test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py index b227a612f7..ff7848126a 100644 --- a/test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py +++ b/test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py @@ -332,5 +332,56 @@ class TestPrefillGraphReplayContext(CustomTestCase): ) +class TestDSV4LateLayerTailLowRatioRows(CustomTestCase): + """An eager low-ratio break inside the prefill graph's late-layer tail must read + the hoisted rows of the tail, not rows rebuilt from the whole extend.""" + + def _run(self, hoisted_req, hoisted_pos, positions): + backend = object.__new__(deepseek_v4_backend.DeepseekV4AttnBackend) + backend.tail_source_kv_layer_id = None + backend.forward_metadata = SimpleNamespace( + low_ratio_req_indices=hoisted_req, low_ratio_pos_i64=hoisted_pos + ) + backend._low_ratio_compress = MagicMock() + backend._low_ratio_index_topk = MagicMock() + # Two requests extend 4 and 6 tokens; the tail keeps their last 2 and 3. + forward_batch = SimpleNamespace( + forward_mode=ForwardMode.EXTEND, + encoder_swa_replay=False, + req_pool_indices=torch.tensor([7, 9]), + extend_seq_lens=torch.tensor([4, 6]), + ) + layer = SimpleNamespace(layer_id=21, compressor=object(), indexer=object()) + with ( + patch.object(deepseek_v4_backend, "dsa_use_prefill_cp", return_value=False), + patch.object( + deepseek_v4_backend.DeepseekV4AttnBackend, + "_low_ratio_in_prefill_graph", + return_value=False, + ), + ): + backend.forward_low_ratio_sources( + layer=layer, + x=torch.zeros(positions.shape[0], 4), + q_lora=torch.zeros(positions.shape[0], 4), + positions=positions, + forward_batch=forward_batch, + ) + compress_args = backend._low_ratio_compress.call_args.args + index_args = backend._low_ratio_index_topk.call_args.args + return compress_args[2], compress_args[3], index_args[3], index_args[4] + + def test_tail_break_uses_live_prefix_of_hoisted_rows(self): + # Bucket of 8 rows, the 5 live tail rows first. + hoisted_req = torch.tensor([7, 7, 9, 9, 9, 0, 0, 0]) + hoisted_pos = torch.tensor([2, 3, 3, 4, 5, 0, 0, 0]) + positions = hoisted_pos[:5].clone() + req, pos, index_req, index_pos = self._run(hoisted_req, hoisted_pos, positions) + self.assertEqual(req.tolist(), [7, 7, 9, 9, 9]) + self.assertEqual(pos.tolist(), [2, 3, 3, 4, 5]) + self.assertEqual(index_req.tolist(), [7, 7, 9, 9, 9]) + self.assertEqual(index_pos.tolist(), [2, 3, 3, 4, 5]) + + if __name__ == "__main__": unittest.main()