diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index 6fbaa06e29..d812abb29f 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -326,6 +326,16 @@ _bcg_prefill_graph_full_topk = eager_on_graph(True)(_prefill_graph_full_topk) _bcg_low_ratio_source_projections = eager_on_graph(True)(_low_ratio_source_projections) +def _first_rows(act, n: int): + """First ``n`` rows of ``act``, cut field by field for HIP's quantized activations + (``Mxfp8Activation``, ``Fp8GridActivation``), which refuse a slice.""" + if isinstance(act, msgspec.Struct): + return msgspec.structs.replace( + act, **{k: getattr(act, k)[:n] for k in act.__struct_fields__} + ) + return act[:n] + + def _as_int_list(values) -> Optional[List[int]]: if values is None: return None @@ -3393,9 +3403,10 @@ class DeepseekV4AttnBackend( # extend past them; an eager break in the late-layer tail sees fewer, and # there extend_seq_lens still spans the whole extend. n = min(hoisted_pos.shape[0], positions.shape[0]) - x, positions = x[:n], positions[:n] - if q_lora is not None: - q_lora = q_lora[:n] + if n < positions.shape[0]: + x, positions = _first_rows(x, n), positions[:n] + if q_lora is not None: + q_lora = _first_rows(q_lora, n) req, pos = hoisted_req[:n], hoisted_pos[:n] if ( forward_batch.forward_mode.is_extend() diff --git a/test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py b/test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py index f89d0c1c69..8f0596b990 100644 --- a/test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py +++ b/test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py @@ -10,6 +10,7 @@ import torch from sglang.kernels.ops.attention.dsv4.attn_metadata_kernels import ( late_layer_tail_layout, ) +from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import Mxfp8Activation from sglang.srt.layers.attention import deepseek_v4_backend from sglang.srt.layers.attention.deepseek_v4_backend import ( SWA_WINDOW, @@ -323,7 +324,7 @@ class TestDSV4LateLayerTailLowRatioRows(CustomTestCase): """An eager low-ratio break inside the prefill graph's late-layer tail must read the hoisted rows of the tail, not rows rebuilt from the whole extend.""" - def _run(self, hoisted_req, hoisted_pos, positions): + def _run(self, hoisted_req, hoisted_pos, positions, q_lora=None): backend = object.__new__(deepseek_v4_backend.DeepseekV4AttnBackend) backend.tail_source_kv_layer_id = None backend.forward_metadata = SimpleNamespace( @@ -350,12 +351,13 @@ class TestDSV4LateLayerTailLowRatioRows(CustomTestCase): backend.forward_low_ratio_sources( layer=layer, x=torch.zeros(positions.shape[0], 4), - q_lora=torch.zeros(positions.shape[0], 4), + q_lora=torch.zeros(positions.shape[0], 4) if q_lora is None else q_lora, positions=positions, forward_batch=forward_batch, ) compress_args = backend._low_ratio_compress.call_args.args index_args = backend._low_ratio_index_topk.call_args.args + self.index_q_lora = index_args[2] return compress_args[2], compress_args[3], index_args[3], index_args[4] def test_tail_break_uses_live_prefix_of_hoisted_rows(self): @@ -369,6 +371,17 @@ class TestDSV4LateLayerTailLowRatioRows(CustomTestCase): self.assertEqual(index_req.tolist(), [7, 7, 9, 9, 9]) self.assertEqual(index_pos.tolist(), [2, 3, 3, 4, 5]) + def test_dp_padding_cuts_a_quantized_q_lora_by_rows(self): + """HIP's quantized q_lora raises a TypeError on a plain slice; DP padding past the + hoisted rows must cut it field by field.""" + q_lora = Mxfp8Activation( + q=torch.arange(320).view(5, 64), scale=torch.arange(10).view(5, 2) + ) + positions = torch.tensor([2, 3, 3, 0, 0]) + self._run(torch.tensor([7, 7, 9]), torch.tensor([2, 3, 3]), positions, q_lora) + self.assertTrue(torch.equal(self.index_q_lora.q, q_lora.q[:3])) + self.assertTrue(torch.equal(self.index_q_lora.scale, q_lora.scale[:3])) + if __name__ == "__main__": unittest.main()