diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/layer.py b/python/sglang/multimodal_gen/runtime/layers/attention/layer.py index 1f6675042a..9b3c871dfd 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/layer.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/layer.py @@ -1740,9 +1740,7 @@ class USPAttention(nn.Module): k_rep, k_shard = k[:, :num_rep], k[:, num_rep:] v_rep, v_shard = v[:, :num_rep], v[:, num_rep:] - q_shard = _usp_input_all_to_all(q_shard, head_dim=2) - k_shard = _usp_input_all_to_all(k_shard, head_dim=2) - v_shard = _usp_input_all_to_all(v_shard, head_dim=2) + q_shard, k_shard, v_shard = _usp_input_all_to_all_qkv(q_shard, k_shard, v_shard) joint_mask = joint_mask_meta = None if attn_mask is not None: @@ -2051,9 +2049,7 @@ class USPAttention(nn.Module): k_shard, k_rep = k[:, :-num_rep], k[:, -num_rep:] v_shard, v_rep = v[:, :-num_rep], v[:, -num_rep:] - q_shard = _usp_input_all_to_all(q_shard, head_dim=2) - k_shard = _usp_input_all_to_all(k_shard, head_dim=2) - v_shard = _usp_input_all_to_all(v_shard, head_dim=2) + q_shard, k_shard, v_shard = _usp_input_all_to_all_qkv(q_shard, k_shard, v_shard) h_local = q_shard.shape[2] kv_h_local = k_shard.shape[2] diff --git a/python/sglang/multimodal_gen/runtime/layers/usp.py b/python/sglang/multimodal_gen/runtime/layers/usp.py index 35dc951245..487a8874ff 100644 --- a/python/sglang/multimodal_gen/runtime/layers/usp.py +++ b/python/sglang/multimodal_gen/runtime/layers/usp.py @@ -413,6 +413,18 @@ def _usp_input_all_to_all_packed_qkv( return q, k, v +def _packed_qkv_row_view_is_free(x: torch.Tensor) -> bool: + """True when ``x.view(b * s_local, h, d)`` costs no copy. + + The pack kernel reads q/k/v through explicit row/head strides, so only the + head_size dim has to be unit-stride; full contiguity is not required. The + batch and sequence dims still have to merge into one, which is free when + batch is 1 or when the two dims are already adjacent in memory. + """ + batch, seq_local = x.shape[0], x.shape[1] + return x.stride(-1) == 1 and (batch == 1 or x.stride(0) == seq_local * x.stride(1)) + + def _can_use_packed_qkv_a2a_4d( q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, world_size: int ) -> bool: @@ -422,9 +434,9 @@ def _can_use_packed_qkv_a2a_4d( and q.shape == k.shape == v.shape and q.dtype == k.dtype == v.dtype and q.dtype in (torch.float16, torch.bfloat16) - and q.is_contiguous() - and k.is_contiguous() - and v.is_contiguous() + and _packed_qkv_row_view_is_free(q) + and _packed_qkv_row_view_is_free(k) + and _packed_qkv_row_view_is_free(v) and q.shape[2] % world_size == 0 and not torch.compiler.is_compiling() ) @@ -441,7 +453,8 @@ def _usp_input_all_to_all_qkv( destination-major by one relayout kernel and exchanged in a single collective instead of three; only data movement changes, so the result is bit-identical to the unpacked path, which stays as the fallback for - ineligible inputs (CPU, GQA-mismatched shapes, non-contiguous layouts). + ineligible inputs (CPU, GQA-mismatched shapes, layouts whose row view + would copy). Adapted from the NVlabs Sana sol-engine branch (Apache-2.0). """ world_size = get_ulysses_parallel_world_size() diff --git a/python/sglang/multimodal_gen/test/unit/test_usp_attention_replicated_prefix.py b/python/sglang/multimodal_gen/test/unit/test_usp_attention_replicated_prefix.py index e2d19b6976..479de77507 100644 --- a/python/sglang/multimodal_gen/test/unit/test_usp_attention_replicated_prefix.py +++ b/python/sglang/multimodal_gen/test/unit/test_usp_attention_replicated_prefix.py @@ -70,7 +70,8 @@ class TestUSPAttentionReplicatedPrefix(unittest.TestCase): patch(f"{_LAYER}.get_ulysses_parallel_rank", return_value=sp_rank), patch(f"{_LAYER}.get_ring_parallel_world_size", return_value=1), patch( - f"{_LAYER}._usp_input_all_to_all", side_effect=_fake_input_all_to_all + f"{_LAYER}._usp_input_all_to_all_qkv", + side_effect=lambda *qkv: tuple(_fake_input_all_to_all(x) for x in qkv), ), patch( f"{_LAYER}._usp_output_all_to_all", @@ -221,7 +222,10 @@ class TestUSPAttentionMaskedReplicatedPrefix(unittest.TestCase): patch(f"{_LAYER}.get_ulysses_parallel_world_size", return_value=_SP), patch(f"{_LAYER}.get_ring_parallel_world_size", return_value=1), patch(f"{_LAYER}.get_ulysses_parallel_rank", return_value=0), - patch(f"{_LAYER}._usp_input_all_to_all", side_effect=fake_input_a2a), + patch( + f"{_LAYER}._usp_input_all_to_all_qkv", + side_effect=lambda *qkv: tuple(fake_input_a2a(x) for x in qkv), + ), patch(f"{_LAYER}._usp_output_all_to_all", side_effect=fake_output_a2a), patch( f"{_LAYER}.sequence_model_parallel_all_gather", diff --git a/python/sglang/multimodal_gen/test/unit/test_usp_packed_qkv_a2a.py b/python/sglang/multimodal_gen/test/unit/test_usp_packed_qkv_a2a.py index 3b0069da56..f9f966c698 100644 --- a/python/sglang/multimodal_gen/test/unit/test_usp_packed_qkv_a2a.py +++ b/python/sglang/multimodal_gen/test/unit/test_usp_packed_qkv_a2a.py @@ -174,5 +174,65 @@ class TestPackedQKVInputA2A(CustomTestCase): self.assertTrue(packed[r][i].is_contiguous()) +@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA") +class TestPackedQKVRowViewPredicate(CustomTestCase): + """`_packed_qkv_row_view_is_free` decides which path the a2a takes. + + It is the whole safety argument for accepting strided input: the pack + kernel reads q/k/v through explicit row/head strides, so only head_size + needs unit stride, but `view(b * s_local, h, d)` still has to be free. If + the predicate admits a layout where that view would copy, the kernel reads + the wrong rows and attention is silently wrong -- no exception. + """ + + H, D, WORLD = 8, 64, 4 + + def _merged_qkv_chunk(self, b, s): + """q/k/v as chunks of one packed projection, the FLUX.2 layout. + + Non-contiguous, unit stride on head_size, batch/seq mergeable -- the + case this PR exists to accept. + """ + inner = self.H * self.D + hidden = torch.randn(b, s, 3 * inner, device="cuda", dtype=torch.bfloat16) + q, k, v = hidden.chunk(3, dim=-1) + return tuple(t.unflatten(-1, (self.H, self.D)) for t in (q, k, v)) + + def test_strided_merged_projection_is_accepted(self): + q, k, v = self._merged_qkv_chunk(1, 32) + self.assertFalse(q.is_contiguous(), "setup: q must be strided") + self.assertTrue(usp_mod._packed_qkv_row_view_is_free(q)) + self.assertTrue(usp_mod._can_use_packed_qkv_a2a_4d(q, k, v, self.WORLD)) + # the row view the pack kernel takes must genuinely not copy + rows = q.shape[0] * q.shape[1] + self.assertEqual(q.view(rows, self.H, self.D).data_ptr(), q.data_ptr()) + + def test_batched_sequence_slice_is_rejected(self): + """b > 1 sliced along seq: stride(0) still spans the unsliced rows, so + merging batch and seq would copy. Must stay on the unpacked path.""" + full = torch.randn(2, 64, self.H, self.D, device="cuda", dtype=torch.bfloat16) + sl = full[:, 32:] + self.assertEqual(sl.stride(-1), 1, "setup: head_size is unit stride") + self.assertNotEqual(sl.stride(0), sl.shape[1] * sl.stride(1)) + self.assertFalse(usp_mod._packed_qkv_row_view_is_free(sl)) + self.assertFalse(usp_mod._can_use_packed_qkv_a2a_4d(sl, sl, sl, self.WORLD)) + + def test_contiguous_still_accepted(self): + """No regression for the layouts that already qualified.""" + t = torch.randn(1, 32, self.H, self.D, device="cuda", dtype=torch.bfloat16) + self.assertTrue(t.is_contiguous()) + self.assertTrue(usp_mod._packed_qkv_row_view_is_free(t)) + self.assertTrue(usp_mod._can_use_packed_qkv_a2a_4d(t, t, t, self.WORLD)) + + def test_non_unit_head_stride_is_rejected(self): + """head_size must be unit-stride; the kernel cannot express a gap.""" + wide = torch.randn( + 1, 32, self.H, 2 * self.D, device="cuda", dtype=torch.bfloat16 + ) + t = wide[..., ::2] + self.assertNotEqual(t.stride(-1), 1) + self.assertFalse(usp_mod._packed_qkv_row_view_is_free(t)) + + if __name__ == "__main__": unittest.main()