[diffusion] feat: accept strided row views in the packed QKV all-to-all (#40588)

Co-authored-by: Mick <mickjagger19@icloud.com>
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Aleksi Vesanto
2026-10-09 13:55:31 +08:00
committed by GitHub
co-authored by Mick Mick Qian
parent 3c1703d2ff
commit e46aaaa789
4 changed files with 85 additions and 12 deletions
@@ -1740,9 +1740,7 @@ class USPAttention(nn.Module):
k_rep, k_shard = k[:, :num_rep], k[:, num_rep:]
v_rep, v_shard = v[:, :num_rep], v[:, num_rep:]
q_shard = _usp_input_all_to_all(q_shard, head_dim=2)
k_shard = _usp_input_all_to_all(k_shard, head_dim=2)
v_shard = _usp_input_all_to_all(v_shard, head_dim=2)
q_shard, k_shard, v_shard = _usp_input_all_to_all_qkv(q_shard, k_shard, v_shard)
joint_mask = joint_mask_meta = None
if attn_mask is not None:
@@ -2051,9 +2049,7 @@ class USPAttention(nn.Module):
k_shard, k_rep = k[:, :-num_rep], k[:, -num_rep:]
v_shard, v_rep = v[:, :-num_rep], v[:, -num_rep:]
q_shard = _usp_input_all_to_all(q_shard, head_dim=2)
k_shard = _usp_input_all_to_all(k_shard, head_dim=2)
v_shard = _usp_input_all_to_all(v_shard, head_dim=2)
q_shard, k_shard, v_shard = _usp_input_all_to_all_qkv(q_shard, k_shard, v_shard)
h_local = q_shard.shape[2]
kv_h_local = k_shard.shape[2]
@@ -413,6 +413,18 @@ def _usp_input_all_to_all_packed_qkv(
return q, k, v
def _packed_qkv_row_view_is_free(x: torch.Tensor) -> bool:
"""True when ``x.view(b * s_local, h, d)`` costs no copy.
The pack kernel reads q/k/v through explicit row/head strides, so only the
head_size dim has to be unit-stride; full contiguity is not required. The
batch and sequence dims still have to merge into one, which is free when
batch is 1 or when the two dims are already adjacent in memory.
"""
batch, seq_local = x.shape[0], x.shape[1]
return x.stride(-1) == 1 and (batch == 1 or x.stride(0) == seq_local * x.stride(1))
def _can_use_packed_qkv_a2a_4d(
q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, world_size: int
) -> bool:
@@ -422,9 +434,9 @@ def _can_use_packed_qkv_a2a_4d(
and q.shape == k.shape == v.shape
and q.dtype == k.dtype == v.dtype
and q.dtype in (torch.float16, torch.bfloat16)
and q.is_contiguous()
and k.is_contiguous()
and v.is_contiguous()
and _packed_qkv_row_view_is_free(q)
and _packed_qkv_row_view_is_free(k)
and _packed_qkv_row_view_is_free(v)
and q.shape[2] % world_size == 0
and not torch.compiler.is_compiling()
)
@@ -441,7 +453,8 @@ def _usp_input_all_to_all_qkv(
destination-major by one relayout kernel and exchanged in a single
collective instead of three; only data movement changes, so the result is
bit-identical to the unpacked path, which stays as the fallback for
ineligible inputs (CPU, GQA-mismatched shapes, non-contiguous layouts).
ineligible inputs (CPU, GQA-mismatched shapes, layouts whose row view
would copy).
Adapted from the NVlabs Sana sol-engine branch (Apache-2.0).
"""
world_size = get_ulysses_parallel_world_size()
@@ -70,7 +70,8 @@ class TestUSPAttentionReplicatedPrefix(unittest.TestCase):
patch(f"{_LAYER}.get_ulysses_parallel_rank", return_value=sp_rank),
patch(f"{_LAYER}.get_ring_parallel_world_size", return_value=1),
patch(
f"{_LAYER}._usp_input_all_to_all", side_effect=_fake_input_all_to_all
f"{_LAYER}._usp_input_all_to_all_qkv",
side_effect=lambda *qkv: tuple(_fake_input_all_to_all(x) for x in qkv),
),
patch(
f"{_LAYER}._usp_output_all_to_all",
@@ -221,7 +222,10 @@ class TestUSPAttentionMaskedReplicatedPrefix(unittest.TestCase):
patch(f"{_LAYER}.get_ulysses_parallel_world_size", return_value=_SP),
patch(f"{_LAYER}.get_ring_parallel_world_size", return_value=1),
patch(f"{_LAYER}.get_ulysses_parallel_rank", return_value=0),
patch(f"{_LAYER}._usp_input_all_to_all", side_effect=fake_input_a2a),
patch(
f"{_LAYER}._usp_input_all_to_all_qkv",
side_effect=lambda *qkv: tuple(fake_input_a2a(x) for x in qkv),
),
patch(f"{_LAYER}._usp_output_all_to_all", side_effect=fake_output_a2a),
patch(
f"{_LAYER}.sequence_model_parallel_all_gather",
@@ -174,5 +174,65 @@ class TestPackedQKVInputA2A(CustomTestCase):
self.assertTrue(packed[r][i].is_contiguous())
@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA")
class TestPackedQKVRowViewPredicate(CustomTestCase):
"""`_packed_qkv_row_view_is_free` decides which path the a2a takes.
It is the whole safety argument for accepting strided input: the pack
kernel reads q/k/v through explicit row/head strides, so only head_size
needs unit stride, but `view(b * s_local, h, d)` still has to be free. If
the predicate admits a layout where that view would copy, the kernel reads
the wrong rows and attention is silently wrong -- no exception.
"""
H, D, WORLD = 8, 64, 4
def _merged_qkv_chunk(self, b, s):
"""q/k/v as chunks of one packed projection, the FLUX.2 layout.
Non-contiguous, unit stride on head_size, batch/seq mergeable -- the
case this PR exists to accept.
"""
inner = self.H * self.D
hidden = torch.randn(b, s, 3 * inner, device="cuda", dtype=torch.bfloat16)
q, k, v = hidden.chunk(3, dim=-1)
return tuple(t.unflatten(-1, (self.H, self.D)) for t in (q, k, v))
def test_strided_merged_projection_is_accepted(self):
q, k, v = self._merged_qkv_chunk(1, 32)
self.assertFalse(q.is_contiguous(), "setup: q must be strided")
self.assertTrue(usp_mod._packed_qkv_row_view_is_free(q))
self.assertTrue(usp_mod._can_use_packed_qkv_a2a_4d(q, k, v, self.WORLD))
# the row view the pack kernel takes must genuinely not copy
rows = q.shape[0] * q.shape[1]
self.assertEqual(q.view(rows, self.H, self.D).data_ptr(), q.data_ptr())
def test_batched_sequence_slice_is_rejected(self):
"""b > 1 sliced along seq: stride(0) still spans the unsliced rows, so
merging batch and seq would copy. Must stay on the unpacked path."""
full = torch.randn(2, 64, self.H, self.D, device="cuda", dtype=torch.bfloat16)
sl = full[:, 32:]
self.assertEqual(sl.stride(-1), 1, "setup: head_size is unit stride")
self.assertNotEqual(sl.stride(0), sl.shape[1] * sl.stride(1))
self.assertFalse(usp_mod._packed_qkv_row_view_is_free(sl))
self.assertFalse(usp_mod._can_use_packed_qkv_a2a_4d(sl, sl, sl, self.WORLD))
def test_contiguous_still_accepted(self):
"""No regression for the layouts that already qualified."""
t = torch.randn(1, 32, self.H, self.D, device="cuda", dtype=torch.bfloat16)
self.assertTrue(t.is_contiguous())
self.assertTrue(usp_mod._packed_qkv_row_view_is_free(t))
self.assertTrue(usp_mod._can_use_packed_qkv_a2a_4d(t, t, t, self.WORLD))
def test_non_unit_head_stride_is_rejected(self):
"""head_size must be unit-stride; the kernel cannot express a gap."""
wide = torch.randn(
1, 32, self.H, 2 * self.D, device="cuda", dtype=torch.bfloat16
)
t = wide[..., ::2]
self.assertNotEqual(t.stride(-1), 1)
self.assertFalse(usp_mod._packed_qkv_row_view_is_free(t))
if __name__ == "__main__":
unittest.main()