[diffusion] feat: accept strided row views in the packed QKV all-to-all (#40588)
Co-authored-by: Mick <mickjagger19@icloud.com> Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
co-authored by
Mick
Mick Qian
parent
3c1703d2ff
commit
e46aaaa789
@@ -1740,9 +1740,7 @@ class USPAttention(nn.Module):
|
||||
k_rep, k_shard = k[:, :num_rep], k[:, num_rep:]
|
||||
v_rep, v_shard = v[:, :num_rep], v[:, num_rep:]
|
||||
|
||||
q_shard = _usp_input_all_to_all(q_shard, head_dim=2)
|
||||
k_shard = _usp_input_all_to_all(k_shard, head_dim=2)
|
||||
v_shard = _usp_input_all_to_all(v_shard, head_dim=2)
|
||||
q_shard, k_shard, v_shard = _usp_input_all_to_all_qkv(q_shard, k_shard, v_shard)
|
||||
|
||||
joint_mask = joint_mask_meta = None
|
||||
if attn_mask is not None:
|
||||
@@ -2051,9 +2049,7 @@ class USPAttention(nn.Module):
|
||||
k_shard, k_rep = k[:, :-num_rep], k[:, -num_rep:]
|
||||
v_shard, v_rep = v[:, :-num_rep], v[:, -num_rep:]
|
||||
|
||||
q_shard = _usp_input_all_to_all(q_shard, head_dim=2)
|
||||
k_shard = _usp_input_all_to_all(k_shard, head_dim=2)
|
||||
v_shard = _usp_input_all_to_all(v_shard, head_dim=2)
|
||||
q_shard, k_shard, v_shard = _usp_input_all_to_all_qkv(q_shard, k_shard, v_shard)
|
||||
|
||||
h_local = q_shard.shape[2]
|
||||
kv_h_local = k_shard.shape[2]
|
||||
|
||||
@@ -413,6 +413,18 @@ def _usp_input_all_to_all_packed_qkv(
|
||||
return q, k, v
|
||||
|
||||
|
||||
def _packed_qkv_row_view_is_free(x: torch.Tensor) -> bool:
|
||||
"""True when ``x.view(b * s_local, h, d)`` costs no copy.
|
||||
|
||||
The pack kernel reads q/k/v through explicit row/head strides, so only the
|
||||
head_size dim has to be unit-stride; full contiguity is not required. The
|
||||
batch and sequence dims still have to merge into one, which is free when
|
||||
batch is 1 or when the two dims are already adjacent in memory.
|
||||
"""
|
||||
batch, seq_local = x.shape[0], x.shape[1]
|
||||
return x.stride(-1) == 1 and (batch == 1 or x.stride(0) == seq_local * x.stride(1))
|
||||
|
||||
|
||||
def _can_use_packed_qkv_a2a_4d(
|
||||
q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, world_size: int
|
||||
) -> bool:
|
||||
@@ -422,9 +434,9 @@ def _can_use_packed_qkv_a2a_4d(
|
||||
and q.shape == k.shape == v.shape
|
||||
and q.dtype == k.dtype == v.dtype
|
||||
and q.dtype in (torch.float16, torch.bfloat16)
|
||||
and q.is_contiguous()
|
||||
and k.is_contiguous()
|
||||
and v.is_contiguous()
|
||||
and _packed_qkv_row_view_is_free(q)
|
||||
and _packed_qkv_row_view_is_free(k)
|
||||
and _packed_qkv_row_view_is_free(v)
|
||||
and q.shape[2] % world_size == 0
|
||||
and not torch.compiler.is_compiling()
|
||||
)
|
||||
@@ -441,7 +453,8 @@ def _usp_input_all_to_all_qkv(
|
||||
destination-major by one relayout kernel and exchanged in a single
|
||||
collective instead of three; only data movement changes, so the result is
|
||||
bit-identical to the unpacked path, which stays as the fallback for
|
||||
ineligible inputs (CPU, GQA-mismatched shapes, non-contiguous layouts).
|
||||
ineligible inputs (CPU, GQA-mismatched shapes, layouts whose row view
|
||||
would copy).
|
||||
Adapted from the NVlabs Sana sol-engine branch (Apache-2.0).
|
||||
"""
|
||||
world_size = get_ulysses_parallel_world_size()
|
||||
|
||||
@@ -70,7 +70,8 @@ class TestUSPAttentionReplicatedPrefix(unittest.TestCase):
|
||||
patch(f"{_LAYER}.get_ulysses_parallel_rank", return_value=sp_rank),
|
||||
patch(f"{_LAYER}.get_ring_parallel_world_size", return_value=1),
|
||||
patch(
|
||||
f"{_LAYER}._usp_input_all_to_all", side_effect=_fake_input_all_to_all
|
||||
f"{_LAYER}._usp_input_all_to_all_qkv",
|
||||
side_effect=lambda *qkv: tuple(_fake_input_all_to_all(x) for x in qkv),
|
||||
),
|
||||
patch(
|
||||
f"{_LAYER}._usp_output_all_to_all",
|
||||
@@ -221,7 +222,10 @@ class TestUSPAttentionMaskedReplicatedPrefix(unittest.TestCase):
|
||||
patch(f"{_LAYER}.get_ulysses_parallel_world_size", return_value=_SP),
|
||||
patch(f"{_LAYER}.get_ring_parallel_world_size", return_value=1),
|
||||
patch(f"{_LAYER}.get_ulysses_parallel_rank", return_value=0),
|
||||
patch(f"{_LAYER}._usp_input_all_to_all", side_effect=fake_input_a2a),
|
||||
patch(
|
||||
f"{_LAYER}._usp_input_all_to_all_qkv",
|
||||
side_effect=lambda *qkv: tuple(fake_input_a2a(x) for x in qkv),
|
||||
),
|
||||
patch(f"{_LAYER}._usp_output_all_to_all", side_effect=fake_output_a2a),
|
||||
patch(
|
||||
f"{_LAYER}.sequence_model_parallel_all_gather",
|
||||
|
||||
@@ -174,5 +174,65 @@ class TestPackedQKVInputA2A(CustomTestCase):
|
||||
self.assertTrue(packed[r][i].is_contiguous())
|
||||
|
||||
|
||||
@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA")
|
||||
class TestPackedQKVRowViewPredicate(CustomTestCase):
|
||||
"""`_packed_qkv_row_view_is_free` decides which path the a2a takes.
|
||||
|
||||
It is the whole safety argument for accepting strided input: the pack
|
||||
kernel reads q/k/v through explicit row/head strides, so only head_size
|
||||
needs unit stride, but `view(b * s_local, h, d)` still has to be free. If
|
||||
the predicate admits a layout where that view would copy, the kernel reads
|
||||
the wrong rows and attention is silently wrong -- no exception.
|
||||
"""
|
||||
|
||||
H, D, WORLD = 8, 64, 4
|
||||
|
||||
def _merged_qkv_chunk(self, b, s):
|
||||
"""q/k/v as chunks of one packed projection, the FLUX.2 layout.
|
||||
|
||||
Non-contiguous, unit stride on head_size, batch/seq mergeable -- the
|
||||
case this PR exists to accept.
|
||||
"""
|
||||
inner = self.H * self.D
|
||||
hidden = torch.randn(b, s, 3 * inner, device="cuda", dtype=torch.bfloat16)
|
||||
q, k, v = hidden.chunk(3, dim=-1)
|
||||
return tuple(t.unflatten(-1, (self.H, self.D)) for t in (q, k, v))
|
||||
|
||||
def test_strided_merged_projection_is_accepted(self):
|
||||
q, k, v = self._merged_qkv_chunk(1, 32)
|
||||
self.assertFalse(q.is_contiguous(), "setup: q must be strided")
|
||||
self.assertTrue(usp_mod._packed_qkv_row_view_is_free(q))
|
||||
self.assertTrue(usp_mod._can_use_packed_qkv_a2a_4d(q, k, v, self.WORLD))
|
||||
# the row view the pack kernel takes must genuinely not copy
|
||||
rows = q.shape[0] * q.shape[1]
|
||||
self.assertEqual(q.view(rows, self.H, self.D).data_ptr(), q.data_ptr())
|
||||
|
||||
def test_batched_sequence_slice_is_rejected(self):
|
||||
"""b > 1 sliced along seq: stride(0) still spans the unsliced rows, so
|
||||
merging batch and seq would copy. Must stay on the unpacked path."""
|
||||
full = torch.randn(2, 64, self.H, self.D, device="cuda", dtype=torch.bfloat16)
|
||||
sl = full[:, 32:]
|
||||
self.assertEqual(sl.stride(-1), 1, "setup: head_size is unit stride")
|
||||
self.assertNotEqual(sl.stride(0), sl.shape[1] * sl.stride(1))
|
||||
self.assertFalse(usp_mod._packed_qkv_row_view_is_free(sl))
|
||||
self.assertFalse(usp_mod._can_use_packed_qkv_a2a_4d(sl, sl, sl, self.WORLD))
|
||||
|
||||
def test_contiguous_still_accepted(self):
|
||||
"""No regression for the layouts that already qualified."""
|
||||
t = torch.randn(1, 32, self.H, self.D, device="cuda", dtype=torch.bfloat16)
|
||||
self.assertTrue(t.is_contiguous())
|
||||
self.assertTrue(usp_mod._packed_qkv_row_view_is_free(t))
|
||||
self.assertTrue(usp_mod._can_use_packed_qkv_a2a_4d(t, t, t, self.WORLD))
|
||||
|
||||
def test_non_unit_head_stride_is_rejected(self):
|
||||
"""head_size must be unit-stride; the kernel cannot express a gap."""
|
||||
wide = torch.randn(
|
||||
1, 32, self.H, 2 * self.D, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
t = wide[..., ::2]
|
||||
self.assertNotEqual(t.stride(-1), 1)
|
||||
self.assertFalse(usp_mod._packed_qkv_row_view_is_free(t))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user