[AMD] Fix Flux2 kv-gather abort on non-contiguous K/V (#43793)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
@@ -447,6 +447,8 @@ class GroupCoordinator:
|
||||
return list(output_tensor.chunk(world_size, dim=dim))
|
||||
return output_tensor
|
||||
else:
|
||||
if not input_.is_contiguous():
|
||||
input_ = input_.contiguous()
|
||||
all_gather_single(output_tensor, input_, group=self.device_group)
|
||||
|
||||
if dim != 0:
|
||||
|
||||
@@ -1680,10 +1680,9 @@ class USPAttention(nn.Module):
|
||||
"Replicated prefix and suffix cannot be used at the same time."
|
||||
)
|
||||
|
||||
# slicing along dim 1 keeps the original row stride, so the shard is
|
||||
# contiguous only while the batch dim is 1; all_gather_into_tensor
|
||||
# requires a contiguous input, so materialize the shard before the
|
||||
# collective
|
||||
# A dim-1 slice keeps the original row stride, and Flux2 K/V are views
|
||||
# from qkv.chunk, so neither form is contiguous. all_gather_into_tensor
|
||||
# rejects a non-contiguous input, so materialize before the collective.
|
||||
if num_replicated_prefix:
|
||||
replicated = tensor[:, :num_replicated_prefix]
|
||||
sharded = tensor[:, num_replicated_prefix:].contiguous()
|
||||
@@ -1696,7 +1695,7 @@ class USPAttention(nn.Module):
|
||||
gathered = sequence_model_parallel_all_gather(sharded, dim=1)
|
||||
return torch.cat([gathered, replicated], dim=1)
|
||||
|
||||
return sequence_model_parallel_all_gather(tensor, dim=1)
|
||||
return sequence_model_parallel_all_gather(tensor.contiguous(), dim=1)
|
||||
|
||||
def _forward_with_kv_gather(
|
||||
self,
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Sequence-parallel K/V gather must materialize non-contiguous views.
|
||||
|
||||
Flux2 packs Q, K, and V with one projection and then ``chunk``s them. That
|
||||
view is not contiguous, and ``all_gather_into_tensor`` rejects it.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.layers.attention.layer import USPAttention
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
|
||||
class TestSequenceParallelKvGatherContiguous(CustomTestCase):
|
||||
def test_noncontiguous_kv_chunk_is_materialized_before_all_gather(self):
|
||||
packed = torch.randn(2, 8, 12)
|
||||
key = packed.chunk(3, dim=-1)[1]
|
||||
self.assertFalse(key.is_contiguous())
|
||||
|
||||
captured = {}
|
||||
|
||||
def fake_all_gather(tensor, dim=-1):
|
||||
captured["tensor"] = tensor
|
||||
captured["dim"] = dim
|
||||
return tensor
|
||||
|
||||
with patch(
|
||||
"sglang.multimodal_gen.runtime.layers.attention.layer.sequence_model_parallel_all_gather",
|
||||
fake_all_gather,
|
||||
):
|
||||
gathered = USPAttention._gather_sharded_sequence(key)
|
||||
|
||||
self.assertEqual(captured["dim"], 1)
|
||||
self.assertTrue(captured["tensor"].is_contiguous())
|
||||
self.assertTrue(torch.equal(captured["tensor"], key))
|
||||
self.assertTrue(torch.equal(gathered, key))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user