[AMD] Fix Flux2 kv-gather abort on non-contiguous K/V (#43793)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
jacky.cheng
2026-10-12 22:20:55 -07:00
committed by GitHub
co-authored by Mick Qian
parent 43356304b4
commit 42e43cf33f
3 changed files with 48 additions and 5 deletions
@@ -447,6 +447,8 @@ class GroupCoordinator:
return list(output_tensor.chunk(world_size, dim=dim))
return output_tensor
else:
if not input_.is_contiguous():
input_ = input_.contiguous()
all_gather_single(output_tensor, input_, group=self.device_group)
if dim != 0:
@@ -1680,10 +1680,9 @@ class USPAttention(nn.Module):
"Replicated prefix and suffix cannot be used at the same time."
)
# slicing along dim 1 keeps the original row stride, so the shard is
# contiguous only while the batch dim is 1; all_gather_into_tensor
# requires a contiguous input, so materialize the shard before the
# collective
# A dim-1 slice keeps the original row stride, and Flux2 K/V are views
# from qkv.chunk, so neither form is contiguous. all_gather_into_tensor
# rejects a non-contiguous input, so materialize before the collective.
if num_replicated_prefix:
replicated = tensor[:, :num_replicated_prefix]
sharded = tensor[:, num_replicated_prefix:].contiguous()
@@ -1696,7 +1695,7 @@ class USPAttention(nn.Module):
gathered = sequence_model_parallel_all_gather(sharded, dim=1)
return torch.cat([gathered, replicated], dim=1)
return sequence_model_parallel_all_gather(tensor, dim=1)
return sequence_model_parallel_all_gather(tensor.contiguous(), dim=1)
def _forward_with_kv_gather(
self,
@@ -0,0 +1,42 @@
"""Sequence-parallel K/V gather must materialize non-contiguous views.
Flux2 packs Q, K, and V with one projection and then ``chunk``s them. That
view is not contiguous, and ``all_gather_into_tensor`` rejects it.
"""
import unittest
from unittest.mock import patch
import torch
from sglang.multimodal_gen.runtime.layers.attention.layer import USPAttention
from sglang.test.test_utils import CustomTestCase
class TestSequenceParallelKvGatherContiguous(CustomTestCase):
def test_noncontiguous_kv_chunk_is_materialized_before_all_gather(self):
packed = torch.randn(2, 8, 12)
key = packed.chunk(3, dim=-1)[1]
self.assertFalse(key.is_contiguous())
captured = {}
def fake_all_gather(tensor, dim=-1):
captured["tensor"] = tensor
captured["dim"] = dim
return tensor
with patch(
"sglang.multimodal_gen.runtime.layers.attention.layer.sequence_model_parallel_all_gather",
fake_all_gather,
):
gathered = USPAttention._gather_sharded_sequence(key)
self.assertEqual(captured["dim"], 1)
self.assertTrue(captured["tensor"].is_contiguous())
self.assertTrue(torch.equal(captured["tensor"], key))
self.assertTrue(torch.equal(gathered, key))
if __name__ == "__main__":
unittest.main()