diff --git a/python/sglang/multimodal_gen/runtime/distributed/group_coordinator.py b/python/sglang/multimodal_gen/runtime/distributed/group_coordinator.py index cb08879d67..af577eba20 100644 --- a/python/sglang/multimodal_gen/runtime/distributed/group_coordinator.py +++ b/python/sglang/multimodal_gen/runtime/distributed/group_coordinator.py @@ -447,6 +447,8 @@ class GroupCoordinator: return list(output_tensor.chunk(world_size, dim=dim)) return output_tensor else: + if not input_.is_contiguous(): + input_ = input_.contiguous() all_gather_single(output_tensor, input_, group=self.device_group) if dim != 0: diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/layer.py b/python/sglang/multimodal_gen/runtime/layers/attention/layer.py index 2f659309be..0c75d17fc6 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/layer.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/layer.py @@ -1680,10 +1680,9 @@ class USPAttention(nn.Module): "Replicated prefix and suffix cannot be used at the same time." ) - # slicing along dim 1 keeps the original row stride, so the shard is - # contiguous only while the batch dim is 1; all_gather_into_tensor - # requires a contiguous input, so materialize the shard before the - # collective + # A dim-1 slice keeps the original row stride, and Flux2 K/V are views + # from qkv.chunk, so neither form is contiguous. all_gather_into_tensor + # rejects a non-contiguous input, so materialize before the collective. if num_replicated_prefix: replicated = tensor[:, :num_replicated_prefix] sharded = tensor[:, num_replicated_prefix:].contiguous() @@ -1696,7 +1695,7 @@ class USPAttention(nn.Module): gathered = sequence_model_parallel_all_gather(sharded, dim=1) return torch.cat([gathered, replicated], dim=1) - return sequence_model_parallel_all_gather(tensor, dim=1) + return sequence_model_parallel_all_gather(tensor.contiguous(), dim=1) def _forward_with_kv_gather( self, diff --git a/python/sglang/multimodal_gen/test/unit/test_sp_kv_gather_contiguous.py b/python/sglang/multimodal_gen/test/unit/test_sp_kv_gather_contiguous.py new file mode 100644 index 0000000000..95b4ab53f1 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_sp_kv_gather_contiguous.py @@ -0,0 +1,42 @@ +"""Sequence-parallel K/V gather must materialize non-contiguous views. + +Flux2 packs Q, K, and V with one projection and then ``chunk``s them. That +view is not contiguous, and ``all_gather_into_tensor`` rejects it. +""" + +import unittest +from unittest.mock import patch + +import torch + +from sglang.multimodal_gen.runtime.layers.attention.layer import USPAttention +from sglang.test.test_utils import CustomTestCase + + +class TestSequenceParallelKvGatherContiguous(CustomTestCase): + def test_noncontiguous_kv_chunk_is_materialized_before_all_gather(self): + packed = torch.randn(2, 8, 12) + key = packed.chunk(3, dim=-1)[1] + self.assertFalse(key.is_contiguous()) + + captured = {} + + def fake_all_gather(tensor, dim=-1): + captured["tensor"] = tensor + captured["dim"] = dim + return tensor + + with patch( + "sglang.multimodal_gen.runtime.layers.attention.layer.sequence_model_parallel_all_gather", + fake_all_gather, + ): + gathered = USPAttention._gather_sharded_sequence(key) + + self.assertEqual(captured["dim"], 1) + self.assertTrue(captured["tensor"].is_contiguous()) + self.assertTrue(torch.equal(captured["tensor"], key)) + self.assertTrue(torch.equal(gathered, key)) + + +if __name__ == "__main__": + unittest.main()