diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/layer.py b/python/sglang/multimodal_gen/runtime/layers/attention/layer.py index b26440e2c7..2f659309be 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/layer.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/layer.py @@ -1510,7 +1510,13 @@ class USPAttention(nn.Module): # Ulysses-style All-to-All for sequence/head sharding if sp_size > 1 and not qkv_pre_all_to_all: # -> [B, S, H_local, D] - if self.enable_packed_qkv_input_a2a and q.device.type == "cuda": + # The overlapped exchange (side stream + async work handles) cannot + # be traced; compiled graphs take the plain all-to-all. + if ( + self.enable_packed_qkv_input_a2a + and q.device.type == "cuda" + and not torch.compiler.is_compiling() + ): q, k, v = async_a2a_communicate( [q, k, v], sp_size, @@ -2172,7 +2178,11 @@ class USPAttention(nn.Module): return self.attn_impl.forward(q_group, k_group, v_group, ctx_attn_metadata) def sequential(): - if q.device.type == "cuda" and get_ulysses_parallel_world_size() > 1: + if ( + q.device.type == "cuda" + and get_ulysses_parallel_world_size() > 1 + and not torch.compiler.is_compiling() + ): q_gathered, k_gathered, v_gathered = async_a2a_communicate( [q, k_shard, v_shard], get_ulysses_parallel_world_size(), diff --git a/python/sglang/multimodal_gen/runtime/layers/usp.py b/python/sglang/multimodal_gen/runtime/layers/usp.py index 8ffc89a3cb..9afe051928 100644 --- a/python/sglang/multimodal_gen/runtime/layers/usp.py +++ b/python/sglang/multimodal_gen/runtime/layers/usp.py @@ -109,7 +109,7 @@ def _usp_all_to_all_single(x: torch.Tensor, role: str | None = None) -> torch.Te assert ulysses_pg is not None, "Ulysses process group is not initialized." x_shape = x.shape x = x.flatten().contiguous() - if x.is_cuda and not _UNCACHED_STAGING: + if x.is_cuda and not _UNCACHED_STAGING and not torch.compiler.is_compiling(): from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a_multi import ( IPC_A2A_MULTI, ipc_a2a_multi_ready, @@ -169,7 +169,9 @@ def _ipc_ready_group(): ipc_a2a_ready, ) - if _UNCACHED_STAGING: + # The transport's custom-class handles cannot be traced; a compiled graph + # keeps the NCCL all-to-all (traceable) instead of breaking here. + if _UNCACHED_STAGING or torch.compiler.is_compiling(): return None group = get_sp_group().ulysses_group return group if ipc_a2a_ready(group) else None diff --git a/python/sglang/multimodal_gen/runtime/models/dits/qwen_image21.py b/python/sglang/multimodal_gen/runtime/models/dits/qwen_image21.py index ad405dfefb..1c0331c88a 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/qwen_image21.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/qwen_image21.py @@ -524,6 +524,10 @@ class QwenImage21Attention(nn.Module): the fused norm + RoPE kernel and SDPA both accept that layout. The first call compares every slice against the module forwards. """ + # Storage-identity checks and the one-shot verification cannot be traced; + # a compiled graph keeps the plain projections instead of breaking here. + if torch.compiler.is_compiling(): + return None layers = (self.to_q, self.to_k, self.to_v) packed = self.packed_qkv_weight() if not ( diff --git a/python/sglang/multimodal_gen/test/unit/test_qwen_image21.py b/python/sglang/multimodal_gen/test/unit/test_qwen_image21.py index 72118d7716..691d86fa47 100644 --- a/python/sglang/multimodal_gen/test/unit/test_qwen_image21.py +++ b/python/sglang/multimodal_gen/test/unit/test_qwen_image21.py @@ -4,7 +4,7 @@ import json import sys from contextlib import nullcontext from types import SimpleNamespace -from unittest.mock import Mock +from unittest.mock import Mock, patch import numpy as np import pytest @@ -709,3 +709,10 @@ def test_packed_qkv_weight_skips_offloaded_placeholders(): ) attention = SimpleNamespace(to_q=projection, to_k=projection, to_v=projection) assert QwenImage21Attention.packed_qkv_weight(attention) is None + + +def test_packed_qkv_projection_is_skipped_under_compile(): + # Its storage-identity checks cannot be traced; compiled graphs keep the + # plain projections instead of breaking in every attention. + with patch.object(torch.compiler, "is_compiling", return_value=True): + assert QwenImage21Attention._project_qkv_packed(object(), None, [], []) is None diff --git a/python/sglang/multimodal_gen/test/unit/test_usp_ipc_a2a_guard.py b/python/sglang/multimodal_gen/test/unit/test_usp_ipc_a2a_guard.py index 840b9818b7..4c921e04fa 100644 --- a/python/sglang/multimodal_gen/test/unit/test_usp_ipc_a2a_guard.py +++ b/python/sglang/multimodal_gen/test/unit/test_usp_ipc_a2a_guard.py @@ -51,5 +51,22 @@ class TestIpcInputA2AQkvGuard(unittest.TestCase): self.assertIsNone(usp._ipc_input_a2a_qkv(q, q.clone(), q.clone())) +class TestIpcSkippedUnderCompile(unittest.TestCase): + """The transport's custom-class handles cannot be traced: a compiled graph + keeps the NCCL all-to-all instead of breaking at every exchange.""" + + def test_ipc_group_is_unavailable_while_compiling(self): + ready = mock.MagicMock(return_value=True) + with ( + mock.patch.object(usp.torch.compiler, "is_compiling", lambda: True), + mock.patch( + "sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a.ipc_a2a_ready", + ready, + ), + ): + self.assertIsNone(usp._ipc_ready_group()) + ready.assert_not_called() + + if __name__ == "__main__": unittest.main()