[diffusion] perf: keep Qwen-Image-2.1 and Ulysses graphs whole (#43586)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Opus 5.5
parent
e1f43c9a61
commit
d45ef0f2d2
@@ -1510,7 +1510,13 @@ class USPAttention(nn.Module):
|
||||
# Ulysses-style All-to-All for sequence/head sharding
|
||||
if sp_size > 1 and not qkv_pre_all_to_all:
|
||||
# -> [B, S, H_local, D]
|
||||
if self.enable_packed_qkv_input_a2a and q.device.type == "cuda":
|
||||
# The overlapped exchange (side stream + async work handles) cannot
|
||||
# be traced; compiled graphs take the plain all-to-all.
|
||||
if (
|
||||
self.enable_packed_qkv_input_a2a
|
||||
and q.device.type == "cuda"
|
||||
and not torch.compiler.is_compiling()
|
||||
):
|
||||
q, k, v = async_a2a_communicate(
|
||||
[q, k, v],
|
||||
sp_size,
|
||||
@@ -2172,7 +2178,11 @@ class USPAttention(nn.Module):
|
||||
return self.attn_impl.forward(q_group, k_group, v_group, ctx_attn_metadata)
|
||||
|
||||
def sequential():
|
||||
if q.device.type == "cuda" and get_ulysses_parallel_world_size() > 1:
|
||||
if (
|
||||
q.device.type == "cuda"
|
||||
and get_ulysses_parallel_world_size() > 1
|
||||
and not torch.compiler.is_compiling()
|
||||
):
|
||||
q_gathered, k_gathered, v_gathered = async_a2a_communicate(
|
||||
[q, k_shard, v_shard],
|
||||
get_ulysses_parallel_world_size(),
|
||||
|
||||
@@ -109,7 +109,7 @@ def _usp_all_to_all_single(x: torch.Tensor, role: str | None = None) -> torch.Te
|
||||
assert ulysses_pg is not None, "Ulysses process group is not initialized."
|
||||
x_shape = x.shape
|
||||
x = x.flatten().contiguous()
|
||||
if x.is_cuda and not _UNCACHED_STAGING:
|
||||
if x.is_cuda and not _UNCACHED_STAGING and not torch.compiler.is_compiling():
|
||||
from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a_multi import (
|
||||
IPC_A2A_MULTI,
|
||||
ipc_a2a_multi_ready,
|
||||
@@ -169,7 +169,9 @@ def _ipc_ready_group():
|
||||
ipc_a2a_ready,
|
||||
)
|
||||
|
||||
if _UNCACHED_STAGING:
|
||||
# The transport's custom-class handles cannot be traced; a compiled graph
|
||||
# keeps the NCCL all-to-all (traceable) instead of breaking here.
|
||||
if _UNCACHED_STAGING or torch.compiler.is_compiling():
|
||||
return None
|
||||
group = get_sp_group().ulysses_group
|
||||
return group if ipc_a2a_ready(group) else None
|
||||
|
||||
@@ -524,6 +524,10 @@ class QwenImage21Attention(nn.Module):
|
||||
the fused norm + RoPE kernel and SDPA both accept that layout. The first
|
||||
call compares every slice against the module forwards.
|
||||
"""
|
||||
# Storage-identity checks and the one-shot verification cannot be traced;
|
||||
# a compiled graph keeps the plain projections instead of breaking here.
|
||||
if torch.compiler.is_compiling():
|
||||
return None
|
||||
layers = (self.to_q, self.to_k, self.to_v)
|
||||
packed = self.packed_qkv_weight()
|
||||
if not (
|
||||
|
||||
@@ -4,7 +4,7 @@ import json
|
||||
import sys
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
@@ -709,3 +709,10 @@ def test_packed_qkv_weight_skips_offloaded_placeholders():
|
||||
)
|
||||
attention = SimpleNamespace(to_q=projection, to_k=projection, to_v=projection)
|
||||
assert QwenImage21Attention.packed_qkv_weight(attention) is None
|
||||
|
||||
|
||||
def test_packed_qkv_projection_is_skipped_under_compile():
|
||||
# Its storage-identity checks cannot be traced; compiled graphs keep the
|
||||
# plain projections instead of breaking in every attention.
|
||||
with patch.object(torch.compiler, "is_compiling", return_value=True):
|
||||
assert QwenImage21Attention._project_qkv_packed(object(), None, [], []) is None
|
||||
|
||||
@@ -51,5 +51,22 @@ class TestIpcInputA2AQkvGuard(unittest.TestCase):
|
||||
self.assertIsNone(usp._ipc_input_a2a_qkv(q, q.clone(), q.clone()))
|
||||
|
||||
|
||||
class TestIpcSkippedUnderCompile(unittest.TestCase):
|
||||
"""The transport's custom-class handles cannot be traced: a compiled graph
|
||||
keeps the NCCL all-to-all instead of breaking at every exchange."""
|
||||
|
||||
def test_ipc_group_is_unavailable_while_compiling(self):
|
||||
ready = mock.MagicMock(return_value=True)
|
||||
with (
|
||||
mock.patch.object(usp.torch.compiler, "is_compiling", lambda: True),
|
||||
mock.patch(
|
||||
"sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a.ipc_a2a_ready",
|
||||
ready,
|
||||
),
|
||||
):
|
||||
self.assertIsNone(usp._ipc_ready_group())
|
||||
ready.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user