[diffusion] fix: share the IPC all-to-all transport between AllToAll4D and USP (#43269)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Opus 5.5
parent
17b2f35ae7
commit
b922f5ca20
@@ -78,6 +78,8 @@ class IpcA2AState:
|
||||
self.calls = 0
|
||||
self.rank = None
|
||||
self.group = None
|
||||
# global ranks of the pair the peer mappings connect
|
||||
self.ranks = None
|
||||
self.failed = False
|
||||
self.inited = False
|
||||
|
||||
@@ -135,6 +137,7 @@ class IpcA2AState:
|
||||
|
||||
self.rank = dist.get_rank(group=group)
|
||||
self.group = group
|
||||
self.ranks = dist.get_process_group_ranks(group)
|
||||
dev = torch.cuda.current_device()
|
||||
peer_dev = _peer_cuda_device(group, self.rank, dev)
|
||||
# Peer access alone also admits PCIe pairs, where this NVLink transport
|
||||
@@ -265,7 +268,13 @@ def ipc_a2a_ready(group) -> bool:
|
||||
|
||||
if not envs.SGLANG_DIFFUSION_IPC_A2A:
|
||||
return False
|
||||
if IPC_A2A.group is not None and IPC_A2A.group is not group:
|
||||
# AllToAll4D passes the SP device group and USP the Ulysses group: distinct
|
||||
# handles over one pair, so only a different pair invalidates the mappings.
|
||||
if (
|
||||
IPC_A2A.group is not None
|
||||
and group is not IPC_A2A.group
|
||||
and dist.get_process_group_ranks(group) != IPC_A2A.ranks
|
||||
):
|
||||
IPC_A2A.reset()
|
||||
if IPC_A2A.failed:
|
||||
return False
|
||||
|
||||
@@ -91,9 +91,9 @@ def _worker() -> int:
|
||||
if not torch.equal(nccl_out, ipc_out):
|
||||
failures.append(f"output a2a {(b, s_local, h_global, d)}")
|
||||
|
||||
# AllToAll4D is a second entry point (stacked-qkv UlyssesAttention: zimage's
|
||||
# secondary attention, wan-VSA, hunyuanvideo) and routes through the same
|
||||
# exchange, so it needs its own parity check.
|
||||
# AllToAll4D is a second entry point (stacked-qkv UlyssesAttention, used by
|
||||
# wan-VSA) and routes through the same exchange, so it needs its own parity
|
||||
# check.
|
||||
from sglang.multimodal_gen.runtime.distributed.communication_op import (
|
||||
sequence_model_parallel_all_to_all_4D as all_to_all_4D,
|
||||
)
|
||||
@@ -101,6 +101,7 @@ def _worker() -> int:
|
||||
# count exchanges, not staging keys: the key is (n_local, n_peer, dtype),
|
||||
# so these shapes reuse the buffers the earlier arms already allocated
|
||||
calls_before_a2a4d = IPC_A2A.calls
|
||||
flag_before_a2a4d = IPC_A2A.flag
|
||||
for b, s_local, h_global, d in shapes:
|
||||
torch.manual_seed(4321)
|
||||
for scatter_dim in (1, 2):
|
||||
@@ -133,8 +134,11 @@ def _worker() -> int:
|
||||
)
|
||||
|
||||
# A parity check that never reached the IPC branch would pass while proving
|
||||
# nothing, so require that exchanges actually happened.
|
||||
if IPC_A2A.calls == calls_before_a2a4d:
|
||||
# nothing, so require that exchanges actually happened. A re-init restarts
|
||||
# the count, so rule it out first.
|
||||
if IPC_A2A.flag is not flag_before_a2a4d:
|
||||
failures.append("AllToAll4D re-initialized the IPC transport USP set up")
|
||||
elif IPC_A2A.calls == calls_before_a2a4d:
|
||||
failures.append(
|
||||
f"AllToAll4D never took the IPC path (exchange count stayed at "
|
||||
f"{calls_before_a2a4d})"
|
||||
|
||||
@@ -19,6 +19,7 @@ def test_ipc_requires_nvlink_before_mapping_peer_memory(monkeypatch, rank, nvlin
|
||||
state = IpcA2AState()
|
||||
with (
|
||||
patch(f"{_IPC}.dist.get_rank", return_value=rank),
|
||||
patch(f"{_IPC}.dist.get_process_group_ranks", return_value=[0, 1]),
|
||||
patch(f"{_IPC}.torch.cuda.current_device", return_value=rank),
|
||||
patch(f"{_IPC}._peer_cuda_device", return_value=1 - rank),
|
||||
patch(
|
||||
@@ -46,12 +47,18 @@ def test_ipc_requires_nvlink_before_mapping_peer_memory(monkeypatch, rank, nvlin
|
||||
topology.assert_called_once_with([1, 3])
|
||||
|
||||
|
||||
def test_reinitializes_ipc_transport_for_replaced_process_group():
|
||||
@pytest.mark.parametrize(
|
||||
"new_ranks", [[0, 2], [0, 1]], ids=["different_pair", "same_pair"]
|
||||
)
|
||||
def test_ipc_transport_reinitializes_only_for_a_different_rank_pair(new_ranks):
|
||||
"""A second group handle over the same pair (the SP device group that
|
||||
AllToAll4D passes vs the Ulysses group USP passes) must reuse the transport."""
|
||||
state = IpcA2AState()
|
||||
old_group = object()
|
||||
new_group = object()
|
||||
state.inited = True
|
||||
state.group = old_group
|
||||
state.ranks = [0, 1]
|
||||
state.calls = 7
|
||||
|
||||
def initialize(group):
|
||||
@@ -61,6 +68,7 @@ def test_reinitializes_ipc_transport_for_replaced_process_group():
|
||||
with (
|
||||
patch(f"{_IPC}.IPC_A2A", state),
|
||||
patch(f"{_IPC}.envs.SGLANG_DIFFUSION_IPC_A2A", True),
|
||||
patch(f"{_IPC}.dist.get_process_group_ranks", return_value=new_ranks),
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.platforms.current_platform.is_cuda",
|
||||
return_value=True,
|
||||
@@ -74,9 +82,10 @@ def test_reinitializes_ipc_transport_for_replaced_process_group():
|
||||
):
|
||||
assert ipc_a2a_ready(new_group)
|
||||
|
||||
init.assert_called_once_with(new_group)
|
||||
assert state.group is new_group
|
||||
assert state.calls == 0
|
||||
replaced = new_ranks != [0, 1]
|
||||
assert init.called == replaced
|
||||
assert state.group is (new_group if replaced else old_group)
|
||||
assert state.calls == (0 if replaced else 7)
|
||||
|
||||
|
||||
def test_peer_cuda_device_uses_the_ulysses_group_mapping():
|
||||
|
||||
Reference in New Issue
Block a user