[Refactor] Drop the reduction-skip mechanisms stage boundaries no longer use (#42312)

This commit is contained in:
Cheng Wan
2026-10-03 02:21:55 -07:00
committed by GitHub
parent 71b04e02cf
commit 7be5e3473c
73 changed files with 291 additions and 430 deletions
@@ -98,8 +98,8 @@ class DeferringLayer(nn.Module):
self.layer_communicator.paths[BatchVariant.INPUT_SCATTERED] = None
self.layer_communicator.paths[BatchVariant.CONTEXT_PARALLEL] = None
self.layer_communicator.output.ffn_reduction_group = lambda forward_batch: GROUP
self.layer_communicator.output._skips_sum_for_reduce_scatter = (
lambda batch, dp_step: False
self.layer_communicator.output._sum_in_reduce_scatter = lambda batch, dp_step: (
False
)
self.layer_communicator.output._complete_now = lambda hidden, residual, **_: (
all_reduce(hidden),
@@ -1,7 +1,6 @@
"""Numerical and interface coverage for boundary/graph integration regressions."""
import unittest
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import Mock, patch
@@ -10,7 +9,6 @@ import torch
from sglang.srt.layers.layer_boundary import (
PLAIN_ADD,
ProducerReduction,
declare_attn,
declare_ffn,
)
@@ -200,12 +198,6 @@ class TestBoundaryIntegrations(unittest.TestCase):
enabled and (lora or shared),
)
def test_unsupported_producer_contracts_fail_at_declaration(self):
with self.assertRaises(ValueError):
declare_ffn(reduction=ProducerReduction.ALWAYS_PARTIAL)
with self.assertRaises(ValueError):
declare_attn(reduction=ProducerReduction.TAIL_AFTER_SUM)
def test_ffn_writing_the_next_stream_hands_on_a_complete_output(self):
# It has each part's sum completed (sum_part) before building the
# stream from the parts, so its exit owes no sum and never defers one.
@@ -261,14 +253,13 @@ class TestBoundaryIntegrations(unittest.TestCase):
capture(hidden)
return hidden
output = SimpleNamespace(finish=lambda hidden: hidden)
layer = SimpleNamespace(
attn_boundary=SimpleNamespace(
prepare=prepare, finish=lambda hidden, fb: hidden
),
ffn_boundary=SimpleNamespace(
prepare=lambda hidden, fb, **kw: hidden,
exit=lambda fb: nullcontext(output),
finish=lambda hidden, fb: hidden,
),
self_attn=lambda **kw: kw["hidden_states"],
mlp=lambda hidden, **kw: hidden,
@@ -213,9 +213,7 @@ class TestBoundaryOutputPolicy(unittest.TestCase):
),
):
for skip in (False, True):
with get_forward().scoped(
fuse_mlp_allreduce=False, mlp_reduce_scatter=skip
):
with get_forward().scoped(mlp_reduce_scatter=skip):
self.assertEqual(
moe_utils.should_skip_post_experts_all_reduce(is_tp_path=True),
skip,
@@ -481,7 +481,7 @@ def test_dual_stream_op_pins_the_deferral_off_under_a_deferring_caller():
return_value=SimpleNamespace(moe_fusions={0: fusion}),
),
):
out = op.redispatch(cuda_key, torch.zeros(4, 8), 0, True, False)
out = op.redispatch(cuda_key, torch.zeros(4, 8), 0, False)
assert get_forward().defer_moe_finalize is True
assert fusion.seen_defer is False
@@ -44,7 +44,6 @@ def layernorm(hidden_states, residual=None):
class Flags:
def __init__(self):
self.fuse_mlp_allreduce = False
self.mlp_reduce_scatter = False
self.defer_moe_finalize = False
self.sp_active = False
@@ -368,7 +368,7 @@ class TestMhcOnTheDeclarations(CustomTestCase):
max_len, communicator.ffn.plan.output.plan.path_for(max_len)
)
self.assertTrue(
communicator.ffn.plan.output._skips_sum_for_reduce_scatter(
communicator.ffn.plan.output._sum_in_reduce_scatter(
communicator.ffn.plan.output.plan.path_for(max_len), step
)
)
@@ -1124,7 +1124,7 @@ class TestPrefillCP(CustomTestCase):
patch_communicator("_batch_shards_over_cp", lambda fb: shards),
):
self.assertIs(
communicator.ffn.plan.output._skips_sum_for_reduce_scatter(
communicator.ffn.plan.output._sum_in_reduce_scatter(
communicator.ffn.plan.output.plan.path_for(
SimpleNamespace(forward_mode=ForwardMode.DECODE)
),
@@ -1590,7 +1590,6 @@ class Flags:
"""The per-forward flags an FFN exit publishes, one set per rank."""
def __init__(self):
self.fuse_mlp_allreduce = False
self.mlp_reduce_scatter = False
self.defer_moe_finalize = False
self.sp_active = False
@@ -89,9 +89,7 @@ def make_communicator(
communicator.output._defers_sum = MagicMock(return_value=fuse)
else:
communicator.output._defers_sum = MagicMock(return_value=False)
communicator.output._skips_sum_for_reduce_scatter = MagicMock(
return_value=reduce_scatter
)
communicator.output._sum_in_reduce_scatter = MagicMock(return_value=reduce_scatter)
communicator.terminal = False
communicator.paths[BatchVariant.SEQUENCE_PARALLEL] = None
communicator.paths[BatchVariant.INPUT_SCATTERED] = None
@@ -109,10 +107,9 @@ def make_communicator(
def published_flags():
"""The skip flags the FFN sees; the boundary completes the sum, so the FFN
"""The skip flag the FFN sees; the boundary completes the sum, so the FFN
never leaves it out."""
forward = get_forward()
return forward.fuse_mlp_allreduce, forward.mlp_reduce_scatter
return (get_forward().mlp_reduce_scatter,)
class TestFfnExit(CustomTestCase):
@@ -150,7 +147,7 @@ class TestFfnExit(CustomTestCase):
scatters_to_local_tokens=True,
)
seen, (hidden_states, residual) = self.run_exit(communicator)
self.assertEqual(seen, (False, False))
self.assertEqual(seen, (False,))
self.assertIsInstance(hidden_states, UnreducedOutput)
bound = hidden_states.reduce_to_dp_local
self.assertIs(bound.func, transport_ops.to_dp_local)
@@ -179,7 +176,7 @@ class TestFfnExit(CustomTestCase):
fuse=False, reduce_scatter=reduce_scatter
)
seen, (hidden_states, residual) = self.run_exit(communicator)
self.assertEqual(seen, (False, False))
self.assertEqual(seen, (False,))
self.assertNotIsInstance(hidden_states, UnreducedOutput)
communicator.output._complete_now.assert_called_once()
# A reduce-scatter on the way back completes the sum; otherwise
@@ -197,7 +194,7 @@ class TestFfnExit(CustomTestCase):
)
seen, (hidden_states, residual) = self.run_exit(communicator)
self.assertEqual(seen, (False, False))
self.assertEqual(seen, (False,))
communicator.output._defers_sum.assert_not_called()
communicator.output._complete_now.assert_called_once()
torch.testing.assert_close(hidden_states, self.hidden_states * 2 + 1)
@@ -216,7 +213,7 @@ class TestFfnExit(CustomTestCase):
decide.return_value = not fuse
hidden_states = self.hidden_states * 2
hidden_states, _ = finish_exit(ffn_exit, hidden_states, self.residual)
self.assertEqual(seen, (False, False))
self.assertEqual(seen, (False,))
self.assertEqual(isinstance(hidden_states, UnreducedOutput), fuse)
self.assertEqual(communicator.output._complete_now.called, not fuse)
@@ -253,7 +250,7 @@ class TestFfnExit(CustomTestCase):
self.forward_batch, stream=ResidualStream()
) as ffn_exit:
seen = published_flags() + (get_forward().defer_moe_finalize,)
self.assertEqual(seen, (False, False, True))
self.assertEqual(seen, (False, True))
self.assertEqual(
finish_exit(ffn_exit, handoff, self.residual), (handoff, self.residual)
)
@@ -354,8 +351,8 @@ class TestSelectFfnCompletion(CustomTestCase):
returns_over_dp=scatters,
)
communicator.output._defers_sum = lambda forward_batch, steps, **_: fuse
communicator.output._skips_sum_for_reduce_scatter = (
lambda forward_batch, dp_step: not fuse
communicator.output._sum_in_reduce_scatter = lambda forward_batch, dp_step: (
not fuse
)
communicator.output._complete_now = lambda h, r, **_: ("now", r)
self.group = make_group()
@@ -193,24 +193,16 @@ class TestIndependentStageConstruction(CustomTestCase):
)
self.assertEqual(edge.incoming.residual, source_input.residual_to)
def test_a_replicated_tail_stays_in_compute_without_a_fusion_object(self):
for local_tail in (False, True):
with fixture.planning(fixture.parallel_of(attn_dp=1, attn_tp=2)):
boundary = make_ffn_stage(
declaration=declare_ffn(
previous=declare_attn(),
sparse=True,
reduction=ProducerReduction.TAIL_AFTER_SUM
if local_tail
else ProducerReduction.EXIT_SCOPED,
),
norm=fixture.Norm(),
)
self.assertIsNone(boundary.plan.fusions)
produced = boundary.plan.paths[BatchVariant.ORDINARY].output
self.assertEqual(produced.may_defer_to_next, not local_tail)
# A local compute tail does not change the existing RS alternative.
self.assertTrue(produced.may_reduce_scatter)
def test_an_ffn_without_a_fusion_object_may_defer_or_reduce_scatter(self):
with fixture.planning(fixture.parallel_of(attn_dp=1, attn_tp=2)):
boundary = make_ffn_stage(
declaration=declare_ffn(previous=declare_attn(), sparse=True),
norm=fixture.Norm(),
)
self.assertIsNone(boundary.plan.fusions)
produced = boundary.plan.paths[BatchVariant.ORDINARY].output
self.assertTrue(produced.may_defer_to_next)
self.assertTrue(produced.may_reduce_scatter)
def test_mixer_successor_comes_from_the_local_sequence(self):
for following in (
@@ -89,7 +89,6 @@ class TestReduceMoeOutput(CustomTestCase):
def test_a_later_step_owns_the_sum(self):
for flags, config in (
({"fuse_mlp_allreduce": True}, {}),
({"mlp_reduce_scatter": True}, {}),
({"mlp_reduce_scatter": True}, {"reduce_scatterv": True}),
):
@@ -136,7 +135,6 @@ class TestReplicatedMoeOutput(CustomTestCase):
def test_a_later_sum_counts_it_once(self):
for flags, config in (
({"fuse_mlp_allreduce": True}, {}),
({"mlp_reduce_scatter": True}, {}),
({"mlp_reduce_scatter": True}, {"reduce_scatterv": True}),
):
@@ -158,7 +156,7 @@ class TestReplicatedMoeOutput(CustomTestCase):
def test_a_single_rank_adds_it(self):
with (
moe_config(tp_size=1, tp_rank=0),
get_forward().scoped(fuse_mlp_allreduce=True),
get_forward().scoped(mlp_reduce_scatter=True),
):
self.assertTrue(should_add_replicated_moe_output())
@@ -243,9 +241,7 @@ class TestModelsWithExplicitDpCompletion(CustomTestCase):
)
with (
moe_config(),
get_forward().scoped(
fuse_mlp_allreduce=False, mlp_reduce_scatter=False
),
get_forward().scoped(mlp_reduce_scatter=False),
patch(
"sglang.srt.distributed.communication_op.tensor_model_parallel_all_reduce",
side_effect=lambda value: trace.append("AR") or value * 2,
@@ -334,7 +334,7 @@ class TestFuseMlpAllReduceGate(CustomTestCase):
self.assertTrue(self._should_fuse(moe_ep_size=4, moe_tp_size=1))
def test_a_layer_gathered_over_moe_cp_does_not_fuse(self):
# Fusion skips finish_complete_output, which holds the CP scatter; a dense
# Fusion skips the exit's completion, which holds the CP scatter; a dense
# layer gathered over the MoE-CP group (moe_dp_size == attn_cp_size) is
# not caught by the is_enable_moe_cp_allgather gate.
self.assertFalse(
@@ -472,7 +472,7 @@ class TestDeferFfnReduction(CustomTestCase):
return communicator.output._defers_sum(
forward_batch,
communicator.output.plan.path_for(forward_batch),
mlp_reduce_scatter=reduce_scatter,
sum_in_reduce_scatter=reduce_scatter,
dp_step=step,
)
@@ -277,9 +277,7 @@ def test_moe_model_handoff(group, rows, dual, defer, shared_tp1):
return hidden, next_pre
with (
get_forward().scoped(
sp_active=False, fuse_mlp_allreduce=False, flashinfer_trtllm_bypass=False
),
get_forward().scoped(sp_active=False, flashinfer_trtllm_bypass=False),
patch(
"sglang.srt.models.deepseek_v2.post_experts_all_reduce",
side_effect=group.all_reduce,
@@ -316,11 +314,11 @@ def test_moe_model_handoff(group, rows, dual, defer, shared_tp1):
torch.testing.assert_close(unfused[1], fused[1], atol=1e-5, rtol=1e-5)
@pytest.mark.parametrize("flag", ["mlp_reduce_scatter", "fuse_mlp_allreduce"])
@pytest.mark.parametrize("flag", ["mlp_reduce_scatter"])
@pytest.mark.parametrize("dual", [False, True], ids=["normal", "dual-stream"])
def test_moe_skipped_reduction_keeps_post(group, dual, flag):
"""A reduce-scattered or elsewhere-fused MoE output must not reach the fused all-reduce
+ mHC post: the pending post stays pending and no collective runs."""
"""A reduce-scattered MoE output must not reach the fused all-reduce + mHC
post: the pending post stays pending and no collective runs."""
from sglang.srt.layers.moe.mhc_post_fusion import (
MhcPostFusion,
use_mhc_post_fusion,
+4 -13
View File
@@ -1414,8 +1414,6 @@ class TestForwardFlags(_IsolatedServerArgs):
x = x + 1
if fwd.is_extend_in_batch:
x = x + 2
if fwd.fuse_mlp_allreduce:
x = x + 4
if fwd.mlp_reduce_scatter:
x = x + 8
if fwd.flashinfer_trtllm_bypass:
@@ -1429,11 +1427,10 @@ class TestForwardFlags(_IsolatedServerArgs):
self.assertEqual(probe(torch.zeros(())).item(), 2)
get_forward().set("is_extend_in_batch", False)
with get_forward().scoped(
fuse_mlp_allreduce=True,
mlp_reduce_scatter=True,
flashinfer_trtllm_bypass=True,
):
self.assertEqual(probe(torch.zeros(())).item(), 28)
self.assertEqual(probe(torch.zeros(())).item(), 24)
self.assertEqual(probe(torch.zeros(())).item(), 0)
def test_parallel_config_leaves_trace_under_torch_compile(self):
@@ -1578,23 +1575,17 @@ class TestForwardFlags(_IsolatedServerArgs):
reset_context()
fwd = get_forward()
self.assertFalse(fwd.fuse_mlp_allreduce)
self.assertFalse(fwd.mlp_reduce_scatter)
self.assertFalse(fwd.flashinfer_trtllm_bypass)
self.assertFalse(should_skip_mlp_all_reduce())
with fwd.scoped(fuse_mlp_allreduce=True):
self.assertTrue(fwd.fuse_mlp_allreduce)
self.assertTrue(should_skip_mlp_all_reduce())
# Fusion alone is enough to skip post-experts AR.
self.assertTrue(should_skip_post_experts_all_reduce(is_tp_path=True))
self.assertFalse(fwd.fuse_mlp_allreduce)
self.assertFalse(should_skip_mlp_all_reduce())
with fwd.scoped(mlp_reduce_scatter=True):
self.assertTrue(fwd.mlp_reduce_scatter)
self.assertTrue(should_skip_mlp_all_reduce())
# The reduce-scatter alone is enough to skip post-experts AR.
self.assertTrue(should_skip_post_experts_all_reduce(is_tp_path=True))
self.assertFalse(fwd.mlp_reduce_scatter)
self.assertFalse(should_skip_mlp_all_reduce())
with fwd.scoped(flashinfer_trtllm_bypass=True):
self.assertTrue(fwd.flashinfer_trtllm_bypass)