[Refactor] Drop the reduction-skip mechanisms stage boundaries no longer use (#42312)
This commit is contained in:
@@ -98,8 +98,8 @@ class DeferringLayer(nn.Module):
|
||||
self.layer_communicator.paths[BatchVariant.INPUT_SCATTERED] = None
|
||||
self.layer_communicator.paths[BatchVariant.CONTEXT_PARALLEL] = None
|
||||
self.layer_communicator.output.ffn_reduction_group = lambda forward_batch: GROUP
|
||||
self.layer_communicator.output._skips_sum_for_reduce_scatter = (
|
||||
lambda batch, dp_step: False
|
||||
self.layer_communicator.output._sum_in_reduce_scatter = lambda batch, dp_step: (
|
||||
False
|
||||
)
|
||||
self.layer_communicator.output._complete_now = lambda hidden, residual, **_: (
|
||||
all_reduce(hidden),
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
"""Numerical and interface coverage for boundary/graph integration regressions."""
|
||||
|
||||
import unittest
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
@@ -10,7 +9,6 @@ import torch
|
||||
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
PLAIN_ADD,
|
||||
ProducerReduction,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
)
|
||||
@@ -200,12 +198,6 @@ class TestBoundaryIntegrations(unittest.TestCase):
|
||||
enabled and (lora or shared),
|
||||
)
|
||||
|
||||
def test_unsupported_producer_contracts_fail_at_declaration(self):
|
||||
with self.assertRaises(ValueError):
|
||||
declare_ffn(reduction=ProducerReduction.ALWAYS_PARTIAL)
|
||||
with self.assertRaises(ValueError):
|
||||
declare_attn(reduction=ProducerReduction.TAIL_AFTER_SUM)
|
||||
|
||||
def test_ffn_writing_the_next_stream_hands_on_a_complete_output(self):
|
||||
# It has each part's sum completed (sum_part) before building the
|
||||
# stream from the parts, so its exit owes no sum and never defers one.
|
||||
@@ -261,14 +253,13 @@ class TestBoundaryIntegrations(unittest.TestCase):
|
||||
capture(hidden)
|
||||
return hidden
|
||||
|
||||
output = SimpleNamespace(finish=lambda hidden: hidden)
|
||||
layer = SimpleNamespace(
|
||||
attn_boundary=SimpleNamespace(
|
||||
prepare=prepare, finish=lambda hidden, fb: hidden
|
||||
),
|
||||
ffn_boundary=SimpleNamespace(
|
||||
prepare=lambda hidden, fb, **kw: hidden,
|
||||
exit=lambda fb: nullcontext(output),
|
||||
finish=lambda hidden, fb: hidden,
|
||||
),
|
||||
self_attn=lambda **kw: kw["hidden_states"],
|
||||
mlp=lambda hidden, **kw: hidden,
|
||||
|
||||
@@ -213,9 +213,7 @@ class TestBoundaryOutputPolicy(unittest.TestCase):
|
||||
),
|
||||
):
|
||||
for skip in (False, True):
|
||||
with get_forward().scoped(
|
||||
fuse_mlp_allreduce=False, mlp_reduce_scatter=skip
|
||||
):
|
||||
with get_forward().scoped(mlp_reduce_scatter=skip):
|
||||
self.assertEqual(
|
||||
moe_utils.should_skip_post_experts_all_reduce(is_tp_path=True),
|
||||
skip,
|
||||
|
||||
@@ -481,7 +481,7 @@ def test_dual_stream_op_pins_the_deferral_off_under_a_deferring_caller():
|
||||
return_value=SimpleNamespace(moe_fusions={0: fusion}),
|
||||
),
|
||||
):
|
||||
out = op.redispatch(cuda_key, torch.zeros(4, 8), 0, True, False)
|
||||
out = op.redispatch(cuda_key, torch.zeros(4, 8), 0, False)
|
||||
assert get_forward().defer_moe_finalize is True
|
||||
|
||||
assert fusion.seen_defer is False
|
||||
|
||||
@@ -44,7 +44,6 @@ def layernorm(hidden_states, residual=None):
|
||||
|
||||
class Flags:
|
||||
def __init__(self):
|
||||
self.fuse_mlp_allreduce = False
|
||||
self.mlp_reduce_scatter = False
|
||||
self.defer_moe_finalize = False
|
||||
self.sp_active = False
|
||||
|
||||
@@ -368,7 +368,7 @@ class TestMhcOnTheDeclarations(CustomTestCase):
|
||||
max_len, communicator.ffn.plan.output.plan.path_for(max_len)
|
||||
)
|
||||
self.assertTrue(
|
||||
communicator.ffn.plan.output._skips_sum_for_reduce_scatter(
|
||||
communicator.ffn.plan.output._sum_in_reduce_scatter(
|
||||
communicator.ffn.plan.output.plan.path_for(max_len), step
|
||||
)
|
||||
)
|
||||
@@ -1124,7 +1124,7 @@ class TestPrefillCP(CustomTestCase):
|
||||
patch_communicator("_batch_shards_over_cp", lambda fb: shards),
|
||||
):
|
||||
self.assertIs(
|
||||
communicator.ffn.plan.output._skips_sum_for_reduce_scatter(
|
||||
communicator.ffn.plan.output._sum_in_reduce_scatter(
|
||||
communicator.ffn.plan.output.plan.path_for(
|
||||
SimpleNamespace(forward_mode=ForwardMode.DECODE)
|
||||
),
|
||||
@@ -1590,7 +1590,6 @@ class Flags:
|
||||
"""The per-forward flags an FFN exit publishes, one set per rank."""
|
||||
|
||||
def __init__(self):
|
||||
self.fuse_mlp_allreduce = False
|
||||
self.mlp_reduce_scatter = False
|
||||
self.defer_moe_finalize = False
|
||||
self.sp_active = False
|
||||
|
||||
@@ -89,9 +89,7 @@ def make_communicator(
|
||||
communicator.output._defers_sum = MagicMock(return_value=fuse)
|
||||
else:
|
||||
communicator.output._defers_sum = MagicMock(return_value=False)
|
||||
communicator.output._skips_sum_for_reduce_scatter = MagicMock(
|
||||
return_value=reduce_scatter
|
||||
)
|
||||
communicator.output._sum_in_reduce_scatter = MagicMock(return_value=reduce_scatter)
|
||||
communicator.terminal = False
|
||||
communicator.paths[BatchVariant.SEQUENCE_PARALLEL] = None
|
||||
communicator.paths[BatchVariant.INPUT_SCATTERED] = None
|
||||
@@ -109,10 +107,9 @@ def make_communicator(
|
||||
|
||||
|
||||
def published_flags():
|
||||
"""The skip flags the FFN sees; the boundary completes the sum, so the FFN
|
||||
"""The skip flag the FFN sees; the boundary completes the sum, so the FFN
|
||||
never leaves it out."""
|
||||
forward = get_forward()
|
||||
return forward.fuse_mlp_allreduce, forward.mlp_reduce_scatter
|
||||
return (get_forward().mlp_reduce_scatter,)
|
||||
|
||||
|
||||
class TestFfnExit(CustomTestCase):
|
||||
@@ -150,7 +147,7 @@ class TestFfnExit(CustomTestCase):
|
||||
scatters_to_local_tokens=True,
|
||||
)
|
||||
seen, (hidden_states, residual) = self.run_exit(communicator)
|
||||
self.assertEqual(seen, (False, False))
|
||||
self.assertEqual(seen, (False,))
|
||||
self.assertIsInstance(hidden_states, UnreducedOutput)
|
||||
bound = hidden_states.reduce_to_dp_local
|
||||
self.assertIs(bound.func, transport_ops.to_dp_local)
|
||||
@@ -179,7 +176,7 @@ class TestFfnExit(CustomTestCase):
|
||||
fuse=False, reduce_scatter=reduce_scatter
|
||||
)
|
||||
seen, (hidden_states, residual) = self.run_exit(communicator)
|
||||
self.assertEqual(seen, (False, False))
|
||||
self.assertEqual(seen, (False,))
|
||||
self.assertNotIsInstance(hidden_states, UnreducedOutput)
|
||||
communicator.output._complete_now.assert_called_once()
|
||||
# A reduce-scatter on the way back completes the sum; otherwise
|
||||
@@ -197,7 +194,7 @@ class TestFfnExit(CustomTestCase):
|
||||
)
|
||||
seen, (hidden_states, residual) = self.run_exit(communicator)
|
||||
|
||||
self.assertEqual(seen, (False, False))
|
||||
self.assertEqual(seen, (False,))
|
||||
communicator.output._defers_sum.assert_not_called()
|
||||
communicator.output._complete_now.assert_called_once()
|
||||
torch.testing.assert_close(hidden_states, self.hidden_states * 2 + 1)
|
||||
@@ -216,7 +213,7 @@ class TestFfnExit(CustomTestCase):
|
||||
decide.return_value = not fuse
|
||||
hidden_states = self.hidden_states * 2
|
||||
hidden_states, _ = finish_exit(ffn_exit, hidden_states, self.residual)
|
||||
self.assertEqual(seen, (False, False))
|
||||
self.assertEqual(seen, (False,))
|
||||
self.assertEqual(isinstance(hidden_states, UnreducedOutput), fuse)
|
||||
self.assertEqual(communicator.output._complete_now.called, not fuse)
|
||||
|
||||
@@ -253,7 +250,7 @@ class TestFfnExit(CustomTestCase):
|
||||
self.forward_batch, stream=ResidualStream()
|
||||
) as ffn_exit:
|
||||
seen = published_flags() + (get_forward().defer_moe_finalize,)
|
||||
self.assertEqual(seen, (False, False, True))
|
||||
self.assertEqual(seen, (False, True))
|
||||
self.assertEqual(
|
||||
finish_exit(ffn_exit, handoff, self.residual), (handoff, self.residual)
|
||||
)
|
||||
@@ -354,8 +351,8 @@ class TestSelectFfnCompletion(CustomTestCase):
|
||||
returns_over_dp=scatters,
|
||||
)
|
||||
communicator.output._defers_sum = lambda forward_batch, steps, **_: fuse
|
||||
communicator.output._skips_sum_for_reduce_scatter = (
|
||||
lambda forward_batch, dp_step: not fuse
|
||||
communicator.output._sum_in_reduce_scatter = lambda forward_batch, dp_step: (
|
||||
not fuse
|
||||
)
|
||||
communicator.output._complete_now = lambda h, r, **_: ("now", r)
|
||||
self.group = make_group()
|
||||
|
||||
@@ -193,24 +193,16 @@ class TestIndependentStageConstruction(CustomTestCase):
|
||||
)
|
||||
self.assertEqual(edge.incoming.residual, source_input.residual_to)
|
||||
|
||||
def test_a_replicated_tail_stays_in_compute_without_a_fusion_object(self):
|
||||
for local_tail in (False, True):
|
||||
with fixture.planning(fixture.parallel_of(attn_dp=1, attn_tp=2)):
|
||||
boundary = make_ffn_stage(
|
||||
declaration=declare_ffn(
|
||||
previous=declare_attn(),
|
||||
sparse=True,
|
||||
reduction=ProducerReduction.TAIL_AFTER_SUM
|
||||
if local_tail
|
||||
else ProducerReduction.EXIT_SCOPED,
|
||||
),
|
||||
norm=fixture.Norm(),
|
||||
)
|
||||
self.assertIsNone(boundary.plan.fusions)
|
||||
produced = boundary.plan.paths[BatchVariant.ORDINARY].output
|
||||
self.assertEqual(produced.may_defer_to_next, not local_tail)
|
||||
# A local compute tail does not change the existing RS alternative.
|
||||
self.assertTrue(produced.may_reduce_scatter)
|
||||
def test_an_ffn_without_a_fusion_object_may_defer_or_reduce_scatter(self):
|
||||
with fixture.planning(fixture.parallel_of(attn_dp=1, attn_tp=2)):
|
||||
boundary = make_ffn_stage(
|
||||
declaration=declare_ffn(previous=declare_attn(), sparse=True),
|
||||
norm=fixture.Norm(),
|
||||
)
|
||||
self.assertIsNone(boundary.plan.fusions)
|
||||
produced = boundary.plan.paths[BatchVariant.ORDINARY].output
|
||||
self.assertTrue(produced.may_defer_to_next)
|
||||
self.assertTrue(produced.may_reduce_scatter)
|
||||
|
||||
def test_mixer_successor_comes_from_the_local_sequence(self):
|
||||
for following in (
|
||||
|
||||
@@ -89,7 +89,6 @@ class TestReduceMoeOutput(CustomTestCase):
|
||||
|
||||
def test_a_later_step_owns_the_sum(self):
|
||||
for flags, config in (
|
||||
({"fuse_mlp_allreduce": True}, {}),
|
||||
({"mlp_reduce_scatter": True}, {}),
|
||||
({"mlp_reduce_scatter": True}, {"reduce_scatterv": True}),
|
||||
):
|
||||
@@ -136,7 +135,6 @@ class TestReplicatedMoeOutput(CustomTestCase):
|
||||
|
||||
def test_a_later_sum_counts_it_once(self):
|
||||
for flags, config in (
|
||||
({"fuse_mlp_allreduce": True}, {}),
|
||||
({"mlp_reduce_scatter": True}, {}),
|
||||
({"mlp_reduce_scatter": True}, {"reduce_scatterv": True}),
|
||||
):
|
||||
@@ -158,7 +156,7 @@ class TestReplicatedMoeOutput(CustomTestCase):
|
||||
def test_a_single_rank_adds_it(self):
|
||||
with (
|
||||
moe_config(tp_size=1, tp_rank=0),
|
||||
get_forward().scoped(fuse_mlp_allreduce=True),
|
||||
get_forward().scoped(mlp_reduce_scatter=True),
|
||||
):
|
||||
self.assertTrue(should_add_replicated_moe_output())
|
||||
|
||||
@@ -243,9 +241,7 @@ class TestModelsWithExplicitDpCompletion(CustomTestCase):
|
||||
)
|
||||
with (
|
||||
moe_config(),
|
||||
get_forward().scoped(
|
||||
fuse_mlp_allreduce=False, mlp_reduce_scatter=False
|
||||
),
|
||||
get_forward().scoped(mlp_reduce_scatter=False),
|
||||
patch(
|
||||
"sglang.srt.distributed.communication_op.tensor_model_parallel_all_reduce",
|
||||
side_effect=lambda value: trace.append("AR") or value * 2,
|
||||
|
||||
@@ -334,7 +334,7 @@ class TestFuseMlpAllReduceGate(CustomTestCase):
|
||||
self.assertTrue(self._should_fuse(moe_ep_size=4, moe_tp_size=1))
|
||||
|
||||
def test_a_layer_gathered_over_moe_cp_does_not_fuse(self):
|
||||
# Fusion skips finish_complete_output, which holds the CP scatter; a dense
|
||||
# Fusion skips the exit's completion, which holds the CP scatter; a dense
|
||||
# layer gathered over the MoE-CP group (moe_dp_size == attn_cp_size) is
|
||||
# not caught by the is_enable_moe_cp_allgather gate.
|
||||
self.assertFalse(
|
||||
@@ -472,7 +472,7 @@ class TestDeferFfnReduction(CustomTestCase):
|
||||
return communicator.output._defers_sum(
|
||||
forward_batch,
|
||||
communicator.output.plan.path_for(forward_batch),
|
||||
mlp_reduce_scatter=reduce_scatter,
|
||||
sum_in_reduce_scatter=reduce_scatter,
|
||||
dp_step=step,
|
||||
)
|
||||
|
||||
|
||||
@@ -277,9 +277,7 @@ def test_moe_model_handoff(group, rows, dual, defer, shared_tp1):
|
||||
return hidden, next_pre
|
||||
|
||||
with (
|
||||
get_forward().scoped(
|
||||
sp_active=False, fuse_mlp_allreduce=False, flashinfer_trtllm_bypass=False
|
||||
),
|
||||
get_forward().scoped(sp_active=False, flashinfer_trtllm_bypass=False),
|
||||
patch(
|
||||
"sglang.srt.models.deepseek_v2.post_experts_all_reduce",
|
||||
side_effect=group.all_reduce,
|
||||
@@ -316,11 +314,11 @@ def test_moe_model_handoff(group, rows, dual, defer, shared_tp1):
|
||||
torch.testing.assert_close(unfused[1], fused[1], atol=1e-5, rtol=1e-5)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("flag", ["mlp_reduce_scatter", "fuse_mlp_allreduce"])
|
||||
@pytest.mark.parametrize("flag", ["mlp_reduce_scatter"])
|
||||
@pytest.mark.parametrize("dual", [False, True], ids=["normal", "dual-stream"])
|
||||
def test_moe_skipped_reduction_keeps_post(group, dual, flag):
|
||||
"""A reduce-scattered or elsewhere-fused MoE output must not reach the fused all-reduce
|
||||
+ mHC post: the pending post stays pending and no collective runs."""
|
||||
"""A reduce-scattered MoE output must not reach the fused all-reduce + mHC
|
||||
post: the pending post stays pending and no collective runs."""
|
||||
from sglang.srt.layers.moe.mhc_post_fusion import (
|
||||
MhcPostFusion,
|
||||
use_mhc_post_fusion,
|
||||
|
||||
@@ -1414,8 +1414,6 @@ class TestForwardFlags(_IsolatedServerArgs):
|
||||
x = x + 1
|
||||
if fwd.is_extend_in_batch:
|
||||
x = x + 2
|
||||
if fwd.fuse_mlp_allreduce:
|
||||
x = x + 4
|
||||
if fwd.mlp_reduce_scatter:
|
||||
x = x + 8
|
||||
if fwd.flashinfer_trtllm_bypass:
|
||||
@@ -1429,11 +1427,10 @@ class TestForwardFlags(_IsolatedServerArgs):
|
||||
self.assertEqual(probe(torch.zeros(())).item(), 2)
|
||||
get_forward().set("is_extend_in_batch", False)
|
||||
with get_forward().scoped(
|
||||
fuse_mlp_allreduce=True,
|
||||
mlp_reduce_scatter=True,
|
||||
flashinfer_trtllm_bypass=True,
|
||||
):
|
||||
self.assertEqual(probe(torch.zeros(())).item(), 28)
|
||||
self.assertEqual(probe(torch.zeros(())).item(), 24)
|
||||
self.assertEqual(probe(torch.zeros(())).item(), 0)
|
||||
|
||||
def test_parallel_config_leaves_trace_under_torch_compile(self):
|
||||
@@ -1578,23 +1575,17 @@ class TestForwardFlags(_IsolatedServerArgs):
|
||||
|
||||
reset_context()
|
||||
fwd = get_forward()
|
||||
self.assertFalse(fwd.fuse_mlp_allreduce)
|
||||
self.assertFalse(fwd.mlp_reduce_scatter)
|
||||
self.assertFalse(fwd.flashinfer_trtllm_bypass)
|
||||
self.assertFalse(should_skip_mlp_all_reduce())
|
||||
|
||||
with fwd.scoped(fuse_mlp_allreduce=True):
|
||||
self.assertTrue(fwd.fuse_mlp_allreduce)
|
||||
self.assertTrue(should_skip_mlp_all_reduce())
|
||||
# Fusion alone is enough to skip post-experts AR.
|
||||
self.assertTrue(should_skip_post_experts_all_reduce(is_tp_path=True))
|
||||
self.assertFalse(fwd.fuse_mlp_allreduce)
|
||||
self.assertFalse(should_skip_mlp_all_reduce())
|
||||
|
||||
with fwd.scoped(mlp_reduce_scatter=True):
|
||||
self.assertTrue(fwd.mlp_reduce_scatter)
|
||||
self.assertTrue(should_skip_mlp_all_reduce())
|
||||
# The reduce-scatter alone is enough to skip post-experts AR.
|
||||
self.assertTrue(should_skip_post_experts_all_reduce(is_tp_path=True))
|
||||
self.assertFalse(fwd.mlp_reduce_scatter)
|
||||
self.assertFalse(should_skip_mlp_all_reduce())
|
||||
|
||||
with fwd.scoped(flashinfer_trtllm_bypass=True):
|
||||
self.assertTrue(fwd.flashinfer_trtllm_bypass)
|
||||
|
||||
Reference in New Issue
Block a user