From b2402ccad63a2f5313f770d46156676ff20ae30e Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Fri, 9 Oct 2026 01:24:42 -0700 Subject: [PATCH] [Fix] Merge DeepSeek-V4 two-batch-overlap microbatches without a residual (#42891) --- .../srt/batch_overlap/two_batch_overlap.py | 42 +++++++------ python/sglang/srt/models/deepseek_v4.py | 10 ++- .../test_deepseek_v4_flash_fp4_b200_tbo.py | 62 +++++++++++++++++++ 3 files changed, 90 insertions(+), 24 deletions(-) create mode 100644 test/registered/e2e/models/test_deepseek_v4_flash_fp4_b200_tbo.py diff --git a/python/sglang/srt/batch_overlap/two_batch_overlap.py b/python/sglang/srt/batch_overlap/two_batch_overlap.py index 5086ea8063..c4e457851e 100644 --- a/python/sglang/srt/batch_overlap/two_batch_overlap.py +++ b/python/sglang/srt/batch_overlap/two_batch_overlap.py @@ -1119,27 +1119,33 @@ def _model_forward_tbo_merge_outputs(output_a, output_b, original_len): output["hidden_states"] ) - def _handle_key(name): - value_a = output_a[name] - value_b = output_b[name] - assert (value_a is None) == (value_b is None) - if value_a is None: - return None - s0, t0 = output_a["forward_batch"].tbo_parent_token_range - s1, t1 = output_b["forward_batch"].tbo_parent_token_range - res = torch.zeros( - (original_len, *value_a.shape[1:]), - dtype=value_a.dtype, - device=value_a.device, - ) - res[slice(s0, t0)] = value_a[: t0 - s0] - res[slice(s1, t1)] = value_b[: t1 - s1] - return res - - hidden, residual = _handle_key("hidden_states"), _handle_key("residual") + hidden = _model_forward_tbo_merge_key( + output_a, output_b, "hidden_states", original_len + ) + residual = _model_forward_tbo_merge_key( + output_a, output_b, "residual", original_len + ) return ResidualStream.from_handoff(hidden, residual, update) +def _model_forward_tbo_merge_key(output_a, output_b, name, original_len): + value_a = output_a[name] + value_b = output_b[name] + assert (value_a is None) == (value_b is None) + if value_a is None: + return None + s0, t0 = output_a["forward_batch"].tbo_parent_token_range + s1, t1 = output_b["forward_batch"].tbo_parent_token_range + res = torch.zeros( + (original_len, *value_a.shape[1:]), + dtype=value_a.dtype, + device=value_a.device, + ) + res[slice(s0, t0)] = value_a[: t0 - s0] + res[slice(s1, t1)] = value_b[: t1 - s1] + return res + + # -------------------------------- Utilities and wrappers --------------------------------------- diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 5234e214a8..5b82f536a5 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -3682,8 +3682,7 @@ class DeepseekV4DecoderLayer(nn.Module): positions=state.positions, hidden_states=hidden_states, # DSV4 non-fused layers carry no residual across layers; the key is - # required by the next layer's op_mhc_prepare_attn (ignored) and by - # _model_forward_tbo_merge_outputs (None -> None). + # required by the next layer's op_mhc_prepare_attn (ignored). residual=None, forward_batch=state.forward_batch, tbo_subbatch_index=state.tbo_subbatch_index, @@ -4163,7 +4162,7 @@ class DeepseekV4Model(nn.Module): from sglang.srt.batch_overlap.operations_strategy import OperationsStrategy from sglang.srt.batch_overlap.two_batch_overlap import ( _model_forward_filter_inputs, - _model_forward_tbo_merge_outputs, + _model_forward_tbo_merge_key, ) layers = [self.layers[i] for i in range(self.start_layer, self.end_layer)] @@ -4237,10 +4236,9 @@ class DeepseekV4Model(nn.Module): delta_stages=[0, operations_strategy.tbo_delta_stages], ) - hidden_states, _ = _model_forward_tbo_merge_outputs( - outputs_arr[0], outputs_arr[1], hidden_states.shape[0] + return _model_forward_tbo_merge_key( + outputs_arr[0], outputs_arr[1], "hidden_states", hidden_states.shape[0] ) - return hidden_states @torch.no_grad() def forward( diff --git a/test/registered/e2e/models/test_deepseek_v4_flash_fp4_b200_tbo.py b/test/registered/e2e/models/test_deepseek_v4_flash_fp4_b200_tbo.py new file mode 100644 index 0000000000..8d222c8832 --- /dev/null +++ b/test/registered/e2e/models/test_deepseek_v4_flash_fp4_b200_tbo.py @@ -0,0 +1,62 @@ +"""B200 nightly CI: DeepSeek-V4-Flash FP4 with two-batch overlap. + +Launches TP=4, DP attention 4 and DeepEP with --enable-two-batch-overlap, so +prefill runs DeepSeek-V4's own two-batch-overlap layer loop. Runs a GSM8K +accuracy gate. + +Registry: nightly-4-gpu-b200 (nightly, 4x B200) +""" + +import unittest + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, + try_cached_model, +) + +register_cuda_ci(est_time=600, stage="nightly", runner_config="4-gpu-b200") + +MODEL = "deepseek-ai/DeepSeek-V4-Flash-0731" +SERVER_LAUNCH_TIMEOUT = 3600 +DEEPEP_CONFIG = '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' + + +class TestDSV4FlashFP4TboB200(GSM8KMixin, CustomTestCase): + gsm8k_accuracy_thres = 0.93 + + @classmethod + def setUpClass(cls): + cls.model = try_cached_model(MODEL) + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=SERVER_LAUNCH_TIMEOUT, + other_args=[ + "--trust-remote-code", + "--tp", + "4", + "--attn-dp-size", + "4", + "--moe-a2a-backend", + "deepep", + "--deepep-config", + DEEPEP_CONFIG, + "--enable-two-batch-overlap", + ], + env={"SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "1024"}, + ) + + @classmethod + def tearDownClass(cls): + if hasattr(cls, "process") and cls.process: + kill_process_tree(cls.process.pid) + + +if __name__ == "__main__": + unittest.main()