[Fix] Merge DeepSeek-V4 two-batch-overlap microbatches without a residual (#42891)

This commit is contained in:
Cheng Wan
2026-10-09 01:24:42 -07:00
committed by GitHub
parent b130f8af02
commit b2402ccad6
3 changed files with 90 additions and 24 deletions
@@ -1119,27 +1119,33 @@ def _model_forward_tbo_merge_outputs(output_a, output_b, original_len):
output["hidden_states"]
)
def _handle_key(name):
value_a = output_a[name]
value_b = output_b[name]
assert (value_a is None) == (value_b is None)
if value_a is None:
return None
s0, t0 = output_a["forward_batch"].tbo_parent_token_range
s1, t1 = output_b["forward_batch"].tbo_parent_token_range
res = torch.zeros(
(original_len, *value_a.shape[1:]),
dtype=value_a.dtype,
device=value_a.device,
)
res[slice(s0, t0)] = value_a[: t0 - s0]
res[slice(s1, t1)] = value_b[: t1 - s1]
return res
hidden, residual = _handle_key("hidden_states"), _handle_key("residual")
hidden = _model_forward_tbo_merge_key(
output_a, output_b, "hidden_states", original_len
)
residual = _model_forward_tbo_merge_key(
output_a, output_b, "residual", original_len
)
return ResidualStream.from_handoff(hidden, residual, update)
def _model_forward_tbo_merge_key(output_a, output_b, name, original_len):
value_a = output_a[name]
value_b = output_b[name]
assert (value_a is None) == (value_b is None)
if value_a is None:
return None
s0, t0 = output_a["forward_batch"].tbo_parent_token_range
s1, t1 = output_b["forward_batch"].tbo_parent_token_range
res = torch.zeros(
(original_len, *value_a.shape[1:]),
dtype=value_a.dtype,
device=value_a.device,
)
res[slice(s0, t0)] = value_a[: t0 - s0]
res[slice(s1, t1)] = value_b[: t1 - s1]
return res
# -------------------------------- Utilities and wrappers ---------------------------------------
+4 -6
View File
@@ -3682,8 +3682,7 @@ class DeepseekV4DecoderLayer(nn.Module):
positions=state.positions,
hidden_states=hidden_states,
# DSV4 non-fused layers carry no residual across layers; the key is
# required by the next layer's op_mhc_prepare_attn (ignored) and by
# _model_forward_tbo_merge_outputs (None -> None).
# required by the next layer's op_mhc_prepare_attn (ignored).
residual=None,
forward_batch=state.forward_batch,
tbo_subbatch_index=state.tbo_subbatch_index,
@@ -4163,7 +4162,7 @@ class DeepseekV4Model(nn.Module):
from sglang.srt.batch_overlap.operations_strategy import OperationsStrategy
from sglang.srt.batch_overlap.two_batch_overlap import (
_model_forward_filter_inputs,
_model_forward_tbo_merge_outputs,
_model_forward_tbo_merge_key,
)
layers = [self.layers[i] for i in range(self.start_layer, self.end_layer)]
@@ -4237,10 +4236,9 @@ class DeepseekV4Model(nn.Module):
delta_stages=[0, operations_strategy.tbo_delta_stages],
)
hidden_states, _ = _model_forward_tbo_merge_outputs(
outputs_arr[0], outputs_arr[1], hidden_states.shape[0]
return _model_forward_tbo_merge_key(
outputs_arr[0], outputs_arr[1], "hidden_states", hidden_states.shape[0]
)
return hidden_states
@torch.no_grad()
def forward(