[Fix] Merge DeepSeek-V4 two-batch-overlap microbatches without a residual (#42891)
This commit is contained in:
@@ -1119,27 +1119,33 @@ def _model_forward_tbo_merge_outputs(output_a, output_b, original_len):
|
||||
output["hidden_states"]
|
||||
)
|
||||
|
||||
def _handle_key(name):
|
||||
value_a = output_a[name]
|
||||
value_b = output_b[name]
|
||||
assert (value_a is None) == (value_b is None)
|
||||
if value_a is None:
|
||||
return None
|
||||
s0, t0 = output_a["forward_batch"].tbo_parent_token_range
|
||||
s1, t1 = output_b["forward_batch"].tbo_parent_token_range
|
||||
res = torch.zeros(
|
||||
(original_len, *value_a.shape[1:]),
|
||||
dtype=value_a.dtype,
|
||||
device=value_a.device,
|
||||
)
|
||||
res[slice(s0, t0)] = value_a[: t0 - s0]
|
||||
res[slice(s1, t1)] = value_b[: t1 - s1]
|
||||
return res
|
||||
|
||||
hidden, residual = _handle_key("hidden_states"), _handle_key("residual")
|
||||
hidden = _model_forward_tbo_merge_key(
|
||||
output_a, output_b, "hidden_states", original_len
|
||||
)
|
||||
residual = _model_forward_tbo_merge_key(
|
||||
output_a, output_b, "residual", original_len
|
||||
)
|
||||
return ResidualStream.from_handoff(hidden, residual, update)
|
||||
|
||||
|
||||
def _model_forward_tbo_merge_key(output_a, output_b, name, original_len):
|
||||
value_a = output_a[name]
|
||||
value_b = output_b[name]
|
||||
assert (value_a is None) == (value_b is None)
|
||||
if value_a is None:
|
||||
return None
|
||||
s0, t0 = output_a["forward_batch"].tbo_parent_token_range
|
||||
s1, t1 = output_b["forward_batch"].tbo_parent_token_range
|
||||
res = torch.zeros(
|
||||
(original_len, *value_a.shape[1:]),
|
||||
dtype=value_a.dtype,
|
||||
device=value_a.device,
|
||||
)
|
||||
res[slice(s0, t0)] = value_a[: t0 - s0]
|
||||
res[slice(s1, t1)] = value_b[: t1 - s1]
|
||||
return res
|
||||
|
||||
|
||||
# -------------------------------- Utilities and wrappers ---------------------------------------
|
||||
|
||||
|
||||
|
||||
@@ -3682,8 +3682,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
positions=state.positions,
|
||||
hidden_states=hidden_states,
|
||||
# DSV4 non-fused layers carry no residual across layers; the key is
|
||||
# required by the next layer's op_mhc_prepare_attn (ignored) and by
|
||||
# _model_forward_tbo_merge_outputs (None -> None).
|
||||
# required by the next layer's op_mhc_prepare_attn (ignored).
|
||||
residual=None,
|
||||
forward_batch=state.forward_batch,
|
||||
tbo_subbatch_index=state.tbo_subbatch_index,
|
||||
@@ -4163,7 +4162,7 @@ class DeepseekV4Model(nn.Module):
|
||||
from sglang.srt.batch_overlap.operations_strategy import OperationsStrategy
|
||||
from sglang.srt.batch_overlap.two_batch_overlap import (
|
||||
_model_forward_filter_inputs,
|
||||
_model_forward_tbo_merge_outputs,
|
||||
_model_forward_tbo_merge_key,
|
||||
)
|
||||
|
||||
layers = [self.layers[i] for i in range(self.start_layer, self.end_layer)]
|
||||
@@ -4237,10 +4236,9 @@ class DeepseekV4Model(nn.Module):
|
||||
delta_stages=[0, operations_strategy.tbo_delta_stages],
|
||||
)
|
||||
|
||||
hidden_states, _ = _model_forward_tbo_merge_outputs(
|
||||
outputs_arr[0], outputs_arr[1], hidden_states.shape[0]
|
||||
return _model_forward_tbo_merge_key(
|
||||
outputs_arr[0], outputs_arr[1], "hidden_states", hidden_states.shape[0]
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(
|
||||
|
||||
Reference in New Issue
Block a user