[OPT]Skip the first delayer to maximize the BS of the decoding. (#19836)

This commit is contained in:
chenxu214
2026-03-06 08:53:19 +08:00
committed by GitHub
parent 261be85ecc
commit b912d7ae19
@@ -66,6 +66,7 @@ class PrefillDelayer:
self._metrics_collector = metrics_collector
self._curr_state: Optional[_State] = None
self.skip_first_delayer = True
assert (
server_args.disaggregation_mode == "null"
@@ -146,8 +147,13 @@ class PrefillDelayer:
if (
max_running_requests - global_running_batch.max().item()
< global_max_prefill_bs.max().item()
and not self.enable_dp_attention
):
# When the "max_decode_bs - running_bs < max_prefill_bs" condition is met,
# the first merge_batch causes the decoding to fail to reach the maximum batch size.
if self.skip_first_delayer:
self.skip_first_delayer = False
pass
else:
next_state = prev_state or _State()
next_state = next_state.bump_delayed_count()
return _NegotiateOutput(