[OPT]Skip the first delayer to maximize the BS of the decoding. (#19836)
This commit is contained in:
@@ -66,6 +66,7 @@ class PrefillDelayer:
|
|||||||
self._metrics_collector = metrics_collector
|
self._metrics_collector = metrics_collector
|
||||||
|
|
||||||
self._curr_state: Optional[_State] = None
|
self._curr_state: Optional[_State] = None
|
||||||
|
self.skip_first_delayer = True
|
||||||
|
|
||||||
assert (
|
assert (
|
||||||
server_args.disaggregation_mode == "null"
|
server_args.disaggregation_mode == "null"
|
||||||
@@ -146,8 +147,13 @@ class PrefillDelayer:
|
|||||||
if (
|
if (
|
||||||
max_running_requests - global_running_batch.max().item()
|
max_running_requests - global_running_batch.max().item()
|
||||||
< global_max_prefill_bs.max().item()
|
< global_max_prefill_bs.max().item()
|
||||||
and not self.enable_dp_attention
|
|
||||||
):
|
):
|
||||||
|
# When the "max_decode_bs - running_bs < max_prefill_bs" condition is met,
|
||||||
|
# the first merge_batch causes the decoding to fail to reach the maximum batch size.
|
||||||
|
if self.skip_first_delayer:
|
||||||
|
self.skip_first_delayer = False
|
||||||
|
pass
|
||||||
|
else:
|
||||||
next_state = prev_state or _State()
|
next_state = prev_state or _State()
|
||||||
next_state = next_state.bump_delayed_count()
|
next_state = next_state.bump_delayed_count()
|
||||||
return _NegotiateOutput(
|
return _NegotiateOutput(
|
||||||
|
|||||||
Reference in New Issue
Block a user