minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

feat(scheduler): add adaptive queue-based prefill delayer trigger (#23189)

Browse Source
This commit is contained in:
YAMY
2026-05-08 16:54:30 -07:00
committed by GitHub
parent 6971a03fe6
commit 560829a171
6 changed files with 246 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/scheduler/test_prefill_delayer.py
+146
View File
@@ -41,6 +41,16 @@ WORLD_SIZE = os.environ.get("SGLANG_TEST_WORLD_SIZE", "8")
class NegotiateCall:
prefillable: List[bool]
token_usage: List[float]
# Optional scheduler state; when None, _run_negotiate_test does not pass
# the kwarg and the delayer falls back to the historical behavior of
# reading kwargs.get(..., 0).
running_batch: Optional[List[int]] = None
max_prefill_bs: Optional[List[int]] = None
waiting_queue_len: Optional[List[int]] = None
max_running_requests: Optional[int] = None
# Inter-call sleep (seconds). Used to exercise the queue-trigger
# wall-clock timeout.
sleep_before_s: float = 0.0
@dataclass
@@ -51,6 +61,10 @@ class NegotiateTestCase:
calls: List[NegotiateCall]
expected_allow: bool
expected_reason: str
# Queue-trigger knobs (new in the queue-based delayer). Leave both None
# to exercise the legacy slot-only code paths.
queue_min_ratio: Optional[float] = None
max_delay_ms: Optional[float] = None
def _run_negotiate_test(rank, test_cases):
@@ -66,15 +80,31 @@ def _run_negotiate_test(rank, test_cases):
enable_dp_attention=True,
disaggregation_mode="null",
disable_overlap_schedule=False,
prefill_delayer_queue_min_ratio=case.queue_min_ratio,
prefill_delayer_max_delay_ms=case.max_delay_ms,
),
max_delay_passes=case.max_delay_passes,
token_usage_low_watermark=case.token_usage_low_watermark,
)
for call in case.calls:
if call.sleep_before_s > 0:
time.sleep(call.sleep_before_s)
extra_kwargs = {}
if call.running_batch is not None:
extra_kwargs["running_batch"] = call.running_batch[rank]
if call.max_prefill_bs is not None:
extra_kwargs["max_prefill_bs"] = call.max_prefill_bs[rank]
if call.waiting_queue_len is not None:
extra_kwargs["waiting_queue_len"] = call.waiting_queue_len[rank]
if call.max_running_requests is not None:
extra_kwargs["max_running_requests"] = call.max_running_requests
result = delayer._negotiate_should_allow_prefill(
local_prefillable=call.prefillable[rank],
token_usage=call.token_usage[rank],
**extra_kwargs,
)
assert (result.output_allow, result.output_reason) == (
@@ -200,6 +230,122 @@ _NEGOTIATE_TEST_CASES = [
expected_allow=True,
expected_reason="wait_timeout",
),
# Queue-based trigger: waiting queue below queue_min = min(running * R,
# max_prefill_bs) should defer prefill. With R=0.5, running=100 and
# max_prefill_bs=80, queue_min = min(50, 80) = 50, and queue_len=10 < 50.
NegotiateTestCase(
name="queue_trigger_delay",
max_delay_passes=100,
token_usage_low_watermark=0.8,
queue_min_ratio=0.5,
max_delay_ms=5000,
calls=[
NegotiateCall(
prefillable=[True, True, True, True],
token_usage=[0.9, 0.9, 0.9, 0.9],
running_batch=[100, 100, 100, 100],
max_prefill_bs=[80, 80, 80, 80],
waiting_queue_len=[10, 10, 10, 10],
max_running_requests=1024,
),
# skip_first_delayer consumes the first would-be delay; a second
# identical call must actually delay.
NegotiateCall(
prefillable=[True, True, True, True],
token_usage=[0.9, 0.9, 0.9, 0.9],
running_batch=[100, 100, 100, 100],
max_prefill_bs=[80, 80, 80, 80],
waiting_queue_len=[10, 10, 10, 10],
max_running_requests=1024,
),
],
expected_allow=False,
expected_reason="delay",
),
# Waiting queue at or above queue_min: queue trigger must not fire.
NegotiateTestCase(
name="queue_trigger_above_threshold",
max_delay_passes=100,
token_usage_low_watermark=0.8,
queue_min_ratio=0.5,
max_delay_ms=5000,
calls=[
NegotiateCall(
prefillable=[True, True, True, True],
token_usage=[0.9, 0.9, 0.9, 0.9],
running_batch=[100, 100, 100, 100],
max_prefill_bs=[80, 80, 80, 80],
waiting_queue_len=[64, 64, 64, 64],
max_running_requests=1024,
)
],
expected_allow=True,
expected_reason="no_wait",
),
# queue_min_ratio unset: queue trigger is opt-in and must stay disabled
# even when running_batch and queue_len would otherwise trigger it.
NegotiateTestCase(
name="queue_trigger_disabled_when_ratio_unset",
max_delay_passes=100,
token_usage_low_watermark=0.8,
queue_min_ratio=None,
max_delay_ms=None,
calls=[
NegotiateCall(
prefillable=[True, True, True, True],
token_usage=[0.9, 0.9, 0.9, 0.9],
running_batch=[100, 100, 100, 100],
max_prefill_bs=[80, 80, 80, 80],
waiting_queue_len=[1, 1, 1, 1],
max_running_requests=1024,
)
],
expected_allow=True,
expected_reason="no_wait",
),
# max_delay_ms wall-clock timeout: once a single queue-trigger delay
# exceeds the cap, prefill must be force-released.
# Call sequence:
# 1) queue_condition holds but skip_first_delayer consumes it
# (no state recorded, falls through to allow)
# 2) queue_condition holds -> delay, records start_time in state
# 3) after sleeping past max_delay_ms, elapsed >= cap -> force release
NegotiateTestCase(
name="queue_trigger_wall_clock_timeout",
max_delay_passes=100,
token_usage_low_watermark=0.8,
queue_min_ratio=0.5,
max_delay_ms=50,
calls=[
NegotiateCall(
prefillable=[True, True, True, True],
token_usage=[0.9, 0.9, 0.9, 0.9],
running_batch=[100, 100, 100, 100],
max_prefill_bs=[80, 80, 80, 80],
waiting_queue_len=[10, 10, 10, 10],
max_running_requests=1024,
),
NegotiateCall(
prefillable=[True, True, True, True],
token_usage=[0.9, 0.9, 0.9, 0.9],
running_batch=[100, 100, 100, 100],
max_prefill_bs=[80, 80, 80, 80],
waiting_queue_len=[10, 10, 10, 10],
max_running_requests=1024,
),
NegotiateCall(
prefillable=[True, True, True, True],
token_usage=[0.9, 0.9, 0.9, 0.9],
running_batch=[100, 100, 100, 100],
max_prefill_bs=[80, 80, 80, 80],
waiting_queue_len=[10, 10, 10, 10],
max_running_requests=1024,
sleep_before_s=0.2, # > max_delay_ms (50ms)
),
],
expected_allow=True,
expected_reason="wait_success",
),
]