[Scheduler] Gather prefill-delayer queue timeout across ranks (#42625)

Co-authored-by: Cheng Wan <54331508+ch-wan@users.noreply.github.com>
This commit is contained in:
Chunan Zeng
2026-10-06 16:15:34 -07:00
committed by GitHub
co-authored by Cheng Wan
parent 0a0fe001f1
commit 90aa1cb0fb
3 changed files with 121 additions and 10 deletions
+13 -7
View File
@@ -124,10 +124,10 @@ class PrefillDelayer:
# Fields packed per rank into the all-gather tensor: prefillable,
# token_watermark_force_allow, running_batch, max_prefill_bs,
# waiting_queue_len. The gather spans the TP group, whose ranks are
# laid out DP-major over attn_cp_size * attn_tp_size.
# waiting_queue_len, queue_timeout_expired. The gather spans the TP
# group, whose ranks are laid out DP-major over attn_cp_size * attn_tp_size.
self._global_info_buffer = torch.empty(
(dp_size_dim, parallel.attn_cp_size * parallel.attn_tp_size, 5),
(dp_size_dim, parallel.attn_cp_size * parallel.attn_tp_size, 6),
dtype=torch.int64,
device=self._gather_device,
)
@@ -187,6 +187,12 @@ class PrefillDelayer:
running_batch=running_batch,
max_prefill_bs=max_prefill_bs,
waiting_queue_len=waiting_queue_len,
queue_timeout_expired=(
self._queue_trigger_enabled
and prev_state is not None
and (time.perf_counter() - prev_state.start_time) * 1000.0
>= self._max_delay_ms
),
)
global_prefillable = tp0_info[:, 0]
global_token_watermark_force_allow = tp0_info[:, 1]
@@ -262,10 +268,8 @@ class PrefillDelayer:
queue_min_effective > 0
and global_waiting_queue_max < queue_min_effective
)
if queue_condition and prev_state is not None:
elapsed_ms = (time.perf_counter() - prev_state.start_time) * 1000.0
if elapsed_ms >= self._max_delay_ms:
queue_condition = False
if queue_condition and tp0_info[:, 5].max().item() > 0:
queue_condition = False
slot_condition = (
max_running_requests - global_running_batch_max
@@ -354,6 +358,7 @@ class PrefillDelayer:
running_batch: int = 0,
max_prefill_bs: int = 0,
waiting_queue_len: int = 0,
queue_timeout_expired: bool = False,
):
local_info = torch.tensor(
[
@@ -362,6 +367,7 @@ class PrefillDelayer:
running_batch,
max_prefill_bs,
waiting_queue_len,
int(queue_timeout_expired),
],
device=self._gather_device,
dtype=torch.int64,
@@ -1,15 +1,18 @@
import os
import tempfile
import time
import unittest
from types import SimpleNamespace
from unittest.mock import patch
import torch
from sglang.srt.managers.prefill_delayer import PrefillDelayer
from sglang.srt.managers.prefill_delayer import PrefillDelayer, _State
from sglang.srt.runtime_context import get_parallel
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase, published_topology
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
register_cpu_ci(est_time=30, suite="base-a-test-cpu")
TOPOLOGIES = {
"tp": dict(tp_size=4),
@@ -55,16 +58,74 @@ class TestPrefillDelayerGather(CustomTestCase):
running_batch=3,
max_prefill_bs=4,
waiting_queue_len=5,
queue_timeout_expired=True,
)
parallel = get_parallel()
dp_groups = parallel.num_dp_ranks if parallel.attn_dp_enabled else 1
ranks_per_dp_group = parallel.tp_size // dp_groups
local = torch.tensor([1, 0, 3, 4, 5])
local = torch.tensor([1, 0, 3, 4, 5, 1])
expected = torch.stack(
[local + 100 * g * ranks_per_dp_group for g in range(dp_groups)]
)
self.assertTrue(torch.equal(info, expected), f"{info} != {expected}")
def _negotiate_with_skewed_clocks(rank, world_size, init_file):
torch.distributed.init_process_group(
"gloo", init_method=f"file://{init_file}", rank=rank, world_size=world_size
)
try:
for name in ("tp", "dp_attention"):
topology = TOPOLOGIES[name]
with (
published_topology(
ranks=dict(world_rank=rank),
prefill_delayer_queue_min_ratio=0.5,
prefill_delayer_max_delay_ms=1000,
**topology,
),
get_parallel().override(
tp_group=SimpleNamespace(
cpu_group=torch.distributed.group.WORLD, device_group=None
)
),
):
delayer = PrefillDelayer(
max_delay_passes=100, token_usage_low_watermark=None
)
delayer.skip_first_delayer = False
for tp0_expired in (True, False):
local_expired = tp0_expired == (rank % 2 == 0)
age = 60.0 if local_expired else 0.0
out = delayer._negotiate_should_allow_prefill_pure(
prev_state=_State(
delayed_count=1, start_time=time.perf_counter() - age
),
local_prefillable=True,
token_usage=0.8,
running_batch=8,
max_prefill_bs=4,
max_running_requests=128,
waiting_queue_len=1,
)
assert out.output_allow == tp0_expired, (
f"{name} rank={rank} local_expired={local_expired} "
f"tp0_expired={tp0_expired} allow={out.output_allow}"
)
finally:
torch.distributed.destroy_process_group()
class TestPrefillDelayerQueueTimeoutMultiRank(CustomTestCase):
def test_ranks_with_skewed_clocks_follow_tp0(self):
world_size = 4
with tempfile.TemporaryDirectory() as tmp:
torch.multiprocessing.spawn(
_negotiate_with_skewed_clocks,
args=(world_size, os.path.join(tmp, "init")),
nprocs=world_size,
)
if __name__ == "__main__":
unittest.main()
@@ -1,10 +1,15 @@
import time
import unittest
from unittest.mock import MagicMock
import torch
from sglang.srt.managers.prefill_delayer import (
PrefillDelayer,
PrefillDelayerSinglePassExecutor,
RecentPrefillBatchSizeTracker,
_NegotiateOutput,
_State,
)
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
@@ -139,5 +144,44 @@ class TestPrefillDelayerHighWatermark(CustomTestCase):
self.assertEqual(tracker.max_prefill_bs, 0)
class TestPrefillDelayerClockSkew(CustomTestCase):
def test_queue_timeout_follows_gathered_flag(self):
delayer = PrefillDelayer.__new__(PrefillDelayer)
delayer.__dict__.update(
_max_delay_passes=100,
_token_usage_low_watermark=None,
_queue_min_ratio=0.5,
_max_delay_ms=500,
_queue_trigger_enabled=True,
_prefill_max_requests=4,
attn_dp_enabled=False,
num_dp_ranks=1,
skip_first_delayer=False,
)
for local_expired in (False, True):
for gathered_expired in (False, True):
with self.subTest(
local_expired=local_expired, gathered_expired=gathered_expired
):
delayer._gather_info = MagicMock(
return_value=torch.tensor(
[[1, 0, 8, 4, 1, int(gathered_expired)]]
)
)
start_time = float("-inf") if local_expired else time.perf_counter()
out = delayer._negotiate_should_allow_prefill_pure(
prev_state=_State(delayed_count=1, start_time=start_time),
local_prefillable=True,
token_usage=0.8,
running_batch=8,
max_prefill_bs=4,
max_running_requests=128,
waiting_queue_len=1,
)
self.assertEqual(out.output_allow, gathered_expired)
kwargs = delayer._gather_info.call_args.kwargs
self.assertEqual(kwargs["queue_timeout_expired"], local_expired)
if __name__ == "__main__":
unittest.main()