[Scheduler] Gather prefill-delayer queue timeout across ranks (#42625)
Co-authored-by: Cheng Wan <54331508+ch-wan@users.noreply.github.com>
This commit is contained in:
@@ -124,10 +124,10 @@ class PrefillDelayer:
|
||||
|
||||
# Fields packed per rank into the all-gather tensor: prefillable,
|
||||
# token_watermark_force_allow, running_batch, max_prefill_bs,
|
||||
# waiting_queue_len. The gather spans the TP group, whose ranks are
|
||||
# laid out DP-major over attn_cp_size * attn_tp_size.
|
||||
# waiting_queue_len, queue_timeout_expired. The gather spans the TP
|
||||
# group, whose ranks are laid out DP-major over attn_cp_size * attn_tp_size.
|
||||
self._global_info_buffer = torch.empty(
|
||||
(dp_size_dim, parallel.attn_cp_size * parallel.attn_tp_size, 5),
|
||||
(dp_size_dim, parallel.attn_cp_size * parallel.attn_tp_size, 6),
|
||||
dtype=torch.int64,
|
||||
device=self._gather_device,
|
||||
)
|
||||
@@ -187,6 +187,12 @@ class PrefillDelayer:
|
||||
running_batch=running_batch,
|
||||
max_prefill_bs=max_prefill_bs,
|
||||
waiting_queue_len=waiting_queue_len,
|
||||
queue_timeout_expired=(
|
||||
self._queue_trigger_enabled
|
||||
and prev_state is not None
|
||||
and (time.perf_counter() - prev_state.start_time) * 1000.0
|
||||
>= self._max_delay_ms
|
||||
),
|
||||
)
|
||||
global_prefillable = tp0_info[:, 0]
|
||||
global_token_watermark_force_allow = tp0_info[:, 1]
|
||||
@@ -262,10 +268,8 @@ class PrefillDelayer:
|
||||
queue_min_effective > 0
|
||||
and global_waiting_queue_max < queue_min_effective
|
||||
)
|
||||
if queue_condition and prev_state is not None:
|
||||
elapsed_ms = (time.perf_counter() - prev_state.start_time) * 1000.0
|
||||
if elapsed_ms >= self._max_delay_ms:
|
||||
queue_condition = False
|
||||
if queue_condition and tp0_info[:, 5].max().item() > 0:
|
||||
queue_condition = False
|
||||
|
||||
slot_condition = (
|
||||
max_running_requests - global_running_batch_max
|
||||
@@ -354,6 +358,7 @@ class PrefillDelayer:
|
||||
running_batch: int = 0,
|
||||
max_prefill_bs: int = 0,
|
||||
waiting_queue_len: int = 0,
|
||||
queue_timeout_expired: bool = False,
|
||||
):
|
||||
local_info = torch.tensor(
|
||||
[
|
||||
@@ -362,6 +367,7 @@ class PrefillDelayer:
|
||||
running_batch,
|
||||
max_prefill_bs,
|
||||
waiting_queue_len,
|
||||
int(queue_timeout_expired),
|
||||
],
|
||||
device=self._gather_device,
|
||||
dtype=torch.int64,
|
||||
|
||||
@@ -1,15 +1,18 @@
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.managers.prefill_delayer import PrefillDelayer
|
||||
from sglang.srt.managers.prefill_delayer import PrefillDelayer, _State
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase, published_topology
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
register_cpu_ci(est_time=30, suite="base-a-test-cpu")
|
||||
|
||||
TOPOLOGIES = {
|
||||
"tp": dict(tp_size=4),
|
||||
@@ -55,16 +58,74 @@ class TestPrefillDelayerGather(CustomTestCase):
|
||||
running_batch=3,
|
||||
max_prefill_bs=4,
|
||||
waiting_queue_len=5,
|
||||
queue_timeout_expired=True,
|
||||
)
|
||||
parallel = get_parallel()
|
||||
dp_groups = parallel.num_dp_ranks if parallel.attn_dp_enabled else 1
|
||||
ranks_per_dp_group = parallel.tp_size // dp_groups
|
||||
local = torch.tensor([1, 0, 3, 4, 5])
|
||||
local = torch.tensor([1, 0, 3, 4, 5, 1])
|
||||
expected = torch.stack(
|
||||
[local + 100 * g * ranks_per_dp_group for g in range(dp_groups)]
|
||||
)
|
||||
self.assertTrue(torch.equal(info, expected), f"{info} != {expected}")
|
||||
|
||||
|
||||
def _negotiate_with_skewed_clocks(rank, world_size, init_file):
|
||||
torch.distributed.init_process_group(
|
||||
"gloo", init_method=f"file://{init_file}", rank=rank, world_size=world_size
|
||||
)
|
||||
try:
|
||||
for name in ("tp", "dp_attention"):
|
||||
topology = TOPOLOGIES[name]
|
||||
with (
|
||||
published_topology(
|
||||
ranks=dict(world_rank=rank),
|
||||
prefill_delayer_queue_min_ratio=0.5,
|
||||
prefill_delayer_max_delay_ms=1000,
|
||||
**topology,
|
||||
),
|
||||
get_parallel().override(
|
||||
tp_group=SimpleNamespace(
|
||||
cpu_group=torch.distributed.group.WORLD, device_group=None
|
||||
)
|
||||
),
|
||||
):
|
||||
delayer = PrefillDelayer(
|
||||
max_delay_passes=100, token_usage_low_watermark=None
|
||||
)
|
||||
delayer.skip_first_delayer = False
|
||||
for tp0_expired in (True, False):
|
||||
local_expired = tp0_expired == (rank % 2 == 0)
|
||||
age = 60.0 if local_expired else 0.0
|
||||
out = delayer._negotiate_should_allow_prefill_pure(
|
||||
prev_state=_State(
|
||||
delayed_count=1, start_time=time.perf_counter() - age
|
||||
),
|
||||
local_prefillable=True,
|
||||
token_usage=0.8,
|
||||
running_batch=8,
|
||||
max_prefill_bs=4,
|
||||
max_running_requests=128,
|
||||
waiting_queue_len=1,
|
||||
)
|
||||
assert out.output_allow == tp0_expired, (
|
||||
f"{name} rank={rank} local_expired={local_expired} "
|
||||
f"tp0_expired={tp0_expired} allow={out.output_allow}"
|
||||
)
|
||||
finally:
|
||||
torch.distributed.destroy_process_group()
|
||||
|
||||
|
||||
class TestPrefillDelayerQueueTimeoutMultiRank(CustomTestCase):
|
||||
def test_ranks_with_skewed_clocks_follow_tp0(self):
|
||||
world_size = 4
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
torch.multiprocessing.spawn(
|
||||
_negotiate_with_skewed_clocks,
|
||||
args=(world_size, os.path.join(tmp, "init")),
|
||||
nprocs=world_size,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,10 +1,15 @@
|
||||
import time
|
||||
import unittest
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.managers.prefill_delayer import (
|
||||
PrefillDelayer,
|
||||
PrefillDelayerSinglePassExecutor,
|
||||
RecentPrefillBatchSizeTracker,
|
||||
_NegotiateOutput,
|
||||
_State,
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
@@ -139,5 +144,44 @@ class TestPrefillDelayerHighWatermark(CustomTestCase):
|
||||
self.assertEqual(tracker.max_prefill_bs, 0)
|
||||
|
||||
|
||||
class TestPrefillDelayerClockSkew(CustomTestCase):
|
||||
def test_queue_timeout_follows_gathered_flag(self):
|
||||
delayer = PrefillDelayer.__new__(PrefillDelayer)
|
||||
delayer.__dict__.update(
|
||||
_max_delay_passes=100,
|
||||
_token_usage_low_watermark=None,
|
||||
_queue_min_ratio=0.5,
|
||||
_max_delay_ms=500,
|
||||
_queue_trigger_enabled=True,
|
||||
_prefill_max_requests=4,
|
||||
attn_dp_enabled=False,
|
||||
num_dp_ranks=1,
|
||||
skip_first_delayer=False,
|
||||
)
|
||||
for local_expired in (False, True):
|
||||
for gathered_expired in (False, True):
|
||||
with self.subTest(
|
||||
local_expired=local_expired, gathered_expired=gathered_expired
|
||||
):
|
||||
delayer._gather_info = MagicMock(
|
||||
return_value=torch.tensor(
|
||||
[[1, 0, 8, 4, 1, int(gathered_expired)]]
|
||||
)
|
||||
)
|
||||
start_time = float("-inf") if local_expired else time.perf_counter()
|
||||
out = delayer._negotiate_should_allow_prefill_pure(
|
||||
prev_state=_State(delayed_count=1, start_time=start_time),
|
||||
local_prefillable=True,
|
||||
token_usage=0.8,
|
||||
running_batch=8,
|
||||
max_prefill_bs=4,
|
||||
max_running_requests=128,
|
||||
waiting_queue_len=1,
|
||||
)
|
||||
self.assertEqual(out.output_allow, gathered_expired)
|
||||
kwargs = delayer._gather_info.call_args.kwargs
|
||||
self.assertEqual(kwargs["queue_timeout_expired"], local_expired)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user