diff --git a/python/sglang/srt/managers/prefill_delayer.py b/python/sglang/srt/managers/prefill_delayer.py index c150e9d90a..d74c934507 100644 --- a/python/sglang/srt/managers/prefill_delayer.py +++ b/python/sglang/srt/managers/prefill_delayer.py @@ -124,10 +124,10 @@ class PrefillDelayer: # Fields packed per rank into the all-gather tensor: prefillable, # token_watermark_force_allow, running_batch, max_prefill_bs, - # waiting_queue_len. The gather spans the TP group, whose ranks are - # laid out DP-major over attn_cp_size * attn_tp_size. + # waiting_queue_len, queue_timeout_expired. The gather spans the TP + # group, whose ranks are laid out DP-major over attn_cp_size * attn_tp_size. self._global_info_buffer = torch.empty( - (dp_size_dim, parallel.attn_cp_size * parallel.attn_tp_size, 5), + (dp_size_dim, parallel.attn_cp_size * parallel.attn_tp_size, 6), dtype=torch.int64, device=self._gather_device, ) @@ -187,6 +187,12 @@ class PrefillDelayer: running_batch=running_batch, max_prefill_bs=max_prefill_bs, waiting_queue_len=waiting_queue_len, + queue_timeout_expired=( + self._queue_trigger_enabled + and prev_state is not None + and (time.perf_counter() - prev_state.start_time) * 1000.0 + >= self._max_delay_ms + ), ) global_prefillable = tp0_info[:, 0] global_token_watermark_force_allow = tp0_info[:, 1] @@ -262,10 +268,8 @@ class PrefillDelayer: queue_min_effective > 0 and global_waiting_queue_max < queue_min_effective ) - if queue_condition and prev_state is not None: - elapsed_ms = (time.perf_counter() - prev_state.start_time) * 1000.0 - if elapsed_ms >= self._max_delay_ms: - queue_condition = False + if queue_condition and tp0_info[:, 5].max().item() > 0: + queue_condition = False slot_condition = ( max_running_requests - global_running_batch_max @@ -354,6 +358,7 @@ class PrefillDelayer: running_batch: int = 0, max_prefill_bs: int = 0, waiting_queue_len: int = 0, + queue_timeout_expired: bool = False, ): local_info = torch.tensor( [ @@ -362,6 +367,7 @@ class PrefillDelayer: running_batch, max_prefill_bs, waiting_queue_len, + int(queue_timeout_expired), ], device=self._gather_device, dtype=torch.int64, diff --git a/test/registered/unit/managers/test_prefill_delayer_gather.py b/test/registered/unit/managers/test_prefill_delayer_gather.py index b2b586876d..adfeda6dae 100644 --- a/test/registered/unit/managers/test_prefill_delayer_gather.py +++ b/test/registered/unit/managers/test_prefill_delayer_gather.py @@ -1,15 +1,18 @@ +import os +import tempfile +import time import unittest from types import SimpleNamespace from unittest.mock import patch import torch -from sglang.srt.managers.prefill_delayer import PrefillDelayer +from sglang.srt.managers.prefill_delayer import PrefillDelayer, _State from sglang.srt.runtime_context import get_parallel from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase, published_topology -register_cpu_ci(est_time=5, suite="base-a-test-cpu") +register_cpu_ci(est_time=30, suite="base-a-test-cpu") TOPOLOGIES = { "tp": dict(tp_size=4), @@ -55,16 +58,74 @@ class TestPrefillDelayerGather(CustomTestCase): running_batch=3, max_prefill_bs=4, waiting_queue_len=5, + queue_timeout_expired=True, ) parallel = get_parallel() dp_groups = parallel.num_dp_ranks if parallel.attn_dp_enabled else 1 ranks_per_dp_group = parallel.tp_size // dp_groups - local = torch.tensor([1, 0, 3, 4, 5]) + local = torch.tensor([1, 0, 3, 4, 5, 1]) expected = torch.stack( [local + 100 * g * ranks_per_dp_group for g in range(dp_groups)] ) self.assertTrue(torch.equal(info, expected), f"{info} != {expected}") +def _negotiate_with_skewed_clocks(rank, world_size, init_file): + torch.distributed.init_process_group( + "gloo", init_method=f"file://{init_file}", rank=rank, world_size=world_size + ) + try: + for name in ("tp", "dp_attention"): + topology = TOPOLOGIES[name] + with ( + published_topology( + ranks=dict(world_rank=rank), + prefill_delayer_queue_min_ratio=0.5, + prefill_delayer_max_delay_ms=1000, + **topology, + ), + get_parallel().override( + tp_group=SimpleNamespace( + cpu_group=torch.distributed.group.WORLD, device_group=None + ) + ), + ): + delayer = PrefillDelayer( + max_delay_passes=100, token_usage_low_watermark=None + ) + delayer.skip_first_delayer = False + for tp0_expired in (True, False): + local_expired = tp0_expired == (rank % 2 == 0) + age = 60.0 if local_expired else 0.0 + out = delayer._negotiate_should_allow_prefill_pure( + prev_state=_State( + delayed_count=1, start_time=time.perf_counter() - age + ), + local_prefillable=True, + token_usage=0.8, + running_batch=8, + max_prefill_bs=4, + max_running_requests=128, + waiting_queue_len=1, + ) + assert out.output_allow == tp0_expired, ( + f"{name} rank={rank} local_expired={local_expired} " + f"tp0_expired={tp0_expired} allow={out.output_allow}" + ) + finally: + torch.distributed.destroy_process_group() + + +class TestPrefillDelayerQueueTimeoutMultiRank(CustomTestCase): + def test_ranks_with_skewed_clocks_follow_tp0(self): + world_size = 4 + with tempfile.TemporaryDirectory() as tmp: + torch.multiprocessing.spawn( + _negotiate_with_skewed_clocks, + args=(world_size, os.path.join(tmp, "init")), + nprocs=world_size, + ) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/managers/test_prefill_delayer_high_watermark.py b/test/registered/unit/managers/test_prefill_delayer_high_watermark.py index 3cb6bba4cb..fbfbd7f287 100644 --- a/test/registered/unit/managers/test_prefill_delayer_high_watermark.py +++ b/test/registered/unit/managers/test_prefill_delayer_high_watermark.py @@ -1,10 +1,15 @@ +import time import unittest from unittest.mock import MagicMock +import torch + from sglang.srt.managers.prefill_delayer import ( + PrefillDelayer, PrefillDelayerSinglePassExecutor, RecentPrefillBatchSizeTracker, _NegotiateOutput, + _State, ) from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -139,5 +144,44 @@ class TestPrefillDelayerHighWatermark(CustomTestCase): self.assertEqual(tracker.max_prefill_bs, 0) +class TestPrefillDelayerClockSkew(CustomTestCase): + def test_queue_timeout_follows_gathered_flag(self): + delayer = PrefillDelayer.__new__(PrefillDelayer) + delayer.__dict__.update( + _max_delay_passes=100, + _token_usage_low_watermark=None, + _queue_min_ratio=0.5, + _max_delay_ms=500, + _queue_trigger_enabled=True, + _prefill_max_requests=4, + attn_dp_enabled=False, + num_dp_ranks=1, + skip_first_delayer=False, + ) + for local_expired in (False, True): + for gathered_expired in (False, True): + with self.subTest( + local_expired=local_expired, gathered_expired=gathered_expired + ): + delayer._gather_info = MagicMock( + return_value=torch.tensor( + [[1, 0, 8, 4, 1, int(gathered_expired)]] + ) + ) + start_time = float("-inf") if local_expired else time.perf_counter() + out = delayer._negotiate_should_allow_prefill_pure( + prev_state=_State(delayed_count=1, start_time=start_time), + local_prefillable=True, + token_usage=0.8, + running_batch=8, + max_prefill_bs=4, + max_running_requests=128, + waiting_queue_len=1, + ) + self.assertEqual(out.output_allow, gathered_expired) + kwargs = delayer._gather_info.call_args.kwargs + self.assertEqual(kwargs["queue_timeout_expired"], local_expired) + + if __name__ == "__main__": unittest.main()