[Fix] Stop PD-decode queue_time from counting decode time before a retraction (#41380)
Signed-off-by: Sohom Chakraborty <sohomchakraborty.iitkgp@gmail.com> Co-authored-by: Shuwen Wang <47200617+alphabetc1@users.noreply.github.com>
This commit is contained in:
co-authored by
Shuwen Wang
parent
bd66ce343e
commit
28c5e7f5cb
@@ -3290,7 +3290,9 @@ class Scheduler(
|
||||
if not is_retracted:
|
||||
req.time_stats.set_decode_prealloc_queue_entry_time()
|
||||
else:
|
||||
req.time_stats.set_retract_time()
|
||||
# Restart the queue wait here;
|
||||
# resume_retracted_reqs() re-queues the request without stamping it.
|
||||
req.time_stats.set_wait_queue_entry_time()
|
||||
else:
|
||||
raise ValueError(f"Invalid {self.disaggregation_mode=}")
|
||||
|
||||
|
||||
@@ -29,6 +29,8 @@ from sglang.srt.managers.schedule_batch import ( # noqa: E402
|
||||
from sglang.srt.managers.scheduler import Scheduler # noqa: E402
|
||||
from sglang.srt.mem_cache.base_prefix_cache import CacheRequestOutcome
|
||||
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache # noqa: E402
|
||||
from sglang.srt.observability import req_time_stats
|
||||
from sglang.srt.observability.req_time_stats import SchedulerReqTimeStats
|
||||
from sglang.srt.runtime_context import get_context, publish, reset_context # noqa: E402
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
@@ -98,6 +100,22 @@ class TestDisaggregationPriorityQueueing(unittest.TestCase):
|
||||
)
|
||||
req.time_stats.set_decode_prealloc_queue_entry_time.assert_called_once()
|
||||
|
||||
def test_decode_retraction_queue_time_excludes_decoding_before_it(self):
|
||||
"""After a PD-decode retraction, queue_time counts waits, not earlier decoding."""
|
||||
scheduler = self._new_scheduler(DisaggregationMode.DECODE)
|
||||
req = self._new_req(priority=0)
|
||||
stats = SchedulerReqTimeStats(disagg_mode=DisaggregationMode.DECODE)
|
||||
req.time_stats = stats
|
||||
t0 = 1000.0
|
||||
stats.set_wait_queue_entry_time(t0)
|
||||
stats.set_forward_entry_time(t0 + 0.1)
|
||||
|
||||
with patch.object(req_time_stats.time, "perf_counter", return_value=t0 + 10.0):
|
||||
scheduler._add_request_to_queue(req, is_retracted=True)
|
||||
stats.set_forward_entry_time(t0 + 12.0)
|
||||
|
||||
self.assertAlmostEqual(stats.get_queueing_time(), 0.1 + 2.0)
|
||||
|
||||
def test_priority_disabled_abort_validation_applies_to_decode_mode(self):
|
||||
scheduler = self._new_scheduler(DisaggregationMode.DECODE)
|
||||
scheduler.enable_priority_scheduling = False
|
||||
|
||||
Reference in New Issue
Block a user