From 2ff52e3ceb0bcfdc8ba1dd0367b39ce0370bbfaf Mon Sep 17 00:00:00 2001 From: Anupa Sajikumar Date: Tue, 29 Sep 2026 10:58:25 +0530 Subject: [PATCH] [Intel][XPU]Enable chunked prefill scnearios for XPU with UT (#33804) Co-authored-by: Ma Mingfei --- .../sglang/test/chunked_prefill_test_utils.py | 18 +++- .../test/scripted_runtime/http_server.py | 49 +++++++++-- .../test/scripted_runtime_chunked_helpers.py | 22 +++++ .../chunked_prefill/test_scripted_abort.py | 86 +++++++------------ .../test_scripted_lifecycle.py | 21 ++--- .../test_scripted_multi_req.py | 3 +- .../chunked_prefill/test_scripted_pp.py | 13 +-- .../test_scripted_regression.py | 62 ++++++------- .../test_scripted_special_case.py | 3 + .../unit/scripted_runtime/test_http_server.py | 26 +++++- 10 files changed, 182 insertions(+), 121 deletions(-) diff --git a/python/sglang/test/chunked_prefill_test_utils.py b/python/sglang/test/chunked_prefill_test_utils.py index 45b79636cf..92193ff177 100644 --- a/python/sglang/test/chunked_prefill_test_utils.py +++ b/python/sglang/test/chunked_prefill_test_utils.py @@ -4,6 +4,10 @@ import time from types import SimpleNamespace from typing import ClassVar, List, Optional +import torch + +from sglang.kernels.ops.kv_canary._dispatch import use_torch_reference +from sglang.srt.utils import get_device from sglang.test.run_eval import run_eval from sglang.test.server_fixtures.disaggregation_fixture import ( PDDisaggregationServerBase, @@ -38,6 +42,16 @@ KV_CANARY_ARGS: List[str] = [ ] +def _canary_args(enabled: bool) -> List[str]: + if not enabled: + return [] + args = list(KV_CANARY_ARGS) + if use_torch_reference(torch.device(get_device())): + # install_canary refuses a captured decode over the torch reference. + args.append("--cuda-graph-backend-decode=disabled") + return args + + class ChunkedGsm8kMixin: __test__ = False use_kv_canary: ClassVar[bool] = True @@ -52,7 +66,7 @@ class ChunkedGsm8kMixin: gsm8k_threshold: ClassVar[float] def build_prefill_side_args(self) -> List[str]: - canary = list(KV_CANARY_ARGS) if self.use_kv_canary else [] + canary = _canary_args(self.use_kv_canary) return ( ["--chunked-prefill-size", str(self.chunked_prefill_size)] + list(self.feature_args) @@ -118,7 +132,7 @@ class ChunkedTestPDBase(ChunkedGsm8kMixin, PDDisaggregationServerBase): cls.extra_prefill_args = cls( "test_mixed_prefix_gsm8k_chunked" ).build_prefill_side_args() - canary = list(KV_CANARY_ARGS) if cls.use_kv_canary else [] + canary = _canary_args(cls.use_kv_canary) cls.extra_decode_args = canary + list(cls.decode_feature_args) PDDisaggregationServerBase.setUpClass() cls.model = try_cached_model(cls.model) diff --git a/python/sglang/test/scripted_runtime/http_server.py b/python/sglang/test/scripted_runtime/http_server.py index 21e99e97c5..ee14d9e198 100644 --- a/python/sglang/test/scripted_runtime/http_server.py +++ b/python/sglang/test/scripted_runtime/http_server.py @@ -8,12 +8,15 @@ from pathlib import Path from typing import Any, Callable, Dict, Optional, Tuple import requests +import torch import zmq +from sglang.kernels.ops.kv_canary._dispatch import use_torch_reference from sglang.srt.entrypoints.http_server import launch_server from sglang.srt.environ import envs from sglang.srt.managers.io_struct import sock_recv, sock_send, wrap_as_pickle from sglang.srt.server_args import ServerArgs +from sglang.srt.utils.common import get_device from sglang.srt.utils.network import get_free_port, get_zmq_socket_on_host from sglang.test.scripted_runtime.io_struct import ( HookReady, @@ -26,12 +29,20 @@ from sglang.test.scripted_runtime.io_struct import ( from sglang.test.scripted_runtime.utils import close_zmq_socket DEFAULT_RUN_TIMEOUT_S: float = 120.0 + +CANARY_REFERENCE_TIMEOUT_SCALE: float = 4.0 SHUTDOWN_JOIN_TIMEOUT_S: float = 60.0 LISTENER_ACCEPT_TIMEOUT_S: float = 300.0 HTTP_READY_TIMEOUT_S: float = 300.0 HTTP_READY_POLL_INTERVAL_S: float = 0.5 SERVER_HOST: str = "127.0.0.1" +CANARY_LAUNCH_DEFAULTS: Dict[str, Any] = dict( + kv_canary="raise", + kv_canary_real_data="partial", + kv_canary_sweep_interval=100, +) + class ScriptedHttpServer: def __init__( @@ -42,6 +53,7 @@ class ScriptedHttpServer: server_process: mp.process.BaseProcess, out_of_band_error_path: Path, http_port: int, + run_timeout_s: float = DEFAULT_RUN_TIMEOUT_S, ) -> None: self._ctx = ctx self._socket = socket @@ -50,6 +62,7 @@ class ScriptedHttpServer: self._base_url = f"http://{SERVER_HOST}:{http_port}" self._shutdown_done = False self._dirty: Optional[str] = None + self._run_timeout_s = run_timeout_s @classmethod def start(cls, **engine_kwargs: Any) -> ScriptedHttpServer: @@ -57,7 +70,7 @@ class ScriptedHttpServer: ctx = zmq.Context() dispatch_port, socket = get_zmq_socket_on_host(ctx, zmq.PAIR, host=SERVER_HOST) - server_process, http_port = _spawn_server_process( + server_process, http_port, run_timeout_s = _spawn_server_process( endpoint=f"tcp://{SERVER_HOST}:{dispatch_port}", out_of_band_error_path=out_of_band_error_path, engine_kwargs=engine_kwargs, @@ -69,6 +82,7 @@ class ScriptedHttpServer: server_process=server_process, out_of_band_error_path=out_of_band_error_path, http_port=http_port, + run_timeout_s=run_timeout_s, ) try: self._await_handshake() @@ -83,8 +97,10 @@ class ScriptedHttpServer: script_fn: Callable, *, args: Tuple[Any, ...] = (), - timeout_s: float = DEFAULT_RUN_TIMEOUT_S, + timeout_s: Optional[float] = None, ) -> None: + if timeout_s is None: + timeout_s = self._run_timeout_s if self._dirty: raise RuntimeError(f"ScriptedHttpServer is dirty: {self._dirty}") @@ -217,23 +233,42 @@ def _create_oob_error_file() -> Path: return Path(err_path) +def _default_run_timeout_s(*, kv_canary: str, device: str) -> float: + if _canary_decode_graph_override(kv_canary=kv_canary, device=device): + return DEFAULT_RUN_TIMEOUT_S * CANARY_REFERENCE_TIMEOUT_SCALE + return DEFAULT_RUN_TIMEOUT_S + + +def _canary_decode_graph_override(*, kv_canary: str, device: str) -> Dict[str, Any]: + if kv_canary != "none" and use_torch_reference(torch.device(device)): + # install_canary refuses a captured decode over the torch reference. + return {"cuda_graph_backend_decode": "disabled"} + return {} + + def _spawn_server_process( *, endpoint: str, out_of_band_error_path: Path, engine_kwargs: Dict[str, Any], -) -> Tuple[mp.process.BaseProcess, int]: +) -> Tuple[mp.process.BaseProcess, int, float]: mp_ctx = mp.get_context("spawn") + device = engine_kwargs.get("device") or get_device() launch_kwargs: Dict[str, Any] = dict( host=SERVER_HOST, port=get_free_port(), - kv_canary="raise", - kv_canary_real_data="partial", - kv_canary_sweep_interval=100, disable_prefill_cuda_graph=True, + **CANARY_LAUNCH_DEFAULTS, ) launch_kwargs.update(engine_kwargs) + for key, value in _canary_decode_graph_override( + kv_canary=launch_kwargs["kv_canary"], device=device.split(":")[0] + ).items(): + launch_kwargs.setdefault(key, value) http_port = launch_kwargs["port"] + run_timeout_s = _default_run_timeout_s( + kv_canary=launch_kwargs["kv_canary"], device=device.split(":")[0] + ) server_process = mp_ctx.Process( target=_launch_scripted_http_server, kwargs=launch_kwargs, @@ -252,7 +287,7 @@ def _spawn_server_process( ): server_process.start() - return server_process, http_port + return server_process, http_port, run_timeout_s def _launch_scripted_http_server(**engine_kwargs: Any) -> None: diff --git a/python/sglang/test/scripted_runtime_chunked_helpers.py b/python/sglang/test/scripted_runtime_chunked_helpers.py index 2324313cdb..7cb840bec4 100644 --- a/python/sglang/test/scripted_runtime_chunked_helpers.py +++ b/python/sglang/test/scripted_runtime_chunked_helpers.py @@ -10,6 +10,10 @@ VERY_LONG_PROMPT_LEN: int = 8 * DEFAULT_CHUNK_SIZE SMALL_MODEL: str = "Qwen/Qwen3-0.6B" +RID_RELEASE_SETTLE_STEPS: int = 40 + +DRAIN_RELEASE_STEPS: int = 12 + def base_engine_kwargs( *, @@ -40,6 +44,24 @@ def run_until_finished(handle, *, max_steps: int = DEFAULT_MAX_STEPS): yield from run_until(handle, lambda h: h.finished, max_steps=max_steps) +def run_until_finished_then_settle(handle, *, max_steps: int = DEFAULT_MAX_STEPS): + yield from run_until_finished(handle, max_steps=max_steps) + for _ in range(RID_RELEASE_SETTLE_STEPS): + yield + + +def drain_until_released(t, *handles, max_steps: int = DRAIN_RELEASE_STEPS): + for _ in range(max_steps): + if all( + h.kv_pages == 0 + and h.lock_refs == 0 + and (h.req is None or h.req.kv.req_pool_idx is None) + for h in handles + ): + return + yield + + def run_until_all_finished(handles: List[Any], *, max_steps: int = DEFAULT_MAX_STEPS): done = [False] * len(handles) for _ in range(max_steps): diff --git a/test/manual/chunked_prefill/test_scripted_abort.py b/test/manual/chunked_prefill/test_scripted_abort.py index 93169c1ba7..06d6e558ec 100644 --- a/test/manual/chunked_prefill/test_scripted_abort.py +++ b/test/manual/chunked_prefill/test_scripted_abort.py @@ -7,29 +7,19 @@ from sglang.test.scripted_runtime.test_case import ScriptedTestCase from sglang.test.scripted_runtime_chunked_helpers import ( DEFAULT_CHUNK_SIZE, DEFAULT_MAX_STEPS, + RID_RELEASE_SETTLE_STEPS, SMALL_KV_POOL_BALLAST_MAX_NEW_TOKENS, SMALL_KV_POOL_BALLAST_PROMPT_LEN, SMALL_KV_POOL_MAX_TOTAL_TOKENS, VERY_LONG_PROMPT_LEN, base_engine_kwargs, + drain_until_released, run_until, run_until_all_finished, run_until_finished, ) -def _drain_until_released(t: ScriptedContext, *handles: ScriptedReqHandle): - for _ in range(12): - if all( - h.kv_pages == 0 - and h.lock_refs == 0 - and (h.req is None or h.req.kv.req_pool_idx is None) - for h in handles - ): - return - yield - - class TestAbortBasic(ScriptedTestCase): ENGINE_KWARGS = base_engine_kwargs(chunked_prefill_size=DEFAULT_CHUNK_SIZE) @@ -47,7 +37,7 @@ class TestAbortBasic(ScriptedTestCase): assert pages_before > 0, "chunked req should own KV pages mid-chunk" t.abort(r) - yield from _drain_until_released(t, r) + yield from drain_until_released(t, r) assert r.status in ( "finished", @@ -75,7 +65,7 @@ class TestAbortBasic(ScriptedTestCase): yield from run_until(r, lambda h: h.is_chunking) t.abort(r) - yield from _drain_until_released(t, r) + yield from drain_until_released(t, r) assert r.kv_pages == 0 assert r.req is None or r.req.kv.req_pool_idx is None @@ -91,7 +81,7 @@ class TestAbortBasic(ScriptedTestCase): yield from run_until(r, lambda h: h.chunks_done >= 2 and h.is_chunking) t.abort(r) - yield from _drain_until_released(t, r) + yield from drain_until_released(t, r) assert r.kv_pages == 0 @@ -109,7 +99,7 @@ class TestAbortBasic(ScriptedTestCase): yield from run_until(r1, lambda h: h.is_chunking) t.abort(r1) - yield from _drain_until_released(t, r1) + yield from drain_until_released(t, r1) assert r1.kv_pages == 0 yield from run_until_finished(r2) @@ -122,7 +112,7 @@ class TestAbortBasic(ScriptedTestCase): def _script_abort_with_zero_yield(t: ScriptedContext): r = t.start_req(prompt_len=VERY_LONG_PROMPT_LEN, max_new_tokens=2) t.abort(r) - yield from _drain_until_released(t, r) + yield from drain_until_released(t, r) assert r.kv_pages == 0 assert r.req is None or r.req.kv.req_pool_idx is None assert r.lock_refs == 0 @@ -135,7 +125,7 @@ class TestAbortBasic(ScriptedTestCase): r = t.start_req(prompt_len=VERY_LONG_PROMPT_LEN, max_new_tokens=2) yield t.abort(r) - yield from _drain_until_released(t, r) + yield from drain_until_released(t, r) assert r.kv_pages == 0 assert r.req is None or r.req.kv.req_pool_idx is None @@ -188,7 +178,7 @@ class TestAbortBasic(ScriptedTestCase): yield from run_until(reqs[0], lambda h: h.is_chunking) for r in reqs: t.abort(r) - yield from _drain_until_released(t, *reqs) + yield from drain_until_released(t, *reqs) for r in reqs: assert r.kv_pages == 0 assert r.req is None or r.req.kv.req_pool_idx is None @@ -248,7 +238,7 @@ class TestAbortBasic(ScriptedTestCase): ) yield from run_until(r, lambda h: h.chunks_done >= 1 and h.is_chunking) t.abort(r) - yield from _drain_until_released(t, r) + yield from drain_until_released(t, r) assert r.kv_pages == 0 assert r.req is None or r.req.inflight_middle_chunks == 0 @@ -262,26 +252,11 @@ class TestAbortBasic(ScriptedTestCase): ) yield from run_until(r, lambda h: h.chunks_done >= 2 and h.is_chunking) t.abort(r) - yield from _drain_until_released(t, r) + yield from drain_until_released(t, r) assert r.kv_pages == 0 assert r.req is None or r.req.kv.req_pool_idx is None assert r.lock_refs == 0 - def test_double_abort_idempotent(self): - self.server.execute_script(self._script_double_abort_idempotent) - - @staticmethod - def _script_double_abort_idempotent(t: ScriptedContext): - r = t.start_req( - prompt_len=VERY_LONG_PROMPT_LEN, max_new_tokens=2, prompt_token=190 - ) - yield from run_until(r, lambda h: h.is_chunking) - t.abort(r) - t.abort(r) - yield from _drain_until_released(t, r) - assert r.kv_pages == 0 - assert r.lock_refs == 0 - def test_abort_during_decode(self): self.server.execute_script(self._script_abort_during_decode) @@ -291,7 +266,7 @@ class TestAbortBasic(ScriptedTestCase): yield from run_until(r, lambda h: h.status == "running") assert r.kv_pages > 0, "decode req must own KV before abort" t.abort(r) - yield from _drain_until_released(t, r) + yield from drain_until_released(t, r) assert r.kv_pages == 0 assert r.lock_refs == 0 @@ -323,7 +298,7 @@ class TestAbortBasic(ScriptedTestCase): for r in reqs: t.abort(r) yield - yield from _drain_until_released(t, *reqs) + yield from drain_until_released(t, *reqs) for r in reqs: assert r.kv_pages == 0 @@ -339,7 +314,7 @@ class TestAbortBasic(ScriptedTestCase): yield from run_until(r, lambda h: h.chunks_done >= 1 and h.is_chunking) t.abort(r) - yield from _drain_until_released(t, r) + yield from drain_until_released(t, r) assert r.kv_pages == 0 @@ -366,7 +341,7 @@ class TestAbortBasic(ScriptedTestCase): yield from run_until(r, lambda h: h.is_chunking) t.abort(r) - yield from _drain_until_released(t, r) + yield from drain_until_released(t, r) assert r.kv_pages == 0 assert r.req is None or r.req.kv.req_pool_idx is None @@ -384,27 +359,30 @@ class TestAbortBasic(ScriptedTestCase): f"node lock_refs={lock_refs_after!r}" ) - def test_abort_then_resubmit_same_rid_same_step(self): - self.server.execute_script(self._script_abort_then_resubmit_same_rid_same_step) + def test_abort_then_resubmit_same_rid_after_settle(self): + self.server.execute_script( + self._script_abort_then_resubmit_same_rid_after_settle + ) @staticmethod - def _script_abort_then_resubmit_same_rid_same_step(t: ScriptedContext): + def _script_abort_then_resubmit_same_rid_after_settle(t: ScriptedContext): r1 = t.start_req( prompt_len=VERY_LONG_PROMPT_LEN, max_new_tokens=2, - rid="abort-resubmit-same-step", + rid="abort-resubmit-same-rid", prompt_token=230, ) yield from run_until(r1, lambda h: h.is_chunking) t.abort(r1) + for _ in range(RID_RELEASE_SETTLE_STEPS): + yield r2 = t.start_req( prompt_len=16, max_new_tokens=2, - rid="abort-resubmit-same-step", + rid="abort-resubmit-same-rid", ) - yield yield from run_until_finished(r2) - assert r2.finished, "resubmit under same rid must complete independently" + assert r2.finished, "resubmit under same rid must complete once the rid settles" assert r1.kv_pages == 0, "aborted r1 must release KV before resubmit" assert r1.req is None or r1.req.kv.req_pool_idx is None assert r1.lock_refs == 0 @@ -426,7 +404,7 @@ class TestAbortBasic(ScriptedTestCase): assert r.req.inflight_middle_chunks > 0 t.abort(r) - yield from _drain_until_released(t, r) + yield from drain_until_released(t, r) assert r.kv_pages == 0 assert r.req is None or r.req.kv.req_pool_idx is None @@ -452,7 +430,7 @@ class TestAbortBasic(ScriptedTestCase): assert (1 if t.scheduler.chunked_req is not None else 0) == 1 t.abort(r) - yield from _drain_until_released(t, r) + yield from drain_until_released(t, r) for _ in range(12): if t.scheduler.chunked_req is None and t.is_idle: @@ -483,7 +461,7 @@ class TestAbortBasic(ScriptedTestCase): t.abort(r1) r2 = t.start_req(prompt_len=16, max_new_tokens=2) - yield from _drain_until_released(t, r1) + yield from drain_until_released(t, r1) cur = ( t.scheduler.chunked_req.rid if t.scheduler.chunked_req is not None else None @@ -506,7 +484,7 @@ class TestAbortBasic(ScriptedTestCase): t.pause_generation(mode="retract") t.abort(r1) - yield from _drain_until_released(t, r1) + yield from drain_until_released(t, r1) assert r1.kv_pages == 0, ( f"force_retract + abort same yield must release KV; got {r1.kv_pages}" @@ -536,7 +514,7 @@ class TestAbortBasic(ScriptedTestCase): assert (1 if t.scheduler.chunked_req is not None else 0) == 1 t.abort(r1) - yield from _drain_until_released(t, r1) + yield from drain_until_released(t, r1) yield from run_until(r2, lambda h: h.is_chunking) assert r1.kv_pages == 0 @@ -567,7 +545,7 @@ class TestAbortPP(ScriptedTestCase): ) t.abort(r) - yield from _drain_until_released(t, r) + yield from drain_until_released(t, r) assert r.kv_pages == 0 assert r.req is None or r.req.kv.req_pool_idx is None @@ -636,7 +614,7 @@ class TestAbortSmallPool(ScriptedTestCase): t.abort(b1) t.abort(b2) - yield from _drain_until_released(t, b1, b2) + yield from drain_until_released(t, b1, b2) if __name__ == "__main__": diff --git a/test/manual/chunked_prefill/test_scripted_lifecycle.py b/test/manual/chunked_prefill/test_scripted_lifecycle.py index f11c7a7c1c..b8cf33880f 100644 --- a/test/manual/chunked_prefill/test_scripted_lifecycle.py +++ b/test/manual/chunked_prefill/test_scripted_lifecycle.py @@ -8,23 +8,12 @@ from sglang.test.scripted_runtime_chunked_helpers import ( DEFAULT_MAX_STEPS, VERY_LONG_PROMPT_LEN, base_engine_kwargs, + drain_until_released, run_until, run_until_finished, ) -def _drain_until_released(t, *handles): - for _ in range(12): - if all( - h.kv_pages == 0 - and h.lock_refs == 0 - and (h.req is None or h.req.kv.req_pool_idx is None) - for h in handles - ): - return - yield - - class TestLifecycleBasic(ScriptedTestCase): ENGINE_KWARGS = base_engine_kwargs(chunked_prefill_size=DEFAULT_CHUNK_SIZE) @@ -234,13 +223,13 @@ class TestLifecycleBasic(ScriptedTestCase): def _script_two_seq_clean_handoff(t: ScriptedContext): r1 = t.start_req(prompt_len=16, max_new_tokens=2, ignore_eos=True) yield from run_until_finished(r1) - yield from _drain_until_released(t, r1) + yield from drain_until_released(t, r1) assert r1.req.kv.req_pool_idx is None and r1.kv_pages == 0 and r1.lock_refs == 0 r1_output_len = len(r1.req.output_ids) r2 = t.start_req(prompt_len=16, max_new_tokens=2, ignore_eos=True) yield from run_until_finished(r2) - yield from _drain_until_released(t, r2) + yield from drain_until_released(t, r2) assert r1.finished and r2.finished assert r1_output_len == 2 and len(r2.req.output_ids) == 2 assert r2.req.kv.req_pool_idx is None and r2.kv_pages == 0 and r2.lock_refs == 0 @@ -319,7 +308,7 @@ class TestLifecycleBasic(ScriptedTestCase): yield from run_until_finished(r) assert r.finished assert len(r.req.output_ids) == 1 - yield from _drain_until_released(t, r) + yield from drain_until_released(t, r) assert r.req is None or r.req.kv.req_pool_idx is None assert r.kv_pages == 0 and r.lock_refs == 0 if L > DEFAULT_CHUNK_SIZE: @@ -341,7 +330,7 @@ class TestLifecycleBasic(ScriptedTestCase): yield from run_until_finished(r) assert r.finished assert len(r.req.output_ids) == 1 - yield from _drain_until_released(t, r) + yield from drain_until_released(t, r) assert r.req is None or r.req.kv.req_pool_idx is None assert r.kv_pages == 0 and r.lock_refs == 0 if L > DEFAULT_CHUNK_SIZE: diff --git a/test/manual/chunked_prefill/test_scripted_multi_req.py b/test/manual/chunked_prefill/test_scripted_multi_req.py index 7a6a5432ef..5fb846500a 100644 --- a/test/manual/chunked_prefill/test_scripted_multi_req.py +++ b/test/manual/chunked_prefill/test_scripted_multi_req.py @@ -11,6 +11,7 @@ from sglang.test.scripted_runtime_chunked_helpers import ( run_until, run_until_all_finished, run_until_finished, + run_until_finished_then_settle, ) @@ -182,7 +183,7 @@ class TestMultiReqBasic(ScriptedTestCase): def _script_rid_reuse_after_finish(t: ScriptedContext): baseline = t.engine_stats() r1 = t.start_req(prompt_len=16, max_new_tokens=2, rid="reuse-rid") - yield from run_until_finished(r1) + yield from run_until_finished_then_settle(r1) r2 = t.start_req(prompt_len=16, max_new_tokens=2, rid="reuse-rid") yield from run_until_finished(r2) assert r1.finished and r2.finished diff --git a/test/manual/chunked_prefill/test_scripted_pp.py b/test/manual/chunked_prefill/test_scripted_pp.py index 55777be311..57efc6ef68 100644 --- a/test/manual/chunked_prefill/test_scripted_pp.py +++ b/test/manual/chunked_prefill/test_scripted_pp.py @@ -7,6 +7,7 @@ from sglang.test.scripted_runtime_chunked_helpers import ( DEFAULT_CHUNK_SIZE, VERY_LONG_PROMPT_LEN, base_engine_kwargs, + drain_until_released, run_until, run_until_all_finished, run_until_finished, @@ -28,16 +29,6 @@ def _expected_chunks(prompt_len: int, chunk_size: int) -> int: return (prompt_len + chunk_size - 1) // chunk_size -def _drain_until_released(t, *handles): - for _ in range(16): - if all( - h.kv_pages == 0 and (h.req is None or h.req.kv.req_pool_idx is None) - for h in handles - ): - return - yield - - class TestPPBasic(ScriptedTestCase): ENGINE_KWARGS = _pp_engine_kwargs() @@ -49,7 +40,7 @@ class TestPPBasic(ScriptedTestCase): r = t.start_req(prompt_len=VERY_LONG_PROMPT_LEN, max_new_tokens=4) yield from run_until(r, lambda h: h.is_chunking and h.chunks_done >= 1) t.abort(r) - yield from _drain_until_released(t, r) + yield from drain_until_released(t, r, max_steps=16) assert r.kv_pages == 0 assert r.lock_refs == 0 diff --git a/test/manual/chunked_prefill/test_scripted_regression.py b/test/manual/chunked_prefill/test_scripted_regression.py index 4a91e0a017..ab6c238242 100644 --- a/test/manual/chunked_prefill/test_scripted_regression.py +++ b/test/manual/chunked_prefill/test_scripted_regression.py @@ -7,6 +7,7 @@ from sglang.test.scripted_runtime_chunked_helpers import ( DEFAULT_MAX_STEPS, VERY_LONG_PROMPT_LEN, base_engine_kwargs, + drain_until_released, run_until, run_until_all_finished, run_until_finished, @@ -14,18 +15,6 @@ from sglang.test.scripted_runtime_chunked_helpers import ( ) -def _drain_until_released(t, *handles): - for _ in range(12): - if all( - h.kv_pages == 0 - and h.lock_refs == 0 - and (h.req is None or h.req.kv.req_pool_idx is None) - for h in handles - ): - return - yield - - class TestRegressionBasic(ScriptedTestCase): ENGINE_KWARGS = base_engine_kwargs(chunked_prefill_size=DEFAULT_CHUNK_SIZE) @@ -34,17 +23,27 @@ class TestRegressionBasic(ScriptedTestCase): @staticmethod def _script_abort_waiting_releases_all(t: ScriptedContext): + baseline = t.engine_stats() r = t.start_req(prompt_len=VERY_LONG_PROMPT_LEN, max_new_tokens=2) yield from run_until(r, lambda h: h.is_chunking) t.abort(r) - yield from _drain_until_released(t, r) + yield from drain_until_released(t, r) - assert r.kv_pages == 0 - assert r.req.kv.req_pool_idx is None - assert r.lock_refs == 0 - assert not r.is_chunking - assert r.req.inflight_middle_chunks == 0 + final = t.engine_stats() + assert final["req_pool_free"] == baseline["req_pool_free"], ( + f"req row leaked: {baseline['req_pool_free']} -> {final['req_pool_free']}" + ) + assert final["kv_pool_free"] == baseline["kv_pool_free"], ( + f"KV leaked: {baseline['kv_pool_free']} -> {final['kv_pool_free']}" + ) + req = r.req + if req is not None: + assert r.kv_pages == 0 + assert req.kv.req_pool_idx is None + assert r.lock_refs == 0 + assert not r.is_chunking + assert req.inflight_middle_chunks == 0 def test_pause_covers_waiting_chunked(self): self.server.execute_script(self._script_pause_covers_waiting_chunked) @@ -230,6 +229,7 @@ class TestRegressionBasic(ScriptedTestCase): ) def test_abort_chunked_resume_releases_all_resources(self): + """Guards 96d4749094: abort releases row, KV, lock_ref.""" self.server.execute_script( self._script_abort_chunked_resume_releases_all_resources ) @@ -237,6 +237,7 @@ class TestRegressionBasic(ScriptedTestCase): @staticmethod def _script_abort_chunked_resume_releases_all_resources(t: ScriptedContext): baseline_refs = sum(t.get_all_node_lock_refs().values()) + baseline = t.engine_stats() r = t.start_req(prompt_len=VERY_LONG_PROMPT_LEN, max_new_tokens=2) yield from run_until(r, lambda h: h.is_chunking and h.chunks_done >= 1) @@ -245,19 +246,22 @@ class TestRegressionBasic(ScriptedTestCase): assert r.lock_refs >= 1, "radix lock_ref must be held mid-chunk" t.abort(r) - yield from _drain_until_released(t, r) + yield from drain_until_released(t, r) - assert r.req.kv.req_pool_idx is None, ( - f"96d4749094: abort must release row; got row_idx={r.req.kv.req_pool_idx!r}" + final = t.engine_stats() + assert final["req_pool_free"] == baseline["req_pool_free"], ( + f"req row leaked: {baseline['req_pool_free']} -> {final['req_pool_free']}" ) - assert r.kv_pages == 0, ( - f"96d4749094: abort must release KV; got kv_pages={r.kv_pages}" + assert final["kv_pool_free"] == baseline["kv_pool_free"], ( + f"KV leaked: {baseline['kv_pool_free']} -> {final['kv_pool_free']}" ) - assert r.lock_refs == 0, ( - f"96d4749094: abort must release lock_ref; got lock_refs={r.lock_refs}" - ) - assert not r.is_chunking - assert r.req.inflight_middle_chunks == 0 + req = r.req + if req is not None: + assert req.kv.req_pool_idx is None, f"row_idx={req.kv.req_pool_idx!r}" + assert r.kv_pages == 0, f"kv_pages={r.kv_pages}" + assert r.lock_refs == 0, f"lock_refs={r.lock_refs}" + assert not r.is_chunking + assert req.inflight_middle_chunks == 0 assert sum(t.get_all_node_lock_refs().values()) == baseline_refs def test_pause_retract_releases_waiting_chunked_resume(self): @@ -348,7 +352,7 @@ class TestRegressionPp(ScriptedTestCase): f"waiting_queue; got {occurrences} occurrences of rid=" f"{r.rid} (pre-fix bug would yield 3)" ) - yield from _drain_until_released(t, r) + yield from drain_until_released(t, r) assert r.finished diff --git a/test/manual/chunked_prefill/test_scripted_special_case.py b/test/manual/chunked_prefill/test_scripted_special_case.py index 29db975671..60ed31f65d 100644 --- a/test/manual/chunked_prefill/test_scripted_special_case.py +++ b/test/manual/chunked_prefill/test_scripted_special_case.py @@ -1,6 +1,7 @@ import unittest from typing import Optional +from sglang.srt.utils import is_cuda, is_flashinfer_available, is_hip from sglang.test.scripted_runtime.context import ScriptedContext from sglang.test.scripted_runtime.test_case import ScriptedTestCase from sglang.test.scripted_runtime_chunked_helpers import ( @@ -755,6 +756,7 @@ class TestSpecialCaseNoChunking(ScriptedTestCase): DETERMINISTIC_ALIGN_SIZE = 4096 +@unittest.skipUnless(is_flashinfer_available(), "flashinfer is NVIDIA-only") class TestSpecialCaseDeterministicFlashInfer(ScriptedTestCase): ENGINE_KWARGS = base_engine_kwargs( chunked_prefill_size=DETERMINISTIC_ALIGN_SIZE, @@ -791,6 +793,7 @@ class TestSpecialCaseDeterministicFlashInfer(ScriptedTestCase): assert saw_chunking, "test must observe the req mid-chunk at least once" +@unittest.skipUnless(is_cuda() or is_hip(), "MHA host-pool movers are CUDA/ROCm only") class TestSpecialCaseHiCache(ScriptedTestCase): ENGINE_KWARGS = base_engine_kwargs( chunked_prefill_size=DEFAULT_CHUNK_SIZE, diff --git a/test/registered/unit/scripted_runtime/test_http_server.py b/test/registered/unit/scripted_runtime/test_http_server.py index e7a1a872ff..8482cd79ec 100644 --- a/test/registered/unit/scripted_runtime/test_http_server.py +++ b/test/registered/unit/scripted_runtime/test_http_server.py @@ -7,7 +7,12 @@ import uuid import zmq from sglang.test.ci.ci_register import register_cpu_ci -from sglang.test.scripted_runtime.http_server import ScriptedHttpServer +from sglang.test.scripted_runtime.http_server import ( + CANARY_LAUNCH_DEFAULTS, + DEFAULT_RUN_TIMEOUT_S, + ScriptedHttpServer, + _canary_decode_graph_override, +) from sglang.test.scripted_runtime.io_struct import ( HookReady, RunScript, @@ -73,6 +78,7 @@ def _make_server(socket: zmq.Socket, process: _FakeProcess) -> ScriptedHttpServe server._socket = socket server._server_process = process server._dirty = None + server._run_timeout_s = DEFAULT_RUN_TIMEOUT_S return server @@ -141,5 +147,23 @@ class TestExecuteScriptDirtyGuard(CustomTestCase): pair.assert_no_sent_message(self) +class TestCanaryLaunchKwargs(CustomTestCase): + def test_canary_is_enabled_without_consulting_the_device(self): + self.assertEqual(CANARY_LAUNCH_DEFAULTS["kv_canary"], "raise") + self.assertEqual(CANARY_LAUNCH_DEFAULTS["kv_canary_real_data"], "partial") + + def test_cuda_keeps_its_decode_graph(self): + self.assertEqual( + _canary_decode_graph_override(kv_canary="raise", device="cuda"), {} + ) + + def test_a_disabled_canary_keeps_the_decode_graph(self): + for device in ("cuda", "xpu", "cpu"): + with self.subTest(device=device): + self.assertEqual( + _canary_decode_graph_override(kv_canary="none", device=device), {} + ) + + if __name__ == "__main__": unittest.main()