[Intel][XPU]Enable chunked prefill scnearios for XPU with UT (#33804)

Co-authored-by: Ma Mingfei <mingfei.ma@intel.com>
This commit is contained in:
Anupa Sajikumar
2026-09-29 13:28:25 +08:00
committed by GitHub
co-authored by Ma Mingfei
parent 9ae43b0eee
commit 2ff52e3ceb
10 changed files with 182 additions and 121 deletions
@@ -4,6 +4,10 @@ import time
from types import SimpleNamespace
from typing import ClassVar, List, Optional
import torch
from sglang.kernels.ops.kv_canary._dispatch import use_torch_reference
from sglang.srt.utils import get_device
from sglang.test.run_eval import run_eval
from sglang.test.server_fixtures.disaggregation_fixture import (
PDDisaggregationServerBase,
@@ -38,6 +42,16 @@ KV_CANARY_ARGS: List[str] = [
]
def _canary_args(enabled: bool) -> List[str]:
if not enabled:
return []
args = list(KV_CANARY_ARGS)
if use_torch_reference(torch.device(get_device())):
# install_canary refuses a captured decode over the torch reference.
args.append("--cuda-graph-backend-decode=disabled")
return args
class ChunkedGsm8kMixin:
__test__ = False
use_kv_canary: ClassVar[bool] = True
@@ -52,7 +66,7 @@ class ChunkedGsm8kMixin:
gsm8k_threshold: ClassVar[float]
def build_prefill_side_args(self) -> List[str]:
canary = list(KV_CANARY_ARGS) if self.use_kv_canary else []
canary = _canary_args(self.use_kv_canary)
return (
["--chunked-prefill-size", str(self.chunked_prefill_size)]
+ list(self.feature_args)
@@ -118,7 +132,7 @@ class ChunkedTestPDBase(ChunkedGsm8kMixin, PDDisaggregationServerBase):
cls.extra_prefill_args = cls(
"test_mixed_prefix_gsm8k_chunked"
).build_prefill_side_args()
canary = list(KV_CANARY_ARGS) if cls.use_kv_canary else []
canary = _canary_args(cls.use_kv_canary)
cls.extra_decode_args = canary + list(cls.decode_feature_args)
PDDisaggregationServerBase.setUpClass()
cls.model = try_cached_model(cls.model)
@@ -8,12 +8,15 @@ from pathlib import Path
from typing import Any, Callable, Dict, Optional, Tuple
import requests
import torch
import zmq
from sglang.kernels.ops.kv_canary._dispatch import use_torch_reference
from sglang.srt.entrypoints.http_server import launch_server
from sglang.srt.environ import envs
from sglang.srt.managers.io_struct import sock_recv, sock_send, wrap_as_pickle
from sglang.srt.server_args import ServerArgs
from sglang.srt.utils.common import get_device
from sglang.srt.utils.network import get_free_port, get_zmq_socket_on_host
from sglang.test.scripted_runtime.io_struct import (
HookReady,
@@ -26,12 +29,20 @@ from sglang.test.scripted_runtime.io_struct import (
from sglang.test.scripted_runtime.utils import close_zmq_socket
DEFAULT_RUN_TIMEOUT_S: float = 120.0
CANARY_REFERENCE_TIMEOUT_SCALE: float = 4.0
SHUTDOWN_JOIN_TIMEOUT_S: float = 60.0
LISTENER_ACCEPT_TIMEOUT_S: float = 300.0
HTTP_READY_TIMEOUT_S: float = 300.0
HTTP_READY_POLL_INTERVAL_S: float = 0.5
SERVER_HOST: str = "127.0.0.1"
CANARY_LAUNCH_DEFAULTS: Dict[str, Any] = dict(
kv_canary="raise",
kv_canary_real_data="partial",
kv_canary_sweep_interval=100,
)
class ScriptedHttpServer:
def __init__(
@@ -42,6 +53,7 @@ class ScriptedHttpServer:
server_process: mp.process.BaseProcess,
out_of_band_error_path: Path,
http_port: int,
run_timeout_s: float = DEFAULT_RUN_TIMEOUT_S,
) -> None:
self._ctx = ctx
self._socket = socket
@@ -50,6 +62,7 @@ class ScriptedHttpServer:
self._base_url = f"http://{SERVER_HOST}:{http_port}"
self._shutdown_done = False
self._dirty: Optional[str] = None
self._run_timeout_s = run_timeout_s
@classmethod
def start(cls, **engine_kwargs: Any) -> ScriptedHttpServer:
@@ -57,7 +70,7 @@ class ScriptedHttpServer:
ctx = zmq.Context()
dispatch_port, socket = get_zmq_socket_on_host(ctx, zmq.PAIR, host=SERVER_HOST)
server_process, http_port = _spawn_server_process(
server_process, http_port, run_timeout_s = _spawn_server_process(
endpoint=f"tcp://{SERVER_HOST}:{dispatch_port}",
out_of_band_error_path=out_of_band_error_path,
engine_kwargs=engine_kwargs,
@@ -69,6 +82,7 @@ class ScriptedHttpServer:
server_process=server_process,
out_of_band_error_path=out_of_band_error_path,
http_port=http_port,
run_timeout_s=run_timeout_s,
)
try:
self._await_handshake()
@@ -83,8 +97,10 @@ class ScriptedHttpServer:
script_fn: Callable,
*,
args: Tuple[Any, ...] = (),
timeout_s: float = DEFAULT_RUN_TIMEOUT_S,
timeout_s: Optional[float] = None,
) -> None:
if timeout_s is None:
timeout_s = self._run_timeout_s
if self._dirty:
raise RuntimeError(f"ScriptedHttpServer is dirty: {self._dirty}")
@@ -217,23 +233,42 @@ def _create_oob_error_file() -> Path:
return Path(err_path)
def _default_run_timeout_s(*, kv_canary: str, device: str) -> float:
if _canary_decode_graph_override(kv_canary=kv_canary, device=device):
return DEFAULT_RUN_TIMEOUT_S * CANARY_REFERENCE_TIMEOUT_SCALE
return DEFAULT_RUN_TIMEOUT_S
def _canary_decode_graph_override(*, kv_canary: str, device: str) -> Dict[str, Any]:
if kv_canary != "none" and use_torch_reference(torch.device(device)):
# install_canary refuses a captured decode over the torch reference.
return {"cuda_graph_backend_decode": "disabled"}
return {}
def _spawn_server_process(
*,
endpoint: str,
out_of_band_error_path: Path,
engine_kwargs: Dict[str, Any],
) -> Tuple[mp.process.BaseProcess, int]:
) -> Tuple[mp.process.BaseProcess, int, float]:
mp_ctx = mp.get_context("spawn")
device = engine_kwargs.get("device") or get_device()
launch_kwargs: Dict[str, Any] = dict(
host=SERVER_HOST,
port=get_free_port(),
kv_canary="raise",
kv_canary_real_data="partial",
kv_canary_sweep_interval=100,
disable_prefill_cuda_graph=True,
**CANARY_LAUNCH_DEFAULTS,
)
launch_kwargs.update(engine_kwargs)
for key, value in _canary_decode_graph_override(
kv_canary=launch_kwargs["kv_canary"], device=device.split(":")[0]
).items():
launch_kwargs.setdefault(key, value)
http_port = launch_kwargs["port"]
run_timeout_s = _default_run_timeout_s(
kv_canary=launch_kwargs["kv_canary"], device=device.split(":")[0]
)
server_process = mp_ctx.Process(
target=_launch_scripted_http_server,
kwargs=launch_kwargs,
@@ -252,7 +287,7 @@ def _spawn_server_process(
):
server_process.start()
return server_process, http_port
return server_process, http_port, run_timeout_s
def _launch_scripted_http_server(**engine_kwargs: Any) -> None:
@@ -10,6 +10,10 @@ VERY_LONG_PROMPT_LEN: int = 8 * DEFAULT_CHUNK_SIZE
SMALL_MODEL: str = "Qwen/Qwen3-0.6B"
RID_RELEASE_SETTLE_STEPS: int = 40
DRAIN_RELEASE_STEPS: int = 12
def base_engine_kwargs(
*,
@@ -40,6 +44,24 @@ def run_until_finished(handle, *, max_steps: int = DEFAULT_MAX_STEPS):
yield from run_until(handle, lambda h: h.finished, max_steps=max_steps)
def run_until_finished_then_settle(handle, *, max_steps: int = DEFAULT_MAX_STEPS):
yield from run_until_finished(handle, max_steps=max_steps)
for _ in range(RID_RELEASE_SETTLE_STEPS):
yield
def drain_until_released(t, *handles, max_steps: int = DRAIN_RELEASE_STEPS):
for _ in range(max_steps):
if all(
h.kv_pages == 0
and h.lock_refs == 0
and (h.req is None or h.req.kv.req_pool_idx is None)
for h in handles
):
return
yield
def run_until_all_finished(handles: List[Any], *, max_steps: int = DEFAULT_MAX_STEPS):
done = [False] * len(handles)
for _ in range(max_steps):
@@ -7,29 +7,19 @@ from sglang.test.scripted_runtime.test_case import ScriptedTestCase
from sglang.test.scripted_runtime_chunked_helpers import (
DEFAULT_CHUNK_SIZE,
DEFAULT_MAX_STEPS,
RID_RELEASE_SETTLE_STEPS,
SMALL_KV_POOL_BALLAST_MAX_NEW_TOKENS,
SMALL_KV_POOL_BALLAST_PROMPT_LEN,
SMALL_KV_POOL_MAX_TOTAL_TOKENS,
VERY_LONG_PROMPT_LEN,
base_engine_kwargs,
drain_until_released,
run_until,
run_until_all_finished,
run_until_finished,
)
def _drain_until_released(t: ScriptedContext, *handles: ScriptedReqHandle):
for _ in range(12):
if all(
h.kv_pages == 0
and h.lock_refs == 0
and (h.req is None or h.req.kv.req_pool_idx is None)
for h in handles
):
return
yield
class TestAbortBasic(ScriptedTestCase):
ENGINE_KWARGS = base_engine_kwargs(chunked_prefill_size=DEFAULT_CHUNK_SIZE)
@@ -47,7 +37,7 @@ class TestAbortBasic(ScriptedTestCase):
assert pages_before > 0, "chunked req should own KV pages mid-chunk"
t.abort(r)
yield from _drain_until_released(t, r)
yield from drain_until_released(t, r)
assert r.status in (
"finished",
@@ -75,7 +65,7 @@ class TestAbortBasic(ScriptedTestCase):
yield from run_until(r, lambda h: h.is_chunking)
t.abort(r)
yield from _drain_until_released(t, r)
yield from drain_until_released(t, r)
assert r.kv_pages == 0
assert r.req is None or r.req.kv.req_pool_idx is None
@@ -91,7 +81,7 @@ class TestAbortBasic(ScriptedTestCase):
yield from run_until(r, lambda h: h.chunks_done >= 2 and h.is_chunking)
t.abort(r)
yield from _drain_until_released(t, r)
yield from drain_until_released(t, r)
assert r.kv_pages == 0
@@ -109,7 +99,7 @@ class TestAbortBasic(ScriptedTestCase):
yield from run_until(r1, lambda h: h.is_chunking)
t.abort(r1)
yield from _drain_until_released(t, r1)
yield from drain_until_released(t, r1)
assert r1.kv_pages == 0
yield from run_until_finished(r2)
@@ -122,7 +112,7 @@ class TestAbortBasic(ScriptedTestCase):
def _script_abort_with_zero_yield(t: ScriptedContext):
r = t.start_req(prompt_len=VERY_LONG_PROMPT_LEN, max_new_tokens=2)
t.abort(r)
yield from _drain_until_released(t, r)
yield from drain_until_released(t, r)
assert r.kv_pages == 0
assert r.req is None or r.req.kv.req_pool_idx is None
assert r.lock_refs == 0
@@ -135,7 +125,7 @@ class TestAbortBasic(ScriptedTestCase):
r = t.start_req(prompt_len=VERY_LONG_PROMPT_LEN, max_new_tokens=2)
yield
t.abort(r)
yield from _drain_until_released(t, r)
yield from drain_until_released(t, r)
assert r.kv_pages == 0
assert r.req is None or r.req.kv.req_pool_idx is None
@@ -188,7 +178,7 @@ class TestAbortBasic(ScriptedTestCase):
yield from run_until(reqs[0], lambda h: h.is_chunking)
for r in reqs:
t.abort(r)
yield from _drain_until_released(t, *reqs)
yield from drain_until_released(t, *reqs)
for r in reqs:
assert r.kv_pages == 0
assert r.req is None or r.req.kv.req_pool_idx is None
@@ -248,7 +238,7 @@ class TestAbortBasic(ScriptedTestCase):
)
yield from run_until(r, lambda h: h.chunks_done >= 1 and h.is_chunking)
t.abort(r)
yield from _drain_until_released(t, r)
yield from drain_until_released(t, r)
assert r.kv_pages == 0
assert r.req is None or r.req.inflight_middle_chunks == 0
@@ -262,26 +252,11 @@ class TestAbortBasic(ScriptedTestCase):
)
yield from run_until(r, lambda h: h.chunks_done >= 2 and h.is_chunking)
t.abort(r)
yield from _drain_until_released(t, r)
yield from drain_until_released(t, r)
assert r.kv_pages == 0
assert r.req is None or r.req.kv.req_pool_idx is None
assert r.lock_refs == 0
def test_double_abort_idempotent(self):
self.server.execute_script(self._script_double_abort_idempotent)
@staticmethod
def _script_double_abort_idempotent(t: ScriptedContext):
r = t.start_req(
prompt_len=VERY_LONG_PROMPT_LEN, max_new_tokens=2, prompt_token=190
)
yield from run_until(r, lambda h: h.is_chunking)
t.abort(r)
t.abort(r)
yield from _drain_until_released(t, r)
assert r.kv_pages == 0
assert r.lock_refs == 0
def test_abort_during_decode(self):
self.server.execute_script(self._script_abort_during_decode)
@@ -291,7 +266,7 @@ class TestAbortBasic(ScriptedTestCase):
yield from run_until(r, lambda h: h.status == "running")
assert r.kv_pages > 0, "decode req must own KV before abort"
t.abort(r)
yield from _drain_until_released(t, r)
yield from drain_until_released(t, r)
assert r.kv_pages == 0
assert r.lock_refs == 0
@@ -323,7 +298,7 @@ class TestAbortBasic(ScriptedTestCase):
for r in reqs:
t.abort(r)
yield
yield from _drain_until_released(t, *reqs)
yield from drain_until_released(t, *reqs)
for r in reqs:
assert r.kv_pages == 0
@@ -339,7 +314,7 @@ class TestAbortBasic(ScriptedTestCase):
yield from run_until(r, lambda h: h.chunks_done >= 1 and h.is_chunking)
t.abort(r)
yield from _drain_until_released(t, r)
yield from drain_until_released(t, r)
assert r.kv_pages == 0
@@ -366,7 +341,7 @@ class TestAbortBasic(ScriptedTestCase):
yield from run_until(r, lambda h: h.is_chunking)
t.abort(r)
yield from _drain_until_released(t, r)
yield from drain_until_released(t, r)
assert r.kv_pages == 0
assert r.req is None or r.req.kv.req_pool_idx is None
@@ -384,27 +359,30 @@ class TestAbortBasic(ScriptedTestCase):
f"node lock_refs={lock_refs_after!r}"
)
def test_abort_then_resubmit_same_rid_same_step(self):
self.server.execute_script(self._script_abort_then_resubmit_same_rid_same_step)
def test_abort_then_resubmit_same_rid_after_settle(self):
self.server.execute_script(
self._script_abort_then_resubmit_same_rid_after_settle
)
@staticmethod
def _script_abort_then_resubmit_same_rid_same_step(t: ScriptedContext):
def _script_abort_then_resubmit_same_rid_after_settle(t: ScriptedContext):
r1 = t.start_req(
prompt_len=VERY_LONG_PROMPT_LEN,
max_new_tokens=2,
rid="abort-resubmit-same-step",
rid="abort-resubmit-same-rid",
prompt_token=230,
)
yield from run_until(r1, lambda h: h.is_chunking)
t.abort(r1)
for _ in range(RID_RELEASE_SETTLE_STEPS):
yield
r2 = t.start_req(
prompt_len=16,
max_new_tokens=2,
rid="abort-resubmit-same-step",
rid="abort-resubmit-same-rid",
)
yield
yield from run_until_finished(r2)
assert r2.finished, "resubmit under same rid must complete independently"
assert r2.finished, "resubmit under same rid must complete once the rid settles"
assert r1.kv_pages == 0, "aborted r1 must release KV before resubmit"
assert r1.req is None or r1.req.kv.req_pool_idx is None
assert r1.lock_refs == 0
@@ -426,7 +404,7 @@ class TestAbortBasic(ScriptedTestCase):
assert r.req.inflight_middle_chunks > 0
t.abort(r)
yield from _drain_until_released(t, r)
yield from drain_until_released(t, r)
assert r.kv_pages == 0
assert r.req is None or r.req.kv.req_pool_idx is None
@@ -452,7 +430,7 @@ class TestAbortBasic(ScriptedTestCase):
assert (1 if t.scheduler.chunked_req is not None else 0) == 1
t.abort(r)
yield from _drain_until_released(t, r)
yield from drain_until_released(t, r)
for _ in range(12):
if t.scheduler.chunked_req is None and t.is_idle:
@@ -483,7 +461,7 @@ class TestAbortBasic(ScriptedTestCase):
t.abort(r1)
r2 = t.start_req(prompt_len=16, max_new_tokens=2)
yield from _drain_until_released(t, r1)
yield from drain_until_released(t, r1)
cur = (
t.scheduler.chunked_req.rid if t.scheduler.chunked_req is not None else None
@@ -506,7 +484,7 @@ class TestAbortBasic(ScriptedTestCase):
t.pause_generation(mode="retract")
t.abort(r1)
yield from _drain_until_released(t, r1)
yield from drain_until_released(t, r1)
assert r1.kv_pages == 0, (
f"force_retract + abort same yield must release KV; got {r1.kv_pages}"
@@ -536,7 +514,7 @@ class TestAbortBasic(ScriptedTestCase):
assert (1 if t.scheduler.chunked_req is not None else 0) == 1
t.abort(r1)
yield from _drain_until_released(t, r1)
yield from drain_until_released(t, r1)
yield from run_until(r2, lambda h: h.is_chunking)
assert r1.kv_pages == 0
@@ -567,7 +545,7 @@ class TestAbortPP(ScriptedTestCase):
)
t.abort(r)
yield from _drain_until_released(t, r)
yield from drain_until_released(t, r)
assert r.kv_pages == 0
assert r.req is None or r.req.kv.req_pool_idx is None
@@ -636,7 +614,7 @@ class TestAbortSmallPool(ScriptedTestCase):
t.abort(b1)
t.abort(b2)
yield from _drain_until_released(t, b1, b2)
yield from drain_until_released(t, b1, b2)
if __name__ == "__main__":
@@ -8,23 +8,12 @@ from sglang.test.scripted_runtime_chunked_helpers import (
DEFAULT_MAX_STEPS,
VERY_LONG_PROMPT_LEN,
base_engine_kwargs,
drain_until_released,
run_until,
run_until_finished,
)
def _drain_until_released(t, *handles):
for _ in range(12):
if all(
h.kv_pages == 0
and h.lock_refs == 0
and (h.req is None or h.req.kv.req_pool_idx is None)
for h in handles
):
return
yield
class TestLifecycleBasic(ScriptedTestCase):
ENGINE_KWARGS = base_engine_kwargs(chunked_prefill_size=DEFAULT_CHUNK_SIZE)
@@ -234,13 +223,13 @@ class TestLifecycleBasic(ScriptedTestCase):
def _script_two_seq_clean_handoff(t: ScriptedContext):
r1 = t.start_req(prompt_len=16, max_new_tokens=2, ignore_eos=True)
yield from run_until_finished(r1)
yield from _drain_until_released(t, r1)
yield from drain_until_released(t, r1)
assert r1.req.kv.req_pool_idx is None and r1.kv_pages == 0 and r1.lock_refs == 0
r1_output_len = len(r1.req.output_ids)
r2 = t.start_req(prompt_len=16, max_new_tokens=2, ignore_eos=True)
yield from run_until_finished(r2)
yield from _drain_until_released(t, r2)
yield from drain_until_released(t, r2)
assert r1.finished and r2.finished
assert r1_output_len == 2 and len(r2.req.output_ids) == 2
assert r2.req.kv.req_pool_idx is None and r2.kv_pages == 0 and r2.lock_refs == 0
@@ -319,7 +308,7 @@ class TestLifecycleBasic(ScriptedTestCase):
yield from run_until_finished(r)
assert r.finished
assert len(r.req.output_ids) == 1
yield from _drain_until_released(t, r)
yield from drain_until_released(t, r)
assert r.req is None or r.req.kv.req_pool_idx is None
assert r.kv_pages == 0 and r.lock_refs == 0
if L > DEFAULT_CHUNK_SIZE:
@@ -341,7 +330,7 @@ class TestLifecycleBasic(ScriptedTestCase):
yield from run_until_finished(r)
assert r.finished
assert len(r.req.output_ids) == 1
yield from _drain_until_released(t, r)
yield from drain_until_released(t, r)
assert r.req is None or r.req.kv.req_pool_idx is None
assert r.kv_pages == 0 and r.lock_refs == 0
if L > DEFAULT_CHUNK_SIZE:
@@ -11,6 +11,7 @@ from sglang.test.scripted_runtime_chunked_helpers import (
run_until,
run_until_all_finished,
run_until_finished,
run_until_finished_then_settle,
)
@@ -182,7 +183,7 @@ class TestMultiReqBasic(ScriptedTestCase):
def _script_rid_reuse_after_finish(t: ScriptedContext):
baseline = t.engine_stats()
r1 = t.start_req(prompt_len=16, max_new_tokens=2, rid="reuse-rid")
yield from run_until_finished(r1)
yield from run_until_finished_then_settle(r1)
r2 = t.start_req(prompt_len=16, max_new_tokens=2, rid="reuse-rid")
yield from run_until_finished(r2)
assert r1.finished and r2.finished
@@ -7,6 +7,7 @@ from sglang.test.scripted_runtime_chunked_helpers import (
DEFAULT_CHUNK_SIZE,
VERY_LONG_PROMPT_LEN,
base_engine_kwargs,
drain_until_released,
run_until,
run_until_all_finished,
run_until_finished,
@@ -28,16 +29,6 @@ def _expected_chunks(prompt_len: int, chunk_size: int) -> int:
return (prompt_len + chunk_size - 1) // chunk_size
def _drain_until_released(t, *handles):
for _ in range(16):
if all(
h.kv_pages == 0 and (h.req is None or h.req.kv.req_pool_idx is None)
for h in handles
):
return
yield
class TestPPBasic(ScriptedTestCase):
ENGINE_KWARGS = _pp_engine_kwargs()
@@ -49,7 +40,7 @@ class TestPPBasic(ScriptedTestCase):
r = t.start_req(prompt_len=VERY_LONG_PROMPT_LEN, max_new_tokens=4)
yield from run_until(r, lambda h: h.is_chunking and h.chunks_done >= 1)
t.abort(r)
yield from _drain_until_released(t, r)
yield from drain_until_released(t, r, max_steps=16)
assert r.kv_pages == 0
assert r.lock_refs == 0
@@ -7,6 +7,7 @@ from sglang.test.scripted_runtime_chunked_helpers import (
DEFAULT_MAX_STEPS,
VERY_LONG_PROMPT_LEN,
base_engine_kwargs,
drain_until_released,
run_until,
run_until_all_finished,
run_until_finished,
@@ -14,18 +15,6 @@ from sglang.test.scripted_runtime_chunked_helpers import (
)
def _drain_until_released(t, *handles):
for _ in range(12):
if all(
h.kv_pages == 0
and h.lock_refs == 0
and (h.req is None or h.req.kv.req_pool_idx is None)
for h in handles
):
return
yield
class TestRegressionBasic(ScriptedTestCase):
ENGINE_KWARGS = base_engine_kwargs(chunked_prefill_size=DEFAULT_CHUNK_SIZE)
@@ -34,17 +23,27 @@ class TestRegressionBasic(ScriptedTestCase):
@staticmethod
def _script_abort_waiting_releases_all(t: ScriptedContext):
baseline = t.engine_stats()
r = t.start_req(prompt_len=VERY_LONG_PROMPT_LEN, max_new_tokens=2)
yield from run_until(r, lambda h: h.is_chunking)
t.abort(r)
yield from _drain_until_released(t, r)
yield from drain_until_released(t, r)
assert r.kv_pages == 0
assert r.req.kv.req_pool_idx is None
assert r.lock_refs == 0
assert not r.is_chunking
assert r.req.inflight_middle_chunks == 0
final = t.engine_stats()
assert final["req_pool_free"] == baseline["req_pool_free"], (
f"req row leaked: {baseline['req_pool_free']} -> {final['req_pool_free']}"
)
assert final["kv_pool_free"] == baseline["kv_pool_free"], (
f"KV leaked: {baseline['kv_pool_free']} -> {final['kv_pool_free']}"
)
req = r.req
if req is not None:
assert r.kv_pages == 0
assert req.kv.req_pool_idx is None
assert r.lock_refs == 0
assert not r.is_chunking
assert req.inflight_middle_chunks == 0
def test_pause_covers_waiting_chunked(self):
self.server.execute_script(self._script_pause_covers_waiting_chunked)
@@ -230,6 +229,7 @@ class TestRegressionBasic(ScriptedTestCase):
)
def test_abort_chunked_resume_releases_all_resources(self):
"""Guards 96d4749094: abort releases row, KV, lock_ref."""
self.server.execute_script(
self._script_abort_chunked_resume_releases_all_resources
)
@@ -237,6 +237,7 @@ class TestRegressionBasic(ScriptedTestCase):
@staticmethod
def _script_abort_chunked_resume_releases_all_resources(t: ScriptedContext):
baseline_refs = sum(t.get_all_node_lock_refs().values())
baseline = t.engine_stats()
r = t.start_req(prompt_len=VERY_LONG_PROMPT_LEN, max_new_tokens=2)
yield from run_until(r, lambda h: h.is_chunking and h.chunks_done >= 1)
@@ -245,19 +246,22 @@ class TestRegressionBasic(ScriptedTestCase):
assert r.lock_refs >= 1, "radix lock_ref must be held mid-chunk"
t.abort(r)
yield from _drain_until_released(t, r)
yield from drain_until_released(t, r)
assert r.req.kv.req_pool_idx is None, (
f"96d4749094: abort must release row; got row_idx={r.req.kv.req_pool_idx!r}"
final = t.engine_stats()
assert final["req_pool_free"] == baseline["req_pool_free"], (
f"req row leaked: {baseline['req_pool_free']} -> {final['req_pool_free']}"
)
assert r.kv_pages == 0, (
f"96d4749094: abort must release KV; got kv_pages={r.kv_pages}"
assert final["kv_pool_free"] == baseline["kv_pool_free"], (
f"KV leaked: {baseline['kv_pool_free']} -> {final['kv_pool_free']}"
)
assert r.lock_refs == 0, (
f"96d4749094: abort must release lock_ref; got lock_refs={r.lock_refs}"
)
assert not r.is_chunking
assert r.req.inflight_middle_chunks == 0
req = r.req
if req is not None:
assert req.kv.req_pool_idx is None, f"row_idx={req.kv.req_pool_idx!r}"
assert r.kv_pages == 0, f"kv_pages={r.kv_pages}"
assert r.lock_refs == 0, f"lock_refs={r.lock_refs}"
assert not r.is_chunking
assert req.inflight_middle_chunks == 0
assert sum(t.get_all_node_lock_refs().values()) == baseline_refs
def test_pause_retract_releases_waiting_chunked_resume(self):
@@ -348,7 +352,7 @@ class TestRegressionPp(ScriptedTestCase):
f"waiting_queue; got {occurrences} occurrences of rid="
f"{r.rid} (pre-fix bug would yield 3)"
)
yield from _drain_until_released(t, r)
yield from drain_until_released(t, r)
assert r.finished
@@ -1,6 +1,7 @@
import unittest
from typing import Optional
from sglang.srt.utils import is_cuda, is_flashinfer_available, is_hip
from sglang.test.scripted_runtime.context import ScriptedContext
from sglang.test.scripted_runtime.test_case import ScriptedTestCase
from sglang.test.scripted_runtime_chunked_helpers import (
@@ -755,6 +756,7 @@ class TestSpecialCaseNoChunking(ScriptedTestCase):
DETERMINISTIC_ALIGN_SIZE = 4096
@unittest.skipUnless(is_flashinfer_available(), "flashinfer is NVIDIA-only")
class TestSpecialCaseDeterministicFlashInfer(ScriptedTestCase):
ENGINE_KWARGS = base_engine_kwargs(
chunked_prefill_size=DETERMINISTIC_ALIGN_SIZE,
@@ -791,6 +793,7 @@ class TestSpecialCaseDeterministicFlashInfer(ScriptedTestCase):
assert saw_chunking, "test must observe the req mid-chunk at least once"
@unittest.skipUnless(is_cuda() or is_hip(), "MHA host-pool movers are CUDA/ROCm only")
class TestSpecialCaseHiCache(ScriptedTestCase):
ENGINE_KWARGS = base_engine_kwargs(
chunked_prefill_size=DEFAULT_CHUNK_SIZE,
@@ -7,7 +7,12 @@ import uuid
import zmq
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.scripted_runtime.http_server import ScriptedHttpServer
from sglang.test.scripted_runtime.http_server import (
CANARY_LAUNCH_DEFAULTS,
DEFAULT_RUN_TIMEOUT_S,
ScriptedHttpServer,
_canary_decode_graph_override,
)
from sglang.test.scripted_runtime.io_struct import (
HookReady,
RunScript,
@@ -73,6 +78,7 @@ def _make_server(socket: zmq.Socket, process: _FakeProcess) -> ScriptedHttpServe
server._socket = socket
server._server_process = process
server._dirty = None
server._run_timeout_s = DEFAULT_RUN_TIMEOUT_S
return server
@@ -141,5 +147,23 @@ class TestExecuteScriptDirtyGuard(CustomTestCase):
pair.assert_no_sent_message(self)
class TestCanaryLaunchKwargs(CustomTestCase):
def test_canary_is_enabled_without_consulting_the_device(self):
self.assertEqual(CANARY_LAUNCH_DEFAULTS["kv_canary"], "raise")
self.assertEqual(CANARY_LAUNCH_DEFAULTS["kv_canary_real_data"], "partial")
def test_cuda_keeps_its_decode_graph(self):
self.assertEqual(
_canary_decode_graph_override(kv_canary="raise", device="cuda"), {}
)
def test_a_disabled_canary_keeps_the_decode_graph(self):
for device in ("cuda", "xpu", "cpu"):
with self.subTest(device=device):
self.assertEqual(
_canary_decode_graph_override(kv_canary="none", device=device), {}
)
if __name__ == "__main__":
unittest.main()