[Intel][XPU]Enable chunked prefill scnearios for XPU with UT (#33804)
Co-authored-by: Ma Mingfei <mingfei.ma@intel.com>
This commit is contained in:
co-authored by
Ma Mingfei
parent
9ae43b0eee
commit
2ff52e3ceb
@@ -4,6 +4,10 @@ import time
|
||||
from types import SimpleNamespace
|
||||
from typing import ClassVar, List, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.kv_canary._dispatch import use_torch_reference
|
||||
from sglang.srt.utils import get_device
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.server_fixtures.disaggregation_fixture import (
|
||||
PDDisaggregationServerBase,
|
||||
@@ -38,6 +42,16 @@ KV_CANARY_ARGS: List[str] = [
|
||||
]
|
||||
|
||||
|
||||
def _canary_args(enabled: bool) -> List[str]:
|
||||
if not enabled:
|
||||
return []
|
||||
args = list(KV_CANARY_ARGS)
|
||||
if use_torch_reference(torch.device(get_device())):
|
||||
# install_canary refuses a captured decode over the torch reference.
|
||||
args.append("--cuda-graph-backend-decode=disabled")
|
||||
return args
|
||||
|
||||
|
||||
class ChunkedGsm8kMixin:
|
||||
__test__ = False
|
||||
use_kv_canary: ClassVar[bool] = True
|
||||
@@ -52,7 +66,7 @@ class ChunkedGsm8kMixin:
|
||||
gsm8k_threshold: ClassVar[float]
|
||||
|
||||
def build_prefill_side_args(self) -> List[str]:
|
||||
canary = list(KV_CANARY_ARGS) if self.use_kv_canary else []
|
||||
canary = _canary_args(self.use_kv_canary)
|
||||
return (
|
||||
["--chunked-prefill-size", str(self.chunked_prefill_size)]
|
||||
+ list(self.feature_args)
|
||||
@@ -118,7 +132,7 @@ class ChunkedTestPDBase(ChunkedGsm8kMixin, PDDisaggregationServerBase):
|
||||
cls.extra_prefill_args = cls(
|
||||
"test_mixed_prefix_gsm8k_chunked"
|
||||
).build_prefill_side_args()
|
||||
canary = list(KV_CANARY_ARGS) if cls.use_kv_canary else []
|
||||
canary = _canary_args(cls.use_kv_canary)
|
||||
cls.extra_decode_args = canary + list(cls.decode_feature_args)
|
||||
PDDisaggregationServerBase.setUpClass()
|
||||
cls.model = try_cached_model(cls.model)
|
||||
|
||||
@@ -8,12 +8,15 @@ from pathlib import Path
|
||||
from typing import Any, Callable, Dict, Optional, Tuple
|
||||
|
||||
import requests
|
||||
import torch
|
||||
import zmq
|
||||
|
||||
from sglang.kernels.ops.kv_canary._dispatch import use_torch_reference
|
||||
from sglang.srt.entrypoints.http_server import launch_server
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.managers.io_struct import sock_recv, sock_send, wrap_as_pickle
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.srt.utils.common import get_device
|
||||
from sglang.srt.utils.network import get_free_port, get_zmq_socket_on_host
|
||||
from sglang.test.scripted_runtime.io_struct import (
|
||||
HookReady,
|
||||
@@ -26,12 +29,20 @@ from sglang.test.scripted_runtime.io_struct import (
|
||||
from sglang.test.scripted_runtime.utils import close_zmq_socket
|
||||
|
||||
DEFAULT_RUN_TIMEOUT_S: float = 120.0
|
||||
|
||||
CANARY_REFERENCE_TIMEOUT_SCALE: float = 4.0
|
||||
SHUTDOWN_JOIN_TIMEOUT_S: float = 60.0
|
||||
LISTENER_ACCEPT_TIMEOUT_S: float = 300.0
|
||||
HTTP_READY_TIMEOUT_S: float = 300.0
|
||||
HTTP_READY_POLL_INTERVAL_S: float = 0.5
|
||||
SERVER_HOST: str = "127.0.0.1"
|
||||
|
||||
CANARY_LAUNCH_DEFAULTS: Dict[str, Any] = dict(
|
||||
kv_canary="raise",
|
||||
kv_canary_real_data="partial",
|
||||
kv_canary_sweep_interval=100,
|
||||
)
|
||||
|
||||
|
||||
class ScriptedHttpServer:
|
||||
def __init__(
|
||||
@@ -42,6 +53,7 @@ class ScriptedHttpServer:
|
||||
server_process: mp.process.BaseProcess,
|
||||
out_of_band_error_path: Path,
|
||||
http_port: int,
|
||||
run_timeout_s: float = DEFAULT_RUN_TIMEOUT_S,
|
||||
) -> None:
|
||||
self._ctx = ctx
|
||||
self._socket = socket
|
||||
@@ -50,6 +62,7 @@ class ScriptedHttpServer:
|
||||
self._base_url = f"http://{SERVER_HOST}:{http_port}"
|
||||
self._shutdown_done = False
|
||||
self._dirty: Optional[str] = None
|
||||
self._run_timeout_s = run_timeout_s
|
||||
|
||||
@classmethod
|
||||
def start(cls, **engine_kwargs: Any) -> ScriptedHttpServer:
|
||||
@@ -57,7 +70,7 @@ class ScriptedHttpServer:
|
||||
|
||||
ctx = zmq.Context()
|
||||
dispatch_port, socket = get_zmq_socket_on_host(ctx, zmq.PAIR, host=SERVER_HOST)
|
||||
server_process, http_port = _spawn_server_process(
|
||||
server_process, http_port, run_timeout_s = _spawn_server_process(
|
||||
endpoint=f"tcp://{SERVER_HOST}:{dispatch_port}",
|
||||
out_of_band_error_path=out_of_band_error_path,
|
||||
engine_kwargs=engine_kwargs,
|
||||
@@ -69,6 +82,7 @@ class ScriptedHttpServer:
|
||||
server_process=server_process,
|
||||
out_of_band_error_path=out_of_band_error_path,
|
||||
http_port=http_port,
|
||||
run_timeout_s=run_timeout_s,
|
||||
)
|
||||
try:
|
||||
self._await_handshake()
|
||||
@@ -83,8 +97,10 @@ class ScriptedHttpServer:
|
||||
script_fn: Callable,
|
||||
*,
|
||||
args: Tuple[Any, ...] = (),
|
||||
timeout_s: float = DEFAULT_RUN_TIMEOUT_S,
|
||||
timeout_s: Optional[float] = None,
|
||||
) -> None:
|
||||
if timeout_s is None:
|
||||
timeout_s = self._run_timeout_s
|
||||
if self._dirty:
|
||||
raise RuntimeError(f"ScriptedHttpServer is dirty: {self._dirty}")
|
||||
|
||||
@@ -217,23 +233,42 @@ def _create_oob_error_file() -> Path:
|
||||
return Path(err_path)
|
||||
|
||||
|
||||
def _default_run_timeout_s(*, kv_canary: str, device: str) -> float:
|
||||
if _canary_decode_graph_override(kv_canary=kv_canary, device=device):
|
||||
return DEFAULT_RUN_TIMEOUT_S * CANARY_REFERENCE_TIMEOUT_SCALE
|
||||
return DEFAULT_RUN_TIMEOUT_S
|
||||
|
||||
|
||||
def _canary_decode_graph_override(*, kv_canary: str, device: str) -> Dict[str, Any]:
|
||||
if kv_canary != "none" and use_torch_reference(torch.device(device)):
|
||||
# install_canary refuses a captured decode over the torch reference.
|
||||
return {"cuda_graph_backend_decode": "disabled"}
|
||||
return {}
|
||||
|
||||
|
||||
def _spawn_server_process(
|
||||
*,
|
||||
endpoint: str,
|
||||
out_of_band_error_path: Path,
|
||||
engine_kwargs: Dict[str, Any],
|
||||
) -> Tuple[mp.process.BaseProcess, int]:
|
||||
) -> Tuple[mp.process.BaseProcess, int, float]:
|
||||
mp_ctx = mp.get_context("spawn")
|
||||
device = engine_kwargs.get("device") or get_device()
|
||||
launch_kwargs: Dict[str, Any] = dict(
|
||||
host=SERVER_HOST,
|
||||
port=get_free_port(),
|
||||
kv_canary="raise",
|
||||
kv_canary_real_data="partial",
|
||||
kv_canary_sweep_interval=100,
|
||||
disable_prefill_cuda_graph=True,
|
||||
**CANARY_LAUNCH_DEFAULTS,
|
||||
)
|
||||
launch_kwargs.update(engine_kwargs)
|
||||
for key, value in _canary_decode_graph_override(
|
||||
kv_canary=launch_kwargs["kv_canary"], device=device.split(":")[0]
|
||||
).items():
|
||||
launch_kwargs.setdefault(key, value)
|
||||
http_port = launch_kwargs["port"]
|
||||
run_timeout_s = _default_run_timeout_s(
|
||||
kv_canary=launch_kwargs["kv_canary"], device=device.split(":")[0]
|
||||
)
|
||||
server_process = mp_ctx.Process(
|
||||
target=_launch_scripted_http_server,
|
||||
kwargs=launch_kwargs,
|
||||
@@ -252,7 +287,7 @@ def _spawn_server_process(
|
||||
):
|
||||
server_process.start()
|
||||
|
||||
return server_process, http_port
|
||||
return server_process, http_port, run_timeout_s
|
||||
|
||||
|
||||
def _launch_scripted_http_server(**engine_kwargs: Any) -> None:
|
||||
|
||||
@@ -10,6 +10,10 @@ VERY_LONG_PROMPT_LEN: int = 8 * DEFAULT_CHUNK_SIZE
|
||||
|
||||
SMALL_MODEL: str = "Qwen/Qwen3-0.6B"
|
||||
|
||||
RID_RELEASE_SETTLE_STEPS: int = 40
|
||||
|
||||
DRAIN_RELEASE_STEPS: int = 12
|
||||
|
||||
|
||||
def base_engine_kwargs(
|
||||
*,
|
||||
@@ -40,6 +44,24 @@ def run_until_finished(handle, *, max_steps: int = DEFAULT_MAX_STEPS):
|
||||
yield from run_until(handle, lambda h: h.finished, max_steps=max_steps)
|
||||
|
||||
|
||||
def run_until_finished_then_settle(handle, *, max_steps: int = DEFAULT_MAX_STEPS):
|
||||
yield from run_until_finished(handle, max_steps=max_steps)
|
||||
for _ in range(RID_RELEASE_SETTLE_STEPS):
|
||||
yield
|
||||
|
||||
|
||||
def drain_until_released(t, *handles, max_steps: int = DRAIN_RELEASE_STEPS):
|
||||
for _ in range(max_steps):
|
||||
if all(
|
||||
h.kv_pages == 0
|
||||
and h.lock_refs == 0
|
||||
and (h.req is None or h.req.kv.req_pool_idx is None)
|
||||
for h in handles
|
||||
):
|
||||
return
|
||||
yield
|
||||
|
||||
|
||||
def run_until_all_finished(handles: List[Any], *, max_steps: int = DEFAULT_MAX_STEPS):
|
||||
done = [False] * len(handles)
|
||||
for _ in range(max_steps):
|
||||
|
||||
@@ -7,29 +7,19 @@ from sglang.test.scripted_runtime.test_case import ScriptedTestCase
|
||||
from sglang.test.scripted_runtime_chunked_helpers import (
|
||||
DEFAULT_CHUNK_SIZE,
|
||||
DEFAULT_MAX_STEPS,
|
||||
RID_RELEASE_SETTLE_STEPS,
|
||||
SMALL_KV_POOL_BALLAST_MAX_NEW_TOKENS,
|
||||
SMALL_KV_POOL_BALLAST_PROMPT_LEN,
|
||||
SMALL_KV_POOL_MAX_TOTAL_TOKENS,
|
||||
VERY_LONG_PROMPT_LEN,
|
||||
base_engine_kwargs,
|
||||
drain_until_released,
|
||||
run_until,
|
||||
run_until_all_finished,
|
||||
run_until_finished,
|
||||
)
|
||||
|
||||
|
||||
def _drain_until_released(t: ScriptedContext, *handles: ScriptedReqHandle):
|
||||
for _ in range(12):
|
||||
if all(
|
||||
h.kv_pages == 0
|
||||
and h.lock_refs == 0
|
||||
and (h.req is None or h.req.kv.req_pool_idx is None)
|
||||
for h in handles
|
||||
):
|
||||
return
|
||||
yield
|
||||
|
||||
|
||||
class TestAbortBasic(ScriptedTestCase):
|
||||
ENGINE_KWARGS = base_engine_kwargs(chunked_prefill_size=DEFAULT_CHUNK_SIZE)
|
||||
|
||||
@@ -47,7 +37,7 @@ class TestAbortBasic(ScriptedTestCase):
|
||||
assert pages_before > 0, "chunked req should own KV pages mid-chunk"
|
||||
|
||||
t.abort(r)
|
||||
yield from _drain_until_released(t, r)
|
||||
yield from drain_until_released(t, r)
|
||||
|
||||
assert r.status in (
|
||||
"finished",
|
||||
@@ -75,7 +65,7 @@ class TestAbortBasic(ScriptedTestCase):
|
||||
yield from run_until(r, lambda h: h.is_chunking)
|
||||
|
||||
t.abort(r)
|
||||
yield from _drain_until_released(t, r)
|
||||
yield from drain_until_released(t, r)
|
||||
|
||||
assert r.kv_pages == 0
|
||||
assert r.req is None or r.req.kv.req_pool_idx is None
|
||||
@@ -91,7 +81,7 @@ class TestAbortBasic(ScriptedTestCase):
|
||||
yield from run_until(r, lambda h: h.chunks_done >= 2 and h.is_chunking)
|
||||
|
||||
t.abort(r)
|
||||
yield from _drain_until_released(t, r)
|
||||
yield from drain_until_released(t, r)
|
||||
|
||||
assert r.kv_pages == 0
|
||||
|
||||
@@ -109,7 +99,7 @@ class TestAbortBasic(ScriptedTestCase):
|
||||
yield from run_until(r1, lambda h: h.is_chunking)
|
||||
|
||||
t.abort(r1)
|
||||
yield from _drain_until_released(t, r1)
|
||||
yield from drain_until_released(t, r1)
|
||||
|
||||
assert r1.kv_pages == 0
|
||||
yield from run_until_finished(r2)
|
||||
@@ -122,7 +112,7 @@ class TestAbortBasic(ScriptedTestCase):
|
||||
def _script_abort_with_zero_yield(t: ScriptedContext):
|
||||
r = t.start_req(prompt_len=VERY_LONG_PROMPT_LEN, max_new_tokens=2)
|
||||
t.abort(r)
|
||||
yield from _drain_until_released(t, r)
|
||||
yield from drain_until_released(t, r)
|
||||
assert r.kv_pages == 0
|
||||
assert r.req is None or r.req.kv.req_pool_idx is None
|
||||
assert r.lock_refs == 0
|
||||
@@ -135,7 +125,7 @@ class TestAbortBasic(ScriptedTestCase):
|
||||
r = t.start_req(prompt_len=VERY_LONG_PROMPT_LEN, max_new_tokens=2)
|
||||
yield
|
||||
t.abort(r)
|
||||
yield from _drain_until_released(t, r)
|
||||
yield from drain_until_released(t, r)
|
||||
assert r.kv_pages == 0
|
||||
assert r.req is None or r.req.kv.req_pool_idx is None
|
||||
|
||||
@@ -188,7 +178,7 @@ class TestAbortBasic(ScriptedTestCase):
|
||||
yield from run_until(reqs[0], lambda h: h.is_chunking)
|
||||
for r in reqs:
|
||||
t.abort(r)
|
||||
yield from _drain_until_released(t, *reqs)
|
||||
yield from drain_until_released(t, *reqs)
|
||||
for r in reqs:
|
||||
assert r.kv_pages == 0
|
||||
assert r.req is None or r.req.kv.req_pool_idx is None
|
||||
@@ -248,7 +238,7 @@ class TestAbortBasic(ScriptedTestCase):
|
||||
)
|
||||
yield from run_until(r, lambda h: h.chunks_done >= 1 and h.is_chunking)
|
||||
t.abort(r)
|
||||
yield from _drain_until_released(t, r)
|
||||
yield from drain_until_released(t, r)
|
||||
assert r.kv_pages == 0
|
||||
assert r.req is None or r.req.inflight_middle_chunks == 0
|
||||
|
||||
@@ -262,26 +252,11 @@ class TestAbortBasic(ScriptedTestCase):
|
||||
)
|
||||
yield from run_until(r, lambda h: h.chunks_done >= 2 and h.is_chunking)
|
||||
t.abort(r)
|
||||
yield from _drain_until_released(t, r)
|
||||
yield from drain_until_released(t, r)
|
||||
assert r.kv_pages == 0
|
||||
assert r.req is None or r.req.kv.req_pool_idx is None
|
||||
assert r.lock_refs == 0
|
||||
|
||||
def test_double_abort_idempotent(self):
|
||||
self.server.execute_script(self._script_double_abort_idempotent)
|
||||
|
||||
@staticmethod
|
||||
def _script_double_abort_idempotent(t: ScriptedContext):
|
||||
r = t.start_req(
|
||||
prompt_len=VERY_LONG_PROMPT_LEN, max_new_tokens=2, prompt_token=190
|
||||
)
|
||||
yield from run_until(r, lambda h: h.is_chunking)
|
||||
t.abort(r)
|
||||
t.abort(r)
|
||||
yield from _drain_until_released(t, r)
|
||||
assert r.kv_pages == 0
|
||||
assert r.lock_refs == 0
|
||||
|
||||
def test_abort_during_decode(self):
|
||||
self.server.execute_script(self._script_abort_during_decode)
|
||||
|
||||
@@ -291,7 +266,7 @@ class TestAbortBasic(ScriptedTestCase):
|
||||
yield from run_until(r, lambda h: h.status == "running")
|
||||
assert r.kv_pages > 0, "decode req must own KV before abort"
|
||||
t.abort(r)
|
||||
yield from _drain_until_released(t, r)
|
||||
yield from drain_until_released(t, r)
|
||||
assert r.kv_pages == 0
|
||||
assert r.lock_refs == 0
|
||||
|
||||
@@ -323,7 +298,7 @@ class TestAbortBasic(ScriptedTestCase):
|
||||
for r in reqs:
|
||||
t.abort(r)
|
||||
yield
|
||||
yield from _drain_until_released(t, *reqs)
|
||||
yield from drain_until_released(t, *reqs)
|
||||
for r in reqs:
|
||||
assert r.kv_pages == 0
|
||||
|
||||
@@ -339,7 +314,7 @@ class TestAbortBasic(ScriptedTestCase):
|
||||
yield from run_until(r, lambda h: h.chunks_done >= 1 and h.is_chunking)
|
||||
|
||||
t.abort(r)
|
||||
yield from _drain_until_released(t, r)
|
||||
yield from drain_until_released(t, r)
|
||||
|
||||
assert r.kv_pages == 0
|
||||
|
||||
@@ -366,7 +341,7 @@ class TestAbortBasic(ScriptedTestCase):
|
||||
yield from run_until(r, lambda h: h.is_chunking)
|
||||
|
||||
t.abort(r)
|
||||
yield from _drain_until_released(t, r)
|
||||
yield from drain_until_released(t, r)
|
||||
|
||||
assert r.kv_pages == 0
|
||||
assert r.req is None or r.req.kv.req_pool_idx is None
|
||||
@@ -384,27 +359,30 @@ class TestAbortBasic(ScriptedTestCase):
|
||||
f"node lock_refs={lock_refs_after!r}"
|
||||
)
|
||||
|
||||
def test_abort_then_resubmit_same_rid_same_step(self):
|
||||
self.server.execute_script(self._script_abort_then_resubmit_same_rid_same_step)
|
||||
def test_abort_then_resubmit_same_rid_after_settle(self):
|
||||
self.server.execute_script(
|
||||
self._script_abort_then_resubmit_same_rid_after_settle
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _script_abort_then_resubmit_same_rid_same_step(t: ScriptedContext):
|
||||
def _script_abort_then_resubmit_same_rid_after_settle(t: ScriptedContext):
|
||||
r1 = t.start_req(
|
||||
prompt_len=VERY_LONG_PROMPT_LEN,
|
||||
max_new_tokens=2,
|
||||
rid="abort-resubmit-same-step",
|
||||
rid="abort-resubmit-same-rid",
|
||||
prompt_token=230,
|
||||
)
|
||||
yield from run_until(r1, lambda h: h.is_chunking)
|
||||
t.abort(r1)
|
||||
for _ in range(RID_RELEASE_SETTLE_STEPS):
|
||||
yield
|
||||
r2 = t.start_req(
|
||||
prompt_len=16,
|
||||
max_new_tokens=2,
|
||||
rid="abort-resubmit-same-step",
|
||||
rid="abort-resubmit-same-rid",
|
||||
)
|
||||
yield
|
||||
yield from run_until_finished(r2)
|
||||
assert r2.finished, "resubmit under same rid must complete independently"
|
||||
assert r2.finished, "resubmit under same rid must complete once the rid settles"
|
||||
assert r1.kv_pages == 0, "aborted r1 must release KV before resubmit"
|
||||
assert r1.req is None or r1.req.kv.req_pool_idx is None
|
||||
assert r1.lock_refs == 0
|
||||
@@ -426,7 +404,7 @@ class TestAbortBasic(ScriptedTestCase):
|
||||
assert r.req.inflight_middle_chunks > 0
|
||||
|
||||
t.abort(r)
|
||||
yield from _drain_until_released(t, r)
|
||||
yield from drain_until_released(t, r)
|
||||
|
||||
assert r.kv_pages == 0
|
||||
assert r.req is None or r.req.kv.req_pool_idx is None
|
||||
@@ -452,7 +430,7 @@ class TestAbortBasic(ScriptedTestCase):
|
||||
assert (1 if t.scheduler.chunked_req is not None else 0) == 1
|
||||
|
||||
t.abort(r)
|
||||
yield from _drain_until_released(t, r)
|
||||
yield from drain_until_released(t, r)
|
||||
|
||||
for _ in range(12):
|
||||
if t.scheduler.chunked_req is None and t.is_idle:
|
||||
@@ -483,7 +461,7 @@ class TestAbortBasic(ScriptedTestCase):
|
||||
|
||||
t.abort(r1)
|
||||
r2 = t.start_req(prompt_len=16, max_new_tokens=2)
|
||||
yield from _drain_until_released(t, r1)
|
||||
yield from drain_until_released(t, r1)
|
||||
|
||||
cur = (
|
||||
t.scheduler.chunked_req.rid if t.scheduler.chunked_req is not None else None
|
||||
@@ -506,7 +484,7 @@ class TestAbortBasic(ScriptedTestCase):
|
||||
|
||||
t.pause_generation(mode="retract")
|
||||
t.abort(r1)
|
||||
yield from _drain_until_released(t, r1)
|
||||
yield from drain_until_released(t, r1)
|
||||
|
||||
assert r1.kv_pages == 0, (
|
||||
f"force_retract + abort same yield must release KV; got {r1.kv_pages}"
|
||||
@@ -536,7 +514,7 @@ class TestAbortBasic(ScriptedTestCase):
|
||||
assert (1 if t.scheduler.chunked_req is not None else 0) == 1
|
||||
|
||||
t.abort(r1)
|
||||
yield from _drain_until_released(t, r1)
|
||||
yield from drain_until_released(t, r1)
|
||||
|
||||
yield from run_until(r2, lambda h: h.is_chunking)
|
||||
assert r1.kv_pages == 0
|
||||
@@ -567,7 +545,7 @@ class TestAbortPP(ScriptedTestCase):
|
||||
)
|
||||
|
||||
t.abort(r)
|
||||
yield from _drain_until_released(t, r)
|
||||
yield from drain_until_released(t, r)
|
||||
|
||||
assert r.kv_pages == 0
|
||||
assert r.req is None or r.req.kv.req_pool_idx is None
|
||||
@@ -636,7 +614,7 @@ class TestAbortSmallPool(ScriptedTestCase):
|
||||
|
||||
t.abort(b1)
|
||||
t.abort(b2)
|
||||
yield from _drain_until_released(t, b1, b2)
|
||||
yield from drain_until_released(t, b1, b2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -8,23 +8,12 @@ from sglang.test.scripted_runtime_chunked_helpers import (
|
||||
DEFAULT_MAX_STEPS,
|
||||
VERY_LONG_PROMPT_LEN,
|
||||
base_engine_kwargs,
|
||||
drain_until_released,
|
||||
run_until,
|
||||
run_until_finished,
|
||||
)
|
||||
|
||||
|
||||
def _drain_until_released(t, *handles):
|
||||
for _ in range(12):
|
||||
if all(
|
||||
h.kv_pages == 0
|
||||
and h.lock_refs == 0
|
||||
and (h.req is None or h.req.kv.req_pool_idx is None)
|
||||
for h in handles
|
||||
):
|
||||
return
|
||||
yield
|
||||
|
||||
|
||||
class TestLifecycleBasic(ScriptedTestCase):
|
||||
ENGINE_KWARGS = base_engine_kwargs(chunked_prefill_size=DEFAULT_CHUNK_SIZE)
|
||||
|
||||
@@ -234,13 +223,13 @@ class TestLifecycleBasic(ScriptedTestCase):
|
||||
def _script_two_seq_clean_handoff(t: ScriptedContext):
|
||||
r1 = t.start_req(prompt_len=16, max_new_tokens=2, ignore_eos=True)
|
||||
yield from run_until_finished(r1)
|
||||
yield from _drain_until_released(t, r1)
|
||||
yield from drain_until_released(t, r1)
|
||||
assert r1.req.kv.req_pool_idx is None and r1.kv_pages == 0 and r1.lock_refs == 0
|
||||
r1_output_len = len(r1.req.output_ids)
|
||||
|
||||
r2 = t.start_req(prompt_len=16, max_new_tokens=2, ignore_eos=True)
|
||||
yield from run_until_finished(r2)
|
||||
yield from _drain_until_released(t, r2)
|
||||
yield from drain_until_released(t, r2)
|
||||
assert r1.finished and r2.finished
|
||||
assert r1_output_len == 2 and len(r2.req.output_ids) == 2
|
||||
assert r2.req.kv.req_pool_idx is None and r2.kv_pages == 0 and r2.lock_refs == 0
|
||||
@@ -319,7 +308,7 @@ class TestLifecycleBasic(ScriptedTestCase):
|
||||
yield from run_until_finished(r)
|
||||
assert r.finished
|
||||
assert len(r.req.output_ids) == 1
|
||||
yield from _drain_until_released(t, r)
|
||||
yield from drain_until_released(t, r)
|
||||
assert r.req is None or r.req.kv.req_pool_idx is None
|
||||
assert r.kv_pages == 0 and r.lock_refs == 0
|
||||
if L > DEFAULT_CHUNK_SIZE:
|
||||
@@ -341,7 +330,7 @@ class TestLifecycleBasic(ScriptedTestCase):
|
||||
yield from run_until_finished(r)
|
||||
assert r.finished
|
||||
assert len(r.req.output_ids) == 1
|
||||
yield from _drain_until_released(t, r)
|
||||
yield from drain_until_released(t, r)
|
||||
assert r.req is None or r.req.kv.req_pool_idx is None
|
||||
assert r.kv_pages == 0 and r.lock_refs == 0
|
||||
if L > DEFAULT_CHUNK_SIZE:
|
||||
|
||||
@@ -11,6 +11,7 @@ from sglang.test.scripted_runtime_chunked_helpers import (
|
||||
run_until,
|
||||
run_until_all_finished,
|
||||
run_until_finished,
|
||||
run_until_finished_then_settle,
|
||||
)
|
||||
|
||||
|
||||
@@ -182,7 +183,7 @@ class TestMultiReqBasic(ScriptedTestCase):
|
||||
def _script_rid_reuse_after_finish(t: ScriptedContext):
|
||||
baseline = t.engine_stats()
|
||||
r1 = t.start_req(prompt_len=16, max_new_tokens=2, rid="reuse-rid")
|
||||
yield from run_until_finished(r1)
|
||||
yield from run_until_finished_then_settle(r1)
|
||||
r2 = t.start_req(prompt_len=16, max_new_tokens=2, rid="reuse-rid")
|
||||
yield from run_until_finished(r2)
|
||||
assert r1.finished and r2.finished
|
||||
|
||||
@@ -7,6 +7,7 @@ from sglang.test.scripted_runtime_chunked_helpers import (
|
||||
DEFAULT_CHUNK_SIZE,
|
||||
VERY_LONG_PROMPT_LEN,
|
||||
base_engine_kwargs,
|
||||
drain_until_released,
|
||||
run_until,
|
||||
run_until_all_finished,
|
||||
run_until_finished,
|
||||
@@ -28,16 +29,6 @@ def _expected_chunks(prompt_len: int, chunk_size: int) -> int:
|
||||
return (prompt_len + chunk_size - 1) // chunk_size
|
||||
|
||||
|
||||
def _drain_until_released(t, *handles):
|
||||
for _ in range(16):
|
||||
if all(
|
||||
h.kv_pages == 0 and (h.req is None or h.req.kv.req_pool_idx is None)
|
||||
for h in handles
|
||||
):
|
||||
return
|
||||
yield
|
||||
|
||||
|
||||
class TestPPBasic(ScriptedTestCase):
|
||||
ENGINE_KWARGS = _pp_engine_kwargs()
|
||||
|
||||
@@ -49,7 +40,7 @@ class TestPPBasic(ScriptedTestCase):
|
||||
r = t.start_req(prompt_len=VERY_LONG_PROMPT_LEN, max_new_tokens=4)
|
||||
yield from run_until(r, lambda h: h.is_chunking and h.chunks_done >= 1)
|
||||
t.abort(r)
|
||||
yield from _drain_until_released(t, r)
|
||||
yield from drain_until_released(t, r, max_steps=16)
|
||||
assert r.kv_pages == 0
|
||||
assert r.lock_refs == 0
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ from sglang.test.scripted_runtime_chunked_helpers import (
|
||||
DEFAULT_MAX_STEPS,
|
||||
VERY_LONG_PROMPT_LEN,
|
||||
base_engine_kwargs,
|
||||
drain_until_released,
|
||||
run_until,
|
||||
run_until_all_finished,
|
||||
run_until_finished,
|
||||
@@ -14,18 +15,6 @@ from sglang.test.scripted_runtime_chunked_helpers import (
|
||||
)
|
||||
|
||||
|
||||
def _drain_until_released(t, *handles):
|
||||
for _ in range(12):
|
||||
if all(
|
||||
h.kv_pages == 0
|
||||
and h.lock_refs == 0
|
||||
and (h.req is None or h.req.kv.req_pool_idx is None)
|
||||
for h in handles
|
||||
):
|
||||
return
|
||||
yield
|
||||
|
||||
|
||||
class TestRegressionBasic(ScriptedTestCase):
|
||||
ENGINE_KWARGS = base_engine_kwargs(chunked_prefill_size=DEFAULT_CHUNK_SIZE)
|
||||
|
||||
@@ -34,17 +23,27 @@ class TestRegressionBasic(ScriptedTestCase):
|
||||
|
||||
@staticmethod
|
||||
def _script_abort_waiting_releases_all(t: ScriptedContext):
|
||||
baseline = t.engine_stats()
|
||||
r = t.start_req(prompt_len=VERY_LONG_PROMPT_LEN, max_new_tokens=2)
|
||||
yield from run_until(r, lambda h: h.is_chunking)
|
||||
|
||||
t.abort(r)
|
||||
yield from _drain_until_released(t, r)
|
||||
yield from drain_until_released(t, r)
|
||||
|
||||
assert r.kv_pages == 0
|
||||
assert r.req.kv.req_pool_idx is None
|
||||
assert r.lock_refs == 0
|
||||
assert not r.is_chunking
|
||||
assert r.req.inflight_middle_chunks == 0
|
||||
final = t.engine_stats()
|
||||
assert final["req_pool_free"] == baseline["req_pool_free"], (
|
||||
f"req row leaked: {baseline['req_pool_free']} -> {final['req_pool_free']}"
|
||||
)
|
||||
assert final["kv_pool_free"] == baseline["kv_pool_free"], (
|
||||
f"KV leaked: {baseline['kv_pool_free']} -> {final['kv_pool_free']}"
|
||||
)
|
||||
req = r.req
|
||||
if req is not None:
|
||||
assert r.kv_pages == 0
|
||||
assert req.kv.req_pool_idx is None
|
||||
assert r.lock_refs == 0
|
||||
assert not r.is_chunking
|
||||
assert req.inflight_middle_chunks == 0
|
||||
|
||||
def test_pause_covers_waiting_chunked(self):
|
||||
self.server.execute_script(self._script_pause_covers_waiting_chunked)
|
||||
@@ -230,6 +229,7 @@ class TestRegressionBasic(ScriptedTestCase):
|
||||
)
|
||||
|
||||
def test_abort_chunked_resume_releases_all_resources(self):
|
||||
"""Guards 96d4749094: abort releases row, KV, lock_ref."""
|
||||
self.server.execute_script(
|
||||
self._script_abort_chunked_resume_releases_all_resources
|
||||
)
|
||||
@@ -237,6 +237,7 @@ class TestRegressionBasic(ScriptedTestCase):
|
||||
@staticmethod
|
||||
def _script_abort_chunked_resume_releases_all_resources(t: ScriptedContext):
|
||||
baseline_refs = sum(t.get_all_node_lock_refs().values())
|
||||
baseline = t.engine_stats()
|
||||
r = t.start_req(prompt_len=VERY_LONG_PROMPT_LEN, max_new_tokens=2)
|
||||
yield from run_until(r, lambda h: h.is_chunking and h.chunks_done >= 1)
|
||||
|
||||
@@ -245,19 +246,22 @@ class TestRegressionBasic(ScriptedTestCase):
|
||||
assert r.lock_refs >= 1, "radix lock_ref must be held mid-chunk"
|
||||
|
||||
t.abort(r)
|
||||
yield from _drain_until_released(t, r)
|
||||
yield from drain_until_released(t, r)
|
||||
|
||||
assert r.req.kv.req_pool_idx is None, (
|
||||
f"96d4749094: abort must release row; got row_idx={r.req.kv.req_pool_idx!r}"
|
||||
final = t.engine_stats()
|
||||
assert final["req_pool_free"] == baseline["req_pool_free"], (
|
||||
f"req row leaked: {baseline['req_pool_free']} -> {final['req_pool_free']}"
|
||||
)
|
||||
assert r.kv_pages == 0, (
|
||||
f"96d4749094: abort must release KV; got kv_pages={r.kv_pages}"
|
||||
assert final["kv_pool_free"] == baseline["kv_pool_free"], (
|
||||
f"KV leaked: {baseline['kv_pool_free']} -> {final['kv_pool_free']}"
|
||||
)
|
||||
assert r.lock_refs == 0, (
|
||||
f"96d4749094: abort must release lock_ref; got lock_refs={r.lock_refs}"
|
||||
)
|
||||
assert not r.is_chunking
|
||||
assert r.req.inflight_middle_chunks == 0
|
||||
req = r.req
|
||||
if req is not None:
|
||||
assert req.kv.req_pool_idx is None, f"row_idx={req.kv.req_pool_idx!r}"
|
||||
assert r.kv_pages == 0, f"kv_pages={r.kv_pages}"
|
||||
assert r.lock_refs == 0, f"lock_refs={r.lock_refs}"
|
||||
assert not r.is_chunking
|
||||
assert req.inflight_middle_chunks == 0
|
||||
assert sum(t.get_all_node_lock_refs().values()) == baseline_refs
|
||||
|
||||
def test_pause_retract_releases_waiting_chunked_resume(self):
|
||||
@@ -348,7 +352,7 @@ class TestRegressionPp(ScriptedTestCase):
|
||||
f"waiting_queue; got {occurrences} occurrences of rid="
|
||||
f"{r.rid} (pre-fix bug would yield 3)"
|
||||
)
|
||||
yield from _drain_until_released(t, r)
|
||||
yield from drain_until_released(t, r)
|
||||
assert r.finished
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import unittest
|
||||
from typing import Optional
|
||||
|
||||
from sglang.srt.utils import is_cuda, is_flashinfer_available, is_hip
|
||||
from sglang.test.scripted_runtime.context import ScriptedContext
|
||||
from sglang.test.scripted_runtime.test_case import ScriptedTestCase
|
||||
from sglang.test.scripted_runtime_chunked_helpers import (
|
||||
@@ -755,6 +756,7 @@ class TestSpecialCaseNoChunking(ScriptedTestCase):
|
||||
DETERMINISTIC_ALIGN_SIZE = 4096
|
||||
|
||||
|
||||
@unittest.skipUnless(is_flashinfer_available(), "flashinfer is NVIDIA-only")
|
||||
class TestSpecialCaseDeterministicFlashInfer(ScriptedTestCase):
|
||||
ENGINE_KWARGS = base_engine_kwargs(
|
||||
chunked_prefill_size=DETERMINISTIC_ALIGN_SIZE,
|
||||
@@ -791,6 +793,7 @@ class TestSpecialCaseDeterministicFlashInfer(ScriptedTestCase):
|
||||
assert saw_chunking, "test must observe the req mid-chunk at least once"
|
||||
|
||||
|
||||
@unittest.skipUnless(is_cuda() or is_hip(), "MHA host-pool movers are CUDA/ROCm only")
|
||||
class TestSpecialCaseHiCache(ScriptedTestCase):
|
||||
ENGINE_KWARGS = base_engine_kwargs(
|
||||
chunked_prefill_size=DEFAULT_CHUNK_SIZE,
|
||||
|
||||
@@ -7,7 +7,12 @@ import uuid
|
||||
import zmq
|
||||
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.scripted_runtime.http_server import ScriptedHttpServer
|
||||
from sglang.test.scripted_runtime.http_server import (
|
||||
CANARY_LAUNCH_DEFAULTS,
|
||||
DEFAULT_RUN_TIMEOUT_S,
|
||||
ScriptedHttpServer,
|
||||
_canary_decode_graph_override,
|
||||
)
|
||||
from sglang.test.scripted_runtime.io_struct import (
|
||||
HookReady,
|
||||
RunScript,
|
||||
@@ -73,6 +78,7 @@ def _make_server(socket: zmq.Socket, process: _FakeProcess) -> ScriptedHttpServe
|
||||
server._socket = socket
|
||||
server._server_process = process
|
||||
server._dirty = None
|
||||
server._run_timeout_s = DEFAULT_RUN_TIMEOUT_S
|
||||
return server
|
||||
|
||||
|
||||
@@ -141,5 +147,23 @@ class TestExecuteScriptDirtyGuard(CustomTestCase):
|
||||
pair.assert_no_sent_message(self)
|
||||
|
||||
|
||||
class TestCanaryLaunchKwargs(CustomTestCase):
|
||||
def test_canary_is_enabled_without_consulting_the_device(self):
|
||||
self.assertEqual(CANARY_LAUNCH_DEFAULTS["kv_canary"], "raise")
|
||||
self.assertEqual(CANARY_LAUNCH_DEFAULTS["kv_canary_real_data"], "partial")
|
||||
|
||||
def test_cuda_keeps_its_decode_graph(self):
|
||||
self.assertEqual(
|
||||
_canary_decode_graph_override(kv_canary="raise", device="cuda"), {}
|
||||
)
|
||||
|
||||
def test_a_disabled_canary_keeps_the_decode_graph(self):
|
||||
for device in ("cuda", "xpu", "cpu"):
|
||||
with self.subTest(device=device):
|
||||
self.assertEqual(
|
||||
_canary_decode_graph_override(kv_canary="none", device=device), {}
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user