Files
sglang/test/registered/mock_model/test_e2e_pd.py
T

181 lines
6.2 KiB
Python

from __future__ import annotations
import time
import unittest
from concurrent.futures import ThreadPoolExecutor, as_completed
from typing import ClassVar, Dict, List
import requests
from sglang.srt.utils import is_hip
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.kv_canary.violation_log_utils import assert_no_violation_in_log
from sglang.test.mock_model.utils import (
MOCK_MODEL_PATH,
mock_model_server_args,
mock_model_server_env,
)
from sglang.test.server_fixtures.disaggregation_fixture import (
PDDisaggregationServerBase,
)
register_cuda_ci(est_time=266, stage="extra-a", runner_config="2-gpu-large")
register_amd_ci(est_time=165, stage="extra-a", runner_config="2-gpu-large-amd")
# DO NOT pass --disable-cuda-graph in canary e2e tests. The canary kernel
# must run inside the cuda graph alongside the real attn kernel; disabling the
# full graph silently bypasses the only path that exercises that invariant
# end-to-end.
#
# --cuda-graph-backend-prefill=disabled is REQUIRED by canary: install_canary
# (api.py) asserts it, and the SingleForwardManager design depends on it.
# mock_model_server_args() already passes it; do not remove it.
_NUM_PROMPTS = 32
_INPUT_LEN = 6144
_OUTPUT_LEN = 1024
def _send_parallel_requests(
base_url: str,
*,
n: int,
max_new_tokens: int,
timeout: float = 60.0,
max_workers: int = 16,
) -> List[Dict[str, object]]:
"""Fire N /generate requests concurrently; return raw response dicts."""
def _one(i: int) -> Dict[str, object]:
payload = {
"input_ids": _make_input_ids(seed=i, length=_INPUT_LEN),
"sampling_params": {"max_new_tokens": max_new_tokens, "temperature": 0.0},
}
try:
resp = requests.post(base_url + "/generate", json=payload, timeout=timeout)
return {"index": i, "status_code": resp.status_code, "text": resp.text}
except requests.exceptions.RequestException as exc:
return {"index": i, "error": repr(exc)}
results: List[Dict[str, object]] = []
with ThreadPoolExecutor(max_workers=max_workers) as pool:
futures = [pool.submit(_one, i) for i in range(n)]
for fut in as_completed(futures):
results.append(fut.result())
results.sort(key=lambda r: r["index"])
return results
def _make_input_ids(*, seed: int, length: int) -> List[int]:
return [((seed + i) % 2048) + 1 for i in range(length)]
class _MockModelPDBase(PDDisaggregationServerBase):
"""PD fixture for mock-model + canary e2e tests."""
capture_per_side_logs = True
model: ClassVar[str] = MOCK_MODEL_PATH
extra_prefill_args: ClassVar[List[str]] = mock_model_server_args(
"--skip-server-warmup"
)
extra_decode_args: ClassVar[List[str]] = mock_model_server_args(
"--skip-server-warmup"
)
extra_prefill_env: ClassVar[Dict[str, str]] = mock_model_server_env(
input_check_enabled=True
)
extra_decode_env: ClassVar[Dict[str, str]] = mock_model_server_env(
input_check_enabled=True
)
@classmethod
def setUpClass(cls) -> None:
super().setUpClass()
cls.launch_all()
def assert_no_canary_violation(self) -> None:
time.sleep(2)
log_text = "".join(
buf.getvalue()
for buf in (
self._prefill_stdout_buf,
self._prefill_stderr_buf,
self._decode_stdout_buf,
self._decode_stderr_buf,
)
if buf is not None
)
assert_no_violation_in_log(log_text)
class TestPdTransferCanaryClean(_MockModelPDBase, unittest.TestCase):
"""PD standard scenario + baseline canary (input-check, no real-KV checksum); no violation expected."""
def test_pd_transfer_canary_clean(self) -> None:
# Step 1: send parallel requests through the LB to exercise PD transfer path.
results = _send_parallel_requests(
self.lb_url,
n=_NUM_PROMPTS,
max_new_tokens=_OUTPUT_LEN,
timeout=240.0,
max_workers=_NUM_PROMPTS,
)
# Step 2: every request must complete with status 200.
for result in results:
self.assertEqual(result.get("status_code"), 200, result)
# Step 3: servers must stay alive.
self.assertIsNone(self.process_prefill.poll(), "Prefill server died")
self.assertIsNone(self.process_decode.poll(), "Decode server died")
self.assert_no_canary_violation()
@unittest.skipIf(
is_hip(),
"ROCm: PD full-real-data KV checksum intermittently trips a "
"verify_real_kv_hash canary violation on the decode-side transferred prefix "
"(see https://github.com/sgl-project/sglang/issues/28971). The baseline PD "
"canary test above stays enabled on AMD.",
)
class TestPdTransferChecksumFullRealData(_MockModelPDBase, unittest.TestCase):
"""--kv-canary-real-data=all + sweep every step, no perturb, no violation."""
extra_prefill_args: ClassVar[List[str]] = mock_model_server_args(
"--skip-server-warmup",
"--kv-canary-real-data",
"all",
"--kv-canary-sweep-interval",
"1",
)
extra_decode_args: ClassVar[List[str]] = mock_model_server_args(
"--skip-server-warmup",
"--kv-canary-real-data",
"all",
"--kv-canary-sweep-interval",
"1",
"--disaggregation-decode-enable-radix-cache",
)
def test_pd_transfer_checksum_full_real_data(self) -> None:
# Step 1: drive traffic through the PD path with full real-KV hashing.
results = _send_parallel_requests(
self.lb_url,
n=_NUM_PROMPTS,
max_new_tokens=_OUTPUT_LEN,
timeout=240.0,
max_workers=_NUM_PROMPTS,
)
# Step 2: all requests must succeed.
for result in results:
self.assertEqual(result.get("status_code"), 200, result)
# Step 3: servers must stay healthy.
self.assertIsNone(self.process_prefill.poll(), "Prefill server died")
self.assertIsNone(self.process_decode.poll(), "Decode server died")
self.assert_no_canary_violation()
if __name__ == "__main__":
unittest.main()