166 lines
5.2 KiB
Python
166 lines
5.2 KiB
Python
import os
|
|
import shutil
|
|
import unittest
|
|
from types import SimpleNamespace
|
|
|
|
from sglang.test.ci.ci_register import register_cuda_ci
|
|
from sglang.test.server_fixtures.disaggregation_fixture import (
|
|
PDDisaggregationServerBase,
|
|
)
|
|
from sglang.test.sgl_eval_utils import run_sgl_eval
|
|
from sglang.test.test_utils import (
|
|
DEFAULT_MODEL_NAME_FOR_TEST,
|
|
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
|
popen_launch_pd_server,
|
|
terminate_and_kill_process_tree,
|
|
)
|
|
|
|
# est_time covers two eval rounds.
|
|
register_cuda_ci(
|
|
est_time=600,
|
|
stage="base-b",
|
|
runner_config="2-gpu-large",
|
|
)
|
|
|
|
|
|
class TestDisaggregationDecodeOffload(PDDisaggregationServerBase):
|
|
"""Decode-side KV cache offloading in a PD disaggregation setup."""
|
|
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
# Set environment variable to make offloading more frequent for testing purposes
|
|
cls.old_stride = os.environ.get("SGLANG_HICACHE_DECODE_OFFLOAD_STRIDE")
|
|
cls.hicache_dir = "/tmp/hicache_test"
|
|
os.environ["SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR"] = cls.hicache_dir
|
|
os.environ["SGLANG_HICACHE_DECODE_OFFLOAD_STRIDE"] = "16"
|
|
|
|
# Ensure a clean cache directory
|
|
if os.path.exists(cls.hicache_dir):
|
|
shutil.rmtree(cls.hicache_dir)
|
|
os.makedirs(cls.hicache_dir, exist_ok=True)
|
|
|
|
super().setUpClass()
|
|
cls.model = DEFAULT_MODEL_NAME_FOR_TEST
|
|
|
|
# Non-blocking start of prefill and decode servers
|
|
cls.start_prefill()
|
|
cls.start_decode()
|
|
|
|
# Wait for both servers to be ready before proceeding
|
|
cls.wait_server_ready(cls.prefill_url + "/health")
|
|
cls.wait_server_ready(cls.decode_url + "/health")
|
|
|
|
cls.launch_lb()
|
|
|
|
@classmethod
|
|
def tearDownClass(cls):
|
|
# Restore the original environment variable state
|
|
super().tearDownClass()
|
|
if cls.old_stride is not None:
|
|
os.environ["SGLANG_HICACHE_DECODE_OFFLOAD_STRIDE"] = cls.old_stride
|
|
else:
|
|
os.environ.pop("SGLANG_HICACHE_DECODE_OFFLOAD_STRIDE", None)
|
|
|
|
os.environ.pop("SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR", None)
|
|
|
|
# Clean up the cache directory
|
|
if os.path.exists(cls.hicache_dir):
|
|
shutil.rmtree(cls.hicache_dir)
|
|
|
|
@classmethod
|
|
def start_prefill(cls):
|
|
prefill_args = [
|
|
"--trust-remote-code",
|
|
"--disaggregation-mode",
|
|
"prefill",
|
|
"--disaggregation-bootstrap-port",
|
|
cls.bootstrap_port,
|
|
"--tp",
|
|
"1",
|
|
"--page-size",
|
|
"16",
|
|
"--enable-hierarchical-cache",
|
|
"--hicache-storage-backend",
|
|
"file",
|
|
"--hicache-ratio",
|
|
"2",
|
|
]
|
|
prefill_args += cls.transfer_backend + cls.rdma_devices_for(range(1))
|
|
cls.process_prefill = popen_launch_pd_server(
|
|
cls.model,
|
|
cls.prefill_url,
|
|
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
|
other_args=prefill_args,
|
|
)
|
|
|
|
@classmethod
|
|
def start_decode(cls):
|
|
decode_args = [
|
|
"--trust-remote-code",
|
|
"--disaggregation-mode",
|
|
"decode",
|
|
"--disaggregation-bootstrap-port",
|
|
cls.bootstrap_port,
|
|
"--tp",
|
|
"1",
|
|
"--base-gpu-id",
|
|
"1",
|
|
"--disaggregation-decode-enable-offload-kvcache",
|
|
"--num-reserved-decode-tokens",
|
|
"128",
|
|
"--hicache-ratio",
|
|
"2",
|
|
"--page-size",
|
|
"16",
|
|
"--hicache-storage-backend",
|
|
"file",
|
|
]
|
|
decode_args += cls.transfer_backend + cls.rdma_devices_for(range(1, 2))
|
|
cls.process_decode = popen_launch_pd_server(
|
|
cls.model,
|
|
cls.decode_url,
|
|
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
|
other_args=decode_args,
|
|
)
|
|
|
|
def test_mmlu_double_eval(self):
|
|
"""Round 1 offloads decode KV to HiCache disk; after restarting all nodes,
|
|
round 2 loads it on prefill. Both rounds must score consistently."""
|
|
args = SimpleNamespace(
|
|
base_url=f"http://{self.base_host}:{self.lb_port}",
|
|
model=self.model,
|
|
eval_name="mmlu",
|
|
num_examples=256,
|
|
num_threads=32,
|
|
)
|
|
|
|
metrics1 = run_sgl_eval(args)
|
|
|
|
# Ensure all offloads are committed to disk
|
|
import time
|
|
|
|
time.sleep(10)
|
|
|
|
terminate_and_kill_process_tree(self.process_prefill)
|
|
terminate_and_kill_process_tree(self.process_decode)
|
|
terminate_and_kill_process_tree(self.process_lb)
|
|
|
|
self.start_prefill()
|
|
self.start_decode()
|
|
self.launch_lb()
|
|
self.wait_server_ready(self.prefill_url + "/health")
|
|
self.wait_server_ready(self.decode_url + "/health")
|
|
|
|
metrics2 = run_sgl_eval(args)
|
|
|
|
# Assert score is above a minimum threshold for both rounds
|
|
self.assertGreater(metrics1["score"], 0.64)
|
|
self.assertGreater(metrics2["score"], 0.64)
|
|
|
|
# Score should be consistent: round 2 should be >= round 1, or at least within a 0.05 margin if slightly lower
|
|
self.assertGreaterEqual(metrics2["score"], metrics1["score"] - 0.05)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|