chore: bump sglang-kernel version to 0.4.1.post1 (#23733)
Co-authored-by: sglang-bot <sglang-bot@users.noreply.github.com> Co-authored-by: Kangyan Zhou <zky314343421@gmail.com> Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
sglang-bot
co-authored by
sglang-bot
Kangyan Zhou
Claude Opus 4.7
sglang-bot
Kangyan Zhou
Claude Opus 4.7
parent
8efa177f1e
commit
9003f24e2b
@@ -1,60 +0,0 @@
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=99, suite="stage-b-test-1-gpu-small")
|
||||
register_amd_ci(est_time=300, suite="stage-b-test-1-gpu-small-amd")
|
||||
|
||||
"""
|
||||
# TODO: Segmentation fault occurs when upgraded to Cu13. Ref: https://github.com/sgl-project/sglang/actions/runs/24603159715/job/71945537414?pr=23119")
|
||||
# Should move back to registered test after it's fixed
|
||||
"""
|
||||
|
||||
import time
|
||||
import unittest
|
||||
|
||||
from sglang.srt.utils import is_hip, kill_process_tree
|
||||
from sglang.test.kits.eval_accuracy_kit import MMLUMixin
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
_is_hip = is_hip()
|
||||
|
||||
|
||||
class TestHiCache(CustomTestCase, MMLUMixin):
|
||||
mmlu_score_threshold = 0.65
|
||||
mmlu_num_examples = 64
|
||||
mmlu_num_threads = 32
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = DEFAULT_MODEL_NAME_FOR_TEST
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=[
|
||||
"--enable-hierarchical-cache",
|
||||
"--mem-fraction-static",
|
||||
0.7,
|
||||
"--hicache-size",
|
||||
100 if not _is_hip else 200,
|
||||
"--page-size",
|
||||
"64",
|
||||
"--hicache-storage-backend",
|
||||
"file",
|
||||
],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
kill_process_tree(cls.process.pid)
|
||||
time.sleep(5)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,94 +0,0 @@
|
||||
"""
|
||||
Benchmark tests for HiCache Storage with 3FS backend.
|
||||
Usage:
|
||||
python3 -m pytest test/registered/hicache/test_hicache_storage_3fs_backend.py -v
|
||||
# TODO: Segmentation fault occurs when upgraded to Cu13. Ref: https://github.com/sgl-project/sglang/actions/runs/24603159715/job/71945537414?pr=23119")
|
||||
# Should move back to registered test after it's fixed
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import unittest
|
||||
|
||||
from test_hicache_storage_file_backend import HiCacheStorageBaseMixin
|
||||
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(est_time=150, suite="stage-b-test-2-gpu-large")
|
||||
register_amd_ci(est_time=300, suite="stage-b-test-2-gpu-large")
|
||||
|
||||
|
||||
class HiCacheStorage3FSBackendBaseMixin(HiCacheStorageBaseMixin):
|
||||
"""Base mixin class with common setup and utilities"""
|
||||
|
||||
@classmethod
|
||||
def _get_additional_server_args_and_env(cls):
|
||||
"""Get additional server arguments specific to configuration - override in subclasses"""
|
||||
# Create a temporary JSON config file for HF3FS
|
||||
hf3fs_config = {
|
||||
"file_path_prefix": os.path.join(cls.temp_dir, "hicache"),
|
||||
"file_size": 1024 * 1024 * 1024 * 2,
|
||||
"numjobs": 2,
|
||||
"entries": 8,
|
||||
"use_mock_hf3fs_client": True,
|
||||
"hicache_storage_pass_prefix_keys": True,
|
||||
}
|
||||
|
||||
# Write config to temporary file
|
||||
config_file = os.path.join(cls.temp_dir, "hf3fs_config.json")
|
||||
with open(config_file, "w") as f:
|
||||
json.dump(hf3fs_config, f, indent=2)
|
||||
|
||||
server_args = {
|
||||
"--tp-size": 1,
|
||||
"--hicache-ratio": 1.2,
|
||||
"--hicache-storage-backend": "hf3fs",
|
||||
"--hicache-storage-backend-extra-config": json.dumps(hf3fs_config),
|
||||
}
|
||||
|
||||
# Set the environment variable to point to our config file
|
||||
env_vars = {
|
||||
"SGLANG_HICACHE_HF3FS_CONFIG_PATH": config_file,
|
||||
}
|
||||
|
||||
return server_args, env_vars
|
||||
|
||||
|
||||
class TestHf3fsBackendLayerFirstLayout(
|
||||
HiCacheStorage3FSBackendBaseMixin, CustomTestCase
|
||||
):
|
||||
"""Layer first layout tests for HiCache-Hf3fs backend"""
|
||||
|
||||
@classmethod
|
||||
def _get_additional_server_args_and_env(cls):
|
||||
"""Get additional server arguments specific to configuration - override in subclasses"""
|
||||
server_args, env_vars = super()._get_additional_server_args_and_env()
|
||||
server_args["--hicache-mem-layout"] = "layer_first"
|
||||
server_args["--hicache-io-backend"] = "direct"
|
||||
server_args["--tp-size"] = 2
|
||||
return server_args, env_vars
|
||||
|
||||
|
||||
class TestHf3fsBackendAccuracy(HiCacheStorage3FSBackendBaseMixin, CustomTestCase):
|
||||
"""Accuracy tests for HiCache-Hf3fs backend"""
|
||||
|
||||
@classmethod
|
||||
def _get_additional_server_args_and_env(cls):
|
||||
"""Get additional server arguments specific to configuration - override in subclasses"""
|
||||
server_args, env_vars = super()._get_additional_server_args_and_env()
|
||||
server_args["--hicache-ratio"] = 1.5
|
||||
server_args["--tp-size"] = 2
|
||||
server_args["--hicache-mem-layout"] = "page_first_direct"
|
||||
server_args["--hicache-io-backend"] = "direct"
|
||||
return server_args, env_vars
|
||||
|
||||
def test_eval_accuracy(self):
|
||||
"""Test eval accuracy with cache persistence across cache flushes"""
|
||||
from test_hicache_storage_file_backend import run_eval_accuracy_test
|
||||
|
||||
run_eval_accuracy_test(self)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
@@ -1,336 +0,0 @@
|
||||
"""
|
||||
E2E tests for HiCache Storage functionality.
|
||||
Usage:
|
||||
python3 -m pytest test/registered/hicache/test_hicache_storage_file_backend.py -v
|
||||
# TODO: Segmentation fault occurs when upgraded to Cu13. Ref: https://github.com/sgl-project/sglang/actions/runs/24603159715/job/71945537414?pr=23119")
|
||||
# Should move back to registered test after it's fixed
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from typing import Dict
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import requests
|
||||
|
||||
from sglang.benchmark.utils import get_tokenizer
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_MLA_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
is_in_ci,
|
||||
popen_launch_server,
|
||||
)
|
||||
from sglang.utils import wait_for_http_ready
|
||||
|
||||
register_cuda_ci(est_time=148, suite="stage-b-test-2-gpu-large")
|
||||
register_amd_ci(est_time=526, suite="stage-b-test-2-gpu-large-amd")
|
||||
|
||||
|
||||
class HiCacheStorageBaseMixin:
|
||||
"""Base mixin class with common setup and utilities"""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
"""Set up test environment and launch server once for all tests"""
|
||||
cls.temp_dir = tempfile.mkdtemp()
|
||||
cls.model = cls._get_model_name()
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
|
||||
parsed_url = urlparse(cls.base_url)
|
||||
cls.base_host = parsed_url.hostname
|
||||
cls.base_port = str(parsed_url.port)
|
||||
|
||||
# Prepare tokenizer for prompt generation
|
||||
cls.tokenizer = get_tokenizer(cls.model)
|
||||
|
||||
# Launch server with HiCache enabled and cache report
|
||||
cls.process = cls._launch_server_with_hicache()
|
||||
cls._wait_for_server_ready(process=cls.process)
|
||||
|
||||
print(f"Test server launched successfully at {cls.base_url}")
|
||||
print(f"Cache directory: {cls.temp_dir}")
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
"""Clean up test environment"""
|
||||
if hasattr(cls, "process") and cls.process:
|
||||
kill_process_tree(cls.process.pid)
|
||||
|
||||
import shutil
|
||||
|
||||
if hasattr(cls, "temp_dir"):
|
||||
shutil.rmtree(cls.temp_dir, ignore_errors=True)
|
||||
|
||||
@classmethod
|
||||
def _get_model_name(cls):
|
||||
"""Get model name for the test configuration - override in subclasses"""
|
||||
return DEFAULT_MODEL_NAME_FOR_TEST
|
||||
|
||||
@classmethod
|
||||
def _get_base_server_args(cls):
|
||||
"""Get base server arguments - can be extended in subclasses"""
|
||||
extra_config = {
|
||||
"hicache_storage_pass_prefix_keys": True,
|
||||
}
|
||||
return {
|
||||
"--enable-hierarchical-cache": True,
|
||||
"--mem-fraction-static": 0.6,
|
||||
"--hicache-ratio": 1.2,
|
||||
"--page-size": 64,
|
||||
"--enable-cache-report": True,
|
||||
"--hicache-storage-prefetch-policy": "wait_complete",
|
||||
"--hicache-storage-backend": "file",
|
||||
"--hicache-storage-backend-extra-config": json.dumps(extra_config),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def _get_additional_server_args_and_env(cls):
|
||||
"""Get additional server arguments specific to configuration - override in subclasses"""
|
||||
return {}, {"SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR": cls.temp_dir}
|
||||
|
||||
@classmethod
|
||||
def _launch_server_with_hicache(cls):
|
||||
"""Launch server with HiCache enabled"""
|
||||
|
||||
additional_server_args, env_vars = cls._get_additional_server_args_and_env()
|
||||
env_vars["SGLANG_ENABLE_DETERMINISTIC_INFERENCE"] = "1"
|
||||
server_args = cls._get_base_server_args()
|
||||
if additional_server_args:
|
||||
server_args.update(additional_server_args)
|
||||
|
||||
final_server_args = []
|
||||
for k, v in server_args.items():
|
||||
if isinstance(v, bool):
|
||||
final_server_args.append(str(k))
|
||||
else:
|
||||
final_server_args.append(str(k))
|
||||
final_server_args.append(str(v))
|
||||
|
||||
print(f"final_server_args: {final_server_args}")
|
||||
|
||||
env_vars = {
|
||||
**os.environ,
|
||||
**env_vars,
|
||||
}
|
||||
|
||||
return popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=final_server_args,
|
||||
env=env_vars,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _wait_for_server_ready(cls, timeout: int = 60, process=None) -> bool:
|
||||
"""Wait for server to be ready"""
|
||||
wait_for_http_ready(
|
||||
url=f"{cls.base_url}/health",
|
||||
timeout=timeout,
|
||||
process=process,
|
||||
)
|
||||
return True
|
||||
|
||||
def send_request(
|
||||
self, prompt: str, max_tokens: int = 100, temperature: float = 0.0
|
||||
) -> Dict:
|
||||
"""Send a generate request and return response"""
|
||||
response = requests.post(
|
||||
f"{self.base_url}/generate",
|
||||
json={
|
||||
"text": prompt,
|
||||
"sampling_params": {
|
||||
"temperature": temperature,
|
||||
"max_new_tokens": max_tokens,
|
||||
"ignore_eos": True,
|
||||
},
|
||||
},
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
response.status_code,
|
||||
200,
|
||||
f"Request failed: {response.status_code} - {response.text}",
|
||||
)
|
||||
return response.json()
|
||||
|
||||
def get_cached_tokens(self, response_json: Dict) -> int:
|
||||
"""Extract cached tokens count from /generate response"""
|
||||
meta = response_json.get("meta_info", {})
|
||||
return int(meta.get("cached_tokens", 0))
|
||||
|
||||
def flush_cache(self):
|
||||
"""Flush device cache to force remote storage access."""
|
||||
res = requests.post(
|
||||
f"{self.base_url}/flush_cache",
|
||||
params={"timeout": 30},
|
||||
timeout=40,
|
||||
)
|
||||
res.raise_for_status()
|
||||
|
||||
def gen_prompt(self, token_num: int) -> str:
|
||||
"""Generate a random prompt of specified token length using tokenizer vocabulary."""
|
||||
all_available_tokens = list(self.tokenizer.get_vocab().values())
|
||||
selected_tokens = random.choices(all_available_tokens, k=token_num)
|
||||
return self.tokenizer.decode(selected_tokens)
|
||||
|
||||
def trigger_offloading_and_flush(self):
|
||||
"""Helper method to trigger offloading and flush cache"""
|
||||
# Trigger offloading
|
||||
self.send_request(self.gen_prompt(1), max_tokens=150)
|
||||
|
||||
# Flush device cache to force remote storage access
|
||||
self.flush_cache()
|
||||
|
||||
def test_basic_backup_and_prefetch(self):
|
||||
"""Test storage and retrieval of large context through remote cache"""
|
||||
print("\n=== Testing Large Context Cache Storage & Retrieval ===")
|
||||
|
||||
# Generate substantial context that will be cached
|
||||
base_prompt = self.gen_prompt(768)
|
||||
|
||||
# First request - populate cache
|
||||
print("Step 1: Populating cache with large context...")
|
||||
response1 = self.send_request(base_prompt, max_tokens=150)
|
||||
self.assertIsNotNone(response1)
|
||||
|
||||
# Flush device cache to force remote storage access
|
||||
self.trigger_offloading_and_flush()
|
||||
|
||||
# Second request with extended prompt - should hit remote cache
|
||||
print("Step 2: Testing cache hit from remote storage...")
|
||||
|
||||
start_time = time.time()
|
||||
response2 = self.send_request(base_prompt, max_tokens=150)
|
||||
retrieval_time = time.time() - start_time
|
||||
|
||||
cached_tokens = self.get_cached_tokens(response2)
|
||||
print(
|
||||
f"Remote cache retrieval time: {retrieval_time:.3f}s, cached_tokens={cached_tokens}"
|
||||
)
|
||||
|
||||
# Assert cached tokens indicate a remote hit
|
||||
self.assertGreater(
|
||||
cached_tokens, 700, "Expected significant cached tokens for remote hit"
|
||||
)
|
||||
|
||||
|
||||
@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.")
|
||||
class TestHiCacheStoragePageFirstLayout(HiCacheStorageBaseMixin, CustomTestCase):
|
||||
"""Page first layout tests for HiCache Storage functionality"""
|
||||
|
||||
@classmethod
|
||||
def _get_additional_server_args_and_env(cls):
|
||||
"""Get additional server arguments specific to configuration - override in subclasses"""
|
||||
server_args = {"--hicache-mem-layout": "page_first"}
|
||||
return server_args, {}
|
||||
|
||||
|
||||
@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.")
|
||||
class TestHiCacheStorageMLA(HiCacheStorageBaseMixin, CustomTestCase):
|
||||
"""MLA Model tests for HiCache Storage functionality"""
|
||||
|
||||
@classmethod
|
||||
def _get_model_name(cls):
|
||||
"""Use MLA model for testing"""
|
||||
return DEFAULT_MLA_MODEL_NAME_FOR_TEST
|
||||
|
||||
@classmethod
|
||||
def _get_additional_server_args_and_env(cls):
|
||||
"""Get additional server arguments specific to configuration - override in subclasses"""
|
||||
server_args = {"--tp-size": 2}
|
||||
return server_args, {}
|
||||
|
||||
|
||||
class TestHiCacheStoragePageFirstDirectIO(HiCacheStorageBaseMixin, CustomTestCase):
|
||||
"""Page first direct tests for HiCache Storage functionality"""
|
||||
|
||||
@classmethod
|
||||
def _get_additional_server_args_and_env(cls):
|
||||
"""Get additional server arguments specific to configuration - override in subclasses"""
|
||||
server_args = {
|
||||
"--hicache-mem-layout": "page_first_direct",
|
||||
"--hicache-io-backend": "direct",
|
||||
"--tp-size": 2,
|
||||
}
|
||||
return server_args, {}
|
||||
|
||||
|
||||
class TestHiCacheStorageAccuracy(HiCacheStorageBaseMixin, CustomTestCase):
|
||||
"""Accuracy tests for HiCache Storage functionality"""
|
||||
|
||||
@classmethod
|
||||
def _get_additional_server_args_and_env(cls):
|
||||
"""Get additional server arguments specific to configuration - override in subclasses"""
|
||||
server_args = {
|
||||
"--tp-size": 2,
|
||||
"--hicache-ratio": 1.5,
|
||||
}
|
||||
|
||||
return server_args, {}
|
||||
|
||||
def test_eval_accuracy(self):
|
||||
"""Test eval accuracy with cache persistence across cache flushes"""
|
||||
run_eval_accuracy_test(self)
|
||||
|
||||
|
||||
def run_eval_accuracy_test(test_instance, accuracy_threshold: float = 0.03):
|
||||
"""Generic eval accuracy test with configurable accuracy threshold
|
||||
|
||||
Args:
|
||||
test_instance: The test class instance that provides base_host, base_port, flush_cache, and assert methods
|
||||
"""
|
||||
print("\n=== Testing Eval Accuracy with Cache Persistence ===")
|
||||
|
||||
# First evaluation - populate cache
|
||||
print("Phase 1: Running initial GSM8K evaluation to populate cache...")
|
||||
args_initial = SimpleNamespace(
|
||||
base_url=f"http://{test_instance.base_host}:{test_instance.base_port}",
|
||||
eval_name="gsm8k",
|
||||
api="completion",
|
||||
max_tokens=512,
|
||||
num_examples=200,
|
||||
num_threads=64,
|
||||
)
|
||||
metrics_initial = run_eval(args_initial)
|
||||
|
||||
# Flush cache to force remote storage access
|
||||
print("Phase 2: Flushing device cache...")
|
||||
test_instance.flush_cache()
|
||||
|
||||
# Second evaluation - should use remote cache
|
||||
print("Phase 3: Running second GSM8K evaluation using remote cache...")
|
||||
metrics_cached = run_eval(args_initial)
|
||||
|
||||
# Verify accuracy consistency
|
||||
accuracy_diff = abs(metrics_initial["score"] - metrics_cached["score"])
|
||||
print(f"Accuracy difference: {accuracy_diff:.4f}")
|
||||
|
||||
# Assertions
|
||||
test_instance.assertGreater(
|
||||
metrics_initial["score"], 0.6, "Initial accuracy should be reasonable"
|
||||
)
|
||||
test_instance.assertGreater(
|
||||
metrics_cached["score"], 0.6, "Cached accuracy should be reasonable"
|
||||
)
|
||||
test_instance.assertLess(
|
||||
accuracy_diff,
|
||||
accuracy_threshold,
|
||||
"Accuracy should be consistent between cache states",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
@@ -1,286 +0,0 @@
|
||||
"""
|
||||
Benchmark tests for HiCache Storage with Mooncake backend.
|
||||
Usage:
|
||||
python3.10 -m pytest test/registered/hicache/test_hicache_storage_mooncake_backend.py -v
|
||||
"""
|
||||
|
||||
# TODO: Segmentation fault occurs when upgraded to Cu13. Ref: https://github.com/sgl-project/sglang/actions/runs/24601791606/job/71942123195?pr=23119")
|
||||
# Should move back to registered test after it's fixed
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import unittest
|
||||
|
||||
import requests
|
||||
from test_hicache_storage_file_backend import HiCacheStorageBaseMixin
|
||||
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_MLA_MODEL_NAME_FOR_TEST,
|
||||
CustomTestCase,
|
||||
find_available_port,
|
||||
is_in_ci,
|
||||
)
|
||||
|
||||
|
||||
class HiCacheStorageMooncakeBackendBaseMixin(HiCacheStorageBaseMixin):
|
||||
"""Base mixin class with common setup and utilities"""
|
||||
|
||||
# Default port ranges for Mooncake services - can be overridden in subclasses
|
||||
mooncake_master_port_base = 50051
|
||||
mooncake_metadata_port_base = 8080
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
"""Set up test environment and launch Mooncake services before server setup"""
|
||||
# Find available ports for Mooncake services to avoid conflicts
|
||||
cls.mooncake_master_port = find_available_port(
|
||||
HiCacheStorageMooncakeBackendBaseMixin.mooncake_master_port_base
|
||||
)
|
||||
cls.mooncake_metadata_port = find_available_port(
|
||||
HiCacheStorageMooncakeBackendBaseMixin.mooncake_metadata_port_base
|
||||
)
|
||||
|
||||
# Start Mooncake services first
|
||||
cls._start_mooncake_services()
|
||||
|
||||
# Call parent setup
|
||||
super().setUpClass()
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
"""Clean up Mooncake services after server teardown"""
|
||||
# Call parent teardown first
|
||||
super().tearDownClass()
|
||||
|
||||
# Stop Mooncake services
|
||||
cls._stop_mooncake_services()
|
||||
|
||||
@classmethod
|
||||
def _start_mooncake_services(cls):
|
||||
"""Start Mooncake metadata and master services with configurable ports and readiness detection"""
|
||||
print("Starting Mooncake services...")
|
||||
print(
|
||||
f"Using master port: {cls.mooncake_master_port}, metadata port: {cls.mooncake_metadata_port}"
|
||||
)
|
||||
|
||||
# Start metadata service with configurable port
|
||||
try:
|
||||
# Start metadata server with port configuration
|
||||
cls.metadata_service_process = subprocess.Popen(
|
||||
[
|
||||
"python3",
|
||||
"-m",
|
||||
"mooncake.http_metadata_server",
|
||||
"--port",
|
||||
str(cls.mooncake_metadata_port),
|
||||
],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
preexec_fn=os.setsid, # Create new process group
|
||||
)
|
||||
print(
|
||||
f"Mooncake metadata service started on port {cls.mooncake_metadata_port}"
|
||||
)
|
||||
except (FileNotFoundError, subprocess.SubprocessError) as e:
|
||||
print(f"Warning: Could not start Mooncake metadata service: {e}")
|
||||
cls.metadata_service_process = None
|
||||
|
||||
# Start master service with configurable port
|
||||
try:
|
||||
# Start master server with port configuration
|
||||
cls.master_service_process = subprocess.Popen(
|
||||
["mooncake_master", "--port", str(cls.mooncake_master_port)],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
preexec_fn=os.setsid, # Create new process group
|
||||
)
|
||||
print(f"Mooncake master service started on port {cls.mooncake_master_port}")
|
||||
except (FileNotFoundError, subprocess.SubprocessError) as e:
|
||||
print(f"Warning: Could not start Mooncake master service: {e}")
|
||||
cls.master_service_process = None
|
||||
|
||||
# Wait for services to be ready instead of fixed sleep
|
||||
cls._wait_for_mooncake_services_ready()
|
||||
|
||||
@classmethod
|
||||
def _wait_for_mooncake_services_ready(cls, timeout: int = 30) -> bool:
|
||||
"""Wait for Mooncake services to be ready by checking their endpoints"""
|
||||
print("Waiting for Mooncake services to be ready...")
|
||||
|
||||
start_time = time.time()
|
||||
services_ready = False
|
||||
|
||||
while time.time() - start_time < timeout:
|
||||
try:
|
||||
# Check metadata service
|
||||
metadata_ready = False
|
||||
if (
|
||||
cls.metadata_service_process
|
||||
and cls.metadata_service_process.poll() is None
|
||||
):
|
||||
try:
|
||||
# Try to connect to the metadata service
|
||||
metadata_url = (
|
||||
f"http://127.0.0.1:{cls.mooncake_metadata_port}/metadata"
|
||||
)
|
||||
response = requests.get(metadata_url, timeout=2)
|
||||
if response.status_code == 200:
|
||||
metadata_ready = True
|
||||
print("Mooncake metadata service is ready")
|
||||
except (requests.RequestException, ConnectionError):
|
||||
# Service might not be fully started yet
|
||||
pass
|
||||
|
||||
# Check master service (if it has a health endpoint)
|
||||
master_ready = False
|
||||
if (
|
||||
cls.master_service_process
|
||||
and cls.master_service_process.poll() is None
|
||||
):
|
||||
# For now, we'll assume master service is ready if process is running
|
||||
# and it's been a few seconds since startup
|
||||
if (
|
||||
time.time() - start_time > 5
|
||||
): # Give master service time to initialize
|
||||
master_ready = True
|
||||
print("Mooncake master service is ready")
|
||||
|
||||
# Both services should be ready
|
||||
if metadata_ready and master_ready:
|
||||
services_ready = True
|
||||
print("All Mooncake services are ready")
|
||||
break
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error checking service readiness: {e}")
|
||||
|
||||
time.sleep(2)
|
||||
|
||||
if not services_ready:
|
||||
print(
|
||||
"Warning: Mooncake services may not be fully ready, continuing anyway..."
|
||||
)
|
||||
|
||||
return services_ready
|
||||
|
||||
@classmethod
|
||||
def _stop_mooncake_services(cls):
|
||||
"""Stop Mooncake services"""
|
||||
print("Stopping Mooncake services...")
|
||||
|
||||
# Stop metadata service
|
||||
if hasattr(cls, "metadata_service_process") and cls.metadata_service_process:
|
||||
try:
|
||||
os.killpg(os.getpgid(cls.metadata_service_process.pid), 9)
|
||||
cls.metadata_service_process.wait(timeout=5)
|
||||
print("Mooncake metadata service stopped")
|
||||
except (ProcessLookupError, subprocess.TimeoutExpired, OSError) as e:
|
||||
print(f"Warning: Could not stop Mooncake metadata service: {e}")
|
||||
|
||||
# Stop master service
|
||||
if hasattr(cls, "master_service_process") and cls.master_service_process:
|
||||
try:
|
||||
os.killpg(os.getpgid(cls.master_service_process.pid), 9)
|
||||
cls.master_service_process.wait(timeout=5)
|
||||
print("Mooncake master service stopped")
|
||||
except (ProcessLookupError, subprocess.TimeoutExpired, OSError) as e:
|
||||
print(f"Warning: Could not stop Mooncake master service: {e}")
|
||||
|
||||
@classmethod
|
||||
def _get_additional_server_args_and_env(cls):
|
||||
"""Get additional server arguments specific to configuration - override in subclasses"""
|
||||
|
||||
server_args = {
|
||||
"--tp-size": 2,
|
||||
"--hicache-ratio": 2,
|
||||
"--hicache-storage-backend": "mooncake",
|
||||
}
|
||||
|
||||
# Set the environment variables for Mooncake using dynamic ports
|
||||
env_vars = {
|
||||
"MOONCAKE_MASTER": f"127.0.0.1:{cls.mooncake_master_port}",
|
||||
"MOONCAKE_PROTOCOL": "tcp",
|
||||
"MC_MS_AUTO_DISC": "0",
|
||||
"MOONCAKE_DEVICE": "",
|
||||
"MOONCAKE_TE_META_DATA_SERVER": f"http://127.0.0.1:{cls.mooncake_metadata_port}/metadata",
|
||||
"MOONCAKE_GLOBAL_SEGMENT_SIZE": "4294967296", # 4 GiB
|
||||
}
|
||||
|
||||
return server_args, env_vars
|
||||
|
||||
|
||||
'''
|
||||
# Same as #10131, layer first layout test TODO(mateng): will make it work
|
||||
class TestMooncakeBackendLayerFirstLayout(
|
||||
HiCacheStorageMooncakeBackendBaseMixin, CustomTestCase
|
||||
):
|
||||
"""Layer first layout tests for HiCache-Mooncake backend"""
|
||||
|
||||
@classmethod
|
||||
def _get_additional_server_args_and_env(cls):
|
||||
"""Get additional server arguments specific to configuration - override in subclasses"""
|
||||
server_args, env_vars = super()._get_additional_server_args_and_env()
|
||||
server_args["--hicache-mem-layout"] = "layer_first"
|
||||
server_args["--hicache-io-backend"] = "direct"
|
||||
return server_args, env_vars
|
||||
'''
|
||||
|
||||
|
||||
@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.")
|
||||
class TestMooncakeBackendPageFirstLayout(
|
||||
HiCacheStorageMooncakeBackendBaseMixin, CustomTestCase
|
||||
):
|
||||
"""Page first layout tests for HiCache-Mooncake backend"""
|
||||
|
||||
@classmethod
|
||||
def _get_additional_server_args_and_env(cls):
|
||||
"""Get additional server arguments specific to configuration - override in subclasses"""
|
||||
server_args, env_vars = super()._get_additional_server_args_and_env()
|
||||
server_args["--hicache-mem-layout"] = "page_first"
|
||||
return server_args, env_vars
|
||||
|
||||
|
||||
class TestMooncakeBackendMLAModel(
|
||||
HiCacheStorageMooncakeBackendBaseMixin, CustomTestCase
|
||||
):
|
||||
"""MLA Model tests for HiCache-Mooncake backend"""
|
||||
|
||||
@classmethod
|
||||
def _get_model_name(cls):
|
||||
"""Use MLA model for testing"""
|
||||
return DEFAULT_MLA_MODEL_NAME_FOR_TEST
|
||||
|
||||
@classmethod
|
||||
def _get_additional_server_args_and_env(cls):
|
||||
"""Get additional server arguments specific to configuration - override in subclasses"""
|
||||
server_args, env_vars = super()._get_additional_server_args_and_env()
|
||||
server_args["--hicache-mem-layout"] = "page_first"
|
||||
server_args["--tp-size"] = 2
|
||||
return server_args, env_vars
|
||||
|
||||
|
||||
class TestMooncakeBackendAccuracy(
|
||||
HiCacheStorageMooncakeBackendBaseMixin, CustomTestCase
|
||||
):
|
||||
"""Accuracy tests for HiCache-Mooncake backend"""
|
||||
|
||||
@classmethod
|
||||
def _get_additional_server_args_and_env(cls):
|
||||
"""Get additional server arguments specific to configuration - override in subclasses"""
|
||||
server_args, env_vars = super()._get_additional_server_args_and_env()
|
||||
server_args["--hicache-ratio"] = 1.5
|
||||
server_args["--tp-size"] = 2
|
||||
server_args["--hicache-mem-layout"] = "page_first_direct"
|
||||
server_args["--hicache-io-backend"] = "direct"
|
||||
return server_args, env_vars
|
||||
|
||||
def test_eval_accuracy(self):
|
||||
"""Test eval accuracy with cache persistence across cache flushes"""
|
||||
from test_hicache_storage_file_backend import run_eval_accuracy_test
|
||||
|
||||
run_eval_accuracy_test(self)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
@@ -1,369 +0,0 @@
|
||||
"""
|
||||
E2E smoke test for HiCache storage runtime attach/detach.
|
||||
|
||||
This test launches an SGLang server with hierarchical cache enabled but WITHOUT
|
||||
any storage backend at startup, then attaches/detaches a storage backend via the
|
||||
HTTP endpoints.
|
||||
|
||||
Usage:
|
||||
python3 -m pytest test/registered/hicache/test_hicache_storage_runtime_attach_detach.py -v
|
||||
# TODO: Segmentation fault occurs when upgraded to Cu13. Ref: https://github.com/sgl-project/sglang/actions/runs/24603159715/job/71945537414?pr=23119")
|
||||
# Should move back to registered test after it's fixed
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from urllib import error, request
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
find_available_port,
|
||||
popen_launch_server,
|
||||
)
|
||||
from sglang.utils import wait_for_http_ready
|
||||
|
||||
register_cuda_ci(est_time=139, suite="stage-b-test-2-gpu-large")
|
||||
|
||||
|
||||
class TestHiCacheStorageRuntimeAttachDetach(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.temp_dir = tempfile.mkdtemp()
|
||||
cls.model = DEFAULT_MODEL_NAME_FOR_TEST
|
||||
# Use a per-test-class available port to reduce flakiness / conflicts.
|
||||
default_port = int(DEFAULT_URL_FOR_TEST.rsplit(":", 1)[1])
|
||||
cls.base_url = f"http://127.0.0.1:{find_available_port(default_port)}"
|
||||
|
||||
cls.other_args = [
|
||||
"--enable-hierarchical-cache",
|
||||
"--mem-fraction-static",
|
||||
"0.6",
|
||||
"--hicache-ratio",
|
||||
"1.2",
|
||||
"--hicache-size",
|
||||
"100",
|
||||
"--page-size",
|
||||
"64",
|
||||
"--enable-cache-report",
|
||||
# NOTE: do NOT pass --hicache-storage-backend* here
|
||||
]
|
||||
|
||||
cls.env = {
|
||||
**os.environ,
|
||||
# File backend uses this env var to decide where to store cache pages.
|
||||
"SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR": cls.temp_dir,
|
||||
# Make runs less flaky for CI/dev.
|
||||
"SGLANG_ENABLE_DETERMINISTIC_INFERENCE": "1",
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
import shutil
|
||||
|
||||
shutil.rmtree(cls.temp_dir, ignore_errors=True)
|
||||
|
||||
@classmethod
|
||||
def _wait_for_server_ready(
|
||||
cls, base_url: str, timeout: int = 60, process=None
|
||||
) -> bool:
|
||||
wait_for_http_ready(
|
||||
url=f"{base_url}/health",
|
||||
timeout=timeout,
|
||||
process=process,
|
||||
)
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _http_get(url: str, timeout: int = 10, headers: dict | None = None):
|
||||
try:
|
||||
req = request.Request(url, headers=headers or {}, method="GET")
|
||||
with request.urlopen(req, timeout=timeout) as resp:
|
||||
return resp.getcode(), resp.read().decode("utf-8", errors="replace")
|
||||
except error.HTTPError as e:
|
||||
body = e.read().decode("utf-8", errors="replace")
|
||||
return e.code, body
|
||||
|
||||
@staticmethod
|
||||
def _http_post_json(url: str, payload: dict | None = None, timeout: int = 30):
|
||||
data = None
|
||||
headers = {}
|
||||
if payload is not None:
|
||||
data = json.dumps(payload).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
req = request.Request(url, data=data, headers=headers, method="POST")
|
||||
try:
|
||||
with request.urlopen(req, timeout=timeout) as resp:
|
||||
return resp.getcode(), resp.read().decode("utf-8", errors="replace")
|
||||
except error.HTTPError as e:
|
||||
body = e.read().decode("utf-8", errors="replace")
|
||||
return e.code, body
|
||||
|
||||
@staticmethod
|
||||
def _http_post_json_with_headers(
|
||||
url: str,
|
||||
payload: dict | None = None,
|
||||
timeout: int = 30,
|
||||
headers: dict | None = None,
|
||||
):
|
||||
data = None
|
||||
all_headers = dict(headers or {})
|
||||
if payload is not None:
|
||||
data = json.dumps(payload).encode("utf-8")
|
||||
all_headers["Content-Type"] = "application/json"
|
||||
req = request.Request(url, data=data, headers=all_headers, method="POST")
|
||||
try:
|
||||
with request.urlopen(req, timeout=timeout) as resp:
|
||||
return resp.getcode(), resp.read().decode("utf-8", errors="replace")
|
||||
except error.HTTPError as e:
|
||||
body = e.read().decode("utf-8", errors="replace")
|
||||
return e.code, body
|
||||
|
||||
@staticmethod
|
||||
def _http_put_json_with_headers(
|
||||
url: str,
|
||||
payload: dict | None = None,
|
||||
timeout: int = 30,
|
||||
headers: dict | None = None,
|
||||
):
|
||||
data = None
|
||||
all_headers = dict(headers or {})
|
||||
if payload is not None:
|
||||
data = json.dumps(payload).encode("utf-8")
|
||||
all_headers["Content-Type"] = "application/json"
|
||||
req = request.Request(url, data=data, headers=all_headers, method="PUT")
|
||||
try:
|
||||
with request.urlopen(req, timeout=timeout) as resp:
|
||||
return resp.getcode(), resp.read().decode("utf-8", errors="replace")
|
||||
except error.HTTPError as e:
|
||||
body = e.read().decode("utf-8", errors="replace")
|
||||
return e.code, body
|
||||
|
||||
@staticmethod
|
||||
def _http_delete_with_headers(
|
||||
url: str, timeout: int = 30, headers: dict | None = None
|
||||
):
|
||||
all_headers = dict(headers or {})
|
||||
req = request.Request(url, headers=all_headers, method="DELETE")
|
||||
try:
|
||||
with request.urlopen(req, timeout=timeout) as resp:
|
||||
return resp.getcode(), resp.read().decode("utf-8", errors="replace")
|
||||
except error.HTTPError as e:
|
||||
body = e.read().decode("utf-8", errors="replace")
|
||||
return e.code, body
|
||||
|
||||
def _get_backend_status(self, base_url: str, headers: dict | None = None):
|
||||
code, body = self._http_get(
|
||||
f"{base_url}/hicache/storage-backend", timeout=10, headers=headers
|
||||
)
|
||||
self.assertEqual(code, 200, body)
|
||||
return json.loads(body)
|
||||
|
||||
def _attach_backend(
|
||||
self,
|
||||
base_url: str,
|
||||
backend: str,
|
||||
extra_cfg: dict,
|
||||
prefetch_policy: str = "timeout",
|
||||
write_policy: str = "write_through",
|
||||
headers: dict | None = None,
|
||||
):
|
||||
payload = {
|
||||
"hicache_storage_backend": backend,
|
||||
"hicache_storage_backend_extra_config_json": json.dumps(extra_cfg),
|
||||
"hicache_storage_prefetch_policy": prefetch_policy,
|
||||
"hicache_write_policy": write_policy,
|
||||
}
|
||||
return self._http_put_json_with_headers(
|
||||
f"{base_url}/hicache/storage-backend",
|
||||
payload,
|
||||
timeout=30,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
def _detach_backend(self, base_url: str, headers: dict | None = None):
|
||||
return self._http_delete_with_headers(
|
||||
f"{base_url}/hicache/storage-backend",
|
||||
timeout=30,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
def test_runtime_attach_detach(self):
|
||||
# Phase A: WITHOUT --admin-api-key, ADMIN_FORCE endpoints must be forbidden (403).
|
||||
process1 = popen_launch_server(
|
||||
self.model,
|
||||
self.base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=self.other_args,
|
||||
env=self.env,
|
||||
)
|
||||
try:
|
||||
self._wait_for_server_ready(self.base_url, process=process1)
|
||||
|
||||
code_info, _body_info = self._http_get(
|
||||
f"{self.base_url}/hicache/storage-backend", timeout=10
|
||||
)
|
||||
self.assertEqual(code_info, 400)
|
||||
code_attach_no_admin, _body_attach_no_admin = self._attach_backend(
|
||||
base_url=self.base_url, backend="file", extra_cfg={}
|
||||
)
|
||||
self.assertEqual(code_attach_no_admin, 400)
|
||||
code_detach_no_admin, _body_detach_no_admin = self._detach_backend(
|
||||
self.base_url
|
||||
)
|
||||
self.assertEqual(code_detach_no_admin, 400)
|
||||
finally:
|
||||
kill_process_tree(process1.pid)
|
||||
time.sleep(2)
|
||||
|
||||
# Phase B: WITH --admin-api-key, must provide Authorization: Bearer <admin_key>.
|
||||
admin_key = "sglang-test-admin-key"
|
||||
base_url2 = f"http://127.0.0.1:{find_available_port(int(self.base_url.rsplit(':', 1)[1]) + 1)}"
|
||||
other_args2 = list(self.other_args) + ["--admin-api-key", admin_key]
|
||||
process2 = popen_launch_server(
|
||||
self.model,
|
||||
base_url2,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=other_args2,
|
||||
env=self.env,
|
||||
)
|
||||
try:
|
||||
self._wait_for_server_ready(base_url2, process=process2)
|
||||
|
||||
# 1) Initially disabled (but unauthorized without admin key)
|
||||
code_info2_unauth, _ = self._http_get(
|
||||
f"{base_url2}/hicache/storage-backend", timeout=10
|
||||
)
|
||||
self.assertEqual(code_info2_unauth, 401)
|
||||
|
||||
admin_headers = {"Authorization": f"Bearer {admin_key}"}
|
||||
status0 = self._get_backend_status(base_url2, headers=admin_headers)
|
||||
self.assertIsNone(status0.get("hicache_storage_backend"))
|
||||
|
||||
# 2) Attach should succeed when idle
|
||||
extra_cfg = {
|
||||
"hicache_storage_pass_prefix_keys": True,
|
||||
# keep knobs small and stable
|
||||
"prefetch_threshold": 256,
|
||||
"prefetch_timeout_base": 3,
|
||||
"prefetch_timeout_per_ki_token": 0.01,
|
||||
}
|
||||
|
||||
# Unauthorized attach must fail.
|
||||
code_attach_unauth, _ = self._attach_backend(
|
||||
base_url=base_url2, backend="file", extra_cfg=extra_cfg
|
||||
)
|
||||
self.assertEqual(code_attach_unauth, 401)
|
||||
|
||||
code_attach, body_attach = self._attach_backend(
|
||||
base_url=base_url2,
|
||||
backend="file",
|
||||
extra_cfg=extra_cfg,
|
||||
prefetch_policy="timeout",
|
||||
write_policy="write_back",
|
||||
headers=admin_headers,
|
||||
)
|
||||
self.assertEqual(code_attach, 200, f"{code_attach} - {body_attach}")
|
||||
|
||||
status1 = self._get_backend_status(base_url2, headers=admin_headers)
|
||||
self.assertEqual(status1.get("hicache_storage_backend"), "file")
|
||||
self.assertEqual(
|
||||
status1.get("hicache_storage_backend_extra_config"),
|
||||
json.dumps(extra_cfg),
|
||||
)
|
||||
self.assertEqual(status1.get("hicache_storage_prefetch_policy"), "timeout")
|
||||
self.assertEqual(status1.get("hicache_write_policy"), "write_back")
|
||||
|
||||
# 3) Attach again succeeds with policies updated
|
||||
code_attach_again, body_attach_again = self._attach_backend(
|
||||
base_url=base_url2,
|
||||
backend="file",
|
||||
extra_cfg=extra_cfg,
|
||||
prefetch_policy="wait_complete",
|
||||
write_policy="write_through_selective",
|
||||
headers=admin_headers,
|
||||
)
|
||||
self.assertEqual(
|
||||
code_attach_again, 200, f"{code_attach_again} - {body_attach_again}"
|
||||
)
|
||||
|
||||
status2 = self._get_backend_status(base_url2, headers=admin_headers)
|
||||
self.assertEqual(
|
||||
status2.get("hicache_storage_backend_extra_config"),
|
||||
json.dumps(extra_cfg),
|
||||
)
|
||||
self.assertEqual(
|
||||
status2.get("hicache_storage_prefetch_policy"), "wait_complete"
|
||||
)
|
||||
self.assertEqual(
|
||||
status2.get("hicache_write_policy"), "write_through_selective"
|
||||
)
|
||||
|
||||
# 4) Attach again with different backend should be rejected
|
||||
code_attach_again, body_attach_again = self._attach_backend(
|
||||
base_url=base_url2,
|
||||
backend="mooncake",
|
||||
extra_cfg=extra_cfg,
|
||||
headers=admin_headers,
|
||||
)
|
||||
self.assertNotEqual(code_attach_again, 200, body_attach_again)
|
||||
|
||||
# 5) Detach should succeed and be idempotent
|
||||
code_detach, body_detach = self._detach_backend(
|
||||
base_url2, headers=admin_headers
|
||||
)
|
||||
self.assertEqual(code_detach, 200, f"{code_detach} - {body_detach}")
|
||||
status3 = self._get_backend_status(base_url2, headers=admin_headers)
|
||||
self.assertIsNone(status3.get("hicache_storage_backend"))
|
||||
self.assertEqual(
|
||||
status3.get("hicache_storage_prefetch_policy"), "wait_complete"
|
||||
)
|
||||
self.assertEqual(
|
||||
status3.get("hicache_write_policy"), "write_through_selective"
|
||||
)
|
||||
|
||||
code_detach_again, body_detach_again = self._detach_backend(
|
||||
base_url2, headers=admin_headers
|
||||
)
|
||||
self.assertEqual(
|
||||
code_detach_again,
|
||||
200,
|
||||
f"{code_detach_again} - {body_detach_again}",
|
||||
)
|
||||
|
||||
# 6) Re-attach after detach should succeed
|
||||
code_attach2, body_attach2 = self._attach_backend(
|
||||
base_url=base_url2,
|
||||
backend="file",
|
||||
extra_cfg=extra_cfg,
|
||||
headers=admin_headers,
|
||||
)
|
||||
self.assertEqual(code_attach2, 200, f"{code_attach2} - {body_attach2}")
|
||||
status4 = self._get_backend_status(base_url2, headers=admin_headers)
|
||||
self.assertEqual(status4.get("hicache_storage_backend"), "file")
|
||||
self.assertEqual(
|
||||
status4.get("hicache_storage_backend_extra_config"),
|
||||
json.dumps(extra_cfg),
|
||||
)
|
||||
self.assertEqual(status4.get("hicache_storage_prefetch_policy"), "timeout")
|
||||
self.assertEqual(status4.get("hicache_write_policy"), "write_through")
|
||||
|
||||
# Cleanup: detach for test isolation
|
||||
code_detach2, body_detach2 = self._detach_backend(
|
||||
base_url2, headers=admin_headers
|
||||
)
|
||||
self.assertEqual(code_detach2, 200, f"{code_detach2} - {body_detach2}")
|
||||
finally:
|
||||
kill_process_tree(process2.pid)
|
||||
time.sleep(2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
@@ -1,138 +0,0 @@
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=450, suite="stage-b-test-1-gpu-large")
|
||||
register_amd_ci(est_time=524, suite="stage-b-test-1-gpu-small-amd")
|
||||
"""
|
||||
Consolidated HiCache variant tests.
|
||||
Tests HiCache with different configurations: standard, MLA, EAGLE, and page size variants.
|
||||
# TODO: Segmentation fault occurs when upgraded to Cu13. Ref: https://github.com/sgl-project/sglang/actions/runs/24603159715/job/71945537414?pr=23119")
|
||||
# Should move back to registered test after it's fixed
|
||||
"""
|
||||
|
||||
import unittest
|
||||
|
||||
from sglang.benchmark.utils import get_tokenizer
|
||||
from sglang.srt.utils import is_hip, kill_process_tree
|
||||
from sglang.test.kits.eval_accuracy_kit import MGSMEnMixin, MMLUMixin
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_DRAFT_MODEL_EAGLE3,
|
||||
DEFAULT_MLA_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_TARGET_MODEL_EAGLE3,
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
_is_hip = is_hip()
|
||||
|
||||
|
||||
class HiCacheBaseServer(CustomTestCase):
|
||||
"""Base class for HiCache tests with configurable server setup"""
|
||||
|
||||
model_name = DEFAULT_MODEL_NAME_FOR_TEST
|
||||
hicache_args = []
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = cls.model_name
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
|
||||
# Setup tokenizer if needed by subclass
|
||||
if hasattr(cls, "needs_tokenizer") and cls.needs_tokenizer:
|
||||
cls.tokenizer = get_tokenizer(cls.model)
|
||||
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=cls.hicache_args,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
kill_process_tree(cls.process.pid)
|
||||
|
||||
|
||||
class TestHiCacheStandard(HiCacheBaseServer, MMLUMixin):
|
||||
"""Standard HiCache configuration tests"""
|
||||
|
||||
model_name = DEFAULT_MODEL_NAME_FOR_TEST
|
||||
hicache_args = [
|
||||
"--enable-hierarchical-cache",
|
||||
"--mem-fraction-static",
|
||||
0.7,
|
||||
"--hicache-size",
|
||||
100 if not _is_hip else 200,
|
||||
]
|
||||
mmlu_score_threshold = 0.65
|
||||
mmlu_num_examples = 64
|
||||
mmlu_num_threads = 32
|
||||
|
||||
|
||||
class TestHiCacheMLA(HiCacheBaseServer, MMLUMixin, MGSMEnMixin):
|
||||
"""HiCache with MLA model tests"""
|
||||
|
||||
model_name = DEFAULT_MLA_MODEL_NAME_FOR_TEST
|
||||
hicache_args = [
|
||||
"--trust-remote-code",
|
||||
"--enable-hierarchical-cache",
|
||||
] + (["--hicache-size", 200] if _is_hip else ["--hicache-ratio", 2])
|
||||
mmlu_score_threshold = 0.5
|
||||
mmlu_num_examples = 64
|
||||
mmlu_num_threads = 32
|
||||
mgsm_en_score_threshold = 0.8
|
||||
|
||||
|
||||
@unittest.skipIf(is_hip(), "Disabled for AMD-aiter")
|
||||
class TestHiCacheEagle(HiCacheBaseServer, MMLUMixin):
|
||||
"""HiCache with EAGLE speculative decoding tests"""
|
||||
|
||||
model_name = DEFAULT_TARGET_MODEL_EAGLE3
|
||||
needs_tokenizer = True
|
||||
hicache_args = [
|
||||
"--enable-hierarchical-cache",
|
||||
"--hicache-ratio",
|
||||
1.2,
|
||||
"--mem-fraction-static",
|
||||
0.7,
|
||||
"--speculative-algorithm",
|
||||
"EAGLE3",
|
||||
"--speculative-draft-model-path",
|
||||
DEFAULT_DRAFT_MODEL_EAGLE3,
|
||||
"--speculative-num-steps",
|
||||
2,
|
||||
"--speculative-eagle-topk",
|
||||
1,
|
||||
"--speculative-num-draft-tokens",
|
||||
3,
|
||||
"--dtype",
|
||||
"float16",
|
||||
"--chunked-prefill-size",
|
||||
1024,
|
||||
]
|
||||
mmlu_score_threshold = 0.72
|
||||
mmlu_num_examples = 64
|
||||
mmlu_num_threads = 32
|
||||
mmlu_accept_length_thres = 2.26
|
||||
|
||||
|
||||
class TestHiCachePage(HiCacheBaseServer, MMLUMixin):
|
||||
"""HiCache with custom page size tests"""
|
||||
|
||||
model_name = DEFAULT_MODEL_NAME_FOR_TEST
|
||||
hicache_args = [
|
||||
"--enable-hierarchical-cache",
|
||||
"--page-size",
|
||||
32,
|
||||
"--hicache-write-policy",
|
||||
"write_back",
|
||||
]
|
||||
mmlu_score_threshold = 0.65
|
||||
mmlu_num_examples = 64
|
||||
mmlu_num_threads = 32
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user