minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

chore: bump sglang-kernel version to 0.4.1.post1 (#23733)

Browse Source 
Co-authored-by: sglang-bot <sglang-bot@users.noreply.github.com>
Co-authored-by: Kangyan Zhou <zky314343421@gmail.com>
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
sglang-bot
2026-04-25 23:23:49 -07:00
committed by GitHub
co-authored by sglang-bot Kangyan Zhou Claude Opus 4.7
parent 8efa177f1e
commit 9003f24e2b
10 changed files with 6 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/manual/4-gpu-models/test_qwen35_hicache.py
-135
View File
@@ -1,135 +0,0 @@
"""
# TODO: Fails on cu13 venv migration. Ref: https://github.com/sgl-project/sglang/actions/runs/24616960626/job/71980705674?pr=23119
# Should move back to registered test after it's fixed
"""
import shutil
import tempfile
import unittest
from types import SimpleNamespace
import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
# This eval harness applies the chat_template, which is critical for qwen3.5
# to get good accuracy on gsm8k
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
register_cuda_ci(est_time=540, suite="stage-c-test-4-gpu-h100")
QWEN35_27B_MODEL = "Qwen/Qwen3.5-27B"
ACC_THRESHOLDS = {QWEN35_27B_MODEL: {"gsm8k": 0.8}}
class TestQwen35WithHiCache(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.model = QWEN35_27B_MODEL
cls.base_url = DEFAULT_URL_FOR_TEST
cls.storage_dir = tempfile.mkdtemp(prefix="qwen35-hicache-")
env = {
"SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR": cls.storage_dir,
}
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
env=env,
other_args=[
"--tp-size",
"4",
"--max-mamba-cache-size",
"500",
"--max-total-tokens",
"120000",
"--chunked-prefill-size",
"2048",
"--mamba-scheduler-strategy",
"extra_buffer",
"--mamba-track-interval",
"128",
"--mamba-ssm-dtype",
"bfloat16",
"--max-running-requests",
"128",
"--reasoning-parser",
"qwen3",
"--model-loader-extra-config",
'{"enable_multithread_load": true,"num_threads": 64}',
"--hicache-mem-layout",
"page_first_direct",
"--enable-hierarchical-cache",
"--hicache-ratio",
"2",
"--hicache-size",
"0",
"--hicache-write-policy",
"write_through",
"--hicache-storage-backend",
"file",
"--hicache-storage-prefetch-policy",
"wait_complete",
],
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
shutil.rmtree(cls.storage_dir, ignore_errors=True)
def _run_gsm8k(self):
args = SimpleNamespace(
model=self.model,
eval_name="gsm8k",
num_shots=5,
num_examples=100,
max_tokens=16000,
num_threads=50,
repeat=1,
temperature=0.6,
top_p=0.95,
top_k=20,
base_url=self.base_url,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
return run_eval(args)
def test_gsm8k(self):
first_metrics = self._run_gsm8k()
print(f"first_metrics={first_metrics}")
self.assertGreaterEqual(
first_metrics["score"], ACC_THRESHOLDS[self.model]["gsm8k"]
)
print(f"flush cache")
res = requests.post(
f"{self.base_url}/flush_cache",
params={"timeout": 30},
timeout=40,
)
res.raise_for_status()
second_metrics = self._run_gsm8k()
print(f"second_metrics={second_metrics}")
self.assertGreaterEqual(
second_metrics["score"], ACC_THRESHOLDS[self.model]["gsm8k"]
)
self.assertLessEqual(
abs(second_metrics["score"] - first_metrics["score"]),
0.05,
f"HiCache prefetch accuracy drift too large: "
f"first={first_metrics['score']}, second={second_metrics['score']}",
)
if __name__ == "__main__":
unittest.main()
test/manual/hicache/test_hicache_storage.py
-60
View File
@@ -1,60 +0,0 @@
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
register_cuda_ci(est_time=99, suite="stage-b-test-1-gpu-small")
register_amd_ci(est_time=300, suite="stage-b-test-1-gpu-small-amd")
"""
# TODO: Segmentation fault occurs when upgraded to Cu13. Ref: https://github.com/sgl-project/sglang/actions/runs/24603159715/job/71945537414?pr=23119")
# Should move back to registered test after it's fixed
"""
import time
import unittest
from sglang.srt.utils import is_hip, kill_process_tree
from sglang.test.kits.eval_accuracy_kit import MMLUMixin
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
_is_hip = is_hip()
class TestHiCache(CustomTestCase, MMLUMixin):
mmlu_score_threshold = 0.65
mmlu_num_examples = 64
mmlu_num_threads = 32
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_MODEL_NAME_FOR_TEST
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--enable-hierarchical-cache",
"--mem-fraction-static",
0.7,
"--hicache-size",
100 if not _is_hip else 200,
"--page-size",
"64",
"--hicache-storage-backend",
"file",
],
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
time.sleep(5)
if __name__ == "__main__":
unittest.main()
test/manual/hicache/test_hicache_storage_3fs_backend.py
-94
View File
@@ -1,94 +0,0 @@
"""
Benchmark tests for HiCache Storage with 3FS backend.
Usage:
python3 -m pytest test/registered/hicache/test_hicache_storage_3fs_backend.py -v
# TODO: Segmentation fault occurs when upgraded to Cu13. Ref: https://github.com/sgl-project/sglang/actions/runs/24603159715/job/71945537414?pr=23119")
# Should move back to registered test after it's fixed
"""
import json
import os
import unittest
from test_hicache_storage_file_backend import HiCacheStorageBaseMixin
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=150, suite="stage-b-test-2-gpu-large")
register_amd_ci(est_time=300, suite="stage-b-test-2-gpu-large")
class HiCacheStorage3FSBackendBaseMixin(HiCacheStorageBaseMixin):
"""Base mixin class with common setup and utilities"""
@classmethod
def _get_additional_server_args_and_env(cls):
"""Get additional server arguments specific to configuration - override in subclasses"""
# Create a temporary JSON config file for HF3FS
hf3fs_config = {
"file_path_prefix": os.path.join(cls.temp_dir, "hicache"),
"file_size": 1024 * 1024 * 1024 * 2,
"numjobs": 2,
"entries": 8,
"use_mock_hf3fs_client": True,
"hicache_storage_pass_prefix_keys": True,
}
# Write config to temporary file
config_file = os.path.join(cls.temp_dir, "hf3fs_config.json")
with open(config_file, "w") as f:
json.dump(hf3fs_config, f, indent=2)
server_args = {
"--tp-size": 1,
"--hicache-ratio": 1.2,
"--hicache-storage-backend": "hf3fs",
"--hicache-storage-backend-extra-config": json.dumps(hf3fs_config),
}
# Set the environment variable to point to our config file
env_vars = {
"SGLANG_HICACHE_HF3FS_CONFIG_PATH": config_file,
}
return server_args, env_vars
class TestHf3fsBackendLayerFirstLayout(
HiCacheStorage3FSBackendBaseMixin, CustomTestCase
):
"""Layer first layout tests for HiCache-Hf3fs backend"""
@classmethod
def _get_additional_server_args_and_env(cls):
"""Get additional server arguments specific to configuration - override in subclasses"""
server_args, env_vars = super()._get_additional_server_args_and_env()
server_args["--hicache-mem-layout"] = "layer_first"
server_args["--hicache-io-backend"] = "direct"
server_args["--tp-size"] = 2
return server_args, env_vars
class TestHf3fsBackendAccuracy(HiCacheStorage3FSBackendBaseMixin, CustomTestCase):
"""Accuracy tests for HiCache-Hf3fs backend"""
@classmethod
def _get_additional_server_args_and_env(cls):
"""Get additional server arguments specific to configuration - override in subclasses"""
server_args, env_vars = super()._get_additional_server_args_and_env()
server_args["--hicache-ratio"] = 1.5
server_args["--tp-size"] = 2
server_args["--hicache-mem-layout"] = "page_first_direct"
server_args["--hicache-io-backend"] = "direct"
return server_args, env_vars
def test_eval_accuracy(self):
"""Test eval accuracy with cache persistence across cache flushes"""
from test_hicache_storage_file_backend import run_eval_accuracy_test
run_eval_accuracy_test(self)
if __name__ == "__main__":
unittest.main(verbosity=2)
test/manual/hicache/test_hicache_storage_file_backend.py
-336
View File
@@ -1,336 +0,0 @@
"""
E2E tests for HiCache Storage functionality.
Usage:
python3 -m pytest test/registered/hicache/test_hicache_storage_file_backend.py -v
# TODO: Segmentation fault occurs when upgraded to Cu13. Ref: https://github.com/sgl-project/sglang/actions/runs/24603159715/job/71945537414?pr=23119")
# Should move back to registered test after it's fixed
"""
import json
import os
import random
import tempfile
import time
import unittest
from types import SimpleNamespace
from typing import Dict
from urllib.parse import urlparse
import requests
from sglang.benchmark.utils import get_tokenizer
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_MLA_MODEL_NAME_FOR_TEST,
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
)
from sglang.utils import wait_for_http_ready
register_cuda_ci(est_time=148, suite="stage-b-test-2-gpu-large")
register_amd_ci(est_time=526, suite="stage-b-test-2-gpu-large-amd")
class HiCacheStorageBaseMixin:
"""Base mixin class with common setup and utilities"""
@classmethod
def setUpClass(cls):
"""Set up test environment and launch server once for all tests"""
cls.temp_dir = tempfile.mkdtemp()
cls.model = cls._get_model_name()
cls.base_url = DEFAULT_URL_FOR_TEST
parsed_url = urlparse(cls.base_url)
cls.base_host = parsed_url.hostname
cls.base_port = str(parsed_url.port)
# Prepare tokenizer for prompt generation
cls.tokenizer = get_tokenizer(cls.model)
# Launch server with HiCache enabled and cache report
cls.process = cls._launch_server_with_hicache()
cls._wait_for_server_ready(process=cls.process)
print(f"Test server launched successfully at {cls.base_url}")
print(f"Cache directory: {cls.temp_dir}")
@classmethod
def tearDownClass(cls):
"""Clean up test environment"""
if hasattr(cls, "process") and cls.process:
kill_process_tree(cls.process.pid)
import shutil
if hasattr(cls, "temp_dir"):
shutil.rmtree(cls.temp_dir, ignore_errors=True)
@classmethod
def _get_model_name(cls):
"""Get model name for the test configuration - override in subclasses"""
return DEFAULT_MODEL_NAME_FOR_TEST
@classmethod
def _get_base_server_args(cls):
"""Get base server arguments - can be extended in subclasses"""
extra_config = {
"hicache_storage_pass_prefix_keys": True,
}
return {
"--enable-hierarchical-cache": True,
"--mem-fraction-static": 0.6,
"--hicache-ratio": 1.2,
"--page-size": 64,
"--enable-cache-report": True,
"--hicache-storage-prefetch-policy": "wait_complete",
"--hicache-storage-backend": "file",
"--hicache-storage-backend-extra-config": json.dumps(extra_config),
}
@classmethod
def _get_additional_server_args_and_env(cls):
"""Get additional server arguments specific to configuration - override in subclasses"""
return {}, {"SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR": cls.temp_dir}
@classmethod
def _launch_server_with_hicache(cls):
"""Launch server with HiCache enabled"""
additional_server_args, env_vars = cls._get_additional_server_args_and_env()
env_vars["SGLANG_ENABLE_DETERMINISTIC_INFERENCE"] = "1"
server_args = cls._get_base_server_args()
if additional_server_args:
server_args.update(additional_server_args)
final_server_args = []
for k, v in server_args.items():
if isinstance(v, bool):
final_server_args.append(str(k))
else:
final_server_args.append(str(k))
final_server_args.append(str(v))
print(f"final_server_args: {final_server_args}")
env_vars = {
**os.environ,
**env_vars,
}
return popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=final_server_args,
env=env_vars,
)
@classmethod
def _wait_for_server_ready(cls, timeout: int = 60, process=None) -> bool:
"""Wait for server to be ready"""
wait_for_http_ready(
url=f"{cls.base_url}/health",
timeout=timeout,
process=process,
)
return True
def send_request(
self, prompt: str, max_tokens: int = 100, temperature: float = 0.0
) -> Dict:
"""Send a generate request and return response"""
response = requests.post(
f"{self.base_url}/generate",
json={
"text": prompt,
"sampling_params": {
"temperature": temperature,
"max_new_tokens": max_tokens,
"ignore_eos": True,
},
},
timeout=60,
)
self.assertEqual(
response.status_code,
200,
f"Request failed: {response.status_code} - {response.text}",
)
return response.json()
def get_cached_tokens(self, response_json: Dict) -> int:
"""Extract cached tokens count from /generate response"""
meta = response_json.get("meta_info", {})
return int(meta.get("cached_tokens", 0))
def flush_cache(self):
"""Flush device cache to force remote storage access."""
res = requests.post(
f"{self.base_url}/flush_cache",
params={"timeout": 30},
timeout=40,
)
res.raise_for_status()
def gen_prompt(self, token_num: int) -> str:
"""Generate a random prompt of specified token length using tokenizer vocabulary."""
all_available_tokens = list(self.tokenizer.get_vocab().values())
selected_tokens = random.choices(all_available_tokens, k=token_num)
return self.tokenizer.decode(selected_tokens)
def trigger_offloading_and_flush(self):
"""Helper method to trigger offloading and flush cache"""
# Trigger offloading
self.send_request(self.gen_prompt(1), max_tokens=150)
# Flush device cache to force remote storage access
self.flush_cache()
def test_basic_backup_and_prefetch(self):
"""Test storage and retrieval of large context through remote cache"""
print("\n=== Testing Large Context Cache Storage & Retrieval ===")
# Generate substantial context that will be cached
base_prompt = self.gen_prompt(768)
# First request - populate cache
print("Step 1: Populating cache with large context...")
response1 = self.send_request(base_prompt, max_tokens=150)
self.assertIsNotNone(response1)
# Flush device cache to force remote storage access
self.trigger_offloading_and_flush()
# Second request with extended prompt - should hit remote cache
print("Step 2: Testing cache hit from remote storage...")
start_time = time.time()
response2 = self.send_request(base_prompt, max_tokens=150)
retrieval_time = time.time() - start_time
cached_tokens = self.get_cached_tokens(response2)
print(
f"Remote cache retrieval time: {retrieval_time:.3f}s, cached_tokens={cached_tokens}"
)
# Assert cached tokens indicate a remote hit
self.assertGreater(
cached_tokens, 700, "Expected significant cached tokens for remote hit"
)
@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.")
class TestHiCacheStoragePageFirstLayout(HiCacheStorageBaseMixin, CustomTestCase):
"""Page first layout tests for HiCache Storage functionality"""
@classmethod
def _get_additional_server_args_and_env(cls):
"""Get additional server arguments specific to configuration - override in subclasses"""
server_args = {"--hicache-mem-layout": "page_first"}
return server_args, {}
@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.")
class TestHiCacheStorageMLA(HiCacheStorageBaseMixin, CustomTestCase):
"""MLA Model tests for HiCache Storage functionality"""
@classmethod
def _get_model_name(cls):
"""Use MLA model for testing"""
return DEFAULT_MLA_MODEL_NAME_FOR_TEST
@classmethod
def _get_additional_server_args_and_env(cls):
"""Get additional server arguments specific to configuration - override in subclasses"""
server_args = {"--tp-size": 2}
return server_args, {}
class TestHiCacheStoragePageFirstDirectIO(HiCacheStorageBaseMixin, CustomTestCase):
"""Page first direct tests for HiCache Storage functionality"""
@classmethod
def _get_additional_server_args_and_env(cls):
"""Get additional server arguments specific to configuration - override in subclasses"""
server_args = {
"--hicache-mem-layout": "page_first_direct",
"--hicache-io-backend": "direct",
"--tp-size": 2,
}
return server_args, {}
class TestHiCacheStorageAccuracy(HiCacheStorageBaseMixin, CustomTestCase):
"""Accuracy tests for HiCache Storage functionality"""
@classmethod
def _get_additional_server_args_and_env(cls):
"""Get additional server arguments specific to configuration - override in subclasses"""
server_args = {
"--tp-size": 2,
"--hicache-ratio": 1.5,
}
return server_args, {}
def test_eval_accuracy(self):
"""Test eval accuracy with cache persistence across cache flushes"""
run_eval_accuracy_test(self)
def run_eval_accuracy_test(test_instance, accuracy_threshold: float = 0.03):
"""Generic eval accuracy test with configurable accuracy threshold
Args:
test_instance: The test class instance that provides base_host, base_port, flush_cache, and assert methods
"""
print("\n=== Testing Eval Accuracy with Cache Persistence ===")
# First evaluation - populate cache
print("Phase 1: Running initial GSM8K evaluation to populate cache...")
args_initial = SimpleNamespace(
base_url=f"http://{test_instance.base_host}:{test_instance.base_port}",
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=200,
num_threads=64,
)
metrics_initial = run_eval(args_initial)
# Flush cache to force remote storage access
print("Phase 2: Flushing device cache...")
test_instance.flush_cache()
# Second evaluation - should use remote cache
print("Phase 3: Running second GSM8K evaluation using remote cache...")
metrics_cached = run_eval(args_initial)
# Verify accuracy consistency
accuracy_diff = abs(metrics_initial["score"] - metrics_cached["score"])
print(f"Accuracy difference: {accuracy_diff:.4f}")
# Assertions
test_instance.assertGreater(
metrics_initial["score"], 0.6, "Initial accuracy should be reasonable"
)
test_instance.assertGreater(
metrics_cached["score"], 0.6, "Cached accuracy should be reasonable"
)
test_instance.assertLess(
accuracy_diff,
accuracy_threshold,
"Accuracy should be consistent between cache states",
)
if __name__ == "__main__":
unittest.main(verbosity=2)
test/manual/hicache/test_hicache_storage_mooncake_backend.py
-286
View File
@@ -1,286 +0,0 @@
"""
Benchmark tests for HiCache Storage with Mooncake backend.
Usage:
python3.10 -m pytest test/registered/hicache/test_hicache_storage_mooncake_backend.py -v
"""
# TODO: Segmentation fault occurs when upgraded to Cu13. Ref: https://github.com/sgl-project/sglang/actions/runs/24601791606/job/71942123195?pr=23119")
# Should move back to registered test after it's fixed
import os
import subprocess
import time
import unittest
import requests
from test_hicache_storage_file_backend import HiCacheStorageBaseMixin
from sglang.test.test_utils import (
DEFAULT_MLA_MODEL_NAME_FOR_TEST,
CustomTestCase,
find_available_port,
is_in_ci,
)
class HiCacheStorageMooncakeBackendBaseMixin(HiCacheStorageBaseMixin):
"""Base mixin class with common setup and utilities"""
# Default port ranges for Mooncake services - can be overridden in subclasses
mooncake_master_port_base = 50051
mooncake_metadata_port_base = 8080
@classmethod
def setUpClass(cls):
"""Set up test environment and launch Mooncake services before server setup"""
# Find available ports for Mooncake services to avoid conflicts
cls.mooncake_master_port = find_available_port(
HiCacheStorageMooncakeBackendBaseMixin.mooncake_master_port_base
)
cls.mooncake_metadata_port = find_available_port(
HiCacheStorageMooncakeBackendBaseMixin.mooncake_metadata_port_base
)
# Start Mooncake services first
cls._start_mooncake_services()
# Call parent setup
super().setUpClass()
@classmethod
def tearDownClass(cls):
"""Clean up Mooncake services after server teardown"""
# Call parent teardown first
super().tearDownClass()
# Stop Mooncake services
cls._stop_mooncake_services()
@classmethod
def _start_mooncake_services(cls):
"""Start Mooncake metadata and master services with configurable ports and readiness detection"""
print("Starting Mooncake services...")
print(
f"Using master port: {cls.mooncake_master_port}, metadata port: {cls.mooncake_metadata_port}"
)
# Start metadata service with configurable port
try:
# Start metadata server with port configuration
cls.metadata_service_process = subprocess.Popen(
[
"python3",
"-m",
"mooncake.http_metadata_server",
"--port",
str(cls.mooncake_metadata_port),
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
preexec_fn=os.setsid, # Create new process group
)
print(
f"Mooncake metadata service started on port {cls.mooncake_metadata_port}"
)
except (FileNotFoundError, subprocess.SubprocessError) as e:
print(f"Warning: Could not start Mooncake metadata service: {e}")
cls.metadata_service_process = None
# Start master service with configurable port
try:
# Start master server with port configuration
cls.master_service_process = subprocess.Popen(
["mooncake_master", "--port", str(cls.mooncake_master_port)],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
preexec_fn=os.setsid, # Create new process group
)
print(f"Mooncake master service started on port {cls.mooncake_master_port}")
except (FileNotFoundError, subprocess.SubprocessError) as e:
print(f"Warning: Could not start Mooncake master service: {e}")
cls.master_service_process = None
# Wait for services to be ready instead of fixed sleep
cls._wait_for_mooncake_services_ready()
@classmethod
def _wait_for_mooncake_services_ready(cls, timeout: int = 30) -> bool:
"""Wait for Mooncake services to be ready by checking their endpoints"""
print("Waiting for Mooncake services to be ready...")
start_time = time.time()
services_ready = False
while time.time() - start_time < timeout:
try:
# Check metadata service
metadata_ready = False
if (
cls.metadata_service_process
and cls.metadata_service_process.poll() is None
):
try:
# Try to connect to the metadata service
metadata_url = (
f"http://127.0.0.1:{cls.mooncake_metadata_port}/metadata"
)
response = requests.get(metadata_url, timeout=2)
if response.status_code == 200:
metadata_ready = True
print("Mooncake metadata service is ready")
except (requests.RequestException, ConnectionError):
# Service might not be fully started yet
pass
# Check master service (if it has a health endpoint)
master_ready = False
if (
cls.master_service_process
and cls.master_service_process.poll() is None
):
# For now, we'll assume master service is ready if process is running
# and it's been a few seconds since startup
if (
time.time() - start_time > 5
): # Give master service time to initialize
master_ready = True
print("Mooncake master service is ready")
# Both services should be ready
if metadata_ready and master_ready:
services_ready = True
print("All Mooncake services are ready")
break
except Exception as e:
print(f"Error checking service readiness: {e}")
time.sleep(2)
if not services_ready:
print(
"Warning: Mooncake services may not be fully ready, continuing anyway..."
)
return services_ready
@classmethod
def _stop_mooncake_services(cls):
"""Stop Mooncake services"""
print("Stopping Mooncake services...")
# Stop metadata service
if hasattr(cls, "metadata_service_process") and cls.metadata_service_process:
try:
os.killpg(os.getpgid(cls.metadata_service_process.pid), 9)
cls.metadata_service_process.wait(timeout=5)
print("Mooncake metadata service stopped")
except (ProcessLookupError, subprocess.TimeoutExpired, OSError) as e:
print(f"Warning: Could not stop Mooncake metadata service: {e}")
# Stop master service
if hasattr(cls, "master_service_process") and cls.master_service_process:
try:
os.killpg(os.getpgid(cls.master_service_process.pid), 9)
cls.master_service_process.wait(timeout=5)
print("Mooncake master service stopped")
except (ProcessLookupError, subprocess.TimeoutExpired, OSError) as e:
print(f"Warning: Could not stop Mooncake master service: {e}")
@classmethod
def _get_additional_server_args_and_env(cls):
"""Get additional server arguments specific to configuration - override in subclasses"""
server_args = {
"--tp-size": 2,
"--hicache-ratio": 2,
"--hicache-storage-backend": "mooncake",
}
# Set the environment variables for Mooncake using dynamic ports
env_vars = {
"MOONCAKE_MASTER": f"127.0.0.1:{cls.mooncake_master_port}",
"MOONCAKE_PROTOCOL": "tcp",
"MC_MS_AUTO_DISC": "0",
"MOONCAKE_DEVICE": "",
"MOONCAKE_TE_META_DATA_SERVER": f"http://127.0.0.1:{cls.mooncake_metadata_port}/metadata",
"MOONCAKE_GLOBAL_SEGMENT_SIZE": "4294967296", # 4 GiB
}
return server_args, env_vars
'''
# Same as #10131, layer first layout test TODO(mateng): will make it work
class TestMooncakeBackendLayerFirstLayout(
HiCacheStorageMooncakeBackendBaseMixin, CustomTestCase
):
"""Layer first layout tests for HiCache-Mooncake backend"""
@classmethod
def _get_additional_server_args_and_env(cls):
"""Get additional server arguments specific to configuration - override in subclasses"""
server_args, env_vars = super()._get_additional_server_args_and_env()
server_args["--hicache-mem-layout"] = "layer_first"
server_args["--hicache-io-backend"] = "direct"
return server_args, env_vars
'''
@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.")
class TestMooncakeBackendPageFirstLayout(
HiCacheStorageMooncakeBackendBaseMixin, CustomTestCase
):
"""Page first layout tests for HiCache-Mooncake backend"""
@classmethod
def _get_additional_server_args_and_env(cls):
"""Get additional server arguments specific to configuration - override in subclasses"""
server_args, env_vars = super()._get_additional_server_args_and_env()
server_args["--hicache-mem-layout"] = "page_first"
return server_args, env_vars
class TestMooncakeBackendMLAModel(
HiCacheStorageMooncakeBackendBaseMixin, CustomTestCase
):
"""MLA Model tests for HiCache-Mooncake backend"""
@classmethod
def _get_model_name(cls):
"""Use MLA model for testing"""
return DEFAULT_MLA_MODEL_NAME_FOR_TEST
@classmethod
def _get_additional_server_args_and_env(cls):
"""Get additional server arguments specific to configuration - override in subclasses"""
server_args, env_vars = super()._get_additional_server_args_and_env()
server_args["--hicache-mem-layout"] = "page_first"
server_args["--tp-size"] = 2
return server_args, env_vars
class TestMooncakeBackendAccuracy(
HiCacheStorageMooncakeBackendBaseMixin, CustomTestCase
):
"""Accuracy tests for HiCache-Mooncake backend"""
@classmethod
def _get_additional_server_args_and_env(cls):
"""Get additional server arguments specific to configuration - override in subclasses"""
server_args, env_vars = super()._get_additional_server_args_and_env()
server_args["--hicache-ratio"] = 1.5
server_args["--tp-size"] = 2
server_args["--hicache-mem-layout"] = "page_first_direct"
server_args["--hicache-io-backend"] = "direct"
return server_args, env_vars
def test_eval_accuracy(self):
"""Test eval accuracy with cache persistence across cache flushes"""
from test_hicache_storage_file_backend import run_eval_accuracy_test
run_eval_accuracy_test(self)
if __name__ == "__main__":
unittest.main(verbosity=2)
test/manual/hicache/test_hicache_storage_runtime_attach_detach.py
-369
View File
@@ -1,369 +0,0 @@
"""
E2E smoke test for HiCache storage runtime attach/detach.
This test launches an SGLang server with hierarchical cache enabled but WITHOUT
any storage backend at startup, then attaches/detaches a storage backend via the
HTTP endpoints.
Usage:
python3 -m pytest test/registered/hicache/test_hicache_storage_runtime_attach_detach.py -v
# TODO: Segmentation fault occurs when upgraded to Cu13. Ref: https://github.com/sgl-project/sglang/actions/runs/24603159715/job/71945537414?pr=23119")
# Should move back to registered test after it's fixed
"""
import json
import os
import tempfile
import time
import unittest
from urllib import error, request
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
find_available_port,
popen_launch_server,
)
from sglang.utils import wait_for_http_ready
register_cuda_ci(est_time=139, suite="stage-b-test-2-gpu-large")
class TestHiCacheStorageRuntimeAttachDetach(CustomTestCase):
@classmethod
def setUpClass(cls):
cls.temp_dir = tempfile.mkdtemp()
cls.model = DEFAULT_MODEL_NAME_FOR_TEST
# Use a per-test-class available port to reduce flakiness / conflicts.
default_port = int(DEFAULT_URL_FOR_TEST.rsplit(":", 1)[1])
cls.base_url = f"http://127.0.0.1:{find_available_port(default_port)}"
cls.other_args = [
"--enable-hierarchical-cache",
"--mem-fraction-static",
"0.6",
"--hicache-ratio",
"1.2",
"--hicache-size",
"100",
"--page-size",
"64",
"--enable-cache-report",
# NOTE: do NOT pass --hicache-storage-backend* here
]
cls.env = {
**os.environ,
# File backend uses this env var to decide where to store cache pages.
"SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR": cls.temp_dir,
# Make runs less flaky for CI/dev.
"SGLANG_ENABLE_DETERMINISTIC_INFERENCE": "1",
}
@classmethod
def tearDownClass(cls):
import shutil
shutil.rmtree(cls.temp_dir, ignore_errors=True)
@classmethod
def _wait_for_server_ready(
cls, base_url: str, timeout: int = 60, process=None
) -> bool:
wait_for_http_ready(
url=f"{base_url}/health",
timeout=timeout,
process=process,
)
return True
@staticmethod
def _http_get(url: str, timeout: int = 10, headers: dict | None = None):
try:
req = request.Request(url, headers=headers or {}, method="GET")
with request.urlopen(req, timeout=timeout) as resp:
return resp.getcode(), resp.read().decode("utf-8", errors="replace")
except error.HTTPError as e:
body = e.read().decode("utf-8", errors="replace")
return e.code, body
@staticmethod
def _http_post_json(url: str, payload: dict | None = None, timeout: int = 30):
data = None
headers = {}
if payload is not None:
data = json.dumps(payload).encode("utf-8")
headers["Content-Type"] = "application/json"
req = request.Request(url, data=data, headers=headers, method="POST")
try:
with request.urlopen(req, timeout=timeout) as resp:
return resp.getcode(), resp.read().decode("utf-8", errors="replace")
except error.HTTPError as e:
body = e.read().decode("utf-8", errors="replace")
return e.code, body
@staticmethod
def _http_post_json_with_headers(
url: str,
payload: dict | None = None,
timeout: int = 30,
headers: dict | None = None,
):
data = None
all_headers = dict(headers or {})
if payload is not None:
data = json.dumps(payload).encode("utf-8")
all_headers["Content-Type"] = "application/json"
req = request.Request(url, data=data, headers=all_headers, method="POST")
try:
with request.urlopen(req, timeout=timeout) as resp:
return resp.getcode(), resp.read().decode("utf-8", errors="replace")
except error.HTTPError as e:
body = e.read().decode("utf-8", errors="replace")
return e.code, body
@staticmethod
def _http_put_json_with_headers(
url: str,
payload: dict | None = None,
timeout: int = 30,
headers: dict | None = None,
):
data = None
all_headers = dict(headers or {})
if payload is not None:
data = json.dumps(payload).encode("utf-8")
all_headers["Content-Type"] = "application/json"
req = request.Request(url, data=data, headers=all_headers, method="PUT")
try:
with request.urlopen(req, timeout=timeout) as resp:
return resp.getcode(), resp.read().decode("utf-8", errors="replace")
except error.HTTPError as e:
body = e.read().decode("utf-8", errors="replace")
return e.code, body
@staticmethod
def _http_delete_with_headers(
url: str, timeout: int = 30, headers: dict | None = None
):
all_headers = dict(headers or {})
req = request.Request(url, headers=all_headers, method="DELETE")
try:
with request.urlopen(req, timeout=timeout) as resp:
return resp.getcode(), resp.read().decode("utf-8", errors="replace")
except error.HTTPError as e:
body = e.read().decode("utf-8", errors="replace")
return e.code, body
def _get_backend_status(self, base_url: str, headers: dict | None = None):
code, body = self._http_get(
f"{base_url}/hicache/storage-backend", timeout=10, headers=headers
)
self.assertEqual(code, 200, body)
return json.loads(body)
def _attach_backend(
self,
base_url: str,
backend: str,
extra_cfg: dict,
prefetch_policy: str = "timeout",
write_policy: str = "write_through",
headers: dict | None = None,
):
payload = {
"hicache_storage_backend": backend,
"hicache_storage_backend_extra_config_json": json.dumps(extra_cfg),
"hicache_storage_prefetch_policy": prefetch_policy,
"hicache_write_policy": write_policy,
}
return self._http_put_json_with_headers(
f"{base_url}/hicache/storage-backend",
payload,
timeout=30,
headers=headers,
)
def _detach_backend(self, base_url: str, headers: dict | None = None):
return self._http_delete_with_headers(
f"{base_url}/hicache/storage-backend",
timeout=30,
headers=headers,
)
def test_runtime_attach_detach(self):
# Phase A: WITHOUT --admin-api-key, ADMIN_FORCE endpoints must be forbidden (403).
process1 = popen_launch_server(
self.model,
self.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=self.other_args,
env=self.env,
)
try:
self._wait_for_server_ready(self.base_url, process=process1)
code_info, _body_info = self._http_get(
f"{self.base_url}/hicache/storage-backend", timeout=10
)
self.assertEqual(code_info, 400)
code_attach_no_admin, _body_attach_no_admin = self._attach_backend(
base_url=self.base_url, backend="file", extra_cfg={}
)
self.assertEqual(code_attach_no_admin, 400)
code_detach_no_admin, _body_detach_no_admin = self._detach_backend(
self.base_url
)
self.assertEqual(code_detach_no_admin, 400)
finally:
kill_process_tree(process1.pid)
time.sleep(2)
# Phase B: WITH --admin-api-key, must provide Authorization: Bearer <admin_key>.
admin_key = "sglang-test-admin-key"
base_url2 = f"http://127.0.0.1:{find_available_port(int(self.base_url.rsplit(':', 1)[1]) + 1)}"
other_args2 = list(self.other_args) + ["--admin-api-key", admin_key]
process2 = popen_launch_server(
self.model,
base_url2,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=other_args2,
env=self.env,
)
try:
self._wait_for_server_ready(base_url2, process=process2)
# 1) Initially disabled (but unauthorized without admin key)
code_info2_unauth, _ = self._http_get(
f"{base_url2}/hicache/storage-backend", timeout=10
)
self.assertEqual(code_info2_unauth, 401)
admin_headers = {"Authorization": f"Bearer {admin_key}"}
status0 = self._get_backend_status(base_url2, headers=admin_headers)
self.assertIsNone(status0.get("hicache_storage_backend"))
# 2) Attach should succeed when idle
extra_cfg = {
"hicache_storage_pass_prefix_keys": True,
# keep knobs small and stable
"prefetch_threshold": 256,
"prefetch_timeout_base": 3,
"prefetch_timeout_per_ki_token": 0.01,
}
# Unauthorized attach must fail.
code_attach_unauth, _ = self._attach_backend(
base_url=base_url2, backend="file", extra_cfg=extra_cfg
)
self.assertEqual(code_attach_unauth, 401)
code_attach, body_attach = self._attach_backend(
base_url=base_url2,
backend="file",
extra_cfg=extra_cfg,
prefetch_policy="timeout",
write_policy="write_back",
headers=admin_headers,
)
self.assertEqual(code_attach, 200, f"{code_attach} - {body_attach}")
status1 = self._get_backend_status(base_url2, headers=admin_headers)
self.assertEqual(status1.get("hicache_storage_backend"), "file")
self.assertEqual(
status1.get("hicache_storage_backend_extra_config"),
json.dumps(extra_cfg),
)
self.assertEqual(status1.get("hicache_storage_prefetch_policy"), "timeout")
self.assertEqual(status1.get("hicache_write_policy"), "write_back")
# 3) Attach again succeeds with policies updated
code_attach_again, body_attach_again = self._attach_backend(
base_url=base_url2,
backend="file",
extra_cfg=extra_cfg,
prefetch_policy="wait_complete",
write_policy="write_through_selective",
headers=admin_headers,
)
self.assertEqual(
code_attach_again, 200, f"{code_attach_again} - {body_attach_again}"
)
status2 = self._get_backend_status(base_url2, headers=admin_headers)
self.assertEqual(
status2.get("hicache_storage_backend_extra_config"),
json.dumps(extra_cfg),
)
self.assertEqual(
status2.get("hicache_storage_prefetch_policy"), "wait_complete"
)
self.assertEqual(
status2.get("hicache_write_policy"), "write_through_selective"
)
# 4) Attach again with different backend should be rejected
code_attach_again, body_attach_again = self._attach_backend(
base_url=base_url2,
backend="mooncake",
extra_cfg=extra_cfg,
headers=admin_headers,
)
self.assertNotEqual(code_attach_again, 200, body_attach_again)
# 5) Detach should succeed and be idempotent
code_detach, body_detach = self._detach_backend(
base_url2, headers=admin_headers
)
self.assertEqual(code_detach, 200, f"{code_detach} - {body_detach}")
status3 = self._get_backend_status(base_url2, headers=admin_headers)
self.assertIsNone(status3.get("hicache_storage_backend"))
self.assertEqual(
status3.get("hicache_storage_prefetch_policy"), "wait_complete"
)
self.assertEqual(
status3.get("hicache_write_policy"), "write_through_selective"
)
code_detach_again, body_detach_again = self._detach_backend(
base_url2, headers=admin_headers
)
self.assertEqual(
code_detach_again,
200,
f"{code_detach_again} - {body_detach_again}",
)
# 6) Re-attach after detach should succeed
code_attach2, body_attach2 = self._attach_backend(
base_url=base_url2,
backend="file",
extra_cfg=extra_cfg,
headers=admin_headers,
)
self.assertEqual(code_attach2, 200, f"{code_attach2} - {body_attach2}")
status4 = self._get_backend_status(base_url2, headers=admin_headers)
self.assertEqual(status4.get("hicache_storage_backend"), "file")
self.assertEqual(
status4.get("hicache_storage_backend_extra_config"),
json.dumps(extra_cfg),
)
self.assertEqual(status4.get("hicache_storage_prefetch_policy"), "timeout")
self.assertEqual(status4.get("hicache_write_policy"), "write_through")
# Cleanup: detach for test isolation
code_detach2, body_detach2 = self._detach_backend(
base_url2, headers=admin_headers
)
self.assertEqual(code_detach2, 200, f"{code_detach2} - {body_detach2}")
finally:
kill_process_tree(process2.pid)
time.sleep(2)
if __name__ == "__main__":
unittest.main(verbosity=2)
test/manual/hicache/test_hicache_variants.py
-138
View File
@@ -1,138 +0,0 @@
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
register_cuda_ci(est_time=450, suite="stage-b-test-1-gpu-large")
register_amd_ci(est_time=524, suite="stage-b-test-1-gpu-small-amd")
"""
Consolidated HiCache variant tests.
Tests HiCache with different configurations: standard, MLA, EAGLE, and page size variants.
# TODO: Segmentation fault occurs when upgraded to Cu13. Ref: https://github.com/sgl-project/sglang/actions/runs/24603159715/job/71945537414?pr=23119")
# Should move back to registered test after it's fixed
"""
import unittest
from sglang.benchmark.utils import get_tokenizer
from sglang.srt.utils import is_hip, kill_process_tree
from sglang.test.kits.eval_accuracy_kit import MGSMEnMixin, MMLUMixin
from sglang.test.test_utils import (
DEFAULT_DRAFT_MODEL_EAGLE3,
DEFAULT_MLA_MODEL_NAME_FOR_TEST,
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_TARGET_MODEL_EAGLE3,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
popen_launch_server,
)
_is_hip = is_hip()
class HiCacheBaseServer(CustomTestCase):
"""Base class for HiCache tests with configurable server setup"""
model_name = DEFAULT_MODEL_NAME_FOR_TEST
hicache_args = []
@classmethod
def setUpClass(cls):
cls.model = cls.model_name
cls.base_url = DEFAULT_URL_FOR_TEST
# Setup tokenizer if needed by subclass
if hasattr(cls, "needs_tokenizer") and cls.needs_tokenizer:
cls.tokenizer = get_tokenizer(cls.model)
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=cls.hicache_args,
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
class TestHiCacheStandard(HiCacheBaseServer, MMLUMixin):
"""Standard HiCache configuration tests"""
model_name = DEFAULT_MODEL_NAME_FOR_TEST
hicache_args = [
"--enable-hierarchical-cache",
"--mem-fraction-static",
0.7,
"--hicache-size",
100 if not _is_hip else 200,
]
mmlu_score_threshold = 0.65
mmlu_num_examples = 64
mmlu_num_threads = 32
class TestHiCacheMLA(HiCacheBaseServer, MMLUMixin, MGSMEnMixin):
"""HiCache with MLA model tests"""
model_name = DEFAULT_MLA_MODEL_NAME_FOR_TEST
hicache_args = [
"--trust-remote-code",
"--enable-hierarchical-cache",
] + (["--hicache-size", 200] if _is_hip else ["--hicache-ratio", 2])
mmlu_score_threshold = 0.5
mmlu_num_examples = 64
mmlu_num_threads = 32
mgsm_en_score_threshold = 0.8
@unittest.skipIf(is_hip(), "Disabled for AMD-aiter")
class TestHiCacheEagle(HiCacheBaseServer, MMLUMixin):
"""HiCache with EAGLE speculative decoding tests"""
model_name = DEFAULT_TARGET_MODEL_EAGLE3
needs_tokenizer = True
hicache_args = [
"--enable-hierarchical-cache",
"--hicache-ratio",
1.2,
"--mem-fraction-static",
0.7,
"--speculative-algorithm",
"EAGLE3",
"--speculative-draft-model-path",
DEFAULT_DRAFT_MODEL_EAGLE3,
"--speculative-num-steps",
2,
"--speculative-eagle-topk",
1,
"--speculative-num-draft-tokens",
3,
"--dtype",
"float16",
"--chunked-prefill-size",
1024,
]
mmlu_score_threshold = 0.72
mmlu_num_examples = 64
mmlu_num_threads = 32
mmlu_accept_length_thres = 2.26
class TestHiCachePage(HiCacheBaseServer, MMLUMixin):
"""HiCache with custom page size tests"""
model_name = DEFAULT_MODEL_NAME_FOR_TEST
hicache_args = [
"--enable-hierarchical-cache",
"--page-size",
32,
"--hicache-write-policy",
"write_back",
]
mmlu_score_threshold = 0.65
mmlu_num_examples = 64
mmlu_num_threads = 32
if __name__ == "__main__":
unittest.main()