ci: migrate 2-GPU tests to test/registered/ (#16529)
This commit is contained in:
Alison Shao
@@ -0,0 +1,91 @@
|
||||
"""
|
||||
Benchmark tests for HiCache Storage with 3FS backend.
|
||||
Usage:
|
||||
python3 -m pytest test/registered/hicache/test_hicache_storage_3fs_backend.py -v
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import unittest
|
||||
|
||||
from test_hicache_storage_file_backend import HiCacheStorageBaseMixin
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(est_time=200, suite="stage-b-test-large-2-gpu")
|
||||
|
||||
|
||||
class HiCacheStorage3FSBackendBaseMixin(HiCacheStorageBaseMixin):
|
||||
"""Base mixin class with common setup and utilities"""
|
||||
|
||||
@classmethod
|
||||
def _get_additional_server_args_and_env(cls):
|
||||
"""Get additional server arguments specific to configuration - override in subclasses"""
|
||||
# Create a temporary JSON config file for HF3FS
|
||||
hf3fs_config = {
|
||||
"file_path_prefix": os.path.join(cls.temp_dir, "hicache"),
|
||||
"file_size": 1024 * 1024 * 1024 * 2,
|
||||
"numjobs": 2,
|
||||
"entries": 8,
|
||||
"use_mock_hf3fs_client": True,
|
||||
"hicache_storage_pass_prefix_keys": True,
|
||||
}
|
||||
|
||||
# Write config to temporary file
|
||||
config_file = os.path.join(cls.temp_dir, "hf3fs_config.json")
|
||||
with open(config_file, "w") as f:
|
||||
json.dump(hf3fs_config, f, indent=2)
|
||||
|
||||
server_args = {
|
||||
"--tp-size": 1,
|
||||
"--hicache-ratio": 1.2,
|
||||
"--hicache-storage-backend": "hf3fs",
|
||||
"--hicache-storage-backend-extra-config": json.dumps(hf3fs_config),
|
||||
}
|
||||
|
||||
# Set the environment variable to point to our config file
|
||||
env_vars = {
|
||||
"SGLANG_HICACHE_HF3FS_CONFIG_PATH": config_file,
|
||||
}
|
||||
|
||||
return server_args, env_vars
|
||||
|
||||
|
||||
class TestHf3fsBackendLayerFirstLayout(
|
||||
HiCacheStorage3FSBackendBaseMixin, CustomTestCase
|
||||
):
|
||||
"""Layer first layout tests for HiCache-Hf3fs backend"""
|
||||
|
||||
@classmethod
|
||||
def _get_additional_server_args_and_env(cls):
|
||||
"""Get additional server arguments specific to configuration - override in subclasses"""
|
||||
server_args, env_vars = super()._get_additional_server_args_and_env()
|
||||
server_args["--hicache-mem-layout"] = "layer_first"
|
||||
server_args["--hicache-io-backend"] = "direct"
|
||||
server_args["--tp-size"] = 2
|
||||
return server_args, env_vars
|
||||
|
||||
|
||||
class TestHf3fsBackendAccuracy(HiCacheStorage3FSBackendBaseMixin, CustomTestCase):
|
||||
"""Accuracy tests for HiCache-Hf3fs backend"""
|
||||
|
||||
@classmethod
|
||||
def _get_additional_server_args_and_env(cls):
|
||||
"""Get additional server arguments specific to configuration - override in subclasses"""
|
||||
server_args, env_vars = super()._get_additional_server_args_and_env()
|
||||
server_args["--hicache-ratio"] = 1.5
|
||||
server_args["--tp-size"] = 2
|
||||
server_args["--hicache-mem-layout"] = "page_first_direct"
|
||||
server_args["--hicache-io-backend"] = "direct"
|
||||
return server_args, env_vars
|
||||
|
||||
def test_eval_accuracy(self):
|
||||
"""Test eval accuracy with cache persistence across cache flushes"""
|
||||
from test_hicache_storage_file_backend import run_eval_accuracy_test
|
||||
|
||||
run_eval_accuracy_test(self)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
@@ -0,0 +1,336 @@
|
||||
"""
|
||||
E2E tests for HiCache Storage functionality.
|
||||
Usage:
|
||||
python3 -m pytest test/registered/hicache/test_hicache_storage_file_backend.py -v
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from typing import Dict
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import requests
|
||||
|
||||
from sglang.bench_serving import get_tokenizer
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_MLA_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
is_in_ci,
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=200, suite="stage-b-test-large-2-gpu")
|
||||
|
||||
|
||||
class HiCacheStorageBaseMixin:
|
||||
"""Base mixin class with common setup and utilities"""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
"""Set up test environment and launch server once for all tests"""
|
||||
cls.temp_dir = tempfile.mkdtemp()
|
||||
cls.model = cls._get_model_name()
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
|
||||
parsed_url = urlparse(cls.base_url)
|
||||
cls.base_host = parsed_url.hostname
|
||||
cls.base_port = str(parsed_url.port)
|
||||
|
||||
# Prepare tokenizer for prompt generation
|
||||
cls.tokenizer = get_tokenizer(cls.model)
|
||||
|
||||
# Launch server with HiCache enabled and cache report
|
||||
cls.process = cls._launch_server_with_hicache()
|
||||
cls._wait_for_server_ready()
|
||||
|
||||
print(f"Test server launched successfully at {cls.base_url}")
|
||||
print(f"Cache directory: {cls.temp_dir}")
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
"""Clean up test environment"""
|
||||
kill_process_tree(cls.process.pid)
|
||||
|
||||
import shutil
|
||||
|
||||
shutil.rmtree(cls.temp_dir, ignore_errors=True)
|
||||
|
||||
@classmethod
|
||||
def _get_model_name(cls):
|
||||
"""Get model name for the test configuration - override in subclasses"""
|
||||
return DEFAULT_MODEL_NAME_FOR_TEST
|
||||
|
||||
@classmethod
|
||||
def _get_base_server_args(cls):
|
||||
"""Get base server arguments - can be extended in subclasses"""
|
||||
extra_config = {
|
||||
"hicache_storage_pass_prefix_keys": True,
|
||||
}
|
||||
return {
|
||||
"--enable-hierarchical-cache": True,
|
||||
"--mem-fraction-static": 0.6,
|
||||
"--hicache-ratio": 1.2,
|
||||
"--page-size": 64,
|
||||
"--enable-cache-report": True,
|
||||
"--hicache-storage-prefetch-policy": "wait_complete",
|
||||
"--hicache-storage-backend": "file",
|
||||
"--hicache-storage-backend-extra-config": json.dumps(extra_config),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def _get_additional_server_args_and_env(cls):
|
||||
"""Get additional server arguments specific to configuration - override in subclasses"""
|
||||
return {}, {"SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR": cls.temp_dir}
|
||||
|
||||
@classmethod
|
||||
def _launch_server_with_hicache(cls):
|
||||
"""Launch server with HiCache enabled"""
|
||||
|
||||
additional_server_args, env_vars = cls._get_additional_server_args_and_env()
|
||||
env_vars["SGLANG_ENABLE_DETERMINISTIC_INFERENCE"] = "1"
|
||||
server_args = cls._get_base_server_args()
|
||||
if additional_server_args:
|
||||
server_args.update(additional_server_args)
|
||||
|
||||
final_server_args = []
|
||||
for k, v in server_args.items():
|
||||
if isinstance(v, bool):
|
||||
final_server_args.append(str(k))
|
||||
else:
|
||||
final_server_args.append(str(k))
|
||||
final_server_args.append(str(v))
|
||||
|
||||
print(f"final_server_args: {final_server_args}")
|
||||
|
||||
env_vars = {
|
||||
**os.environ,
|
||||
**env_vars,
|
||||
}
|
||||
|
||||
return popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=final_server_args,
|
||||
env=env_vars,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _wait_for_server_ready(cls, timeout: int = 60) -> bool:
|
||||
"""Wait for server to be ready"""
|
||||
start_time = time.time()
|
||||
while time.time() - start_time < timeout:
|
||||
try:
|
||||
response = requests.get(f"{cls.base_url}/health", timeout=5)
|
||||
if response.status_code == 200:
|
||||
return True
|
||||
except requests.RequestException:
|
||||
pass
|
||||
time.sleep(2)
|
||||
raise TimeoutError("Server failed to start within timeout")
|
||||
|
||||
def send_request(
|
||||
self, prompt: str, max_tokens: int = 100, temperature: float = 0.0
|
||||
) -> Dict:
|
||||
"""Send a generate request and return response"""
|
||||
response = requests.post(
|
||||
f"{self.base_url}/generate",
|
||||
json={
|
||||
"text": prompt,
|
||||
"sampling_params": {
|
||||
"temperature": temperature,
|
||||
"max_new_tokens": max_tokens,
|
||||
"ignore_eos": True,
|
||||
},
|
||||
},
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
response.status_code,
|
||||
200,
|
||||
f"Request failed: {response.status_code} - {response.text}",
|
||||
)
|
||||
return response.json()
|
||||
|
||||
def get_cached_tokens(self, response_json: Dict) -> int:
|
||||
"""Extract cached tokens count from /generate response"""
|
||||
meta = response_json.get("meta_info", {})
|
||||
return int(meta.get("cached_tokens", 0))
|
||||
|
||||
def flush_cache(self) -> bool:
|
||||
"""Flush device cache to force remote storage access"""
|
||||
try:
|
||||
response = requests.post(f"{self.base_url}/flush_cache", timeout=10)
|
||||
return response.status_code == 200
|
||||
except requests.RequestException:
|
||||
return False
|
||||
|
||||
def gen_prompt(self, token_num: int) -> str:
|
||||
"""Generate a random prompt of specified token length using tokenizer vocabulary."""
|
||||
all_available_tokens = list(self.tokenizer.get_vocab().values())
|
||||
selected_tokens = random.choices(all_available_tokens, k=token_num)
|
||||
return self.tokenizer.decode(selected_tokens)
|
||||
|
||||
def trigger_offloading_and_flush(self):
|
||||
"""Helper method to trigger offloading and flush cache"""
|
||||
# Trigger offloading
|
||||
self.send_request(self.gen_prompt(1), max_tokens=150)
|
||||
|
||||
# Flush device cache to force remote storage access
|
||||
time.sleep(2)
|
||||
self.assertTrue(self.flush_cache(), "Cache flush should succeed")
|
||||
|
||||
def test_basic_backup_and_prefetch(self):
|
||||
"""Test storage and retrieval of large context through remote cache"""
|
||||
print("\n=== Testing Large Context Cache Storage & Retrieval ===")
|
||||
|
||||
# Generate substantial context that will be cached
|
||||
base_prompt = self.gen_prompt(768)
|
||||
|
||||
# First request - populate cache
|
||||
print("Step 1: Populating cache with large context...")
|
||||
response1 = self.send_request(base_prompt, max_tokens=150)
|
||||
self.assertIsNotNone(response1)
|
||||
|
||||
# Flush device cache to force remote storage access
|
||||
self.trigger_offloading_and_flush()
|
||||
|
||||
# Second request with extended prompt - should hit remote cache
|
||||
print("Step 2: Testing cache hit from remote storage...")
|
||||
|
||||
start_time = time.time()
|
||||
response2 = self.send_request(base_prompt, max_tokens=150)
|
||||
retrieval_time = time.time() - start_time
|
||||
|
||||
cached_tokens = self.get_cached_tokens(response2)
|
||||
print(
|
||||
f"Remote cache retrieval time: {retrieval_time:.3f}s, cached_tokens={cached_tokens}"
|
||||
)
|
||||
|
||||
# Assert cached tokens indicate a remote hit
|
||||
self.assertGreater(
|
||||
cached_tokens, 700, "Expected significant cached tokens for remote hit"
|
||||
)
|
||||
|
||||
|
||||
@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.")
|
||||
class TestHiCacheStoragePageFirstLayout(HiCacheStorageBaseMixin, CustomTestCase):
|
||||
"""Page first layout tests for HiCache Storage functionality"""
|
||||
|
||||
@classmethod
|
||||
def _get_additional_server_args_and_env(cls):
|
||||
"""Get additional server arguments specific to configuration - override in subclasses"""
|
||||
server_args = {"--hicache-mem-layout": "page_first"}
|
||||
return server_args, {}
|
||||
|
||||
|
||||
@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.")
|
||||
class TestHiCacheStorageMLA(HiCacheStorageBaseMixin, CustomTestCase):
|
||||
"""MLA Model tests for HiCache Storage functionality"""
|
||||
|
||||
@classmethod
|
||||
def _get_model_name(cls):
|
||||
"""Use MLA model for testing"""
|
||||
return DEFAULT_MLA_MODEL_NAME_FOR_TEST
|
||||
|
||||
@classmethod
|
||||
def _get_additional_server_args_and_env(cls):
|
||||
"""Get additional server arguments specific to configuration - override in subclasses"""
|
||||
server_args = {"--tp-size": 2}
|
||||
return server_args, {}
|
||||
|
||||
|
||||
class TestHiCacheStoragePageFirstDirectIO(HiCacheStorageBaseMixin, CustomTestCase):
|
||||
"""Page first direct tests for HiCache Storage functionality"""
|
||||
|
||||
@classmethod
|
||||
def _get_additional_server_args_and_env(cls):
|
||||
"""Get additional server arguments specific to configuration - override in subclasses"""
|
||||
server_args = {
|
||||
"--hicache-mem-layout": "page_first_direct",
|
||||
"--hicache-io-backend": "direct",
|
||||
"--tp-size": 2,
|
||||
}
|
||||
return server_args, {}
|
||||
|
||||
|
||||
class TestHiCacheStorageAccuracy(HiCacheStorageBaseMixin, CustomTestCase):
|
||||
"""Accuracy tests for HiCache Storage functionality"""
|
||||
|
||||
@classmethod
|
||||
def _get_additional_server_args_and_env(cls):
|
||||
"""Get additional server arguments specific to configuration - override in subclasses"""
|
||||
server_args = {
|
||||
"--tp-size": 2,
|
||||
"--hicache-ratio": 1.5,
|
||||
}
|
||||
|
||||
return server_args, {}
|
||||
|
||||
def test_eval_accuracy(self):
|
||||
"""Test eval accuracy with cache persistence across cache flushes"""
|
||||
run_eval_accuracy_test(self)
|
||||
|
||||
|
||||
def run_eval_accuracy_test(test_instance, accuracy_threshold: float = 0.03):
|
||||
"""Generic eval accuracy test with configurable accuracy threshold
|
||||
|
||||
Args:
|
||||
test_instance: The test class instance that provides base_host, base_port, flush_cache, and assert methods
|
||||
"""
|
||||
print("\n=== Testing Eval Accuracy with Cache Persistence ===")
|
||||
|
||||
# First evaluation - populate cache
|
||||
print("Phase 1: Running initial GSM8K evaluation to populate cache...")
|
||||
args_initial = SimpleNamespace(
|
||||
num_shots=5,
|
||||
data_path=None,
|
||||
num_questions=50,
|
||||
max_new_tokens=512,
|
||||
parallel=10,
|
||||
host=f"http://{test_instance.base_host}",
|
||||
port=int(test_instance.base_port),
|
||||
)
|
||||
metrics_initial = run_eval_few_shot_gsm8k(args_initial)
|
||||
|
||||
# Flush cache to force remote storage access
|
||||
print("Phase 2: Flushing device cache...")
|
||||
test_instance.assertTrue(test_instance.flush_cache(), "Cache flush should succeed")
|
||||
time.sleep(2)
|
||||
|
||||
# Second evaluation - should use remote cache
|
||||
print("Phase 3: Running second GSM8K evaluation using remote cache...")
|
||||
metrics_cached = run_eval_few_shot_gsm8k(args_initial)
|
||||
|
||||
# Verify accuracy consistency
|
||||
accuracy_diff = abs(metrics_initial["accuracy"] - metrics_cached["accuracy"])
|
||||
print(f"Accuracy difference: {accuracy_diff:.4f}")
|
||||
|
||||
# Assertions
|
||||
test_instance.assertGreater(
|
||||
metrics_initial["accuracy"], 0.6, "Initial accuracy should be reasonable"
|
||||
)
|
||||
test_instance.assertGreater(
|
||||
metrics_cached["accuracy"], 0.6, "Cached accuracy should be reasonable"
|
||||
)
|
||||
test_instance.assertLess(
|
||||
accuracy_diff,
|
||||
accuracy_threshold,
|
||||
"Accuracy should be consistent between cache states",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
@@ -0,0 +1,286 @@
|
||||
"""
|
||||
Benchmark tests for HiCache Storage with Mooncake backend.
|
||||
Usage:
|
||||
python3.10 -m pytest test/registered/hicache/test_hicache_storage_mooncake_backend.py -v
|
||||
"""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import unittest
|
||||
|
||||
import requests
|
||||
from test_hicache_storage_file_backend import HiCacheStorageBaseMixin
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_MLA_MODEL_NAME_FOR_TEST,
|
||||
CustomTestCase,
|
||||
find_available_port,
|
||||
is_in_ci,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=300, suite="stage-b-test-large-2-gpu")
|
||||
|
||||
|
||||
class HiCacheStorageMooncakeBackendBaseMixin(HiCacheStorageBaseMixin):
|
||||
"""Base mixin class with common setup and utilities"""
|
||||
|
||||
# Default port ranges for Mooncake services - can be overridden in subclasses
|
||||
mooncake_master_port_base = 50051
|
||||
mooncake_metadata_port_base = 8080
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
"""Set up test environment and launch Mooncake services before server setup"""
|
||||
# Find available ports for Mooncake services to avoid conflicts
|
||||
cls.mooncake_master_port = find_available_port(
|
||||
HiCacheStorageMooncakeBackendBaseMixin.mooncake_master_port_base
|
||||
)
|
||||
cls.mooncake_metadata_port = find_available_port(
|
||||
HiCacheStorageMooncakeBackendBaseMixin.mooncake_metadata_port_base
|
||||
)
|
||||
|
||||
# Start Mooncake services first
|
||||
cls._start_mooncake_services()
|
||||
|
||||
# Call parent setup
|
||||
super().setUpClass()
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
"""Clean up Mooncake services after server teardown"""
|
||||
# Call parent teardown first
|
||||
super().tearDownClass()
|
||||
|
||||
# Stop Mooncake services
|
||||
cls._stop_mooncake_services()
|
||||
|
||||
@classmethod
|
||||
def _start_mooncake_services(cls):
|
||||
"""Start Mooncake metadata and master services with configurable ports and readiness detection"""
|
||||
print("Starting Mooncake services...")
|
||||
print(
|
||||
f"Using master port: {cls.mooncake_master_port}, metadata port: {cls.mooncake_metadata_port}"
|
||||
)
|
||||
|
||||
# Start metadata service with configurable port
|
||||
try:
|
||||
# Start metadata server with port configuration
|
||||
cls.metadata_service_process = subprocess.Popen(
|
||||
[
|
||||
"python3",
|
||||
"-m",
|
||||
"mooncake.http_metadata_server",
|
||||
"--port",
|
||||
str(cls.mooncake_metadata_port),
|
||||
],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
preexec_fn=os.setsid, # Create new process group
|
||||
)
|
||||
print(
|
||||
f"Mooncake metadata service started on port {cls.mooncake_metadata_port}"
|
||||
)
|
||||
except (FileNotFoundError, subprocess.SubprocessError) as e:
|
||||
print(f"Warning: Could not start Mooncake metadata service: {e}")
|
||||
cls.metadata_service_process = None
|
||||
|
||||
# Start master service with configurable port
|
||||
try:
|
||||
# Start master server with port configuration
|
||||
cls.master_service_process = subprocess.Popen(
|
||||
["mooncake_master", "--port", str(cls.mooncake_master_port)],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
preexec_fn=os.setsid, # Create new process group
|
||||
)
|
||||
print(f"Mooncake master service started on port {cls.mooncake_master_port}")
|
||||
except (FileNotFoundError, subprocess.SubprocessError) as e:
|
||||
print(f"Warning: Could not start Mooncake master service: {e}")
|
||||
cls.master_service_process = None
|
||||
|
||||
# Wait for services to be ready instead of fixed sleep
|
||||
cls._wait_for_mooncake_services_ready()
|
||||
|
||||
@classmethod
|
||||
def _wait_for_mooncake_services_ready(cls, timeout: int = 30) -> bool:
|
||||
"""Wait for Mooncake services to be ready by checking their endpoints"""
|
||||
print("Waiting for Mooncake services to be ready...")
|
||||
|
||||
start_time = time.time()
|
||||
services_ready = False
|
||||
|
||||
while time.time() - start_time < timeout:
|
||||
try:
|
||||
# Check metadata service
|
||||
metadata_ready = False
|
||||
if (
|
||||
cls.metadata_service_process
|
||||
and cls.metadata_service_process.poll() is None
|
||||
):
|
||||
try:
|
||||
# Try to connect to the metadata service
|
||||
metadata_url = (
|
||||
f"http://127.0.0.1:{cls.mooncake_metadata_port}/metadata"
|
||||
)
|
||||
response = requests.get(metadata_url, timeout=2)
|
||||
if response.status_code == 200:
|
||||
metadata_ready = True
|
||||
print("Mooncake metadata service is ready")
|
||||
except (requests.RequestException, ConnectionError):
|
||||
# Service might not be fully started yet
|
||||
pass
|
||||
|
||||
# Check master service (if it has a health endpoint)
|
||||
master_ready = False
|
||||
if (
|
||||
cls.master_service_process
|
||||
and cls.master_service_process.poll() is None
|
||||
):
|
||||
# For now, we'll assume master service is ready if process is running
|
||||
# and it's been a few seconds since startup
|
||||
if (
|
||||
time.time() - start_time > 5
|
||||
): # Give master service time to initialize
|
||||
master_ready = True
|
||||
print("Mooncake master service is ready")
|
||||
|
||||
# Both services should be ready
|
||||
if metadata_ready and master_ready:
|
||||
services_ready = True
|
||||
print("All Mooncake services are ready")
|
||||
break
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error checking service readiness: {e}")
|
||||
|
||||
time.sleep(2)
|
||||
|
||||
if not services_ready:
|
||||
print(
|
||||
"Warning: Mooncake services may not be fully ready, continuing anyway..."
|
||||
)
|
||||
|
||||
return services_ready
|
||||
|
||||
@classmethod
|
||||
def _stop_mooncake_services(cls):
|
||||
"""Stop Mooncake services"""
|
||||
print("Stopping Mooncake services...")
|
||||
|
||||
# Stop metadata service
|
||||
if hasattr(cls, "metadata_service_process") and cls.metadata_service_process:
|
||||
try:
|
||||
os.killpg(os.getpgid(cls.metadata_service_process.pid), 9)
|
||||
cls.metadata_service_process.wait(timeout=5)
|
||||
print("Mooncake metadata service stopped")
|
||||
except (ProcessLookupError, subprocess.TimeoutExpired, OSError) as e:
|
||||
print(f"Warning: Could not stop Mooncake metadata service: {e}")
|
||||
|
||||
# Stop master service
|
||||
if hasattr(cls, "master_service_process") and cls.master_service_process:
|
||||
try:
|
||||
os.killpg(os.getpgid(cls.master_service_process.pid), 9)
|
||||
cls.master_service_process.wait(timeout=5)
|
||||
print("Mooncake master service stopped")
|
||||
except (ProcessLookupError, subprocess.TimeoutExpired, OSError) as e:
|
||||
print(f"Warning: Could not stop Mooncake master service: {e}")
|
||||
|
||||
@classmethod
|
||||
def _get_additional_server_args_and_env(cls):
|
||||
"""Get additional server arguments specific to configuration - override in subclasses"""
|
||||
|
||||
server_args = {
|
||||
"--tp-size": 2,
|
||||
"--hicache-ratio": 2,
|
||||
"--hicache-storage-backend": "mooncake",
|
||||
}
|
||||
|
||||
# Set the environment variables for Mooncake using dynamic ports
|
||||
env_vars = {
|
||||
"MOONCAKE_MASTER": f"127.0.0.1:{cls.mooncake_master_port}",
|
||||
"MOONCAKE_PROTOCOL": "tcp",
|
||||
"MC_MS_AUTO_DISC": "0",
|
||||
"MOONCAKE_DEVICE": "",
|
||||
"MOONCAKE_TE_META_DATA_SERVER": f"http://127.0.0.1:{cls.mooncake_metadata_port}/metadata",
|
||||
"MOONCAKE_GLOBAL_SEGMENT_SIZE": "4294967296", # 4 GiB
|
||||
}
|
||||
|
||||
return server_args, env_vars
|
||||
|
||||
|
||||
'''
|
||||
# Same as #10131, layer first layout test TODO(mateng): will make it work
|
||||
class TestMooncakeBackendLayerFirstLayout(
|
||||
HiCacheStorageMooncakeBackendBaseMixin, CustomTestCase
|
||||
):
|
||||
"""Layer first layout tests for HiCache-Mooncake backend"""
|
||||
|
||||
@classmethod
|
||||
def _get_additional_server_args_and_env(cls):
|
||||
"""Get additional server arguments specific to configuration - override in subclasses"""
|
||||
server_args, env_vars = super()._get_additional_server_args_and_env()
|
||||
server_args["--hicache-mem-layout"] = "layer_first"
|
||||
server_args["--hicache-io-backend"] = "direct"
|
||||
return server_args, env_vars
|
||||
'''
|
||||
|
||||
|
||||
@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.")
|
||||
class TestMooncakeBackendPageFirstLayout(
|
||||
HiCacheStorageMooncakeBackendBaseMixin, CustomTestCase
|
||||
):
|
||||
"""Page first layout tests for HiCache-Mooncake backend"""
|
||||
|
||||
@classmethod
|
||||
def _get_additional_server_args_and_env(cls):
|
||||
"""Get additional server arguments specific to configuration - override in subclasses"""
|
||||
server_args, env_vars = super()._get_additional_server_args_and_env()
|
||||
server_args["--hicache-mem-layout"] = "page_first"
|
||||
return server_args, env_vars
|
||||
|
||||
|
||||
class TestMooncakeBackendMLAModel(
|
||||
HiCacheStorageMooncakeBackendBaseMixin, CustomTestCase
|
||||
):
|
||||
"""MLA Model tests for HiCache-Mooncake backend"""
|
||||
|
||||
@classmethod
|
||||
def _get_model_name(cls):
|
||||
"""Use MLA model for testing"""
|
||||
return DEFAULT_MLA_MODEL_NAME_FOR_TEST
|
||||
|
||||
@classmethod
|
||||
def _get_additional_server_args_and_env(cls):
|
||||
"""Get additional server arguments specific to configuration - override in subclasses"""
|
||||
server_args, env_vars = super()._get_additional_server_args_and_env()
|
||||
server_args["--hicache-mem-layout"] = "page_first"
|
||||
server_args["--tp-size"] = 2
|
||||
return server_args, env_vars
|
||||
|
||||
|
||||
class TestMooncakeBackendAccuracy(
|
||||
HiCacheStorageMooncakeBackendBaseMixin, CustomTestCase
|
||||
):
|
||||
"""Accuracy tests for HiCache-Mooncake backend"""
|
||||
|
||||
@classmethod
|
||||
def _get_additional_server_args_and_env(cls):
|
||||
"""Get additional server arguments specific to configuration - override in subclasses"""
|
||||
server_args, env_vars = super()._get_additional_server_args_and_env()
|
||||
server_args["--hicache-ratio"] = 1.5
|
||||
server_args["--tp-size"] = 2
|
||||
server_args["--hicache-mem-layout"] = "page_first_direct"
|
||||
server_args["--hicache-io-backend"] = "direct"
|
||||
return server_args, env_vars
|
||||
|
||||
def test_eval_accuracy(self):
|
||||
"""Test eval accuracy with cache persistence across cache flushes"""
|
||||
from test_hicache_storage_file_backend import run_eval_accuracy_test
|
||||
|
||||
run_eval_accuracy_test(self)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
Reference in New Issue
Block a user