minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

[CI] Key scheduled CUDA suites by runner_config instead of hand-written jobs (#34186)

Browse Source
This commit is contained in:
Liangsheng Yin
2026-08-09 16:44:53 -07:00
committed by GitHub
parent 4a5d7d3c67
commit 7c90840bad
94 changed files with 531 additions and 869 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/test/ci/ci_utils.py
+28 -9
View File
@@ -137,9 +137,23 @@ def _repo_relative_path(p: str) -> str:
return p[idx + len(marker) :] if idx >= 0 else p
# Slow-run variance is largely additive (cold HF cache, slow server launch), so
# the multiplier alone under-provisions at both ends: test_encoder_dp runs
# 200-426s but once took over 1185s against a 1.5x budget of 765s, and
# test_lora_deepseek_v3_base_logprob_diff (est 1800) landed on exactly 1.5 * est.
# Every file gets the same absolute slack on top of the proportional one.
DERIVED_TIMEOUT_SLACK = 1800.0
DERIVED_TIMEOUT_FACTOR = 1.5
def derive_timeout_per_file(est_time: float) -> float:
est = float(est_time)
return max(est * DERIVED_TIMEOUT_FACTOR, est + DERIVED_TIMEOUT_SLACK)
def run_unittest_files(
files: Union[List[TestFile], List[CIRegistry]],
timeout_per_file: float,
timeout_per_file: Optional[float] = None,
continue_on_error: bool = False,
enable_retry: bool = False,
max_attempts: int = 2,
@@ -150,7 +164,8 @@ def run_unittest_files(
Args:
files: List of TestFile objects to run
timeout_per_file: Timeout in seconds for each test file
timeout_per_file: Fixed timeout in seconds for every test file, or None
to derive each file's budget from its own est_time.
continue_on_error: If True, continue running remaining tests even if one fails.
If False, stop at first failure (default behavior for PR tests).
enable_retry: If True, retry failed tests that appear to be accuracy/performance
@@ -178,6 +193,12 @@ def run_unittest_files(
# FIXME: remove this branch after migrating all tests to use CIRegistry
filename, estimated_time = file.name, file.estimated_time
file_timeout = (
timeout_per_file
if timeout_per_file is not None
else derive_timeout_per_file(estimated_time)
)
process = None
output_lines = []
@@ -235,7 +256,7 @@ def run_unittest_files(
run_one_file,
args=(filename,),
kwargs={"capture_output": enable_retry},
timeout=timeout_per_file,
timeout=file_timeout,
)
if ret_code == 0:
@@ -281,24 +302,22 @@ def run_unittest_files(
# TimeoutError aborts run_one_file before its elapsed write;
# record the timeout cap as an upper bound so the file still
# appears in the TIMINGS block below.
file_elapsed[filename] = float(timeout_per_file)
file_elapsed[filename] = float(file_timeout)
# Retry once on timeout: usually a stuck server / hung device.
# A real hang times out again and is reported.
if enable_retry and attempt < max_attempts:
logger.info(
f"\n[CI Retry] {filename} timed out after "
f"{timeout_per_file}s; waiting {retry_wait_seconds}s "
f"{file_timeout}s; waiting {retry_wait_seconds}s "
f"before retry (attempt {attempt + 1}/{max_attempts})\n"
)
time.sleep(retry_wait_seconds)
attempt += 1
continue
logger.info(
f"\n✗ TIMEOUT: {filename} after {timeout_per_file} seconds\n"
)
logger.info(f"\n✗ TIMEOUT: {filename} after {file_timeout} seconds\n")
if was_retried:
retried_tests.append((filename, attempt, "timeout"))
failed_tests.append((filename, f"timeout after {timeout_per_file}s"))
failed_tests.append((filename, f"timeout after {file_timeout}s"))
break
if not file_passed:
python/sglang/test/precision_baseline_store.py
+15 -3
View File
@@ -26,6 +26,16 @@ from huggingface_hub.errors import (
)
def _store_token() -> Optional[str]:
"""Write token for the baseline dataset repo.
Deliberately not HF_TOKEN: that name already carries the runner's
gated-model read token, so writing the store token there would shadow it
and turn every gated model on the job into a 401.
"""
return os.environ.get("SGLANG_PRECISION_HF_TOKEN") or None
@dataclass
class HfStoreConfig:
repo: str
@@ -38,7 +48,7 @@ class HfStoreConfig:
raise RuntimeError(
"SGLANG_PRECISION_HF_REPO is not set. The precision baseline "
"store is required (there is no local-only mode); set the repo "
"and HF_TOKEN_PRECISION_STORE."
"and SGLANG_PRECISION_HF_TOKEN."
)
revision = os.environ.get("SGLANG_PRECISION_HF_REVISION", "main")
return cls(repo=repo, revision=revision)
@@ -148,6 +158,7 @@ def fetch_latest_baseline(
repo_type="dataset",
revision=config.revision,
allow_patterns=[f"{run_path}/tensors/*"],
token=_store_token(),
),
what="snapshot download",
)
@@ -171,6 +182,7 @@ def _read_manifest(config: HfStoreConfig) -> tuple[list[dict[str, Any]], str]:
repo_type="dataset",
filename="manifest.jsonl",
revision=config.revision,
token=_store_token(),
),
what="manifest fetch",
)
@@ -215,7 +227,7 @@ def push_run(
# Dedup: same model+date+sha → skip tensor upload but still refresh meta
# + comparator_report + append a new manifest row, so pass-1 baseline and
# pass-2 stats both land. force=True re-uploads tensors too.
api = HfApi()
api = HfApi(token=_store_token())
date_str, date_path = _today_path()
model_sanitized = _sanitize_model_name(model)
sha7 = (
@@ -306,7 +318,7 @@ def prune_old_runs(
# dry_run defaults True because model=None+keep_days=0 would wipe the
# store. Live mode rewrites the manifest before deleting folders so a
# mid-run failure leaves manifest pointing at the kept rows only.
api = HfApi()
api = HfApi(token=_store_token())
rows, _ = _read_manifest(config)
if not rows:
return {"kept": [], "pruned": []}