minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[CI] ci-coverage-overview: schedule + manual only, include XPU/MUSA/multimodal_gen (#26619)

Browse Source
This commit is contained in:
Alison Shao
2026-05-30 16:39:18 +08:00
committed by GitHub
parent b4bf489dea
commit edfe8d34e8
3 changed files with 205 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci-coverage-overview.yml
+11 -6
View File
@@ -3,11 +3,6 @@ name: CI Coverage Overview
on:
schedule:
- cron: '0 6 * * *' # Daily at 6 AM UTC
pull_request:
paths:
- '.github/workflows/ci-coverage-overview.yml'
- 'scripts/ci/utils/ci_coverage_report.py'
- 'test/registered/**'
workflow_dispatch:
inputs:
output_format:
@@ -70,13 +65,23 @@ jobs:
unit-test-coverage:
name: Unit Test Code Coverage
if: github.event_name != 'pull_request'
runs-on: 1-gpu-h100
timeout-minutes: 30
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Install Rust toolchain + protoc
# Editable install of sglang triggers `build_rust` for the bundled
# native gRPC extension (rust/sglang-grpc). The 1-gpu-h100 runner
# image has no Rust toolchain by default, which is why this job has
# been failing on every scheduled run. Use the same helper that
# ci_install_dependency.sh uses so the install path matches the
# rest of CI.
run: |
bash scripts/ci/utils/install_rust_protoc.sh
echo "${CARGO_HOME:-$HOME/.cargo}/bin" >> $GITHUB_PATH
- name: Install dependencies
timeout-minutes: 10
run: |
python/sglang/test/ci/ci_register.py
+16
View File
@@ -14,6 +14,7 @@ __all__ = [
"register_amd_ci",
"register_npu_ci",
"register_xpu_ci",
"register_musa_ci",
"ut_parse_one_file",
]
@@ -32,6 +33,7 @@ class HWBackend(Enum):
AMD = auto()
NPU = auto()
XPU = auto()
MUSA = auto()
@dataclass
@@ -119,12 +121,26 @@ def register_xpu_ci(
return None
def register_musa_ci(
est_time: float,
suite: Optional[str] = None,
nightly: bool = False,
disabled: Optional[str] = None,
*,
stage: Optional[str] = None,
runner_config: Optional[str] = None,
):
"""Marker for MUSA CI registration (parsed via AST; runtime no-op)."""
return None
REGISTER_MAPPING = {
"register_cpu_ci": HWBackend.CPU,
"register_cuda_ci": HWBackend.CUDA,
"register_amd_ci": HWBackend.AMD,
"register_npu_ci": HWBackend.NPU,
"register_xpu_ci": HWBackend.XPU,
"register_musa_ci": HWBackend.MUSA,
}
scripts/ci/utils/ci_coverage_report.py
+178 -23
View File
@@ -27,6 +27,63 @@ sys.path.insert(
from ci_register import CIRegistry, HWBackend, ut_parse_one_file
# Display order for backend tables / sections. The list is sourced from
# HWBackend so a newly-added enum member can never be silently dropped from
# the report -- if the assert below fires, add the new backend name here in
# the right display slot. Order isn't alphabetical: CUDA/AMD/NPU/CPU lead
# (highest test volume historically), then accelerators that have been
# wired into the registry more recently (XPU, MUSA).
BACKEND_DISPLAY_ORDER = ("CUDA", "AMD", "NPU", "CPU", "XPU", "MUSA")
assert set(BACKEND_DISPLAY_ORDER) == {
b.name for b in HWBackend
}, "BACKEND_DISPLAY_ORDER is out of sync with HWBackend"
# --------------------------------------------------------------------------- #
# multimodal_gen test coverage
#
# multimodal_gen tests live under python/sglang/multimodal_gen/test/ and use
# their own run_suite.py / partitioning framework, NOT the register_*_ci()
# registry used under test/registered/. To surface them in the daily
# overview we synthesize CIRegistry records from file paths + filename
# tokens, using the rules below. These are heuristics derived from how the
# multimodal_gen workflows (pr-test-{musa,npu,amd}.yml, pr-test-multimodal-
# gen.yml) currently invoke each file -- they MAY drift if a workflow
# stops/starts running a directory.
# --------------------------------------------------------------------------- #
MULTIMODAL_GEN_TEST_DIR = "python/sglang/multimodal_gen/test"
# Subdirectory (relative to MULTIMODAL_GEN_TEST_DIR) -> backends those files
# run on by default. Empty string is the top-level. Files whose tokenized
# filename contains an explicit backend marker (see below) override this.
_MM_GEN_SUBDIR_BACKENDS = {
# Top-level helper-style tests (e.g. test_consistency_metrics.py).
"": ("CUDA",),
# server/ top-level: pr-test-multimodal-gen.yml drives CUDA, pr-test-amd
# mirrors the same suite on AMD runners.
"server": ("CUDA", "AMD"),
"server/musa": ("MUSA",),
"server/ascend": ("NPU",),
"layers": ("CUDA",),
"unit": ("CUDA",),
"cli": ("CUDA",),
"manual": ("CUDA",),
}
# Filenames that match `test_*.py` by convention but contain no real tests
# (utility / fixture modules). Skipped before classification.
_MM_GEN_HELPER_FILENAMES = frozenset({"test_utils.py"})
# Filename token -> backend override. Tokenization is `stem.split("_")`, so a
# file like `test_server_1_gpu_musa.py` yields tokens
# {test, server, 1, gpu, musa} and matches `musa`. This correctly catches
# both `test_musa_*.py` (musa-named kernels) and `test_*_musa*.py` (musa
# variants of generic server tests). `nightly` is detected the same way and
# flips the nightly flag without changing the backend.
_MM_GEN_FILENAME_BACKEND_TOKENS = {
"musa": ("MUSA",),
"npu": ("NPU",),
}
def collect_all_tests(registered_dir: str) -> list[CIRegistry]:
"""Collect all CI registrations from registered directory."""
@@ -43,10 +100,92 @@ def collect_all_tests(registered_dir: str) -> list[CIRegistry]:
return all_tests
def collect_multimodal_gen_tests(
mm_gen_dir: str = MULTIMODAL_GEN_TEST_DIR,
) -> list[CIRegistry]:
"""Synthesize CIRegistry records for multimodal_gen tests.
multimodal_gen doesn't use register_*_ci(); see the module-level comment
above MULTIMODAL_GEN_TEST_DIR for the rules. Returns one record per
(file, backend) pair, matching the convention used for registered tests
that target multiple backends.
"""
mm_gen_path = Path(mm_gen_dir)
if not mm_gen_path.is_dir():
return []
# Discover test files: anything matching test_*.py anywhere under the
# tree. The csrc/ and apps/ subtrees aren't part of the test/ directory
# so we don't need to exclude them.
test_files = sorted(mm_gen_path.glob("**/test_*.py"))
records: list[CIRegistry] = []
for file in test_files:
rel = file.relative_to(mm_gen_path)
subdir = "/".join(rel.parts[:-1]) # "" for top-level
filename_only = rel.parts[-1]
if filename_only in _MM_GEN_HELPER_FILENAMES:
continue # test_utils.py and similar -- helper modules, not tests
# Tokenize stem to look for explicit backend / nightly markers.
stem_tokens = set(filename_only[:-3].split("_"))
nightly = "nightly" in stem_tokens
backends: tuple[str, ...] = ()
for token, override in _MM_GEN_FILENAME_BACKEND_TOKENS.items():
if token in stem_tokens:
backends = override
break
if not backends:
backends = _MM_GEN_SUBDIR_BACKENDS.get(subdir, ())
if not backends:
print(
f"Warning: multimodal_gen file {file} matches no backend "
f"rule (subdir={subdir!r}, tokens={sorted(stem_tokens)}); "
f"add a rule to _MM_GEN_SUBDIR_BACKENDS.",
file=sys.stderr,
)
continue
for backend_name in backends:
records.append(
CIRegistry(
backend=HWBackend[backend_name],
filename=str(file),
# mm_gen does its own per-case partitioning -- file-level
# estimates aren't available. Surfaced as 0 in the
# by-suite section, which is correct (we don't know).
est_time=0.0,
suite=f"mm-gen-{backend_name.lower()}",
nightly=nightly,
disabled=None,
)
)
return records
def get_folder_name(filename: str) -> str:
"""Extract folder name from test filename."""
# e.g., "registered/models/test_foo.py" -> "models"
"""Extract folder name from test filename.
Registered tests use test/registered/<folder>/test_*.py and map to
<folder>. multimodal_gen tests live outside that tree and get a virtual
`mm_gen/<subdir>` folder so they show up as their own rows in the
Folder Summary table.
"""
parts = Path(filename).parts
if "multimodal_gen" in parts:
try:
mg_idx = parts.index("multimodal_gen")
test_idx = parts.index("test", mg_idx)
sub_parts = parts[test_idx + 1 : -1] # strip 'test' anchor + file
if sub_parts:
return "mm_gen/" + "/".join(sub_parts)
return "mm_gen"
except ValueError:
pass # fall through to default
if "registered" in parts:
idx = parts.index("registered")
if idx + 1 < len(parts) - 1: # Has subfolder
@@ -113,7 +252,7 @@ def generate_summary_section(data: dict) -> str:
lines.append("| Backend | Total | Enabled | Disabled | Per-Commit | Nightly |")
lines.append("|---------|-------|---------|----------|------------|---------|")
for backend in ["CUDA", "AMD", "NPU", "CPU"]:
for backend in BACKEND_DISPLAY_ORDER:
backend_tests = by_backend.get(backend, [])
if not backend_tests:
continue
@@ -128,21 +267,25 @@ def generate_summary_section(data: dict) -> str:
lines.append("\n</details>\n")
# Folder summary (collapsible)
# Folder summary (collapsible). Only show columns for backends that
# have at least one registered test across the whole report -- otherwise
# adding scaffolding for an unused backend would widen every row with a
# column of zeros.
active_backends = [b for b in BACKEND_DISPLAY_ORDER if by_backend.get(b)]
lines.append("<details>")
lines.append("<summary><h2>Folder Summary</h2></summary>\n")
lines.append("| Folder | CUDA | AMD | NPU | CPU | Total |")
lines.append("|--------|------|-----|-----|-----|-------|")
header_cells = ["Folder", *active_backends, "Total"]
lines.append("| " + " | ".join(header_cells) + " |")
lines.append("|" + "|".join(["-" * max(len(c), 3) for c in header_cells]) + "|")
for folder in sorted(by_folder.keys()):
folder_tests = by_folder[folder]
cuda = sum(1 for t in folder_tests if t.backend == HWBackend.CUDA)
amd = sum(1 for t in folder_tests if t.backend == HWBackend.AMD)
npu = sum(1 for t in folder_tests if t.backend == HWBackend.NPU)
cpu = sum(1 for t in folder_tests if t.backend == HWBackend.CPU)
lines.append(
f"| {folder} | {cuda} | {amd} | {npu} | {cpu} | {len(folder_tests)} |"
)
backend_counts = {b.name: 0 for b in HWBackend}
for t in folder_tests:
backend_counts[t.backend.name] += 1
row = [folder] + [str(backend_counts[b]) for b in active_backends]
row.append(str(len(folder_tests)))
lines.append("| " + " | ".join(row) + " |")
lines.append("\n</details>\n")
@@ -182,7 +325,7 @@ def generate_by_folder_section(data: dict) -> str:
for t in folder_tests:
folder_by_backend[t.backend.name].append(t)
for backend in ["CUDA", "AMD", "NPU", "CPU"]:
for backend in BACKEND_DISPLAY_ORDER:
backend_tests = folder_by_backend.get(backend, [])
if not backend_tests:
continue
@@ -216,7 +359,7 @@ def generate_by_suite_section(data: dict) -> str:
lines.append("# All Tests by Test Suite\n")
for backend in ["CUDA", "AMD", "NPU", "CPU"]:
for backend in BACKEND_DISPLAY_ORDER:
backend_tests = by_backend.get(backend, [])
if not backend_tests:
continue
@@ -332,7 +475,7 @@ def generate_json_report(tests: list[CIRegistry]) -> str:
"backends": {},
}
for backend in ["CUDA", "AMD", "NPU", "CPU"]:
for backend in BACKEND_DISPLAY_ORDER:
backend_tests = folder_by_backend.get(backend, [])
if backend_tests:
data["tests_by_folder"][folder]["backends"][backend] = [
@@ -350,7 +493,7 @@ def generate_json_report(tests: list[CIRegistry]) -> str:
]
# Section 2: Tests by Suite (Backend -> Suite)
for backend in ["CUDA", "AMD", "NPU", "CPU"]:
for backend in BACKEND_DISPLAY_ORDER:
backend_tests = by_backend.get(backend, [])
if not backend_tests:
continue
@@ -393,7 +536,7 @@ def generate_json_report(tests: list[CIRegistry]) -> str:
}
# Backend summary
for backend in ["CUDA", "AMD", "NPU", "CPU"]:
for backend in BACKEND_DISPLAY_ORDER:
backend_tests = by_backend.get(backend, [])
if backend_tests:
data["backend_summary"][backend] = {
@@ -408,14 +551,16 @@ def generate_json_report(tests: list[CIRegistry]) -> str:
),
}
# Folder summary
# Folder summary -- one count per backend in HWBackend, in display
# order, so every registered backend shows up regardless of whether
# this folder has tests for it.
for folder in sorted(by_folder.keys()):
folder_tests = by_folder[folder]
backend_counts = {b: 0 for b in BACKEND_DISPLAY_ORDER}
for t in folder_tests:
backend_counts[t.backend.name] += 1
data["folder_summary"][folder] = {
"CUDA": sum(1 for t in folder_tests if t.backend == HWBackend.CUDA),
"AMD": sum(1 for t in folder_tests if t.backend == HWBackend.AMD),
"NPU": sum(1 for t in folder_tests if t.backend == HWBackend.NPU),
"CPU": sum(1 for t in folder_tests if t.backend == HWBackend.CPU),
**backend_counts,
"total": len(folder_tests),
}
@@ -452,6 +597,14 @@ def main():
default="test/registered",
help="Path to registered test directory",
)
parser.add_argument(
"--multimodal-gen-dir",
default=MULTIMODAL_GEN_TEST_DIR,
help=(
"Path to multimodal_gen test directory. Pass empty string to "
"skip multimodal_gen tests entirely."
),
)
args = parser.parse_args()
# Change to repo root if needed
@@ -460,6 +613,8 @@ def main():
os.chdir(repo_root)
tests = collect_all_tests(args.registered_dir)
if args.multimodal_gen_dir:
tests.extend(collect_multimodal_gen_tests(args.multimodal_gen_dir))
if args.output_format == "markdown":
report = generate_markdown_report(tests, section=args.section)