diff --git a/.github/workflows/ci-coverage-overview.yml b/.github/workflows/ci-coverage-overview.yml index 9a9f84fda..121a68764 100644 --- a/.github/workflows/ci-coverage-overview.yml +++ b/.github/workflows/ci-coverage-overview.yml @@ -3,11 +3,6 @@ name: CI Coverage Overview on: schedule: - cron: '0 6 * * *' # Daily at 6 AM UTC - pull_request: - paths: - - '.github/workflows/ci-coverage-overview.yml' - - 'scripts/ci/utils/ci_coverage_report.py' - - 'test/registered/**' workflow_dispatch: inputs: output_format: @@ -70,13 +65,23 @@ jobs: unit-test-coverage: name: Unit Test Code Coverage - if: github.event_name != 'pull_request' runs-on: 1-gpu-h100 timeout-minutes: 30 steps: - name: Checkout code uses: actions/checkout@v4 + - name: Install Rust toolchain + protoc + # Editable install of sglang triggers `build_rust` for the bundled + # native gRPC extension (rust/sglang-grpc). The 1-gpu-h100 runner + # image has no Rust toolchain by default, which is why this job has + # been failing on every scheduled run. Use the same helper that + # ci_install_dependency.sh uses so the install path matches the + # rest of CI. + run: | + bash scripts/ci/utils/install_rust_protoc.sh + echo "${CARGO_HOME:-$HOME/.cargo}/bin" >> $GITHUB_PATH + - name: Install dependencies timeout-minutes: 10 run: | diff --git a/python/sglang/test/ci/ci_register.py b/python/sglang/test/ci/ci_register.py index d832527c9..d42c310eb 100644 --- a/python/sglang/test/ci/ci_register.py +++ b/python/sglang/test/ci/ci_register.py @@ -14,6 +14,7 @@ __all__ = [ "register_amd_ci", "register_npu_ci", "register_xpu_ci", + "register_musa_ci", "ut_parse_one_file", ] @@ -32,6 +33,7 @@ class HWBackend(Enum): AMD = auto() NPU = auto() XPU = auto() + MUSA = auto() @dataclass @@ -119,12 +121,26 @@ def register_xpu_ci( return None +def register_musa_ci( + est_time: float, + suite: Optional[str] = None, + nightly: bool = False, + disabled: Optional[str] = None, + *, + stage: Optional[str] = None, + runner_config: Optional[str] = None, +): + """Marker for MUSA CI registration (parsed via AST; runtime no-op).""" + return None + + REGISTER_MAPPING = { "register_cpu_ci": HWBackend.CPU, "register_cuda_ci": HWBackend.CUDA, "register_amd_ci": HWBackend.AMD, "register_npu_ci": HWBackend.NPU, "register_xpu_ci": HWBackend.XPU, + "register_musa_ci": HWBackend.MUSA, } diff --git a/scripts/ci/utils/ci_coverage_report.py b/scripts/ci/utils/ci_coverage_report.py index 80a5f9239..e04218f18 100755 --- a/scripts/ci/utils/ci_coverage_report.py +++ b/scripts/ci/utils/ci_coverage_report.py @@ -27,6 +27,63 @@ sys.path.insert( from ci_register import CIRegistry, HWBackend, ut_parse_one_file +# Display order for backend tables / sections. The list is sourced from +# HWBackend so a newly-added enum member can never be silently dropped from +# the report -- if the assert below fires, add the new backend name here in +# the right display slot. Order isn't alphabetical: CUDA/AMD/NPU/CPU lead +# (highest test volume historically), then accelerators that have been +# wired into the registry more recently (XPU, MUSA). +BACKEND_DISPLAY_ORDER = ("CUDA", "AMD", "NPU", "CPU", "XPU", "MUSA") +assert set(BACKEND_DISPLAY_ORDER) == { + b.name for b in HWBackend +}, "BACKEND_DISPLAY_ORDER is out of sync with HWBackend" + +# --------------------------------------------------------------------------- # +# multimodal_gen test coverage +# +# multimodal_gen tests live under python/sglang/multimodal_gen/test/ and use +# their own run_suite.py / partitioning framework, NOT the register_*_ci() +# registry used under test/registered/. To surface them in the daily +# overview we synthesize CIRegistry records from file paths + filename +# tokens, using the rules below. These are heuristics derived from how the +# multimodal_gen workflows (pr-test-{musa,npu,amd}.yml, pr-test-multimodal- +# gen.yml) currently invoke each file -- they MAY drift if a workflow +# stops/starts running a directory. +# --------------------------------------------------------------------------- # +MULTIMODAL_GEN_TEST_DIR = "python/sglang/multimodal_gen/test" + +# Subdirectory (relative to MULTIMODAL_GEN_TEST_DIR) -> backends those files +# run on by default. Empty string is the top-level. Files whose tokenized +# filename contains an explicit backend marker (see below) override this. +_MM_GEN_SUBDIR_BACKENDS = { + # Top-level helper-style tests (e.g. test_consistency_metrics.py). + "": ("CUDA",), + # server/ top-level: pr-test-multimodal-gen.yml drives CUDA, pr-test-amd + # mirrors the same suite on AMD runners. + "server": ("CUDA", "AMD"), + "server/musa": ("MUSA",), + "server/ascend": ("NPU",), + "layers": ("CUDA",), + "unit": ("CUDA",), + "cli": ("CUDA",), + "manual": ("CUDA",), +} + +# Filenames that match `test_*.py` by convention but contain no real tests +# (utility / fixture modules). Skipped before classification. +_MM_GEN_HELPER_FILENAMES = frozenset({"test_utils.py"}) + +# Filename token -> backend override. Tokenization is `stem.split("_")`, so a +# file like `test_server_1_gpu_musa.py` yields tokens +# {test, server, 1, gpu, musa} and matches `musa`. This correctly catches +# both `test_musa_*.py` (musa-named kernels) and `test_*_musa*.py` (musa +# variants of generic server tests). `nightly` is detected the same way and +# flips the nightly flag without changing the backend. +_MM_GEN_FILENAME_BACKEND_TOKENS = { + "musa": ("MUSA",), + "npu": ("NPU",), +} + def collect_all_tests(registered_dir: str) -> list[CIRegistry]: """Collect all CI registrations from registered directory.""" @@ -43,10 +100,92 @@ def collect_all_tests(registered_dir: str) -> list[CIRegistry]: return all_tests +def collect_multimodal_gen_tests( + mm_gen_dir: str = MULTIMODAL_GEN_TEST_DIR, +) -> list[CIRegistry]: + """Synthesize CIRegistry records for multimodal_gen tests. + + multimodal_gen doesn't use register_*_ci(); see the module-level comment + above MULTIMODAL_GEN_TEST_DIR for the rules. Returns one record per + (file, backend) pair, matching the convention used for registered tests + that target multiple backends. + """ + mm_gen_path = Path(mm_gen_dir) + if not mm_gen_path.is_dir(): + return [] + + # Discover test files: anything matching test_*.py anywhere under the + # tree. The csrc/ and apps/ subtrees aren't part of the test/ directory + # so we don't need to exclude them. + test_files = sorted(mm_gen_path.glob("**/test_*.py")) + records: list[CIRegistry] = [] + + for file in test_files: + rel = file.relative_to(mm_gen_path) + subdir = "/".join(rel.parts[:-1]) # "" for top-level + filename_only = rel.parts[-1] + + if filename_only in _MM_GEN_HELPER_FILENAMES: + continue # test_utils.py and similar -- helper modules, not tests + + # Tokenize stem to look for explicit backend / nightly markers. + stem_tokens = set(filename_only[:-3].split("_")) + nightly = "nightly" in stem_tokens + + backends: tuple[str, ...] = () + for token, override in _MM_GEN_FILENAME_BACKEND_TOKENS.items(): + if token in stem_tokens: + backends = override + break + if not backends: + backends = _MM_GEN_SUBDIR_BACKENDS.get(subdir, ()) + + if not backends: + print( + f"Warning: multimodal_gen file {file} matches no backend " + f"rule (subdir={subdir!r}, tokens={sorted(stem_tokens)}); " + f"add a rule to _MM_GEN_SUBDIR_BACKENDS.", + file=sys.stderr, + ) + continue + + for backend_name in backends: + records.append( + CIRegistry( + backend=HWBackend[backend_name], + filename=str(file), + # mm_gen does its own per-case partitioning -- file-level + # estimates aren't available. Surfaced as 0 in the + # by-suite section, which is correct (we don't know). + est_time=0.0, + suite=f"mm-gen-{backend_name.lower()}", + nightly=nightly, + disabled=None, + ) + ) + + return records + + def get_folder_name(filename: str) -> str: - """Extract folder name from test filename.""" - # e.g., "registered/models/test_foo.py" -> "models" + """Extract folder name from test filename. + + Registered tests use test/registered//test_*.py and map to + . multimodal_gen tests live outside that tree and get a virtual + `mm_gen/` folder so they show up as their own rows in the + Folder Summary table. + """ parts = Path(filename).parts + if "multimodal_gen" in parts: + try: + mg_idx = parts.index("multimodal_gen") + test_idx = parts.index("test", mg_idx) + sub_parts = parts[test_idx + 1 : -1] # strip 'test' anchor + file + if sub_parts: + return "mm_gen/" + "/".join(sub_parts) + return "mm_gen" + except ValueError: + pass # fall through to default if "registered" in parts: idx = parts.index("registered") if idx + 1 < len(parts) - 1: # Has subfolder @@ -113,7 +252,7 @@ def generate_summary_section(data: dict) -> str: lines.append("| Backend | Total | Enabled | Disabled | Per-Commit | Nightly |") lines.append("|---------|-------|---------|----------|------------|---------|") - for backend in ["CUDA", "AMD", "NPU", "CPU"]: + for backend in BACKEND_DISPLAY_ORDER: backend_tests = by_backend.get(backend, []) if not backend_tests: continue @@ -128,21 +267,25 @@ def generate_summary_section(data: dict) -> str: lines.append("\n\n") - # Folder summary (collapsible) + # Folder summary (collapsible). Only show columns for backends that + # have at least one registered test across the whole report -- otherwise + # adding scaffolding for an unused backend would widen every row with a + # column of zeros. + active_backends = [b for b in BACKEND_DISPLAY_ORDER if by_backend.get(b)] lines.append("
") lines.append("

Folder Summary

\n") - lines.append("| Folder | CUDA | AMD | NPU | CPU | Total |") - lines.append("|--------|------|-----|-----|-----|-------|") + header_cells = ["Folder", *active_backends, "Total"] + lines.append("| " + " | ".join(header_cells) + " |") + lines.append("|" + "|".join(["-" * max(len(c), 3) for c in header_cells]) + "|") for folder in sorted(by_folder.keys()): folder_tests = by_folder[folder] - cuda = sum(1 for t in folder_tests if t.backend == HWBackend.CUDA) - amd = sum(1 for t in folder_tests if t.backend == HWBackend.AMD) - npu = sum(1 for t in folder_tests if t.backend == HWBackend.NPU) - cpu = sum(1 for t in folder_tests if t.backend == HWBackend.CPU) - lines.append( - f"| {folder} | {cuda} | {amd} | {npu} | {cpu} | {len(folder_tests)} |" - ) + backend_counts = {b.name: 0 for b in HWBackend} + for t in folder_tests: + backend_counts[t.backend.name] += 1 + row = [folder] + [str(backend_counts[b]) for b in active_backends] + row.append(str(len(folder_tests))) + lines.append("| " + " | ".join(row) + " |") lines.append("\n
\n") @@ -182,7 +325,7 @@ def generate_by_folder_section(data: dict) -> str: for t in folder_tests: folder_by_backend[t.backend.name].append(t) - for backend in ["CUDA", "AMD", "NPU", "CPU"]: + for backend in BACKEND_DISPLAY_ORDER: backend_tests = folder_by_backend.get(backend, []) if not backend_tests: continue @@ -216,7 +359,7 @@ def generate_by_suite_section(data: dict) -> str: lines.append("# All Tests by Test Suite\n") - for backend in ["CUDA", "AMD", "NPU", "CPU"]: + for backend in BACKEND_DISPLAY_ORDER: backend_tests = by_backend.get(backend, []) if not backend_tests: continue @@ -332,7 +475,7 @@ def generate_json_report(tests: list[CIRegistry]) -> str: "backends": {}, } - for backend in ["CUDA", "AMD", "NPU", "CPU"]: + for backend in BACKEND_DISPLAY_ORDER: backend_tests = folder_by_backend.get(backend, []) if backend_tests: data["tests_by_folder"][folder]["backends"][backend] = [ @@ -350,7 +493,7 @@ def generate_json_report(tests: list[CIRegistry]) -> str: ] # Section 2: Tests by Suite (Backend -> Suite) - for backend in ["CUDA", "AMD", "NPU", "CPU"]: + for backend in BACKEND_DISPLAY_ORDER: backend_tests = by_backend.get(backend, []) if not backend_tests: continue @@ -393,7 +536,7 @@ def generate_json_report(tests: list[CIRegistry]) -> str: } # Backend summary - for backend in ["CUDA", "AMD", "NPU", "CPU"]: + for backend in BACKEND_DISPLAY_ORDER: backend_tests = by_backend.get(backend, []) if backend_tests: data["backend_summary"][backend] = { @@ -408,14 +551,16 @@ def generate_json_report(tests: list[CIRegistry]) -> str: ), } - # Folder summary + # Folder summary -- one count per backend in HWBackend, in display + # order, so every registered backend shows up regardless of whether + # this folder has tests for it. for folder in sorted(by_folder.keys()): folder_tests = by_folder[folder] + backend_counts = {b: 0 for b in BACKEND_DISPLAY_ORDER} + for t in folder_tests: + backend_counts[t.backend.name] += 1 data["folder_summary"][folder] = { - "CUDA": sum(1 for t in folder_tests if t.backend == HWBackend.CUDA), - "AMD": sum(1 for t in folder_tests if t.backend == HWBackend.AMD), - "NPU": sum(1 for t in folder_tests if t.backend == HWBackend.NPU), - "CPU": sum(1 for t in folder_tests if t.backend == HWBackend.CPU), + **backend_counts, "total": len(folder_tests), } @@ -452,6 +597,14 @@ def main(): default="test/registered", help="Path to registered test directory", ) + parser.add_argument( + "--multimodal-gen-dir", + default=MULTIMODAL_GEN_TEST_DIR, + help=( + "Path to multimodal_gen test directory. Pass empty string to " + "skip multimodal_gen tests entirely." + ), + ) args = parser.parse_args() # Change to repo root if needed @@ -460,6 +613,8 @@ def main(): os.chdir(repo_root) tests = collect_all_tests(args.registered_dir) + if args.multimodal_gen_dir: + tests.extend(collect_multimodal_gen_tests(args.multimodal_gen_dir)) if args.output_format == "markdown": report = generate_markdown_report(tests, section=args.section)