From d010e50ad662ffd03fa220a8a0d0bda2d98325a7 Mon Sep 17 00:00:00 2001 From: yunzshi <40199309+yunzshi@users.noreply.github.com> Date: Thu, 1 Oct 2026 15:01:17 +0800 Subject: [PATCH] [PPU][1/N] CI: Add backend registration and runner preflight (#39788) Co-authored-by: Yihao Wang <42559837+AgainstEntropy@users.noreply.github.com> Co-authored-by: Alex Nails --- .github/workflows/pr-test-ppu.yml | 169 ++++++++++++++++ python/sglang/test/ci/ci_register.py | 16 ++ scripts/ci/ppu/check_ppu_environment.py | 184 ++++++++++++++++++ scripts/ci/utils/ci_coverage_report.py | 4 +- scripts/ci/utils/slash_command_handler.py | 1 + .../check_no_registered_tests_in_package.py | 1 + test/run_suite.py | 10 + 7 files changed, 383 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/pr-test-ppu.yml create mode 100644 scripts/ci/ppu/check_ppu_environment.py diff --git a/.github/workflows/pr-test-ppu.yml b/.github/workflows/pr-test-ppu.yml new file mode 100644 index 0000000000..b1f5c19f54 --- /dev/null +++ b/.github/workflows/pr-test-ppu.yml @@ -0,0 +1,169 @@ +name: PR Test (PPU) + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + types: [opened, synchronize, reopened, labeled] + workflow_dispatch: + inputs: + run_all_tests: + description: "Run the preflight even if the last commit touched no PPU path" + required: false + type: boolean + default: true + workflow_call: + inputs: + ref: + description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.' + required: false + type: string + default: '' + run_all_tests: + description: "Run all tests (for releasing or testing purpose)" + required: false + type: boolean + default: false + +concurrency: + # Newer runs cancel older runs for the same ref; unrelated label events + # use isolated groups to avoid interfering with active runs. + group: >- + pr-test-ppu-${{ + (github.event_name == 'pull_request' && + github.event.action == 'labeled' && + github.event.label.name != 'run-ci' && + github.event.label.name != 'run-ci-ppu') && + format('ignored-label-{0}', github.run_id) || + inputs.ref || github.ref + }} + cancel-in-progress: true + +jobs: + # ==================== Check Changes ==================== # + check-changes: + # Give skipped checks from unrelated labels distinct names so they cannot + # replace the corresponding checks from an active run. + name: >- + ${{ (github.event_name == 'pull_request' && github.event.action == 'labeled' && + github.event.label.name != 'run-ci' && github.event.label.name != 'run-ci-ppu') && + 'ignored-label-check-changes' || 'check-changes' }} + if: | + github.event_name != 'pull_request' || + github.event.action != 'labeled' || + github.event.label.name == 'run-ci' || + github.event.label.name == 'run-ci-ppu' + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + outputs: + # Scope only; the shared pr-gate checks both required labels. + ppu: ${{ steps.filter.outputs.ppu == 'true' || steps.run-mode.outputs.run_all_tests == 'true' }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + # github.sha (not github.ref) so a runner that dequeues after main + # advances still validates the SHA that triggered the run. + ref: ${{ inputs.ref || github.sha }} + + - name: Determine run mode + id: run-mode + run: | + if [[ "${{ inputs.run_all_tests }}" == "true" ]]; then + echo "run_all_tests=true" >> "$GITHUB_OUTPUT" + echo "Run mode: ALL TESTS" + else + echo "run_all_tests=false" >> "$GITHUB_OUTPUT" + echo "Run mode: FILTERED (triggered by ${{ github.event_name }})" + fi + + - name: Detect file changes + id: filter + uses: dorny/paths-filter@v3 + if: steps.run-mode.outputs.run_all_tests != 'true' + with: + # PPU-specific paths only. + filters: | + ppu: + - ".github/workflows/pr-test-ppu.yml" + - "scripts/ci/ppu/**" + - "python/sglang/srt/platforms/ppu.py" + - "python/sglang/srt/hardware_backend/ppu/**" + - "python/sglang/kernels/aot/csrc/ppu/**" + + # ==================== PR Gate ==================== # + pr-gate: + name: >- + ${{ (github.event_name == 'pull_request' && github.event.action == 'labeled' && + github.event.label.name != 'run-ci' && github.event.label.name != 'run-ci-ppu') && + 'ignored-label-pr-gate' || 'pr-gate' }} + needs: check-changes + if: needs.check-changes.outputs.ppu == 'true' + uses: ./.github/workflows/pr-gate.yml + with: + require-run-ci: true + require-label: run-ci-ppu + secrets: inherit + + # ==================== PPU Runner Preflight ==================== # + # Validates the runner contract only; the SRT platform and AOT kernels land + # in later PRs and build their first real test on top of this job. + ppu-preflight: + name: >- + ${{ (github.event_name == 'pull_request' && github.event.action == 'labeled' && + github.event.label.name != 'run-ci' && github.event.label.name != 'run-ci-ppu') && + 'ignored-label-ppu-preflight' || 'ppu-preflight' }} + needs: [check-changes, pr-gate] + if: needs.check-changes.outputs.ppu == 'true' + runs-on: 1-ppu-m890p + timeout-minutes: 60 + env: + SGLANG_IS_IN_CI: true + steps: + - name: Checkout code + timeout-minutes: 10 + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Validate PPU runner contract + timeout-minutes: 10 + run: python3 scripts/ci/ppu/check_ppu_environment.py + + # ==================== Finish ==================== # + pr-test-ppu-finish: + name: >- + ${{ (github.event_name == 'pull_request' && github.event.action == 'labeled' && + github.event.label.name != 'run-ci' && github.event.label.name != 'run-ci-ppu') && + 'ignored-label-pr-test-ppu-finish' || 'pr-test-ppu-finish' }} + needs: [pr-gate, check-changes, ppu-preflight] + if: always() && needs.check-changes.result != 'skipped' + runs-on: ubuntu-latest + steps: + - name: Check all dependent job statuses + run: | + json_needs='${{ toJson(needs) }}' + for job in $(echo "$json_needs" | jq -r 'keys_unsorted[]'); do + result=$(echo "$json_needs" | jq -r --arg j "$job" '.[$j].result') + echo "$job: $result" + if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then + echo "The above jobs failed." + exit 1 + fi + done + echo "All jobs completed successfully" + + - name: Require the preflight when PPU is in scope + run: | + in_scope='${{ needs.check-changes.outputs.ppu }}' + preflight='${{ needs.ppu-preflight.result }}' + echo "PPU in scope: '$in_scope' / ppu-preflight: '$preflight'" + + if [[ "$in_scope" == "true" && "$preflight" != "success" ]]; then + echo "PPU was in scope but ppu-preflight did not succeed." + exit 1 + fi + echo "Preflight status is consistent with the requested scope." diff --git a/python/sglang/test/ci/ci_register.py b/python/sglang/test/ci/ci_register.py index 227fd57013..9d946f5aae 100644 --- a/python/sglang/test/ci/ci_register.py +++ b/python/sglang/test/ci/ci_register.py @@ -17,6 +17,7 @@ __all__ = [ "register_xpu_ci", "register_musa_ci", "register_mlx_ci", + "register_ppu_ci", "ut_parse_one_file", ] @@ -37,6 +38,7 @@ class HWBackend(Enum): XPU = auto() MUSA = auto() MLX = auto() + PPU = auto() @dataclass @@ -150,6 +152,19 @@ def register_mlx_ci( return None +def register_ppu_ci( + est_time: float, + suite: Optional[str] = None, + nightly: bool = False, + disabled: Optional[str] = None, + *, + stage: Optional[str] = None, + runner_config: Optional[str] = None, +): + """Marker for PPU CI registration (parsed via AST; runtime no-op).""" + return None + + REGISTER_MAPPING = { "register_cpu_ci": HWBackend.CPU, "register_cuda_ci": HWBackend.CUDA, @@ -159,6 +174,7 @@ REGISTER_MAPPING = { "register_xpu_ci": HWBackend.XPU, "register_musa_ci": HWBackend.MUSA, "register_mlx_ci": HWBackend.MLX, + "register_ppu_ci": HWBackend.PPU, } diff --git a/scripts/ci/ppu/check_ppu_environment.py b/scripts/ci/ppu/check_ppu_environment.py new file mode 100644 index 0000000000..893c3be29a --- /dev/null +++ b/scripts/ci/ppu/check_ppu_environment.py @@ -0,0 +1,184 @@ +"""Validate the PPU runner environment without importing SGLang.""" + +import os +import platform +import re +import shutil +import stat +import subprocess +import traceback +from pathlib import Path + + +class CheckFailure(RuntimeError): + """A failed PPU environment contract check.""" + + +def require(condition: bool, message: str) -> None: + """Raise a concise contract failure when a condition is false.""" + if not condition: + raise CheckFailure(message) + + +def print_host_summary() -> None: + """Print capacity information without hostnames or network identifiers.""" + disk_usage = shutil.disk_usage("/") + + print(f"Runner OS: {platform.system()}") + print(f"Runner architecture: {platform.machine()}") + print(f"Python version: {platform.python_version()}") + + # Memory reporting is diagnostic-only; it must not fail the preflight. + try: + import psutil + + print(f"Memory total (bytes): {psutil.virtual_memory().total}") + except Exception: + print("Memory total (bytes): unavailable") + print(f"Root disk total (bytes): {disk_usage.total}") + print(f"Root disk available (bytes): {disk_usage.free}") + + +def check_ppu_inventory() -> int: + """Return the number of PPUs reported by the management interface.""" + ppu_smi = shutil.which("ppu-smi") + require(ppu_smi is not None, "ppu-smi is not available on PATH") + + try: + result = subprocess.run( + [ppu_smi, "--query-ppu=index", "--format=csv,noheader"], + check=False, + capture_output=True, + text=True, + timeout=15, + ) + except (OSError, subprocess.TimeoutExpired) as error: + raise CheckFailure( + f"ppu-smi could not query the PPU inventory: {error}" + ) from error + + if result.returncode != 0: + stderr = result.stderr.strip() + detail = f" (rc={result.returncode})" + if stderr: + detail += f": {stderr}" + raise CheckFailure(f"ppu-smi failed to query the PPU inventory{detail}") + + inventory_lines = result.stdout.splitlines() + indices = [line.strip() for line in inventory_lines if line.strip()] + require(indices, "ppu-smi did not report any PPU devices") + require( + all(re.fullmatch(r"[0-9]+", index) for index in indices), + "ppu-smi returned an unexpected inventory format", + ) + + print(f"PPU devices reported: {len(indices)}") + return len(indices) + + +def check_device_nodes() -> None: + """Check the character devices used by the current PPU runtime.""" + for device_node in (Path("/dev/alixpu_ctl"), Path("/dev/alixpu")): + try: + is_character_device = stat.S_ISCHR(device_node.stat().st_mode) + except OSError as error: + raise CheckFailure( + f"Required PPU character device is unavailable: {device_node}" + ) from error + require( + is_character_device, + f"Required PPU device is not a character device: {device_node}", + ) + + print("Required PPU device nodes are present.") + + +def check_ppu_sdk() -> None: + """Verify that PPU_SDK names an installed SDK directory.""" + ppu_sdk = os.environ.get("PPU_SDK", "") + require(ppu_sdk, "PPU_SDK is not set") + require( + Path(ppu_sdk).is_dir(), + f"PPU_SDK must name an existing directory, got {ppu_sdk!r}", + ) + print(f"PPU SDK directory is available: {ppu_sdk}") + + +def check_torch_compute() -> int: + """Run a minimal PPU computation using PyTorch's CUDA-compatible API.""" + try: + import torch + except Exception as error: + raise CheckFailure(f"PyTorch cannot be imported: {error}") from error + + require( + torch.cuda.is_available(), + "torch.cuda.is_available() returned false; PPU is reached through the " + "CUDA-compatible device type, so this means no PPU is usable", + ) + device_count = torch.cuda.device_count() + require( + device_count > 0, + "torch.cuda.device_count() did not report a device", + ) + + # Device-name lookup is optional and must not block the compute check. + try: + print(f"torch.cuda.get_device_name(0): {torch.cuda.get_device_name(0)}") + except Exception: + print("torch.cuda.get_device_name(0): unavailable") + + device = torch.device("cuda:0") + try: + source = torch.tensor([1.0, 2.0, 3.0], device=device) + actual = source * 2.0 + 1.0 + torch.cuda.synchronize(device) + actual_cpu = actual.cpu() + expected = torch.tensor([3.0, 5.0, 7.0]) + torch.testing.assert_close(actual_cpu, expected) + except Exception as error: + raise CheckFailure(f"Tensor compute on cuda:0 failed: {error}") from error + + print(f"PyTorch version: {torch.__version__}") + print(f"PPU devices visible through torch.cuda: {device_count}") + print("Tensor compute on cuda:0 passed.") + return device_count + + +def check_visibility_consistency(reported: int, visible: int) -> None: + """Require visible devices not to exceed the ppu-smi inventory. + + Device visibility restrictions may expose fewer devices to PyTorch. + """ + require( + visible <= reported, + f"torch reports {visible} device(s) but ppu-smi reports only {reported}", + ) + if visible < reported: + print( + f"Note: {visible} of {reported} PPU(s) visible to this job " + f"(device visibility is restricted per runner job)." + ) + else: + print(f"ppu-smi and torch agree on {reported} device(s).") + + +def main() -> int: + """Run every check and return a process-compatible status code.""" + try: + print_host_summary() + reported = check_ppu_inventory() + check_device_nodes() + check_ppu_sdk() + visible = check_torch_compute() + check_visibility_consistency(reported, visible) + except Exception: + traceback.print_exc() + return 1 + + print("PPU runner environment preflight passed.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci/utils/ci_coverage_report.py b/scripts/ci/utils/ci_coverage_report.py index 0862f048c7..7bda046ae9 100755 --- a/scripts/ci/utils/ci_coverage_report.py +++ b/scripts/ci/utils/ci_coverage_report.py @@ -34,8 +34,8 @@ from ci_register import CIRegistry, HWBackend, ut_parse_one_file # the report -- if the assert below fires, add the new backend name here in # the right display slot. Order isn't alphabetical: CUDA/AMD/NPU/CPU lead # (highest test volume historically), then accelerators that have been -# wired into the registry more recently (XPU, MUSA, MLX). -BACKEND_DISPLAY_ORDER = ("CUDA", "AMD", "NPU", "CPU", "XPU", "MUSA", "MLX") +# wired into the registry more recently (XPU, MUSA, MLX, PPU). +BACKEND_DISPLAY_ORDER = ("CUDA", "AMD", "NPU", "CPU", "XPU", "MUSA", "MLX", "PPU") assert set(BACKEND_DISPLAY_ORDER) == {b.name for b in HWBackend}, ( "BACKEND_DISPLAY_ORDER is out of sync with HWBackend" ) diff --git a/scripts/ci/utils/slash_command_handler.py b/scripts/ci/utils/slash_command_handler.py index d613f68565..abfe8d54fa 100644 --- a/scripts/ci/utils/slash_command_handler.py +++ b/scripts/ci/utils/slash_command_handler.py @@ -942,6 +942,7 @@ _OTHER_BACKEND_REGISTERS = { "register_xpu_ci": "XPU", "register_musa_ci": "MUSA", "register_mlx_ci": "MLX", + "register_ppu_ci": "PPU", } diff --git a/scripts/lint/check_no_registered_tests_in_package.py b/scripts/lint/check_no_registered_tests_in_package.py index d1e8e4f1fd..799b0cc102 100755 --- a/scripts/lint/check_no_registered_tests_in_package.py +++ b/scripts/lint/check_no_registered_tests_in_package.py @@ -30,6 +30,7 @@ _MARKERS = ( "register_npu_ci", "register_xpu_ci", "register_musa_ci", + "register_ppu_ci", ) diff --git a/test/run_suite.py b/test/run_suite.py index e69f35d26a..b048a18bd4 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -23,6 +23,7 @@ HW_MAPPING = { "npu": HWBackend.NPU, "xpu": HWBackend.XPU, "mlx": HWBackend.MLX, + "ppu": HWBackend.PPU, } # Per-commit test suites (run on every PR). @@ -124,6 +125,14 @@ PER_COMMIT_SUITES = { "stage-a-unit-test-mlx", "stage-b-e2e-mlx", ], + # PPU has no suite in any of the three dicts yet: pr-test-ppu.yml only runs + # the runner preflight until the PPU SRT platform and AOT kernels land. + # Declaring a name here before a workflow dispatches it would let a test + # register, validate, and count as covered while never running, so each PPU + # suite name arrives with the job that runs it. Until then PPU is in + # _SUITE_CHECKED_BACKENDS with no valid suite, so validate_all_suites() + # rejects any register_ppu_ci() outright. + HWBackend.PPU: [], } # Nightly test suites (run nightly, organized by GPU configuration) @@ -231,6 +240,7 @@ _SUITE_CHECKED_BACKENDS = { HWBackend.MUSA, HWBackend.XPU, HWBackend.MLX, + HWBackend.PPU, }