[PPU][1/N] CI: Add backend registration and runner preflight (#39788)

Co-authored-by: Yihao Wang <42559837+AgainstEntropy@users.noreply.github.com>
Co-authored-by: Alex Nails <alex.nails@radixark.ai>
This commit is contained in:
yunzshi
2026-10-01 00:01:17 -07:00
committed by GitHub
co-authored by Yihao Wang Alex Nails
parent 9c3262d855
commit d010e50ad6
7 changed files with 383 additions and 2 deletions
+169
View File
@@ -0,0 +1,169 @@
name: PR Test (PPU)
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
types: [opened, synchronize, reopened, labeled]
workflow_dispatch:
inputs:
run_all_tests:
description: "Run the preflight even if the last commit touched no PPU path"
required: false
type: boolean
default: true
workflow_call:
inputs:
ref:
description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
required: false
type: string
default: ''
run_all_tests:
description: "Run all tests (for releasing or testing purpose)"
required: false
type: boolean
default: false
concurrency:
# Newer runs cancel older runs for the same ref; unrelated label events
# use isolated groups to avoid interfering with active runs.
group: >-
pr-test-ppu-${{
(github.event_name == 'pull_request' &&
github.event.action == 'labeled' &&
github.event.label.name != 'run-ci' &&
github.event.label.name != 'run-ci-ppu') &&
format('ignored-label-{0}', github.run_id) ||
inputs.ref || github.ref
}}
cancel-in-progress: true
jobs:
# ==================== Check Changes ==================== #
check-changes:
# Give skipped checks from unrelated labels distinct names so they cannot
# replace the corresponding checks from an active run.
name: >-
${{ (github.event_name == 'pull_request' && github.event.action == 'labeled' &&
github.event.label.name != 'run-ci' && github.event.label.name != 'run-ci-ppu') &&
'ignored-label-check-changes' || 'check-changes' }}
if: |
github.event_name != 'pull_request' ||
github.event.action != 'labeled' ||
github.event.label.name == 'run-ci' ||
github.event.label.name == 'run-ci-ppu'
runs-on: ubuntu-latest
permissions:
contents: read
pull-requests: read
outputs:
# Scope only; the shared pr-gate checks both required labels.
ppu: ${{ steps.filter.outputs.ppu == 'true' || steps.run-mode.outputs.run_all_tests == 'true' }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
# github.sha (not github.ref) so a runner that dequeues after main
# advances still validates the SHA that triggered the run.
ref: ${{ inputs.ref || github.sha }}
- name: Determine run mode
id: run-mode
run: |
if [[ "${{ inputs.run_all_tests }}" == "true" ]]; then
echo "run_all_tests=true" >> "$GITHUB_OUTPUT"
echo "Run mode: ALL TESTS"
else
echo "run_all_tests=false" >> "$GITHUB_OUTPUT"
echo "Run mode: FILTERED (triggered by ${{ github.event_name }})"
fi
- name: Detect file changes
id: filter
uses: dorny/paths-filter@v3
if: steps.run-mode.outputs.run_all_tests != 'true'
with:
# PPU-specific paths only.
filters: |
ppu:
- ".github/workflows/pr-test-ppu.yml"
- "scripts/ci/ppu/**"
- "python/sglang/srt/platforms/ppu.py"
- "python/sglang/srt/hardware_backend/ppu/**"
- "python/sglang/kernels/aot/csrc/ppu/**"
# ==================== PR Gate ==================== #
pr-gate:
name: >-
${{ (github.event_name == 'pull_request' && github.event.action == 'labeled' &&
github.event.label.name != 'run-ci' && github.event.label.name != 'run-ci-ppu') &&
'ignored-label-pr-gate' || 'pr-gate' }}
needs: check-changes
if: needs.check-changes.outputs.ppu == 'true'
uses: ./.github/workflows/pr-gate.yml
with:
require-run-ci: true
require-label: run-ci-ppu
secrets: inherit
# ==================== PPU Runner Preflight ==================== #
# Validates the runner contract only; the SRT platform and AOT kernels land
# in later PRs and build their first real test on top of this job.
ppu-preflight:
name: >-
${{ (github.event_name == 'pull_request' && github.event.action == 'labeled' &&
github.event.label.name != 'run-ci' && github.event.label.name != 'run-ci-ppu') &&
'ignored-label-ppu-preflight' || 'ppu-preflight' }}
needs: [check-changes, pr-gate]
if: needs.check-changes.outputs.ppu == 'true'
runs-on: 1-ppu-m890p
timeout-minutes: 60
env:
SGLANG_IS_IN_CI: true
steps:
- name: Checkout code
timeout-minutes: 10
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.sha }}
- name: Validate PPU runner contract
timeout-minutes: 10
run: python3 scripts/ci/ppu/check_ppu_environment.py
# ==================== Finish ==================== #
pr-test-ppu-finish:
name: >-
${{ (github.event_name == 'pull_request' && github.event.action == 'labeled' &&
github.event.label.name != 'run-ci' && github.event.label.name != 'run-ci-ppu') &&
'ignored-label-pr-test-ppu-finish' || 'pr-test-ppu-finish' }}
needs: [pr-gate, check-changes, ppu-preflight]
if: always() && needs.check-changes.result != 'skipped'
runs-on: ubuntu-latest
steps:
- name: Check all dependent job statuses
run: |
json_needs='${{ toJson(needs) }}'
for job in $(echo "$json_needs" | jq -r 'keys_unsorted[]'); do
result=$(echo "$json_needs" | jq -r --arg j "$job" '.[$j].result')
echo "$job: $result"
if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then
echo "The above jobs failed."
exit 1
fi
done
echo "All jobs completed successfully"
- name: Require the preflight when PPU is in scope
run: |
in_scope='${{ needs.check-changes.outputs.ppu }}'
preflight='${{ needs.ppu-preflight.result }}'
echo "PPU in scope: '$in_scope' / ppu-preflight: '$preflight'"
if [[ "$in_scope" == "true" && "$preflight" != "success" ]]; then
echo "PPU was in scope but ppu-preflight did not succeed."
exit 1
fi
echo "Preflight status is consistent with the requested scope."
+16
View File
@@ -17,6 +17,7 @@ __all__ = [
"register_xpu_ci",
"register_musa_ci",
"register_mlx_ci",
"register_ppu_ci",
"ut_parse_one_file",
]
@@ -37,6 +38,7 @@ class HWBackend(Enum):
XPU = auto()
MUSA = auto()
MLX = auto()
PPU = auto()
@dataclass
@@ -150,6 +152,19 @@ def register_mlx_ci(
return None
def register_ppu_ci(
est_time: float,
suite: Optional[str] = None,
nightly: bool = False,
disabled: Optional[str] = None,
*,
stage: Optional[str] = None,
runner_config: Optional[str] = None,
):
"""Marker for PPU CI registration (parsed via AST; runtime no-op)."""
return None
REGISTER_MAPPING = {
"register_cpu_ci": HWBackend.CPU,
"register_cuda_ci": HWBackend.CUDA,
@@ -159,6 +174,7 @@ REGISTER_MAPPING = {
"register_xpu_ci": HWBackend.XPU,
"register_musa_ci": HWBackend.MUSA,
"register_mlx_ci": HWBackend.MLX,
"register_ppu_ci": HWBackend.PPU,
}
+184
View File
@@ -0,0 +1,184 @@
"""Validate the PPU runner environment without importing SGLang."""
import os
import platform
import re
import shutil
import stat
import subprocess
import traceback
from pathlib import Path
class CheckFailure(RuntimeError):
"""A failed PPU environment contract check."""
def require(condition: bool, message: str) -> None:
"""Raise a concise contract failure when a condition is false."""
if not condition:
raise CheckFailure(message)
def print_host_summary() -> None:
"""Print capacity information without hostnames or network identifiers."""
disk_usage = shutil.disk_usage("/")
print(f"Runner OS: {platform.system()}")
print(f"Runner architecture: {platform.machine()}")
print(f"Python version: {platform.python_version()}")
# Memory reporting is diagnostic-only; it must not fail the preflight.
try:
import psutil
print(f"Memory total (bytes): {psutil.virtual_memory().total}")
except Exception:
print("Memory total (bytes): unavailable")
print(f"Root disk total (bytes): {disk_usage.total}")
print(f"Root disk available (bytes): {disk_usage.free}")
def check_ppu_inventory() -> int:
"""Return the number of PPUs reported by the management interface."""
ppu_smi = shutil.which("ppu-smi")
require(ppu_smi is not None, "ppu-smi is not available on PATH")
try:
result = subprocess.run(
[ppu_smi, "--query-ppu=index", "--format=csv,noheader"],
check=False,
capture_output=True,
text=True,
timeout=15,
)
except (OSError, subprocess.TimeoutExpired) as error:
raise CheckFailure(
f"ppu-smi could not query the PPU inventory: {error}"
) from error
if result.returncode != 0:
stderr = result.stderr.strip()
detail = f" (rc={result.returncode})"
if stderr:
detail += f": {stderr}"
raise CheckFailure(f"ppu-smi failed to query the PPU inventory{detail}")
inventory_lines = result.stdout.splitlines()
indices = [line.strip() for line in inventory_lines if line.strip()]
require(indices, "ppu-smi did not report any PPU devices")
require(
all(re.fullmatch(r"[0-9]+", index) for index in indices),
"ppu-smi returned an unexpected inventory format",
)
print(f"PPU devices reported: {len(indices)}")
return len(indices)
def check_device_nodes() -> None:
"""Check the character devices used by the current PPU runtime."""
for device_node in (Path("/dev/alixpu_ctl"), Path("/dev/alixpu")):
try:
is_character_device = stat.S_ISCHR(device_node.stat().st_mode)
except OSError as error:
raise CheckFailure(
f"Required PPU character device is unavailable: {device_node}"
) from error
require(
is_character_device,
f"Required PPU device is not a character device: {device_node}",
)
print("Required PPU device nodes are present.")
def check_ppu_sdk() -> None:
"""Verify that PPU_SDK names an installed SDK directory."""
ppu_sdk = os.environ.get("PPU_SDK", "")
require(ppu_sdk, "PPU_SDK is not set")
require(
Path(ppu_sdk).is_dir(),
f"PPU_SDK must name an existing directory, got {ppu_sdk!r}",
)
print(f"PPU SDK directory is available: {ppu_sdk}")
def check_torch_compute() -> int:
"""Run a minimal PPU computation using PyTorch's CUDA-compatible API."""
try:
import torch
except Exception as error:
raise CheckFailure(f"PyTorch cannot be imported: {error}") from error
require(
torch.cuda.is_available(),
"torch.cuda.is_available() returned false; PPU is reached through the "
"CUDA-compatible device type, so this means no PPU is usable",
)
device_count = torch.cuda.device_count()
require(
device_count > 0,
"torch.cuda.device_count() did not report a device",
)
# Device-name lookup is optional and must not block the compute check.
try:
print(f"torch.cuda.get_device_name(0): {torch.cuda.get_device_name(0)}")
except Exception:
print("torch.cuda.get_device_name(0): unavailable")
device = torch.device("cuda:0")
try:
source = torch.tensor([1.0, 2.0, 3.0], device=device)
actual = source * 2.0 + 1.0
torch.cuda.synchronize(device)
actual_cpu = actual.cpu()
expected = torch.tensor([3.0, 5.0, 7.0])
torch.testing.assert_close(actual_cpu, expected)
except Exception as error:
raise CheckFailure(f"Tensor compute on cuda:0 failed: {error}") from error
print(f"PyTorch version: {torch.__version__}")
print(f"PPU devices visible through torch.cuda: {device_count}")
print("Tensor compute on cuda:0 passed.")
return device_count
def check_visibility_consistency(reported: int, visible: int) -> None:
"""Require visible devices not to exceed the ppu-smi inventory.
Device visibility restrictions may expose fewer devices to PyTorch.
"""
require(
visible <= reported,
f"torch reports {visible} device(s) but ppu-smi reports only {reported}",
)
if visible < reported:
print(
f"Note: {visible} of {reported} PPU(s) visible to this job "
f"(device visibility is restricted per runner job)."
)
else:
print(f"ppu-smi and torch agree on {reported} device(s).")
def main() -> int:
"""Run every check and return a process-compatible status code."""
try:
print_host_summary()
reported = check_ppu_inventory()
check_device_nodes()
check_ppu_sdk()
visible = check_torch_compute()
check_visibility_consistency(reported, visible)
except Exception:
traceback.print_exc()
return 1
print("PPU runner environment preflight passed.")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+2 -2
View File
@@ -34,8 +34,8 @@ from ci_register import CIRegistry, HWBackend, ut_parse_one_file
# the report -- if the assert below fires, add the new backend name here in
# the right display slot. Order isn't alphabetical: CUDA/AMD/NPU/CPU lead
# (highest test volume historically), then accelerators that have been
# wired into the registry more recently (XPU, MUSA, MLX).
BACKEND_DISPLAY_ORDER = ("CUDA", "AMD", "NPU", "CPU", "XPU", "MUSA", "MLX")
# wired into the registry more recently (XPU, MUSA, MLX, PPU).
BACKEND_DISPLAY_ORDER = ("CUDA", "AMD", "NPU", "CPU", "XPU", "MUSA", "MLX", "PPU")
assert set(BACKEND_DISPLAY_ORDER) == {b.name for b in HWBackend}, (
"BACKEND_DISPLAY_ORDER is out of sync with HWBackend"
)
@@ -942,6 +942,7 @@ _OTHER_BACKEND_REGISTERS = {
"register_xpu_ci": "XPU",
"register_musa_ci": "MUSA",
"register_mlx_ci": "MLX",
"register_ppu_ci": "PPU",
}
@@ -30,6 +30,7 @@ _MARKERS = (
"register_npu_ci",
"register_xpu_ci",
"register_musa_ci",
"register_ppu_ci",
)
+10
View File
@@ -23,6 +23,7 @@ HW_MAPPING = {
"npu": HWBackend.NPU,
"xpu": HWBackend.XPU,
"mlx": HWBackend.MLX,
"ppu": HWBackend.PPU,
}
# Per-commit test suites (run on every PR).
@@ -124,6 +125,14 @@ PER_COMMIT_SUITES = {
"stage-a-unit-test-mlx",
"stage-b-e2e-mlx",
],
# PPU has no suite in any of the three dicts yet: pr-test-ppu.yml only runs
# the runner preflight until the PPU SRT platform and AOT kernels land.
# Declaring a name here before a workflow dispatches it would let a test
# register, validate, and count as covered while never running, so each PPU
# suite name arrives with the job that runs it. Until then PPU is in
# _SUITE_CHECKED_BACKENDS with no valid suite, so validate_all_suites()
# rejects any register_ppu_ci() outright.
HWBackend.PPU: [],
}
# Nightly test suites (run nightly, organized by GPU configuration)
@@ -231,6 +240,7 @@ _SUITE_CHECKED_BACKENDS = {
HWBackend.MUSA,
HWBackend.XPU,
HWBackend.MLX,
HWBackend.PPU,
}