[PPU][1/N] CI: Add backend registration and runner preflight (#39788)
Co-authored-by: Yihao Wang <42559837+AgainstEntropy@users.noreply.github.com> Co-authored-by: Alex Nails <alex.nails@radixark.ai>
This commit is contained in:
co-authored by
Yihao Wang
Alex Nails
parent
9c3262d855
commit
d010e50ad6
@@ -0,0 +1,169 @@
|
||||
name: PR Test (PPU)
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main ]
|
||||
pull_request:
|
||||
branches: [ main ]
|
||||
types: [opened, synchronize, reopened, labeled]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
run_all_tests:
|
||||
description: "Run the preflight even if the last commit touched no PPU path"
|
||||
required: false
|
||||
type: boolean
|
||||
default: true
|
||||
workflow_call:
|
||||
inputs:
|
||||
ref:
|
||||
description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
|
||||
required: false
|
||||
type: string
|
||||
default: ''
|
||||
run_all_tests:
|
||||
description: "Run all tests (for releasing or testing purpose)"
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
concurrency:
|
||||
# Newer runs cancel older runs for the same ref; unrelated label events
|
||||
# use isolated groups to avoid interfering with active runs.
|
||||
group: >-
|
||||
pr-test-ppu-${{
|
||||
(github.event_name == 'pull_request' &&
|
||||
github.event.action == 'labeled' &&
|
||||
github.event.label.name != 'run-ci' &&
|
||||
github.event.label.name != 'run-ci-ppu') &&
|
||||
format('ignored-label-{0}', github.run_id) ||
|
||||
inputs.ref || github.ref
|
||||
}}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# ==================== Check Changes ==================== #
|
||||
check-changes:
|
||||
# Give skipped checks from unrelated labels distinct names so they cannot
|
||||
# replace the corresponding checks from an active run.
|
||||
name: >-
|
||||
${{ (github.event_name == 'pull_request' && github.event.action == 'labeled' &&
|
||||
github.event.label.name != 'run-ci' && github.event.label.name != 'run-ci-ppu') &&
|
||||
'ignored-label-check-changes' || 'check-changes' }}
|
||||
if: |
|
||||
github.event_name != 'pull_request' ||
|
||||
github.event.action != 'labeled' ||
|
||||
github.event.label.name == 'run-ci' ||
|
||||
github.event.label.name == 'run-ci-ppu'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
outputs:
|
||||
# Scope only; the shared pr-gate checks both required labels.
|
||||
ppu: ${{ steps.filter.outputs.ppu == 'true' || steps.run-mode.outputs.run_all_tests == 'true' }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# github.sha (not github.ref) so a runner that dequeues after main
|
||||
# advances still validates the SHA that triggered the run.
|
||||
ref: ${{ inputs.ref || github.sha }}
|
||||
|
||||
- name: Determine run mode
|
||||
id: run-mode
|
||||
run: |
|
||||
if [[ "${{ inputs.run_all_tests }}" == "true" ]]; then
|
||||
echo "run_all_tests=true" >> "$GITHUB_OUTPUT"
|
||||
echo "Run mode: ALL TESTS"
|
||||
else
|
||||
echo "run_all_tests=false" >> "$GITHUB_OUTPUT"
|
||||
echo "Run mode: FILTERED (triggered by ${{ github.event_name }})"
|
||||
fi
|
||||
|
||||
- name: Detect file changes
|
||||
id: filter
|
||||
uses: dorny/paths-filter@v3
|
||||
if: steps.run-mode.outputs.run_all_tests != 'true'
|
||||
with:
|
||||
# PPU-specific paths only.
|
||||
filters: |
|
||||
ppu:
|
||||
- ".github/workflows/pr-test-ppu.yml"
|
||||
- "scripts/ci/ppu/**"
|
||||
- "python/sglang/srt/platforms/ppu.py"
|
||||
- "python/sglang/srt/hardware_backend/ppu/**"
|
||||
- "python/sglang/kernels/aot/csrc/ppu/**"
|
||||
|
||||
# ==================== PR Gate ==================== #
|
||||
pr-gate:
|
||||
name: >-
|
||||
${{ (github.event_name == 'pull_request' && github.event.action == 'labeled' &&
|
||||
github.event.label.name != 'run-ci' && github.event.label.name != 'run-ci-ppu') &&
|
||||
'ignored-label-pr-gate' || 'pr-gate' }}
|
||||
needs: check-changes
|
||||
if: needs.check-changes.outputs.ppu == 'true'
|
||||
uses: ./.github/workflows/pr-gate.yml
|
||||
with:
|
||||
require-run-ci: true
|
||||
require-label: run-ci-ppu
|
||||
secrets: inherit
|
||||
|
||||
# ==================== PPU Runner Preflight ==================== #
|
||||
# Validates the runner contract only; the SRT platform and AOT kernels land
|
||||
# in later PRs and build their first real test on top of this job.
|
||||
ppu-preflight:
|
||||
name: >-
|
||||
${{ (github.event_name == 'pull_request' && github.event.action == 'labeled' &&
|
||||
github.event.label.name != 'run-ci' && github.event.label.name != 'run-ci-ppu') &&
|
||||
'ignored-label-ppu-preflight' || 'ppu-preflight' }}
|
||||
needs: [check-changes, pr-gate]
|
||||
if: needs.check-changes.outputs.ppu == 'true'
|
||||
runs-on: 1-ppu-m890p
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
SGLANG_IS_IN_CI: true
|
||||
steps:
|
||||
- name: Checkout code
|
||||
timeout-minutes: 10
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.sha }}
|
||||
|
||||
- name: Validate PPU runner contract
|
||||
timeout-minutes: 10
|
||||
run: python3 scripts/ci/ppu/check_ppu_environment.py
|
||||
|
||||
# ==================== Finish ==================== #
|
||||
pr-test-ppu-finish:
|
||||
name: >-
|
||||
${{ (github.event_name == 'pull_request' && github.event.action == 'labeled' &&
|
||||
github.event.label.name != 'run-ci' && github.event.label.name != 'run-ci-ppu') &&
|
||||
'ignored-label-pr-test-ppu-finish' || 'pr-test-ppu-finish' }}
|
||||
needs: [pr-gate, check-changes, ppu-preflight]
|
||||
if: always() && needs.check-changes.result != 'skipped'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check all dependent job statuses
|
||||
run: |
|
||||
json_needs='${{ toJson(needs) }}'
|
||||
for job in $(echo "$json_needs" | jq -r 'keys_unsorted[]'); do
|
||||
result=$(echo "$json_needs" | jq -r --arg j "$job" '.[$j].result')
|
||||
echo "$job: $result"
|
||||
if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then
|
||||
echo "The above jobs failed."
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "All jobs completed successfully"
|
||||
|
||||
- name: Require the preflight when PPU is in scope
|
||||
run: |
|
||||
in_scope='${{ needs.check-changes.outputs.ppu }}'
|
||||
preflight='${{ needs.ppu-preflight.result }}'
|
||||
echo "PPU in scope: '$in_scope' / ppu-preflight: '$preflight'"
|
||||
|
||||
if [[ "$in_scope" == "true" && "$preflight" != "success" ]]; then
|
||||
echo "PPU was in scope but ppu-preflight did not succeed."
|
||||
exit 1
|
||||
fi
|
||||
echo "Preflight status is consistent with the requested scope."
|
||||
@@ -17,6 +17,7 @@ __all__ = [
|
||||
"register_xpu_ci",
|
||||
"register_musa_ci",
|
||||
"register_mlx_ci",
|
||||
"register_ppu_ci",
|
||||
"ut_parse_one_file",
|
||||
]
|
||||
|
||||
@@ -37,6 +38,7 @@ class HWBackend(Enum):
|
||||
XPU = auto()
|
||||
MUSA = auto()
|
||||
MLX = auto()
|
||||
PPU = auto()
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -150,6 +152,19 @@ def register_mlx_ci(
|
||||
return None
|
||||
|
||||
|
||||
def register_ppu_ci(
|
||||
est_time: float,
|
||||
suite: Optional[str] = None,
|
||||
nightly: bool = False,
|
||||
disabled: Optional[str] = None,
|
||||
*,
|
||||
stage: Optional[str] = None,
|
||||
runner_config: Optional[str] = None,
|
||||
):
|
||||
"""Marker for PPU CI registration (parsed via AST; runtime no-op)."""
|
||||
return None
|
||||
|
||||
|
||||
REGISTER_MAPPING = {
|
||||
"register_cpu_ci": HWBackend.CPU,
|
||||
"register_cuda_ci": HWBackend.CUDA,
|
||||
@@ -159,6 +174,7 @@ REGISTER_MAPPING = {
|
||||
"register_xpu_ci": HWBackend.XPU,
|
||||
"register_musa_ci": HWBackend.MUSA,
|
||||
"register_mlx_ci": HWBackend.MLX,
|
||||
"register_ppu_ci": HWBackend.PPU,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
"""Validate the PPU runner environment without importing SGLang."""
|
||||
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class CheckFailure(RuntimeError):
|
||||
"""A failed PPU environment contract check."""
|
||||
|
||||
|
||||
def require(condition: bool, message: str) -> None:
|
||||
"""Raise a concise contract failure when a condition is false."""
|
||||
if not condition:
|
||||
raise CheckFailure(message)
|
||||
|
||||
|
||||
def print_host_summary() -> None:
|
||||
"""Print capacity information without hostnames or network identifiers."""
|
||||
disk_usage = shutil.disk_usage("/")
|
||||
|
||||
print(f"Runner OS: {platform.system()}")
|
||||
print(f"Runner architecture: {platform.machine()}")
|
||||
print(f"Python version: {platform.python_version()}")
|
||||
|
||||
# Memory reporting is diagnostic-only; it must not fail the preflight.
|
||||
try:
|
||||
import psutil
|
||||
|
||||
print(f"Memory total (bytes): {psutil.virtual_memory().total}")
|
||||
except Exception:
|
||||
print("Memory total (bytes): unavailable")
|
||||
print(f"Root disk total (bytes): {disk_usage.total}")
|
||||
print(f"Root disk available (bytes): {disk_usage.free}")
|
||||
|
||||
|
||||
def check_ppu_inventory() -> int:
|
||||
"""Return the number of PPUs reported by the management interface."""
|
||||
ppu_smi = shutil.which("ppu-smi")
|
||||
require(ppu_smi is not None, "ppu-smi is not available on PATH")
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[ppu_smi, "--query-ppu=index", "--format=csv,noheader"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=15,
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired) as error:
|
||||
raise CheckFailure(
|
||||
f"ppu-smi could not query the PPU inventory: {error}"
|
||||
) from error
|
||||
|
||||
if result.returncode != 0:
|
||||
stderr = result.stderr.strip()
|
||||
detail = f" (rc={result.returncode})"
|
||||
if stderr:
|
||||
detail += f": {stderr}"
|
||||
raise CheckFailure(f"ppu-smi failed to query the PPU inventory{detail}")
|
||||
|
||||
inventory_lines = result.stdout.splitlines()
|
||||
indices = [line.strip() for line in inventory_lines if line.strip()]
|
||||
require(indices, "ppu-smi did not report any PPU devices")
|
||||
require(
|
||||
all(re.fullmatch(r"[0-9]+", index) for index in indices),
|
||||
"ppu-smi returned an unexpected inventory format",
|
||||
)
|
||||
|
||||
print(f"PPU devices reported: {len(indices)}")
|
||||
return len(indices)
|
||||
|
||||
|
||||
def check_device_nodes() -> None:
|
||||
"""Check the character devices used by the current PPU runtime."""
|
||||
for device_node in (Path("/dev/alixpu_ctl"), Path("/dev/alixpu")):
|
||||
try:
|
||||
is_character_device = stat.S_ISCHR(device_node.stat().st_mode)
|
||||
except OSError as error:
|
||||
raise CheckFailure(
|
||||
f"Required PPU character device is unavailable: {device_node}"
|
||||
) from error
|
||||
require(
|
||||
is_character_device,
|
||||
f"Required PPU device is not a character device: {device_node}",
|
||||
)
|
||||
|
||||
print("Required PPU device nodes are present.")
|
||||
|
||||
|
||||
def check_ppu_sdk() -> None:
|
||||
"""Verify that PPU_SDK names an installed SDK directory."""
|
||||
ppu_sdk = os.environ.get("PPU_SDK", "")
|
||||
require(ppu_sdk, "PPU_SDK is not set")
|
||||
require(
|
||||
Path(ppu_sdk).is_dir(),
|
||||
f"PPU_SDK must name an existing directory, got {ppu_sdk!r}",
|
||||
)
|
||||
print(f"PPU SDK directory is available: {ppu_sdk}")
|
||||
|
||||
|
||||
def check_torch_compute() -> int:
|
||||
"""Run a minimal PPU computation using PyTorch's CUDA-compatible API."""
|
||||
try:
|
||||
import torch
|
||||
except Exception as error:
|
||||
raise CheckFailure(f"PyTorch cannot be imported: {error}") from error
|
||||
|
||||
require(
|
||||
torch.cuda.is_available(),
|
||||
"torch.cuda.is_available() returned false; PPU is reached through the "
|
||||
"CUDA-compatible device type, so this means no PPU is usable",
|
||||
)
|
||||
device_count = torch.cuda.device_count()
|
||||
require(
|
||||
device_count > 0,
|
||||
"torch.cuda.device_count() did not report a device",
|
||||
)
|
||||
|
||||
# Device-name lookup is optional and must not block the compute check.
|
||||
try:
|
||||
print(f"torch.cuda.get_device_name(0): {torch.cuda.get_device_name(0)}")
|
||||
except Exception:
|
||||
print("torch.cuda.get_device_name(0): unavailable")
|
||||
|
||||
device = torch.device("cuda:0")
|
||||
try:
|
||||
source = torch.tensor([1.0, 2.0, 3.0], device=device)
|
||||
actual = source * 2.0 + 1.0
|
||||
torch.cuda.synchronize(device)
|
||||
actual_cpu = actual.cpu()
|
||||
expected = torch.tensor([3.0, 5.0, 7.0])
|
||||
torch.testing.assert_close(actual_cpu, expected)
|
||||
except Exception as error:
|
||||
raise CheckFailure(f"Tensor compute on cuda:0 failed: {error}") from error
|
||||
|
||||
print(f"PyTorch version: {torch.__version__}")
|
||||
print(f"PPU devices visible through torch.cuda: {device_count}")
|
||||
print("Tensor compute on cuda:0 passed.")
|
||||
return device_count
|
||||
|
||||
|
||||
def check_visibility_consistency(reported: int, visible: int) -> None:
|
||||
"""Require visible devices not to exceed the ppu-smi inventory.
|
||||
|
||||
Device visibility restrictions may expose fewer devices to PyTorch.
|
||||
"""
|
||||
require(
|
||||
visible <= reported,
|
||||
f"torch reports {visible} device(s) but ppu-smi reports only {reported}",
|
||||
)
|
||||
if visible < reported:
|
||||
print(
|
||||
f"Note: {visible} of {reported} PPU(s) visible to this job "
|
||||
f"(device visibility is restricted per runner job)."
|
||||
)
|
||||
else:
|
||||
print(f"ppu-smi and torch agree on {reported} device(s).")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Run every check and return a process-compatible status code."""
|
||||
try:
|
||||
print_host_summary()
|
||||
reported = check_ppu_inventory()
|
||||
check_device_nodes()
|
||||
check_ppu_sdk()
|
||||
visible = check_torch_compute()
|
||||
check_visibility_consistency(reported, visible)
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
|
||||
print("PPU runner environment preflight passed.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -34,8 +34,8 @@ from ci_register import CIRegistry, HWBackend, ut_parse_one_file
|
||||
# the report -- if the assert below fires, add the new backend name here in
|
||||
# the right display slot. Order isn't alphabetical: CUDA/AMD/NPU/CPU lead
|
||||
# (highest test volume historically), then accelerators that have been
|
||||
# wired into the registry more recently (XPU, MUSA, MLX).
|
||||
BACKEND_DISPLAY_ORDER = ("CUDA", "AMD", "NPU", "CPU", "XPU", "MUSA", "MLX")
|
||||
# wired into the registry more recently (XPU, MUSA, MLX, PPU).
|
||||
BACKEND_DISPLAY_ORDER = ("CUDA", "AMD", "NPU", "CPU", "XPU", "MUSA", "MLX", "PPU")
|
||||
assert set(BACKEND_DISPLAY_ORDER) == {b.name for b in HWBackend}, (
|
||||
"BACKEND_DISPLAY_ORDER is out of sync with HWBackend"
|
||||
)
|
||||
|
||||
@@ -942,6 +942,7 @@ _OTHER_BACKEND_REGISTERS = {
|
||||
"register_xpu_ci": "XPU",
|
||||
"register_musa_ci": "MUSA",
|
||||
"register_mlx_ci": "MLX",
|
||||
"register_ppu_ci": "PPU",
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -30,6 +30,7 @@ _MARKERS = (
|
||||
"register_npu_ci",
|
||||
"register_xpu_ci",
|
||||
"register_musa_ci",
|
||||
"register_ppu_ci",
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ HW_MAPPING = {
|
||||
"npu": HWBackend.NPU,
|
||||
"xpu": HWBackend.XPU,
|
||||
"mlx": HWBackend.MLX,
|
||||
"ppu": HWBackend.PPU,
|
||||
}
|
||||
|
||||
# Per-commit test suites (run on every PR).
|
||||
@@ -124,6 +125,14 @@ PER_COMMIT_SUITES = {
|
||||
"stage-a-unit-test-mlx",
|
||||
"stage-b-e2e-mlx",
|
||||
],
|
||||
# PPU has no suite in any of the three dicts yet: pr-test-ppu.yml only runs
|
||||
# the runner preflight until the PPU SRT platform and AOT kernels land.
|
||||
# Declaring a name here before a workflow dispatches it would let a test
|
||||
# register, validate, and count as covered while never running, so each PPU
|
||||
# suite name arrives with the job that runs it. Until then PPU is in
|
||||
# _SUITE_CHECKED_BACKENDS with no valid suite, so validate_all_suites()
|
||||
# rejects any register_ppu_ci() outright.
|
||||
HWBackend.PPU: [],
|
||||
}
|
||||
|
||||
# Nightly test suites (run nightly, organized by GPU configuration)
|
||||
@@ -231,6 +240,7 @@ _SUITE_CHECKED_BACKENDS = {
|
||||
HWBackend.MUSA,
|
||||
HWBackend.XPU,
|
||||
HWBackend.MLX,
|
||||
HWBackend.PPU,
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user