[CI] Rename basic CI stage-a/b/c -> base-a/b/c for symmetry with extra CI (#25420)
This commit is contained in:
Liangsheng Yin
@@ -435,7 +435,7 @@ if torch.cuda.get_device_capability()[0] < 9:
|
||||
|
||||
JIT kernel tests live under `python/sglang/jit_kernel/tests/`. **CI does not run `pytest` in that directory directly.** The unified runner `test/run_suite.py` discovers every `test_*.py` there (and every `bench_*.py` under `benchmark/`), collects `register_*_ci(...)` calls by **statically parsing each file's AST**, and executes the selected suite. Every test file must register at least one CUDA entry or the collector fails its sanity check.
|
||||
|
||||
- **PR / per-commit CUDA suites** (see `test/run_suite.py` → `PER_COMMIT_SUITES`): JIT unit tests use `stage-b-kernel-unit-1-gpu-large` on H100 and `stage-b-kernel-unit-1-gpu-b200` on B200/SM100 paths (see `.github/workflows/pr-test-jit-kernel.yml`). Multi-GPU JIT tests use `stage-b-kernel-unit-8-gpu-h200`.
|
||||
- **PR / per-commit CUDA suites** (see `test/run_suite.py` → `PER_COMMIT_SUITES`): JIT unit tests use `base-b-kernel-unit-1-gpu-large` on H100 and `base-b-kernel-unit-1-gpu-b200` on B200/SM100 paths (see `.github/workflows/pr-test-jit-kernel.yml`). Multi-GPU JIT tests use `base-b-kernel-unit-8-gpu-h200`.
|
||||
- **Nightly kernel suite**: `nightly-kernel-1-gpu` with `--nightly` — typically used with `SGLANG_JIT_KERNEL_RUN_FULL_TESTS=1` in CI for expanded parameter grids (see `python/sglang/jit_kernel/utils.py` → `should_run_full_tests` / `get_ci_test_range`). Wired in `.github/workflows/nightly-test-nvidia.yml` (e.g. `python3 run_suite.py --hw cuda --suite nightly-kernel-1-gpu --nightly --continue-on-error`).
|
||||
|
||||
Registration pattern (module level, **literal** `est_time` and `suite` strings — required for AST parsing):
|
||||
@@ -443,9 +443,9 @@ Registration pattern (module level, **literal** `est_time` and `suite` strings
|
||||
```python
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=30, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=30, suite="base-b-kernel-unit-1-gpu-large")
|
||||
# Optional B200/SM100 registration for tests that cover Blackwell-specific code paths
|
||||
# register_cuda_ci(est_time=30, suite="stage-b-kernel-unit-1-gpu-b200")
|
||||
# register_cuda_ci(est_time=30, suite="base-b-kernel-unit-1-gpu-b200")
|
||||
# Optional second registration: same file also listed under the nightly kernel suite
|
||||
# register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
```
|
||||
@@ -457,9 +457,9 @@ Use `register_cuda_ci(..., disabled="reason")` if the file must stay in-tree but
|
||||
**Run like CI** (from repo root):
|
||||
|
||||
```bash
|
||||
(cd test && python3 run_suite.py --hw cuda --suite stage-b-kernel-unit-1-gpu-large)
|
||||
(cd test && python3 run_suite.py --hw cuda --suite base-b-kernel-unit-1-gpu-large)
|
||||
# For B200/SM100-specific coverage:
|
||||
(cd test && python3 run_suite.py --hw cuda --suite stage-b-kernel-unit-1-gpu-b200)
|
||||
(cd test && python3 run_suite.py --hw cuda --suite base-b-kernel-unit-1-gpu-b200)
|
||||
```
|
||||
|
||||
For fast iteration you can still run `pytest` on a single file locally; CI coverage is via `run_suite.py`.
|
||||
@@ -472,7 +472,7 @@ import torch
|
||||
from sglang.jit_kernel.scale import scale
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=30, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=30, suite="base-b-kernel-unit-1-gpu-large")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
|
||||
@@ -517,7 +517,7 @@ if __name__ == "__main__":
|
||||
|
||||
## Step 5: Add a benchmark (required)
|
||||
|
||||
Benchmarks are `bench_*.py` files under `python/sglang/jit_kernel/benchmark/`. They are picked up by the same `run_suite.py` machinery as unit tests. Register them for **`stage-b-kernel-benchmark-1-gpu-large`** (PR JIT benchmark job: `python3 run_suite.py --hw cuda --suite stage-b-kernel-benchmark-1-gpu-large`).
|
||||
Benchmarks are `bench_*.py` files under `python/sglang/jit_kernel/benchmark/`. They are picked up by the same `run_suite.py` machinery as unit tests. Register them for **`base-b-kernel-benchmark-1-gpu-large`** (PR JIT benchmark job: `python3 run_suite.py --hw cuda --suite base-b-kernel-benchmark-1-gpu-large`).
|
||||
|
||||
Create `python/sglang/jit_kernel/benchmark/bench_scale.py`:
|
||||
|
||||
@@ -537,7 +537,7 @@ from sglang.jit_kernel.benchmark.utils import (
|
||||
from sglang.jit_kernel.scale import scale as jit_scale
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=6, suite="stage-b-kernel-benchmark-1-gpu-large")
|
||||
register_cuda_ci(est_time=6, suite="base-b-kernel-benchmark-1-gpu-large")
|
||||
|
||||
SIZE_LIST = get_benchmark_range(
|
||||
full_range=[2**n for n in range(10, 20)], # 1K … 512K elements
|
||||
@@ -585,7 +585,7 @@ python python/sglang/jit_kernel/benchmark/bench_scale.py
|
||||
Run the benchmark suite the way CI does:
|
||||
|
||||
```bash
|
||||
cd test && python3 run_suite.py --hw cuda --suite stage-b-kernel-benchmark-1-gpu-large
|
||||
cd test && python3 run_suite.py --hw cuda --suite base-b-kernel-benchmark-1-gpu-large
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
@@ -11,7 +11,7 @@ This skill covers the CI **infrastructure** layer — how tests are dispatched,
|
||||
|
||||
## Naming Conventions
|
||||
|
||||
- **Suite**: `stage-{a,b,c}-test-{gpu_count}-gpu-{hardware}` (e.g., `stage-b-test-1-gpu-small`)
|
||||
- **Suite**: `base-{a,b,c}-test-{gpu_count}-gpu-{hardware}` (e.g., `base-b-test-1-gpu-small`)
|
||||
- **Test group**: Directory-level registered test group under `test/registered/` (e.g., `hicache` maps to `test/registered/hicache/test_*.py`)
|
||||
- **CI runner**: `{gpu_count}-gpu-{hardware}` (e.g., `1-gpu-5090`, `4-gpu-h100`, `8-gpu-h200`)
|
||||
|
||||
@@ -23,7 +23,7 @@ This skill covers the CI **infrastructure** layer — how tests are dispatched,
|
||||
|------|------|
|
||||
| `.github/workflows/pr-test.yml` | Main workflow — all stages, jobs, conditions, matrix definitions |
|
||||
| `.github/workflows/pr-gate.yml` | PR gating: draft check, `run-ci` label, per-user rate limiting |
|
||||
| `.github/actions/check-stage-health/action.yml` | Cross-job fast-fail: queries API for any failed job |
|
||||
| `.github/actions/check-pr-test-health/action.yml` | Cross-job fast-fail: queries API for any failed job |
|
||||
| `.github/actions/wait-for-jobs/action.yml` | Stage gating: polls API until stage jobs complete |
|
||||
| `.github/actions/check-maintenance/action.yml` | Maintenance mode check |
|
||||
| `test/run_suite.py` | Suite runner: collects, filters, partitions, executes tests |
|
||||
@@ -49,53 +49,53 @@ This skill covers the CI **infrastructure** layer — how tests are dispatched,
|
||||
│ │
|
||||
▼ │
|
||||
┌─────────────────────────────────────┐ │
|
||||
│ Stage A (~3 min) │ │
|
||||
│ Base A (~3 min) │ │
|
||||
│ pre-flight check │ │
|
||||
│ │ │
|
||||
│ ┌─────────────────────────────┐ │ │
|
||||
│ │ stage-a-test-1-gpu-small │ │ │
|
||||
│ │ base-a-test-1-gpu-small │ │ │
|
||||
│ │ (small GPUs) │ │ │
|
||||
│ └─────────────────────────────┘ │ │
|
||||
│ ┌─────────────────────────────┐ │ │
|
||||
│ │ stage-a-test-cpu │ │ │
|
||||
│ │ base-a-test-cpu │ │ │
|
||||
│ │ (CPU) │ │ │
|
||||
│ └─────────────────────────────┘ │ │
|
||||
└──────┬──────────────────────────────┘ │
|
||||
│ │
|
||||
▼ ▼
|
||||
┌─────────────────────────────────────┐ ┌──────────────────────────┐
|
||||
│ Stage B (~30 min) │ │ kernel test │
|
||||
│ basic tests │ └──────────────────────────┘
|
||||
│ Base B (~30 min) │ │ kernel test │
|
||||
│ base tests │ └──────────────────────────┘
|
||||
│ │ ┌──────────────────────────┐
|
||||
│ ┌─────────────────────────────┐ │ │ multimodal gen test │
|
||||
│ │ stage-b-test-1-gpu-small │ │ └──────────────────────────┘
|
||||
│ │ base-b-test-1-gpu-small │ │ └──────────────────────────┘
|
||||
│ │ (small GPUs, e.g. 5090) │ │
|
||||
│ └─────────────────────────────┘ │
|
||||
│ ┌─────────────────────────────┐ │
|
||||
│ │ stage-b-test-1-gpu-large │ │
|
||||
│ │ base-b-test-1-gpu-large │ │
|
||||
│ │ (large GPUs, e.g. H100) │ │
|
||||
│ └─────────────────────────────┘ │
|
||||
│ ┌─────────────────────────────┐ │
|
||||
│ │ stage-b-test-2-gpu-large │ │
|
||||
│ │ base-b-test-2-gpu-large │ │
|
||||
│ │ (large GPUs, e.g. H100) │ │
|
||||
│ └─────────────────────────────┘ │
|
||||
└──────┬──────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────┐
|
||||
│ Stage C (~30 min) │
|
||||
│ Base C (~30 min) │
|
||||
│ advanced tests │
|
||||
│ │
|
||||
│ ┌─────────────────────────────┐ │
|
||||
│ │ stage-c-test-4-gpu-h100 │ │
|
||||
│ │ base-c-test-4-gpu-h100 │ │
|
||||
│ │ (H100 GPUs) │ │
|
||||
│ └─────────────────────────────┘ │
|
||||
│ ┌─────────────────────────────┐ │
|
||||
│ │ stage-c-test-8-gpu-h200 │ │
|
||||
│ │ base-c-test-8-gpu-h200 │ │
|
||||
│ │ (8 x H200 GPUs) │ │
|
||||
│ └─────────────────────────────┘ │
|
||||
│ ┌─────────────────────────────┐ │
|
||||
│ │ stage-c-test-4-gpu-b200 │ │
|
||||
│ │ base-c-test-4-gpu-b200 │ │
|
||||
│ │ (4 x B200 GPUs) │ │
|
||||
│ └─────────────────────────────┘ │
|
||||
│ ┌─────────────────────────────┐ │
|
||||
@@ -112,9 +112,9 @@ This skill covers the CI **infrastructure** layer — how tests are dispatched,
|
||||
└─────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**Every stage test job** includes a `check-stage-health` step after checkout — if any job in the run has already failed, the job fast-fails (red X) with a root cause annotation.
|
||||
**Every stage test job** includes a `check-pr-test-health` step after checkout — if any job in the run has already failed, the job fast-fails (red X) with a root cause annotation.
|
||||
|
||||
**Scheduled runs** skip `wait-for-stage-*` jobs, running all stages in parallel. Fast-fail is also disabled.
|
||||
**Scheduled runs** skip `wait-for-base-*` jobs, running all stages in parallel. Fast-fail is also disabled.
|
||||
|
||||
---
|
||||
|
||||
@@ -126,13 +126,13 @@ This skill covers the CI **infrastructure** layer — how tests are dispatched,
|
||||
|-------|-----------|-------------|----------------------|
|
||||
| **1. Test method → file** | `unittest -f` (failfast) | One test method fails → entire test file stops immediately | Yes |
|
||||
| **2. File → suite** | `run_unittest_files()` default | One test file fails → entire suite stops (`--continue-on-error` off) | Yes |
|
||||
| **3. Job → job (same stage)** | `check-stage-health` action | One job fails → other waiting jobs in same stage fast-fail (red X) | Yes |
|
||||
| **4. Stage → stage (cross-stage)** | `wait-for-stage` + `needs` | Stage A fails → stage B/C jobs skip entirely (never get a runner) | Yes (wait jobs skipped) |
|
||||
| **3. Job → job (same stage)** | `check-pr-test-health` action | One job fails → other waiting jobs in same stage fast-fail (red X) | Yes |
|
||||
| **4. Stage → stage (cross-stage)** | `wait-for-base-*` + `needs` | Base A fails → base B/C jobs skip entirely (never get a runner) | Yes (wait jobs skipped) |
|
||||
|
||||
- **Layer 1**: `-f` flag appended to all `python3 -m pytest` / `unittest` invocations in `ci_utils.py`
|
||||
- **Layer 2**: `--continue-on-error` flag in `run_suite.py` — off for PRs, on for scheduled runs
|
||||
- **Layer 3**: `check-stage-health` auto-detects `schedule` event and skips; filters out cascade failures to show only root cause jobs
|
||||
- **Layer 4**: `wait-for-stage-*` jobs are conditioned on `github.event_name == 'pull_request'` — skipped for scheduled runs
|
||||
- **Layer 3**: `check-pr-test-health` auto-detects `schedule` event and skips; filters out cascade failures to show only root cause jobs
|
||||
- **Layer 4**: `wait-for-base-*` jobs are conditioned on `github.event_name == 'pull_request'` — skipped for scheduled runs
|
||||
|
||||
---
|
||||
|
||||
@@ -140,8 +140,8 @@ This skill covers the CI **infrastructure** layer — how tests are dispatched,
|
||||
|
||||
| Aspect | PR (`pull_request`) | Scheduled (`cron`, every 6h) | `/rerun-stage` (`workflow_dispatch`) |
|
||||
|--------|---------------------|------------------------------|--------------------------------------|
|
||||
| **Stage ordering** | Sequential: A → B → C via `wait-for-stage-*` | Parallel (all at once) | Single target stage only |
|
||||
| **Cross-job fast-fail** | Yes (`check-stage-health`) | Yes | Yes |
|
||||
| **Stage ordering** | Sequential: A → B → C via `wait-for-base-*` | Parallel (all at once) | Single target stage only |
|
||||
| **Cross-job fast-fail** | Yes (`check-pr-test-health`) | Yes | Yes |
|
||||
| **continue-on-error** | No (stop at first failure within suite) | Yes (run all tests) | No |
|
||||
| **Retry** | Enabled | Enabled | Enabled |
|
||||
| **max_parallel** | 3 (default), 14 if `high priority` label | 14 | 3 (default), 14 if `high priority` |
|
||||
@@ -152,23 +152,23 @@ This skill covers the CI **infrastructure** layer — how tests are dispatched,
|
||||
|
||||
## Stage Gating (`wait-for-jobs` action)
|
||||
|
||||
`wait-for-stage-a` and `wait-for-stage-b` are lightweight `ubuntu-latest` jobs that poll the GitHub Actions API.
|
||||
`wait-for-base-a` and `wait-for-base-b` are lightweight `ubuntu-latest` jobs that poll the GitHub Actions API.
|
||||
|
||||
**How it works:**
|
||||
1. Calls `listJobsForWorkflowRun` to list all jobs in the current run
|
||||
2. Matches jobs by exact name or prefix (for matrix jobs, e.g., `stage-b-test-1-gpu-small (3)`)
|
||||
2. Matches jobs by exact name or prefix (for matrix jobs, e.g., `base-b-test-1-gpu-small (3)`)
|
||||
3. If any matched job has `conclusion === 'failure'` → fail immediately (fast-fail)
|
||||
4. If all matched jobs are completed and count matches `expected_count` → success
|
||||
5. Otherwise → sleep `poll-interval-seconds` (default: 60s) and retry
|
||||
6. Timeout after `max-wait-minutes` (240 min for stage-a, 480 min for stage-b)
|
||||
6. Timeout after `max-wait-minutes` (240 min for base-a, 480 min for base-b)
|
||||
|
||||
**Job specs example** (stage-b):
|
||||
**Job specs example** (base-b):
|
||||
```json
|
||||
[
|
||||
{"prefix": "stage-b-test-1-gpu-small", "expected_count": 8},
|
||||
{"prefix": "stage-b-test-1-gpu-large", "expected_count": 14},
|
||||
{"prefix": "stage-b-test-2-gpu-large", "expected_count": 4},
|
||||
{"prefix": "stage-b-test-4-gpu-b200", "expected_count": 1}
|
||||
{"prefix": "base-b-test-1-gpu-small", "expected_count": 8},
|
||||
{"prefix": "base-b-test-1-gpu-large", "expected_count": 14},
|
||||
{"prefix": "base-b-test-2-gpu-large", "expected_count": 4},
|
||||
{"prefix": "base-b-test-4-gpu-b200", "expected_count": 1}
|
||||
]
|
||||
```
|
||||
|
||||
@@ -178,17 +178,17 @@ This skill covers the CI **infrastructure** layer — how tests are dispatched,
|
||||
|
||||
---
|
||||
|
||||
## Cross-Job Fast-Fail (`check-stage-health` action)
|
||||
## Cross-Job Fast-Fail (`check-pr-test-health` action)
|
||||
|
||||
Composite action called after checkout in every stage test job (21 jobs total across `pr-test.yml`, `pr-test-multimodal-gen.yml`, `pr-test-sgl-kernel.yml`, `pr-test-jit-kernel.yml`).
|
||||
|
||||
**How it works:**
|
||||
1. Queries `listJobsForWorkflowRun` for the current workflow run
|
||||
2. Filters for **root cause failures only** — jobs with `conclusion === 'failure'` whose failing step is NOT `check-stage-health` (excludes cascade failures)
|
||||
2. Filters for **root cause failures only** — jobs with `conclusion === 'failure'` whose failing step is NOT `check-pr-test-health` (excludes cascade failures)
|
||||
3. If root cause failures found → calls `core.setFailed()` with the list of root cause job names
|
||||
4. If none → does nothing (step succeeds)
|
||||
|
||||
**Cascade filtering**: When job A fast-fails due to health check, it also has `conclusion: failure`. Without filtering, job B would list both the original failure AND job A's fast-fail. The filter checks each failed job's `steps` array — if the failing step name contains `check-stage-health` or `Check stage health`, it's excluded from the root cause list.
|
||||
**Cascade filtering**: When job A fast-fails due to health check, it also has `conclusion: failure`. Without filtering, job B would list both the original failure AND job A's fast-fail. The filter checks each failed job's `steps` array — if the failing step name contains `check-pr-test-health` or `Check PR test health`, it's excluded from the root cause list.
|
||||
|
||||
**Usage pattern:**
|
||||
```yaml
|
||||
@@ -197,8 +197,8 @@ steps:
|
||||
uses: actions/checkout@v4
|
||||
...
|
||||
|
||||
- uses: ./.github/actions/check-stage-health
|
||||
id: stage-health
|
||||
- uses: ./.github/actions/check-pr-test-health
|
||||
id: pr-test-health
|
||||
|
||||
- name: Install dependencies # skipped automatically if health check failed
|
||||
... # (default if: success() is false)
|
||||
@@ -213,7 +213,7 @@ steps:
|
||||
|
||||
**Error message example:**
|
||||
```
|
||||
Fast-fail: skipping — root cause job(s): stage-b-test-1-gpu-small (0), stage-b-test-1-gpu-small (1)
|
||||
Fast-fail: skipping — root cause job(s): base-b-test-1-gpu-small (0), base-b-test-1-gpu-small (1)
|
||||
```
|
||||
|
||||
---
|
||||
@@ -270,27 +270,26 @@ Large suites are split across matrix jobs using the **LPT (Longest Processing Ti
|
||||
|
||||
| Suite | Partitions | Runner | max_parallel |
|
||||
|-------|-----------|--------|-------------|
|
||||
| `stage-a-test-1-gpu-small` | 1 (no matrix) | `1-gpu-5090` | — |
|
||||
| `stage-a-test-cpu` | 4 | `ubuntu-latest` | — |
|
||||
| `stage-b-test-1-gpu-small` | 8 | `1-gpu-5090` | 8 |
|
||||
| `stage-b-test-1-gpu-large` | 14 | `1-gpu-h100` | dynamic (3 or 14) |
|
||||
| `stage-b-test-2-gpu-large` | 4 | `2-gpu-h100` | — |
|
||||
| `stage-b-test-4-gpu-b200` | 1 (no matrix) | `4-gpu-b200` | — |
|
||||
| `stage-b-kernel-unit-1-gpu-large` | 1 (no matrix) | `1-gpu-h100` | — |
|
||||
| `stage-b-kernel-unit-1-gpu-b200` | 1 (no matrix) | `4-gpu-b200` | — |
|
||||
| `stage-b-kernel-unit-8-gpu-h200` | 1 (no matrix) | `8-gpu-h200` | — |
|
||||
| `stage-b-kernel-benchmark-1-gpu-large` | 1 (no matrix) | `1-gpu-h100` | — |
|
||||
| `stage-c-test-4-gpu-h100` | 3 | `4-gpu-h100` | — |
|
||||
| `stage-c-test-8-gpu-h200` | 4 | `8-gpu-h200` | — |
|
||||
| `stage-c-test-8-gpu-h20` | 2 | `8-gpu-h20` | — |
|
||||
| `stage-c-test-deepep-4-gpu-h100` | 1 (no matrix) | `4-gpu-h100` | — |
|
||||
| `stage-c-test-deepep-8-gpu-h200` | 1 (no matrix) | `8-gpu-h200` | — |
|
||||
| `stage-c-test-4-gpu-b200` | 3 | `4-gpu-b200` | — |
|
||||
| `stage-c-test-4-gpu-b200-small` | 3 | `4-gpu-b200-low-disk` | — |
|
||||
| `stage-c-test-8-gpu-b200` | registered only | `8-gpu-b200` | — |
|
||||
| `stage-c-test-4-gpu-gb200` | registered only | `4-gpu-gb200` | — |
|
||||
| `base-a-test-1-gpu-small` | 1 (no matrix) | `1-gpu-5090` | — |
|
||||
| `base-a-test-cpu` | 4 | `ubuntu-latest` | — |
|
||||
| `base-b-test-1-gpu-small` | 8 | `1-gpu-5090` | 8 |
|
||||
| `base-b-test-1-gpu-large` | 14 | `1-gpu-h100` | dynamic (3 or 14) |
|
||||
| `base-b-test-2-gpu-large` | 4 | `2-gpu-h100` | — |
|
||||
| `base-b-test-4-gpu-b200` | 1 (no matrix) | `4-gpu-b200` | — |
|
||||
| `base-b-kernel-unit-1-gpu-large` | 1 (no matrix) | `1-gpu-h100` | — |
|
||||
| `base-b-kernel-unit-1-gpu-b200` | 1 (no matrix) | `4-gpu-b200` | — |
|
||||
| `base-b-kernel-unit-8-gpu-h200` | 1 (no matrix) | `8-gpu-h200` | — |
|
||||
| `base-b-kernel-benchmark-1-gpu-large` | 1 (no matrix) | `1-gpu-h100` | — |
|
||||
| `base-c-test-4-gpu-h100` | 3 | `4-gpu-h100` | — |
|
||||
| `base-c-test-8-gpu-h200` | 4 | `8-gpu-h200` | — |
|
||||
| `base-c-test-8-gpu-h20` | 2 | `8-gpu-h20` | — |
|
||||
| `base-c-test-deepep-4-gpu-h100` | 1 (no matrix) | `4-gpu-h100` | — |
|
||||
| `base-c-test-4-gpu-b200` | 3 | `4-gpu-b200` | — |
|
||||
| `base-c-test-4-gpu-b200-small` | 3 | `4-gpu-b200-low-disk` | — |
|
||||
| `base-c-test-8-gpu-b200` | registered only | `8-gpu-b200` | — |
|
||||
| `base-c-test-4-gpu-gb200` | registered only | `4-gpu-gb200` | — |
|
||||
|
||||
> **Note**: Kernel suites (`stage-b-kernel-*`) run via `pr-test-jit-kernel.yml` and `pr-test-sgl-kernel.yml`, not the main `pr-test.yml`. `stage-c-test-8-gpu-b200` is registered in `test/run_suite.py` but not wired to PR CI. The GB200 job is currently commented out in `pr-test.yml` until a company-owned runner is provisioned. Multimodal diffusion uses `python/sglang/multimodal_gen/test/run_suite.py`, not `test/run_suite.py`.
|
||||
> **Note**: Kernel suites (`base-b-kernel-*`) run via `pr-test-jit-kernel.yml` and `pr-test-sgl-kernel.yml`, not the main `pr-test.yml`. `base-c-test-8-gpu-b200` is registered in `test/run_suite.py` but not wired to PR CI. The GB200 job is currently commented out in `pr-test.yml` until a company-owned runner is provisioned. Multimodal diffusion uses `python/sglang/multimodal_gen/test/run_suite.py`, not `test/run_suite.py`.
|
||||
|
||||
**Workflow usage:**
|
||||
```yaml
|
||||
@@ -298,7 +297,7 @@ strategy:
|
||||
matrix:
|
||||
partition: [0, 1, 2, 3, 4, 5, 6, 7]
|
||||
steps:
|
||||
- run: python3 run_suite.py --hw cuda --suite stage-b-test-1-gpu-small \
|
||||
- run: python3 run_suite.py --hw cuda --suite base-b-test-1-gpu-small \
|
||||
--auto-partition-id ${{ matrix.partition }} --auto-partition-size 8
|
||||
```
|
||||
|
||||
@@ -320,7 +319,7 @@ Determines which test suites to run based on file changes.
|
||||
|
||||
| Output | Triggers |
|
||||
|--------|----------|
|
||||
| `main_package` | Stage A/B/C test suites |
|
||||
| `main_package` | Base A/B/C test suites |
|
||||
| `sgl_kernel` | Kernel wheel builds + kernel test suites; also switches B200 jobs to kernel-build runner labels outside `target_stage` mode |
|
||||
| `jit_kernel` | JIT kernel test workflow |
|
||||
| `multimodal_gen` | Multimodal-gen test workflow |
|
||||
@@ -348,10 +347,10 @@ group: pr-test-{event_name}-{branch}-{pr_sha}-{stage}
|
||||
|
||||
## How To: Add a New Stage Job
|
||||
|
||||
1. Define the job in `pr-test.yml` with `needs: [check-changes, call-gate, wait-for-stage-X, ...]`
|
||||
1. Define the job in `pr-test.yml` with `needs: [check-changes, call-gate, wait-for-base-X, ...]`
|
||||
2. Copy the `if:` condition pattern from an existing same-stage job (handles `target_stage`, `schedule`, `main_package`)
|
||||
3. Add `checkout` step
|
||||
4. Add `check-stage-health` step (after checkout) — if any prior job failed, `core.setFailed()` fires and all subsequent steps auto-skip via default `if: success()`
|
||||
4. Add `check-pr-test-health` step (after checkout) — if any prior job failed, `core.setFailed()` fires and all subsequent steps auto-skip via default `if: success()`
|
||||
5. Add `check-maintenance` step
|
||||
6. Add `download-artifact` step if `sgl_kernel` changed
|
||||
7. Add `install dependencies` step
|
||||
@@ -359,7 +358,7 @@ group: pr-test-{event_name}-{branch}-{pr_sha}-{stage}
|
||||
9. Add `upload-cuda-coredumps` step with `if: always()`
|
||||
10. Register the suite name in `PER_COMMIT_SUITES` in `test/run_suite.py`
|
||||
11. If using matrix, add `--auto-partition-id` and `--auto-partition-size` to the run command
|
||||
12. **Update `wait-for-stage-X`** job spec with the new job name and `expected_count` (if matrix)
|
||||
12. **Update `wait-for-base-X`** job spec with the new job name and `expected_count` (if matrix)
|
||||
13. **Add the job to `pr-test-finish.needs`** list
|
||||
|
||||
---
|
||||
@@ -368,8 +367,8 @@ group: pr-test-{event_name}-{branch}-{pr_sha}-{stage}
|
||||
|
||||
| Symptom | Likely cause | What to check |
|
||||
|---------|-------------|---------------|
|
||||
| All stage-B/C jobs green but steps skipped | Earlier job failed, `check-stage-health` triggered | Find the actual failed job (red X) |
|
||||
| `wait-for-stage-b` timeout | `expected_count` doesn't match matrix size | Verify job spec counts match `matrix:` array length |
|
||||
| All stage-B/C jobs green but steps skipped | Earlier job failed, `check-pr-test-health` triggered | Find the actual failed job (red X) |
|
||||
| `wait-for-base-b` timeout | `expected_count` doesn't match matrix size | Verify job spec counts match `matrix:` array length |
|
||||
| `pr-test-finish` fails but all jobs green | A job was `cancelled` (counts as failure in finish) | Check concurrency cancellation |
|
||||
| Tests pass locally but fail in CI | Partition assignment, runner GPU type, or `est_time` inaccuracy | Check which partition the test lands in; verify runner label |
|
||||
| Flaky test retried and passed | Retriable failure (accuracy/perf) | Check `[CI Retry]` markers in job logs |
|
||||
|
||||
@@ -27,10 +27,10 @@ JIT kernel exception:
|
||||
|
||||
| Scenario | Model | CI Registration | Suite |
|
||||
|----------|-------|-----------------|-------|
|
||||
| **Unit tests** (no server / engine launch) | None | `register_cpu_ci` (prefer) or `register_cuda_ci` | `stage-a-test-cpu` or `stage-b-test-1-gpu-small` |
|
||||
| **Common / backend-independent** (middleware, abort, routing, config, arg parsing) | `DEFAULT_SMALL_MODEL_NAME_FOR_TEST` (1B) | `register_cuda_ci` only | `stage-b-test-1-gpu-small` |
|
||||
| **Model-agnostic functionality** (sampling, session, OpenAI API features) | `DEFAULT_SMALL_MODEL_NAME_FOR_TEST` (1B) | `register_cuda_ci` (+ AMD if relevant) | `stage-b-test-1-gpu-small` |
|
||||
| **General performance** (single node, no spec/DP/parallelism) | `DEFAULT_MODEL_NAME_FOR_TEST` (8B) | `register_cuda_ci` | `stage-b-test-1-gpu-large` |
|
||||
| **Unit tests** (no server / engine launch) | None | `register_cpu_ci` (prefer) or `register_cuda_ci` | `base-a-test-cpu` or `base-b-test-1-gpu-small` |
|
||||
| **Common / backend-independent** (middleware, abort, routing, config, arg parsing) | `DEFAULT_SMALL_MODEL_NAME_FOR_TEST` (1B) | `register_cuda_ci` only | `base-b-test-1-gpu-small` |
|
||||
| **Model-agnostic functionality** (sampling, session, OpenAI API features) | `DEFAULT_SMALL_MODEL_NAME_FOR_TEST` (1B) | `register_cuda_ci` (+ AMD if relevant) | `base-b-test-1-gpu-small` |
|
||||
| **General performance** (single node, no spec/DP/parallelism) | `DEFAULT_MODEL_NAME_FOR_TEST` (8B) | `register_cuda_ci` | `base-b-test-1-gpu-large` |
|
||||
| **Bigger features** (spec, DP, TP, disaggregation) | Case by case | Case by case | See suite table below |
|
||||
|
||||
**Key principle for E2E tests**: Do NOT add `register_amd_ci` unless the test specifically exercises AMD/ROCm code paths. Common E2E tests just need any GPU to run — duplicating across backends wastes CI time with no extra coverage.
|
||||
@@ -50,7 +50,7 @@ Defined in `python/sglang/test/test_utils.py`:
|
||||
|
||||
### Naming Conventions
|
||||
|
||||
- **Suite**: `stage-{a,b,c}-test-{gpu_count}-gpu-{hardware}` (e.g., `stage-b-test-1-gpu-small`)
|
||||
- **Suite**: `base-{a,b,c}-test-{gpu_count}-gpu-{hardware}` (e.g., `base-b-test-1-gpu-small`)
|
||||
- **CI runner**: `{gpu_count}-gpu-{hardware}` (e.g., `1-gpu-5090`, `4-gpu-h100`, `8-gpu-h200`)
|
||||
|
||||
### All CI Suites
|
||||
@@ -59,25 +59,24 @@ Defined in `python/sglang/test/test_utils.py`:
|
||||
|
||||
| Suite | Runner (label) | Description |
|
||||
|-------|----------------|-------------|
|
||||
| `stage-a-test-1-gpu-small` | `1-gpu-5090` | Quick checks on a small NVIDIA GPU before heavier stages |
|
||||
| `stage-a-test-cpu` | `ubuntu-latest` | CPU-only unit tests |
|
||||
| `stage-b-test-1-gpu-small` | `1-gpu-5090` | Core engine tests that fit a 5090-class card |
|
||||
| `stage-b-test-1-gpu-large` | `1-gpu-h100` | Tests that need H100-class memory or kernels (e.g. FA3) |
|
||||
| `stage-b-test-2-gpu-large` | `2-gpu-h100` | Two-GPU correctness and parallelism (TP/PP) on H100 |
|
||||
| `stage-b-test-4-gpu-b200` | `4-gpu-b200` | Early Blackwell coverage (SM100+ paths) on four GPUs |
|
||||
| `stage-b-kernel-unit-1-gpu-large` | `1-gpu-h100` | JIT kernel correctness tests under `python/sglang/jit_kernel/tests/` |
|
||||
| `stage-b-kernel-unit-1-gpu-b200` | `4-gpu-b200` | JIT kernel correctness tests for Blackwell / SM100-specific paths |
|
||||
| `stage-b-kernel-unit-8-gpu-h200` | `8-gpu-h200` | Multi-GPU JIT kernel correctness tests under `python/sglang/jit_kernel/tests/` |
|
||||
| `stage-b-kernel-benchmark-1-gpu-large` | `1-gpu-h100` | JIT kernel benchmark files under `python/sglang/jit_kernel/benchmark/` |
|
||||
| `stage-c-test-4-gpu-h100` | `4-gpu-h100` | Large 4-GPU H100 integration and scaling tests |
|
||||
| `stage-c-test-8-gpu-h200` | `8-gpu-h200` | Large 8-GPU H200 runs for big models and parallelism |
|
||||
| `stage-c-test-8-gpu-h20` | `8-gpu-h20` | Large 8-GPU H20 runs for big models |
|
||||
| `stage-c-test-deepep-4-gpu-h100` | `4-gpu-h100` | DeepEP expert-parallel and networking on four H100s |
|
||||
| `stage-c-test-deepep-8-gpu-h200` | `8-gpu-h200` | DeepEP at 8-GPU H200 scale |
|
||||
| `stage-c-test-8-gpu-b200` | `8-gpu-b200` | 8-GPU B200 suite (registered but not yet wired to a workflow) |
|
||||
| `stage-c-test-4-gpu-b200` | `4-gpu-b200` | 4-GPU B200 suite for large models on Blackwell |
|
||||
| `stage-c-test-4-gpu-b200-small` | `4-gpu-b200` | Smaller 4-GPU B200 suite split onto low-disk B200 runners |
|
||||
| `stage-c-test-4-gpu-gb200` | `4-gpu-gb200` | 4-GPU GB200 suite for Grace Blackwell; registered in `run_suite.py`, but the PR workflow is currently disabled until a runner is provisioned |
|
||||
| `base-a-test-1-gpu-small` | `1-gpu-5090` | Quick checks on a small NVIDIA GPU before heavier stages |
|
||||
| `base-a-test-cpu` | `ubuntu-latest` | CPU-only unit tests |
|
||||
| `base-b-test-1-gpu-small` | `1-gpu-5090` | Core engine tests that fit a 5090-class card |
|
||||
| `base-b-test-1-gpu-large` | `1-gpu-h100` | Tests that need H100-class memory or kernels (e.g. FA3) |
|
||||
| `base-b-test-2-gpu-large` | `2-gpu-h100` | Two-GPU correctness and parallelism (TP/PP) on H100 |
|
||||
| `base-b-test-4-gpu-b200` | `4-gpu-b200` | Early Blackwell coverage (SM100+ paths) on four GPUs |
|
||||
| `base-b-kernel-unit-1-gpu-large` | `1-gpu-h100` | JIT kernel correctness tests under `python/sglang/jit_kernel/tests/` |
|
||||
| `base-b-kernel-unit-1-gpu-b200` | `4-gpu-b200` | JIT kernel correctness tests for Blackwell / SM100-specific paths |
|
||||
| `base-b-kernel-unit-8-gpu-h200` | `8-gpu-h200` | Multi-GPU JIT kernel correctness tests under `python/sglang/jit_kernel/tests/` |
|
||||
| `base-b-kernel-benchmark-1-gpu-large` | `1-gpu-h100` | JIT kernel benchmark files under `python/sglang/jit_kernel/benchmark/` |
|
||||
| `base-c-test-4-gpu-h100` | `4-gpu-h100` | Large 4-GPU H100 integration and scaling tests |
|
||||
| `base-c-test-8-gpu-h200` | `8-gpu-h200` | Large 8-GPU H200 runs for big models and parallelism |
|
||||
| `base-c-test-8-gpu-h20` | `8-gpu-h20` | Large 8-GPU H20 runs for big models |
|
||||
| `base-c-test-deepep-4-gpu-h100` | `4-gpu-h100` | DeepEP expert-parallel and networking on four H100s |
|
||||
| `base-c-test-8-gpu-b200` | `8-gpu-b200` | 8-GPU B200 suite (registered but not yet wired to a workflow) |
|
||||
| `base-c-test-4-gpu-b200` | `4-gpu-b200` | 4-GPU B200 suite for large models on Blackwell |
|
||||
| `base-c-test-4-gpu-b200-small` | `4-gpu-b200` | Smaller 4-GPU B200 suite split onto low-disk B200 runners |
|
||||
| `base-c-test-4-gpu-gb200` | `4-gpu-gb200` | 4-GPU GB200 suite for Grace Blackwell; registered in `run_suite.py`, but the PR workflow is currently disabled until a runner is provisioned |
|
||||
|
||||
#### Per-commit (AMD)
|
||||
|
||||
@@ -130,12 +129,12 @@ Nightly suites are listed in `NIGHTLY_SUITES` in [`test/run_suite.py`](../../../
|
||||
|
||||
Use the lightest suite that meets your test's needs:
|
||||
|
||||
- **No GPU required** → `stage-a-test-cpu`
|
||||
- **Most small GPU tests** → `stage-b-test-1-gpu-small` (default choice)
|
||||
- **Need H100 memory or Hopper features** → `stage-b-test-1-gpu-large`
|
||||
- **JIT kernel correctness** → `stage-b-kernel-unit-1-gpu-large`
|
||||
- **JIT kernel correctness for B200 / SM100 paths** → `stage-b-kernel-unit-1-gpu-b200`
|
||||
- **JIT kernel benchmarks** → `stage-b-kernel-benchmark-1-gpu-large`
|
||||
- **No GPU required** → `base-a-test-cpu`
|
||||
- **Most small GPU tests** → `base-b-test-1-gpu-small` (default choice)
|
||||
- **Need H100 memory or Hopper features** → `base-b-test-1-gpu-large`
|
||||
- **JIT kernel correctness** → `base-b-kernel-unit-1-gpu-large`
|
||||
- **JIT kernel correctness for B200 / SM100 paths** → `base-b-kernel-unit-1-gpu-b200`
|
||||
- **JIT kernel benchmarks** → `base-b-kernel-benchmark-1-gpu-large`
|
||||
- **Multi-GPU** → only when the test actually needs multiple GPUs
|
||||
|
||||
---
|
||||
@@ -156,7 +155,7 @@ from sglang.srt.<module> import TargetClass
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=5, suite="stage-a-test-cpu")
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
# Prefer CPU. Only use register_cuda_ci when the test truly needs a GPU.
|
||||
|
||||
class TestTargetClass(CustomTestCase):
|
||||
@@ -196,7 +195,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=60, suite="stage-b-test-1-gpu-small")
|
||||
register_cuda_ci(est_time=60, suite="base-b-test-1-gpu-small")
|
||||
|
||||
|
||||
class TestMyFeature(CustomTestCase):
|
||||
@@ -246,7 +245,7 @@ from sglang.test.test_utils import (
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=300, suite="stage-b-test-1-gpu-large")
|
||||
register_cuda_ci(est_time=300, suite="base-b-test-1-gpu-large")
|
||||
|
||||
|
||||
class TestMyFeaturePerf(CustomTestCase):
|
||||
@@ -321,21 +320,21 @@ from sglang.test.ci.ci_register import (
|
||||
)
|
||||
|
||||
# Per-commit test (small 1-gpu, runs on 5090)
|
||||
register_cuda_ci(est_time=80, suite="stage-b-test-1-gpu-small")
|
||||
register_cuda_ci(est_time=80, suite="base-b-test-1-gpu-small")
|
||||
|
||||
# Per-commit test (large 1-gpu, runs on H100)
|
||||
register_cuda_ci(est_time=120, suite="stage-b-test-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="base-b-test-1-gpu-large")
|
||||
|
||||
# Nightly-only test
|
||||
register_cuda_ci(est_time=200, suite="nightly-1-gpu", nightly=True)
|
||||
|
||||
# Multi-backend test (only when testing backend-specific code paths)
|
||||
register_cuda_ci(est_time=80, suite="stage-a-test-1-gpu-small")
|
||||
register_cuda_ci(est_time=80, suite="base-a-test-1-gpu-small")
|
||||
register_amd_ci(est_time=120, suite="stage-a-test-1-gpu-small-amd")
|
||||
register_npu_ci(est_time=400, suite="nightly-8-npu-a3", nightly=True)
|
||||
|
||||
# Temporarily disabled test
|
||||
register_cuda_ci(est_time=80, suite="stage-b-test-1-gpu-small", disabled="flaky - see #12345")
|
||||
register_cuda_ci(est_time=80, suite="base-b-test-1-gpu-small", disabled="flaky - see #12345")
|
||||
```
|
||||
|
||||
Parameters:
|
||||
@@ -354,12 +353,12 @@ JIT kernel files live outside `test/registered/` but still use registration:
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
# Correctness tests in python/sglang/jit_kernel/tests/
|
||||
register_cuda_ci(est_time=30, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=30, suite="stage-b-kernel-unit-1-gpu-b200")
|
||||
register_cuda_ci(est_time=120, suite="stage-b-kernel-unit-8-gpu-h200")
|
||||
register_cuda_ci(est_time=30, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=30, suite="base-b-kernel-unit-1-gpu-b200")
|
||||
register_cuda_ci(est_time=120, suite="base-b-kernel-unit-8-gpu-h200")
|
||||
|
||||
# Benchmarks in python/sglang/jit_kernel/benchmark/
|
||||
register_cuda_ci(est_time=6, suite="stage-b-kernel-benchmark-1-gpu-large")
|
||||
register_cuda_ci(est_time=6, suite="base-b-kernel-benchmark-1-gpu-large")
|
||||
|
||||
# Optional nightly registration
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
+7
-7
@@ -1,4 +1,4 @@
|
||||
name: Check Stage Health
|
||||
name: Check PR Test Health
|
||||
description: Fail fast if any job in the current workflow run has already failed, or if the lint check (from lint.yml) has failed. Auto-skips for scheduled runs. The jobs-failed check (but not the lint check) is bypassed when the PR carries the `bypass-fastfail` label.
|
||||
|
||||
inputs:
|
||||
@@ -10,16 +10,16 @@ inputs:
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Check stage health
|
||||
- name: Check PR test health
|
||||
uses: actions/github-script@v7
|
||||
env:
|
||||
SKIP_STAGE_HEALTH_CHECK: ${{ env.SKIP_STAGE_HEALTH_CHECK }}
|
||||
SKIP_PR_TEST_HEALTH_CHECK: ${{ env.SKIP_PR_TEST_HEALTH_CHECK }}
|
||||
with:
|
||||
github-token: ${{ inputs.github-token }}
|
||||
script: |
|
||||
// Skip when explicitly requested via env var (e.g. release branch cut)
|
||||
if (process.env.SKIP_STAGE_HEALTH_CHECK === 'true') {
|
||||
core.info('Skipping health check (SKIP_STAGE_HEALTH_CHECK=true)');
|
||||
if (process.env.SKIP_PR_TEST_HEALTH_CHECK === 'true') {
|
||||
core.info('Skipping health check (SKIP_PR_TEST_HEALTH_CHECK=true)');
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -83,12 +83,12 @@ runs:
|
||||
// match so we cover both inline + reusable forms without confusing
|
||||
// 'h20' with the 'h200' prefix.
|
||||
const baseName = j.name.split(/[ /]/)[0];
|
||||
if (baseName === 'stage-c-test-8-gpu-h20') {
|
||||
if (baseName === 'base-c-test-8-gpu-h20') {
|
||||
return false;
|
||||
}
|
||||
// If the failing step is the health check, it's a cascade — skip it
|
||||
const failedStep = (j.steps || []).find(s => s.conclusion === 'failure');
|
||||
if (failedStep && (failedStep.name.includes('check-stage-health') || failedStep.name.includes('Check stage health'))) {
|
||||
if (failedStep && (failedStep.name.includes('check-pr-test-health') || failedStep.name.includes('Check PR test health'))) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
@@ -3,12 +3,12 @@ description: Poll and wait for specified jobs in the current workflow run to com
|
||||
|
||||
inputs:
|
||||
stage-name:
|
||||
description: 'Human-readable stage name for log messages (e.g. "stage-a")'
|
||||
description: 'Human-readable stage name for log messages (e.g. "base-a")'
|
||||
required: true
|
||||
jobs:
|
||||
description: |
|
||||
JSON array of job specs to wait for. Each element is either:
|
||||
- a string: exact job name (e.g. "stage-a-test-1-gpu-small")
|
||||
- a string: exact job name (e.g. "base-a-test-1-gpu-small")
|
||||
- an object { "prefix": "...", "expected_count": N }: for matrix jobs
|
||||
required: true
|
||||
max-wait-minutes:
|
||||
|
||||
@@ -24,7 +24,7 @@ on:
|
||||
git_ref:
|
||||
type: string
|
||||
default: ''
|
||||
skip_stage_health_check:
|
||||
skip_pr_test_health_check:
|
||||
description: 'Forwarded from the caller workflow input of the same name. Not consumed inside this reusable workflow today, but mirrored so the resulting env context matches the inline-job baseline.'
|
||||
type: boolean
|
||||
default: false
|
||||
@@ -39,7 +39,7 @@ env:
|
||||
SGLANG_IS_IN_CI: true
|
||||
SGLANG_CUDA_COREDUMP: "1"
|
||||
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true
|
||||
SKIP_STAGE_HEALTH_CHECK: ${{ inputs.skip_stage_health_check && 'true' || 'false' }}
|
||||
SKIP_PR_TEST_HEALTH_CHECK: ${{ inputs.skip_pr_test_health_check && 'true' || 'false' }}
|
||||
FORCE_REBUILD_DEEPEP: '1'
|
||||
PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }}
|
||||
USE_VENV: false
|
||||
|
||||
@@ -4,7 +4,7 @@ name: PR Test Stage
|
||||
# `caller_inputs` = toJson(inputs) as bundles. `partitions` is forwarded
|
||||
# separately to keep matrix expressions single-fromJson — its value is itself
|
||||
# a JSON string, so reading it via `check_changes` would need double fromJson,
|
||||
# and matrix can't use step-resolved values. stage-a-test-cpu stays inline in
|
||||
# and matrix can't use step-resolved values. base-a-test-cpu stays inline in
|
||||
# pr-test.yml (bespoke uv pip / protoc / rust-cache install).
|
||||
|
||||
on:
|
||||
@@ -23,7 +23,7 @@ on:
|
||||
type: string
|
||||
required: true
|
||||
caller_inputs:
|
||||
description: 'toJson(inputs) from pr-test.yml. Read via fromJson(...).git_ref / skip_stage_health_check / test_parallel_dispatch.'
|
||||
description: 'toJson(inputs) from pr-test.yml. Read via fromJson(...).git_ref / skip_pr_test_health_check / test_parallel_dispatch.'
|
||||
type: string
|
||||
required: true
|
||||
partitions:
|
||||
@@ -62,7 +62,7 @@ env:
|
||||
SGLANG_IS_IN_CI: true
|
||||
SGLANG_CUDA_COREDUMP: "1"
|
||||
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true
|
||||
SKIP_STAGE_HEALTH_CHECK: ${{ fromJson(inputs.caller_inputs).skip_stage_health_check && 'true' || 'false' }}
|
||||
SKIP_PR_TEST_HEALTH_CHECK: ${{ fromJson(inputs.caller_inputs).skip_pr_test_health_check && 'true' || 'false' }}
|
||||
FORCE_REBUILD_DEEPEP: '1'
|
||||
PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }}
|
||||
USE_VENV: false
|
||||
@@ -97,7 +97,7 @@ jobs:
|
||||
- name: Export rdma_devices to job env
|
||||
run: echo "SGLANG_CI_RDMA_ALL_DEVICES=${{ steps.rc.outputs.rdma_devices || '' }}" >> "$GITHUB_ENV"
|
||||
|
||||
- uses: ./.github/actions/check-stage-health
|
||||
- uses: ./.github/actions/check-pr-test-health
|
||||
|
||||
- uses: ./.github/actions/check-maintenance
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ name: CI Auto Bisect
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: ["PR Test"]
|
||||
workflows: ["PR Test Base"]
|
||||
types: [completed]
|
||||
branches: [main]
|
||||
workflow_dispatch: {}
|
||||
|
||||
@@ -85,7 +85,7 @@ jobs:
|
||||
'---',
|
||||
'### CI States',
|
||||
'',
|
||||
`Latest PR Test: ${ptStart}${ptText}${ptEnd}`,
|
||||
`Latest PR Test (Base): ${ptStart}${ptText}${ptEnd}`,
|
||||
`Latest PR Test (Extra): ${peStart}${peText}${peEnd}`,
|
||||
outerEnd,
|
||||
].join('\n');
|
||||
|
||||
@@ -36,8 +36,8 @@ on:
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
skip_stage_health_check:
|
||||
description: "Skip stage health check fast-fail (e.g. for release branch cuts)"
|
||||
skip_pr_test_health_check:
|
||||
description: "Skip PR test health check fast-fail (e.g. for release branch cuts)"
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
@@ -50,7 +50,7 @@ env:
|
||||
SGLANG_IS_IN_CI: true
|
||||
SGLANG_CUDA_COREDUMP: "1"
|
||||
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true
|
||||
SKIP_STAGE_HEALTH_CHECK: ${{ (inputs.skip_stage_health_check == true || inputs.run_all_tests == true) && 'true' || 'false' }}
|
||||
SKIP_PR_TEST_HEALTH_CHECK: ${{ (inputs.skip_pr_test_health_check == true || inputs.run_all_tests == true) && 'true' || 'false' }}
|
||||
FORCE_REBUILD_DEEPEP: '1'
|
||||
PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }}
|
||||
USE_VENV: false
|
||||
@@ -96,7 +96,7 @@ jobs:
|
||||
runs_on: x64-kernel-build-node
|
||||
job_display_name: Build Wheel
|
||||
git_ref: ${{ inputs.git_ref || '' }}
|
||||
skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }}
|
||||
skip_pr_test_health_check: ${{ inputs.skip_pr_test_health_check == true }}
|
||||
secrets: inherit
|
||||
|
||||
# =============================================== extra-a (1-/2-gpu) ===============================================
|
||||
|
||||
@@ -23,7 +23,7 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
default: 'false'
|
||||
skip_stage_health_check:
|
||||
skip_pr_test_health_check:
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
@@ -35,7 +35,7 @@ env:
|
||||
SGLANG_CUDA_COREDUMP: "1"
|
||||
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true
|
||||
PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }}
|
||||
SKIP_STAGE_HEALTH_CHECK: ${{ inputs.skip_stage_health_check == true && 'true' || 'false' }}
|
||||
SKIP_PR_TEST_HEALTH_CHECK: ${{ inputs.skip_pr_test_health_check == true && 'true' || 'false' }}
|
||||
|
||||
jobs:
|
||||
jit-kernel-unit-test:
|
||||
@@ -49,7 +49,7 @@ jobs:
|
||||
with:
|
||||
ref: ${{ inputs.git_ref || github.sha }}
|
||||
|
||||
- uses: ./.github/actions/check-stage-health
|
||||
- uses: ./.github/actions/check-pr-test-health
|
||||
|
||||
- uses: ./.github/actions/check-maintenance
|
||||
|
||||
@@ -76,7 +76,7 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
cd test/
|
||||
python3 run_suite.py --hw cuda --suite stage-b-kernel-unit-1-gpu-large
|
||||
python3 run_suite.py --hw cuda --suite base-b-kernel-unit-1-gpu-large
|
||||
|
||||
jit-kernel-multigpu-unit-test:
|
||||
if: |
|
||||
@@ -114,7 +114,7 @@ jobs:
|
||||
timeout-minutes: 45
|
||||
run: |
|
||||
cd test/
|
||||
python3 run_suite.py --hw cuda --suite stage-b-kernel-unit-8-gpu-h200
|
||||
python3 run_suite.py --hw cuda --suite base-b-kernel-unit-8-gpu-h200
|
||||
|
||||
jit-kernel-benchmark-test:
|
||||
if: |
|
||||
@@ -127,7 +127,7 @@ jobs:
|
||||
with:
|
||||
ref: ${{ inputs.git_ref || github.sha }}
|
||||
|
||||
- uses: ./.github/actions/check-stage-health
|
||||
- uses: ./.github/actions/check-pr-test-health
|
||||
|
||||
- uses: ./.github/actions/check-maintenance
|
||||
|
||||
@@ -154,7 +154,7 @@ jobs:
|
||||
timeout-minutes: 45
|
||||
run: |
|
||||
cd test/
|
||||
python3 run_suite.py --hw cuda --suite stage-b-kernel-benchmark-1-gpu-large
|
||||
python3 run_suite.py --hw cuda --suite base-b-kernel-benchmark-1-gpu-large
|
||||
|
||||
jit-kernel-b200-test:
|
||||
if: |
|
||||
@@ -167,7 +167,7 @@ jobs:
|
||||
with:
|
||||
ref: ${{ inputs.git_ref || github.sha }}
|
||||
|
||||
- uses: ./.github/actions/check-stage-health
|
||||
- uses: ./.github/actions/check-pr-test-health
|
||||
|
||||
- uses: ./.github/actions/check-maintenance
|
||||
|
||||
@@ -194,4 +194,4 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
cd test/
|
||||
python3 run_suite.py --hw cuda --suite stage-b-kernel-unit-1-gpu-b200
|
||||
python3 run_suite.py --hw cuda --suite base-b-kernel-unit-1-gpu-b200
|
||||
|
||||
@@ -31,7 +31,7 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
default: 'false'
|
||||
skip_stage_health_check:
|
||||
skip_pr_test_health_check:
|
||||
required: false
|
||||
type: string
|
||||
default: 'false'
|
||||
@@ -42,7 +42,7 @@ env:
|
||||
SGLANG_IS_IN_CI: true
|
||||
SGLANG_CUDA_COREDUMP: "1"
|
||||
PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }}
|
||||
SKIP_STAGE_HEALTH_CHECK: ${{ inputs.skip_stage_health_check == 'true' }}
|
||||
SKIP_PR_TEST_HEALTH_CHECK: ${{ inputs.skip_pr_test_health_check == 'true' }}
|
||||
|
||||
jobs:
|
||||
compute-diffusion-partitions:
|
||||
@@ -90,7 +90,7 @@ jobs:
|
||||
with:
|
||||
ref: ${{ inputs.git_ref || github.sha }}
|
||||
|
||||
- uses: ./.github/actions/check-stage-health
|
||||
- uses: ./.github/actions/check-pr-test-health
|
||||
|
||||
- uses: ./.github/actions/check-maintenance
|
||||
|
||||
@@ -163,7 +163,7 @@ jobs:
|
||||
with:
|
||||
ref: ${{ inputs.git_ref || github.sha }}
|
||||
|
||||
- uses: ./.github/actions/check-stage-health
|
||||
- uses: ./.github/actions/check-pr-test-health
|
||||
|
||||
- uses: ./.github/actions/check-maintenance
|
||||
|
||||
@@ -230,7 +230,7 @@ jobs:
|
||||
with:
|
||||
ref: ${{ inputs.git_ref || github.sha }}
|
||||
|
||||
- uses: ./.github/actions/check-stage-health
|
||||
- uses: ./.github/actions/check-pr-test-health
|
||||
|
||||
- uses: ./.github/actions/check-maintenance
|
||||
|
||||
@@ -275,7 +275,7 @@ jobs:
|
||||
with:
|
||||
ref: ${{ inputs.git_ref || github.sha }}
|
||||
|
||||
- uses: ./.github/actions/check-stage-health
|
||||
- uses: ./.github/actions/check-pr-test-health
|
||||
|
||||
- uses: ./.github/actions/check-maintenance
|
||||
|
||||
@@ -327,7 +327,7 @@ jobs:
|
||||
with:
|
||||
ref: ${{ inputs.git_ref || github.sha }}
|
||||
|
||||
- uses: ./.github/actions/check-stage-health
|
||||
- uses: ./.github/actions/check-pr-test-health
|
||||
|
||||
- uses: ./.github/actions/check-maintenance
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
default: ''
|
||||
skip_stage_health_check:
|
||||
skip_pr_test_health_check:
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
@@ -27,7 +27,7 @@ env:
|
||||
SGLANG_IS_IN_CI: true
|
||||
SGLANG_CUDA_COREDUMP: "1"
|
||||
PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }}
|
||||
SKIP_STAGE_HEALTH_CHECK: ${{ inputs.skip_stage_health_check == true && 'true' || 'false' }}
|
||||
SKIP_PR_TEST_HEALTH_CHECK: ${{ inputs.skip_pr_test_health_check == true && 'true' || 'false' }}
|
||||
|
||||
jobs:
|
||||
sgl-kernel-unit-test:
|
||||
@@ -38,7 +38,7 @@ jobs:
|
||||
with:
|
||||
ref: ${{ inputs.git_ref || github.sha }}
|
||||
|
||||
- uses: ./.github/actions/check-stage-health
|
||||
- uses: ./.github/actions/check-pr-test-health
|
||||
|
||||
- uses: ./.github/actions/check-maintenance
|
||||
|
||||
@@ -73,7 +73,7 @@ jobs:
|
||||
with:
|
||||
ref: ${{ inputs.git_ref || github.sha }}
|
||||
|
||||
- uses: ./.github/actions/check-stage-health
|
||||
- uses: ./.github/actions/check-pr-test-health
|
||||
|
||||
- uses: ./.github/actions/check-maintenance
|
||||
|
||||
@@ -120,7 +120,7 @@ jobs:
|
||||
with:
|
||||
ref: ${{ inputs.git_ref || github.sha }}
|
||||
|
||||
- uses: ./.github/actions/check-stage-health
|
||||
- uses: ./.github/actions/check-pr-test-health
|
||||
|
||||
- uses: ./.github/actions/check-maintenance
|
||||
|
||||
|
||||
@@ -115,7 +115,7 @@ jobs:
|
||||
timeout-minutes: 36
|
||||
run: |
|
||||
docker exec -w /sglang-checkout/ ci_sglang_xeon \
|
||||
bash -c "source /opt/.venv/bin/activate && cd ./test && python3 run_suite.py --hw cpu --suite stage-b-test-cpu"
|
||||
bash -c "source /opt/.venv/bin/activate && cd ./test && python3 run_suite.py --hw cpu --suite base-b-test-cpu"
|
||||
|
||||
- name: Change permission
|
||||
timeout-minutes: 2
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
name: PR Test
|
||||
name: PR Test Base
|
||||
|
||||
on:
|
||||
schedule:
|
||||
@@ -34,8 +34,8 @@ on:
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
skip_stage_health_check:
|
||||
description: "Skip stage health check fast-fail (e.g. for release branch cuts)"
|
||||
skip_pr_test_health_check:
|
||||
description: "Skip PR test health check fast-fail (e.g. for release branch cuts)"
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
@@ -52,7 +52,7 @@ env:
|
||||
SGLANG_IS_IN_CI: true
|
||||
SGLANG_CUDA_COREDUMP: "1"
|
||||
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true
|
||||
SKIP_STAGE_HEALTH_CHECK: ${{ (inputs.skip_stage_health_check == true || inputs.test_parallel_dispatch == true || inputs.run_all_tests == true) && 'true' || 'false' }}
|
||||
SKIP_PR_TEST_HEALTH_CHECK: ${{ (inputs.skip_pr_test_health_check == true || inputs.test_parallel_dispatch == true || inputs.run_all_tests == true) && 'true' || 'false' }}
|
||||
# TEMP: rebuild deepep against the new torch for torch-211-merge PR only — revert before merging to main.
|
||||
FORCE_REBUILD_DEEPEP: '1'
|
||||
# Schedule / main-branch dispatch / workflow_call from main use refs/heads/main; PR events use refs/pull/*/merge
|
||||
@@ -89,7 +89,7 @@ jobs:
|
||||
# For PRs with the `bypass-fastfail` label: wait jobs run but return success immediately
|
||||
# (handled inside the wait-for-jobs action), so downstream stages dispatch in parallel.
|
||||
|
||||
wait-for-stage-a:
|
||||
wait-for-base-a:
|
||||
needs: [check-changes, call-gate]
|
||||
if: |
|
||||
always() &&
|
||||
@@ -100,7 +100,7 @@ jobs:
|
||||
(needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped')
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
stage_a_result: ${{ steps.wait.outputs.result }}
|
||||
base_a_result: ${{ steps.wait.outputs.result }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
@@ -109,27 +109,27 @@ jobs:
|
||||
- uses: ./.github/actions/wait-for-jobs
|
||||
id: wait
|
||||
with:
|
||||
stage-name: stage-a
|
||||
stage-name: base-a
|
||||
jobs: |
|
||||
[
|
||||
{"prefix": "stage-a-test-1-gpu-small", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['stage-a-test-1-gpu-small'].size }}},
|
||||
{"prefix": "stage-a-test-cpu", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['stage-a-test-cpu'].size }}}
|
||||
{"prefix": "base-a-test-1-gpu-small", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['base-a-test-1-gpu-small'].size }}},
|
||||
{"prefix": "base-a-test-cpu", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['base-a-test-cpu'].size }}}
|
||||
]
|
||||
max-wait-minutes: '240'
|
||||
|
||||
wait-for-stage-b:
|
||||
needs: [check-changes, call-gate, wait-for-stage-a]
|
||||
wait-for-base-b:
|
||||
needs: [check-changes, call-gate, wait-for-base-a]
|
||||
if: |
|
||||
always() &&
|
||||
!cancelled() &&
|
||||
github.event_name == 'pull_request' &&
|
||||
inputs.test_parallel_dispatch != true &&
|
||||
(needs.check-changes.outputs.main_package == 'true' || needs.check-changes.outputs.sgl_kernel == 'true') &&
|
||||
(needs.wait-for-stage-a.result == 'success' || needs.wait-for-stage-a.result == 'skipped') &&
|
||||
(needs.wait-for-base-a.result == 'success' || needs.wait-for-base-a.result == 'skipped') &&
|
||||
(needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped')
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
stage_b_result: ${{ steps.wait.outputs.result }}
|
||||
base_b_result: ${{ steps.wait.outputs.result }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
@@ -138,13 +138,13 @@ jobs:
|
||||
- uses: ./.github/actions/wait-for-jobs
|
||||
id: wait
|
||||
with:
|
||||
stage-name: stage-b
|
||||
stage-name: base-b
|
||||
jobs: |
|
||||
[
|
||||
{"prefix": "stage-b-test-1-gpu-small", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['stage-b-test-1-gpu-small'].size }}},
|
||||
{"prefix": "stage-b-test-1-gpu-large", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['stage-b-test-1-gpu-large'].size }}},
|
||||
{"prefix": "stage-b-test-2-gpu-large", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['stage-b-test-2-gpu-large'].size }}},
|
||||
{"prefix": "stage-b-test-4-gpu-b200", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['stage-b-test-4-gpu-b200'].size }}}
|
||||
{"prefix": "base-b-test-1-gpu-small", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['base-b-test-1-gpu-small'].size }}},
|
||||
{"prefix": "base-b-test-1-gpu-large", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['base-b-test-1-gpu-large'].size }}},
|
||||
{"prefix": "base-b-test-2-gpu-large", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['base-b-test-2-gpu-large'].size }}},
|
||||
{"prefix": "base-b-test-4-gpu-b200", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['base-b-test-4-gpu-b200'].size }}}
|
||||
]
|
||||
max-wait-minutes: '480'
|
||||
|
||||
@@ -180,7 +180,7 @@ jobs:
|
||||
runs_on: x64-kernel-build-node
|
||||
job_display_name: Build Wheel
|
||||
git_ref: ${{ inputs.git_ref || '' }}
|
||||
skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }}
|
||||
skip_pr_test_health_check: ${{ inputs.skip_pr_test_health_check == true }}
|
||||
secrets: inherit
|
||||
|
||||
sgl-kernel-build-wheels-arm:
|
||||
@@ -198,7 +198,7 @@ jobs:
|
||||
job_display_name: Build Wheel Arm
|
||||
arch_suffix: '-aarch64'
|
||||
git_ref: ${{ inputs.git_ref || '' }}
|
||||
skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }}
|
||||
skip_pr_test_health_check: ${{ inputs.skip_pr_test_health_check == true }}
|
||||
secrets: inherit
|
||||
|
||||
call-sgl-kernel-tests:
|
||||
@@ -213,7 +213,7 @@ jobs:
|
||||
runs_on_map: ${{ needs.check-changes.outputs.runs_on_map }}
|
||||
sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }}
|
||||
git_ref: ${{ inputs.git_ref || '' }}
|
||||
skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }}
|
||||
skip_pr_test_health_check: ${{ inputs.skip_pr_test_health_check == true }}
|
||||
secrets: inherit
|
||||
|
||||
# =============================================== jit-kernel ====================================================
|
||||
@@ -234,18 +234,18 @@ jobs:
|
||||
sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }}
|
||||
git_ref: ${{ inputs.git_ref || '' }}
|
||||
test_parallel_dispatch: ${{ inputs.test_parallel_dispatch == true && 'true' || 'false' }}
|
||||
skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }}
|
||||
skip_pr_test_health_check: ${{ inputs.skip_pr_test_health_check == true }}
|
||||
secrets: inherit
|
||||
|
||||
# =============================================== primary ====================================================
|
||||
|
||||
# Runs on 5090 (32GB, SM120)
|
||||
stage-a-test-1-gpu-small:
|
||||
base-a-test-1-gpu-small:
|
||||
needs: [check-changes, call-gate, sgl-kernel-build-wheels]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/_pr-test-stage.yml
|
||||
with:
|
||||
self_name: stage-a-test-1-gpu-small
|
||||
self_name: base-a-test-1-gpu-small
|
||||
runner_config: 1-gpu-small
|
||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
||||
caller_inputs: ${{ toJson(inputs) }}
|
||||
@@ -253,7 +253,7 @@ jobs:
|
||||
run_timeout_minutes: '10'
|
||||
secrets: inherit
|
||||
|
||||
stage-a-test-cpu:
|
||||
base-a-test-cpu:
|
||||
needs: [check-changes, call-gate]
|
||||
if: |
|
||||
always() &&
|
||||
@@ -263,9 +263,9 @@ jobs:
|
||||
timeout-minutes: 240
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: ${{ fromJson(needs.check-changes.outputs.partitions)['stage-a-test-cpu'].max_parallel }}
|
||||
max-parallel: ${{ fromJson(needs.check-changes.outputs.partitions)['base-a-test-cpu'].max_parallel }}
|
||||
matrix:
|
||||
partition: ${{ fromJson(needs.check-changes.outputs.partitions)['stage-a-test-cpu'].arr }}
|
||||
partition: ${{ fromJson(needs.check-changes.outputs.partitions)['base-a-test-cpu'].arr }}
|
||||
steps:
|
||||
- name: Free disk space
|
||||
run: |
|
||||
@@ -277,7 +277,7 @@ jobs:
|
||||
with:
|
||||
ref: ${{ inputs.git_ref || github.sha }}
|
||||
|
||||
- uses: ./.github/actions/check-stage-health
|
||||
- uses: ./.github/actions/check-pr-test-health
|
||||
|
||||
- uses: ./.github/actions/check-maintenance
|
||||
|
||||
@@ -316,15 +316,15 @@ jobs:
|
||||
CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
run: |
|
||||
cd test/
|
||||
python3 run_suite.py --hw cpu --suite stage-a-test-cpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size ${{ fromJson(needs.check-changes.outputs.partitions)['stage-a-test-cpu'].size }} $CONTINUE_ON_ERROR_FLAG
|
||||
python3 run_suite.py --hw cpu --suite base-a-test-cpu --auto-partition-id ${{ matrix.partition }} --auto-partition-size ${{ fromJson(needs.check-changes.outputs.partitions)['base-a-test-cpu'].size }} $CONTINUE_ON_ERROR_FLAG
|
||||
|
||||
# Runs on 5090 (32GB, SM120)
|
||||
stage-b-test-1-gpu-small:
|
||||
needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels]
|
||||
base-b-test-1-gpu-small:
|
||||
needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/_pr-test-stage.yml
|
||||
with:
|
||||
self_name: stage-b-test-1-gpu-small
|
||||
self_name: base-b-test-1-gpu-small
|
||||
runner_config: 1-gpu-small
|
||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
||||
caller_inputs: ${{ toJson(inputs) }}
|
||||
@@ -333,12 +333,12 @@ jobs:
|
||||
secrets: inherit
|
||||
|
||||
# Runs on H100 (80GB, SM90) - tests that don't pass on 5090 (FA3, FP8, high VRAM, etc.)
|
||||
stage-b-test-1-gpu-large:
|
||||
needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels]
|
||||
base-b-test-1-gpu-large:
|
||||
needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/_pr-test-stage.yml
|
||||
with:
|
||||
self_name: stage-b-test-1-gpu-large
|
||||
self_name: base-b-test-1-gpu-large
|
||||
runner_config: 1-gpu-large
|
||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
||||
caller_inputs: ${{ toJson(inputs) }}
|
||||
@@ -347,12 +347,12 @@ jobs:
|
||||
timeout_per_file: '1800'
|
||||
secrets: inherit
|
||||
|
||||
stage-b-test-2-gpu-large:
|
||||
needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels]
|
||||
base-b-test-2-gpu-large:
|
||||
needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/_pr-test-stage.yml
|
||||
with:
|
||||
self_name: stage-b-test-2-gpu-large
|
||||
self_name: base-b-test-2-gpu-large
|
||||
runner_config: 2-gpu-large
|
||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
||||
caller_inputs: ${{ toJson(inputs) }}
|
||||
@@ -360,12 +360,12 @@ jobs:
|
||||
run_timeout_minutes: '30'
|
||||
secrets: inherit
|
||||
|
||||
stage-b-test-4-gpu-b200:
|
||||
needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels]
|
||||
base-b-test-4-gpu-b200:
|
||||
needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/_pr-test-stage.yml
|
||||
with:
|
||||
self_name: stage-b-test-4-gpu-b200
|
||||
self_name: base-b-test-4-gpu-b200
|
||||
runner_config: 4-gpu-b200
|
||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
||||
caller_inputs: ${{ toJson(inputs) }}
|
||||
@@ -391,15 +391,15 @@ jobs:
|
||||
git_ref: ${{ inputs.git_ref || '' }}
|
||||
test_parallel_dispatch: ${{ inputs.test_parallel_dispatch == true && 'true' || 'false' }}
|
||||
caller_needs_failure: ${{ (needs.call-gate.result == 'failure' || needs.sgl-kernel-build-wheels.result == 'failure' || needs.check-changes.result == 'failure') && 'true' || 'false' }}
|
||||
skip_stage_health_check: ${{ inputs.skip_stage_health_check == true && 'true' || 'false' }}
|
||||
skip_pr_test_health_check: ${{ inputs.skip_pr_test_health_check == true && 'true' || 'false' }}
|
||||
secrets: inherit
|
||||
|
||||
stage-c-test-4-gpu-h100:
|
||||
needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels]
|
||||
base-c-test-4-gpu-h100:
|
||||
needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/_pr-test-stage.yml
|
||||
with:
|
||||
self_name: stage-c-test-4-gpu-h100
|
||||
self_name: base-c-test-4-gpu-h100
|
||||
runner_config: 4-gpu-h100
|
||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
||||
caller_inputs: ${{ toJson(inputs) }}
|
||||
@@ -407,12 +407,12 @@ jobs:
|
||||
run_timeout_minutes: '30'
|
||||
secrets: inherit
|
||||
|
||||
stage-c-test-8-gpu-h200:
|
||||
needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels]
|
||||
base-c-test-8-gpu-h200:
|
||||
needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/_pr-test-stage.yml
|
||||
with:
|
||||
self_name: stage-c-test-8-gpu-h200
|
||||
self_name: base-c-test-8-gpu-h200
|
||||
runner_config: 8-gpu-h200
|
||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
||||
caller_inputs: ${{ toJson(inputs) }}
|
||||
@@ -427,12 +427,12 @@ jobs:
|
||||
warmup_timeout_minutes: '60'
|
||||
secrets: inherit
|
||||
|
||||
stage-c-test-8-gpu-h20:
|
||||
needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels]
|
||||
base-c-test-8-gpu-h20:
|
||||
needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/_pr-test-stage.yml
|
||||
with:
|
||||
self_name: stage-c-test-8-gpu-h20
|
||||
self_name: base-c-test-8-gpu-h20
|
||||
runner_config: 8-gpu-h20
|
||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
||||
caller_inputs: ${{ toJson(inputs) }}
|
||||
@@ -440,12 +440,12 @@ jobs:
|
||||
run_timeout_minutes: '30'
|
||||
secrets: inherit
|
||||
|
||||
stage-c-test-deepep-4-gpu-h100:
|
||||
needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels]
|
||||
base-c-test-deepep-4-gpu-h100:
|
||||
needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/_pr-test-stage.yml
|
||||
with:
|
||||
self_name: stage-c-test-deepep-4-gpu-h100
|
||||
self_name: base-c-test-deepep-4-gpu-h100
|
||||
runner_config: deepep-4-gpu-h100
|
||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
||||
caller_inputs: ${{ toJson(inputs) }}
|
||||
@@ -455,12 +455,12 @@ jobs:
|
||||
warmup_server_models: 'lmsys/sglang-ci-dsv3-test:4'
|
||||
secrets: inherit
|
||||
|
||||
stage-c-test-4-gpu-b200:
|
||||
needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels]
|
||||
base-c-test-4-gpu-b200:
|
||||
needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/_pr-test-stage.yml
|
||||
with:
|
||||
self_name: stage-c-test-4-gpu-b200
|
||||
self_name: base-c-test-4-gpu-b200
|
||||
runner_config: 4-gpu-b200
|
||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
||||
caller_inputs: ${{ toJson(inputs) }}
|
||||
@@ -469,12 +469,12 @@ jobs:
|
||||
timeout_per_file: '1800'
|
||||
secrets: inherit
|
||||
|
||||
stage-c-test-dsv4-4-gpu-b200:
|
||||
needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels]
|
||||
base-c-test-dsv4-4-gpu-b200:
|
||||
needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/_pr-test-stage.yml
|
||||
with:
|
||||
self_name: stage-c-test-dsv4-4-gpu-b200
|
||||
self_name: base-c-test-dsv4-4-gpu-b200
|
||||
runner_config: dsv4-4-gpu-b200
|
||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
||||
caller_inputs: ${{ toJson(inputs) }}
|
||||
@@ -483,12 +483,12 @@ jobs:
|
||||
timeout_per_file: '1800'
|
||||
secrets: inherit
|
||||
|
||||
stage-c-test-dsv4-8-gpu-h200:
|
||||
needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels]
|
||||
base-c-test-dsv4-8-gpu-h200:
|
||||
needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/_pr-test-stage.yml
|
||||
with:
|
||||
self_name: stage-c-test-dsv4-8-gpu-h200
|
||||
self_name: base-c-test-dsv4-8-gpu-h200
|
||||
runner_config: dsv4-8-gpu-h200
|
||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
||||
caller_inputs: ${{ toJson(inputs) }}
|
||||
@@ -507,27 +507,27 @@ jobs:
|
||||
sgl-kernel-build-wheels-arm,
|
||||
call-sgl-kernel-tests,
|
||||
|
||||
wait-for-stage-a,
|
||||
wait-for-stage-b,
|
||||
wait-for-base-a,
|
||||
wait-for-base-b,
|
||||
|
||||
call-jit-kernel-tests,
|
||||
|
||||
call-multimodal-gen-tests,
|
||||
|
||||
stage-a-test-1-gpu-small,
|
||||
stage-a-test-cpu,
|
||||
stage-b-test-1-gpu-small,
|
||||
stage-b-test-1-gpu-large,
|
||||
stage-b-test-2-gpu-large,
|
||||
stage-b-test-4-gpu-b200,
|
||||
stage-c-test-4-gpu-h100,
|
||||
stage-c-test-8-gpu-h20,
|
||||
stage-c-test-8-gpu-h200,
|
||||
stage-c-test-deepep-4-gpu-h100,
|
||||
stage-c-test-4-gpu-b200,
|
||||
stage-c-test-dsv4-4-gpu-b200,
|
||||
stage-c-test-dsv4-8-gpu-h200,
|
||||
# stage-c-test-4-gpu-gb200, # Temporarily disabled — no GB200 runner
|
||||
base-a-test-1-gpu-small,
|
||||
base-a-test-cpu,
|
||||
base-b-test-1-gpu-small,
|
||||
base-b-test-1-gpu-large,
|
||||
base-b-test-2-gpu-large,
|
||||
base-b-test-4-gpu-b200,
|
||||
base-c-test-4-gpu-h100,
|
||||
base-c-test-8-gpu-h20,
|
||||
base-c-test-8-gpu-h200,
|
||||
base-c-test-deepep-4-gpu-h100,
|
||||
base-c-test-4-gpu-b200,
|
||||
base-c-test-dsv4-4-gpu-b200,
|
||||
base-c-test-dsv4-8-gpu-h200,
|
||||
# base-c-test-4-gpu-gb200, # Temporarily disabled — no GB200 runner
|
||||
]
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -151,7 +151,7 @@ jobs:
|
||||
with:
|
||||
git_ref: ${{ needs.cut-release-branch.outputs.branch_name }}
|
||||
run_all_tests: true
|
||||
skip_stage_health_check: true
|
||||
skip_pr_test_health_check: true
|
||||
secrets: inherit
|
||||
|
||||
run-pr-tests-amd:
|
||||
|
||||
@@ -19,7 +19,7 @@ from sglang.jit_kernel.benchmark.utils import (
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=30, suite="stage-b-kernel-benchmark-1-gpu-large")
|
||||
register_cuda_ci(est_time=30, suite="base-b-kernel-benchmark-1-gpu-large")
|
||||
|
||||
|
||||
@torch.compile
|
||||
|
||||
@@ -9,7 +9,7 @@ from sglang.jit_kernel.benchmark.utils import run_benchmark
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.utils import is_in_ci
|
||||
|
||||
register_cuda_ci(est_time=5, suite="stage-b-kernel-benchmark-1-gpu-large")
|
||||
register_cuda_ci(est_time=5, suite="base-b-kernel-benchmark-1-gpu-large")
|
||||
|
||||
try:
|
||||
from sgl_kernel import awq_dequantize as aot_awq_dequantize
|
||||
|
||||
@@ -10,7 +10,7 @@ from sglang.jit_kernel.benchmark.utils import (
|
||||
from sglang.jit_kernel.cast import downcast_fp8 as downcast_fp8_jit
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=10, suite="stage-b-kernel-benchmark-1-gpu-large")
|
||||
register_cuda_ci(est_time=10, suite="base-b-kernel-benchmark-1-gpu-large")
|
||||
|
||||
DEVICE = DEFAULT_DEVICE
|
||||
DTYPE = torch.bfloat16
|
||||
|
||||
@@ -13,7 +13,7 @@ from sglang.jit_kernel.clamp_position import clamp_position_cuda
|
||||
from sglang.srt.utils import get_compiler_backend
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=13, suite="stage-b-kernel-benchmark-1-gpu-large")
|
||||
register_cuda_ci(est_time=13, suite="base-b-kernel-benchmark-1-gpu-large")
|
||||
register_amd_ci(est_time=16, suite="jit-kernel-unit-test-amd")
|
||||
|
||||
SIZE_LIST = get_benchmark_range(
|
||||
|
||||
@@ -12,7 +12,7 @@ from sglang.jit_kernel.concat_mla import concat_mla_k as jit_k
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.utils import is_in_ci
|
||||
|
||||
register_cuda_ci(est_time=6, suite="stage-b-kernel-benchmark-1-gpu-large")
|
||||
register_cuda_ci(est_time=6, suite="base-b-kernel-benchmark-1-gpu-large")
|
||||
|
||||
IS_CI = is_in_ci()
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(
|
||||
est_time=120,
|
||||
suite="stage-b-kernel-benchmark-1-gpu-large",
|
||||
suite="base-b-kernel-benchmark-1-gpu-large",
|
||||
disabled="requires multi-GPU, self-skips in CI",
|
||||
)
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ from sglang.jit_kernel.fused_qknorm_rope import (
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=6, suite="stage-b-kernel-benchmark-1-gpu-large")
|
||||
register_cuda_ci(est_time=6, suite="base-b-kernel-benchmark-1-gpu-large")
|
||||
|
||||
try:
|
||||
from sgl_kernel import fused_qk_norm_rope as fused_qk_norm_rope_aot
|
||||
|
||||
@@ -16,7 +16,7 @@ from sglang.jit_kernel.benchmark.utils import (
|
||||
from sglang.jit_kernel.hadamard import hadamard_transform
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=5, suite="stage-b-kernel-benchmark-1-gpu-large")
|
||||
register_cuda_ci(est_time=5, suite="base-b-kernel-benchmark-1-gpu-large")
|
||||
|
||||
# AOT kernel: might not be available in all environments.
|
||||
# This is used for performance baseline comparison.
|
||||
|
||||
@@ -33,7 +33,7 @@ from sglang.jit_kernel.hicache import (
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=29, suite="stage-b-kernel-benchmark-1-gpu-large")
|
||||
register_cuda_ci(est_time=29, suite="base-b-kernel-benchmark-1-gpu-large")
|
||||
|
||||
DISABLE_TORCH = os.environ.get("DISABLE_TORCH", "0") == "1"
|
||||
PAGE_SIZE = 1
|
||||
|
||||
@@ -9,7 +9,7 @@ from sglang.jit_kernel.benchmark.utils import DEFAULT_DEVICE, DEFAULT_DTYPE
|
||||
from sglang.jit_kernel.hisparse import load_cache_to_device_buffer_mla
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=12, suite="stage-b-kernel-benchmark-1-gpu-large")
|
||||
register_cuda_ci(est_time=12, suite="base-b-kernel-benchmark-1-gpu-large")
|
||||
|
||||
DEVICE = DEFAULT_DEVICE
|
||||
DTYPE = DEFAULT_DTYPE
|
||||
|
||||
@@ -20,7 +20,7 @@ from sglang.jit_kernel.mla_kv_pack_quantize_fp8 import (
|
||||
from sglang.jit_kernel.utils import is_arch_support_pdl
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=15, suite="stage-b-kernel-benchmark-1-gpu-large")
|
||||
register_cuda_ci(est_time=15, suite="base-b-kernel-benchmark-1-gpu-large")
|
||||
|
||||
|
||||
@triton.jit
|
||||
|
||||
@@ -13,7 +13,7 @@ from sglang.jit_kernel.mxfp8 import (
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=5, suite="stage-b-kernel-benchmark-1-gpu-large")
|
||||
register_cuda_ci(est_time=5, suite="base-b-kernel-benchmark-1-gpu-large")
|
||||
|
||||
|
||||
def is_sm100_supported(device=None) -> bool:
|
||||
|
||||
@@ -11,7 +11,7 @@ from sglang.jit_kernel.norm import fused_add_rmsnorm as jit_fused_add_rmsnorm
|
||||
from sglang.jit_kernel.norm import rmsnorm as jit_rmsnorm
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=30, suite="stage-b-kernel-benchmark-1-gpu-large")
|
||||
register_cuda_ci(est_time=30, suite="base-b-kernel-benchmark-1-gpu-large")
|
||||
|
||||
|
||||
DTYPE = torch.bfloat16
|
||||
|
||||
@@ -15,7 +15,7 @@ from sglang.jit_kernel.nvfp4 import (
|
||||
from sglang.srt.utils import is_sm100_supported
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=5, suite="stage-b-kernel-benchmark-1-gpu-large")
|
||||
register_cuda_ci(est_time=5, suite="base-b-kernel-benchmark-1-gpu-large")
|
||||
|
||||
FLOAT4_E2M1_MAX = 6.0
|
||||
FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max
|
||||
|
||||
@@ -10,7 +10,7 @@ from sglang.jit_kernel.nvfp4 import scaled_fp4_quant
|
||||
from sglang.srt.utils import is_sm100_supported
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=5, suite="stage-b-kernel-benchmark-1-gpu-large")
|
||||
register_cuda_ci(est_time=5, suite="base-b-kernel-benchmark-1-gpu-large")
|
||||
|
||||
FLOAT4_E2M1_MAX = 6.0
|
||||
FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max
|
||||
|
||||
@@ -10,7 +10,7 @@ from sglang.jit_kernel.nvfp4 import cutlass_scaled_fp4_mm, scaled_fp4_quant
|
||||
from sglang.srt.utils import is_sm100_supported, is_sm120_supported
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=5, suite="stage-b-kernel-benchmark-1-gpu-large")
|
||||
register_cuda_ci(est_time=5, suite="base-b-kernel-benchmark-1-gpu-large")
|
||||
|
||||
FLOAT4_E2M1_MAX = 6.0
|
||||
FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max
|
||||
|
||||
@@ -8,7 +8,7 @@ from sglang.jit_kernel.benchmark.utils import get_benchmark_range, run_benchmark
|
||||
from sglang.jit_kernel.per_tensor_quant_fp8 import per_tensor_quant_fp8
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=5, suite="stage-b-kernel-benchmark-1-gpu-large")
|
||||
register_cuda_ci(est_time=5, suite="base-b-kernel-benchmark-1-gpu-large")
|
||||
|
||||
try:
|
||||
from vllm import _custom_ops as ops
|
||||
|
||||
@@ -20,7 +20,7 @@ from sglang.srt.utils.bench_utils import bench_kineto
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.utils import is_in_ci
|
||||
|
||||
register_cuda_ci(est_time=13, suite="stage-b-kernel-benchmark-1-gpu-large")
|
||||
register_cuda_ci(est_time=13, suite="base-b-kernel-benchmark-1-gpu-large")
|
||||
|
||||
IS_CI = is_in_ci()
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ from sglang.jit_kernel.norm import fused_inplace_qknorm
|
||||
from sglang.srt.utils import get_current_device_stream_fast
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=10, suite="stage-b-kernel-benchmark-1-gpu-large")
|
||||
register_cuda_ci(est_time=10, suite="base-b-kernel-benchmark-1-gpu-large")
|
||||
|
||||
alt_stream = torch.cuda.Stream()
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ from sglang.srt.utils import get_current_device_stream_fast
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.utils import is_in_ci
|
||||
|
||||
register_cuda_ci(est_time=12, suite="stage-b-kernel-benchmark-1-gpu-large")
|
||||
register_cuda_ci(est_time=12, suite="base-b-kernel-benchmark-1-gpu-large")
|
||||
|
||||
IS_CI = is_in_ci()
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ from sglang.jit_kernel.benchmark.utils import run_benchmark_no_cudagraph
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.utils import is_in_ci
|
||||
|
||||
register_cuda_ci(est_time=5, suite="stage-b-kernel-benchmark-1-gpu-large")
|
||||
register_cuda_ci(est_time=5, suite="base-b-kernel-benchmark-1-gpu-large")
|
||||
|
||||
|
||||
def torch_top_k_renorm_probs(probs, top_k):
|
||||
|
||||
@@ -13,7 +13,7 @@ from sglang.jit_kernel.resolve_future_token_ids import resolve_future_token_ids_
|
||||
from sglang.srt.utils import get_compiler_backend
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=10, suite="stage-b-kernel-benchmark-1-gpu-large")
|
||||
register_cuda_ci(est_time=10, suite="base-b-kernel-benchmark-1-gpu-large")
|
||||
register_amd_ci(est_time=10, suite="jit-kernel-unit-test-amd")
|
||||
|
||||
SIZE_LIST = get_benchmark_range(
|
||||
|
||||
@@ -12,7 +12,7 @@ from sglang.jit_kernel.benchmark.utils import (
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=6, suite="stage-b-kernel-benchmark-1-gpu-large")
|
||||
register_cuda_ci(est_time=6, suite="base-b-kernel-benchmark-1-gpu-large")
|
||||
|
||||
MAX_SEQ_LEN = 131072
|
||||
ROPE_BASE = 10000.0
|
||||
|
||||
@@ -26,7 +26,7 @@ from sglang.srt.mem_cache.utils import set_mla_kv_buffer_kernel as sglang_triton
|
||||
from sglang.srt.mem_cache.utils import set_mla_kv_buffer_triton as sglang_wrapper
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=9, suite="stage-b-kernel-benchmark-1-gpu-large")
|
||||
register_cuda_ci(est_time=9, suite="base-b-kernel-benchmark-1-gpu-large")
|
||||
|
||||
|
||||
def _triton_baseline(kv_buffer, loc, cache_k_nope, cache_k_rope):
|
||||
|
||||
@@ -14,7 +14,7 @@ from sglang.jit_kernel.benchmark.utils import (
|
||||
from sglang.jit_kernel.kvcache import store_cache
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=9, suite="stage-b-kernel-benchmark-1-gpu-large")
|
||||
register_cuda_ci(est_time=9, suite="base-b-kernel-benchmark-1-gpu-large")
|
||||
|
||||
|
||||
def sglang_jit_store_cache(
|
||||
|
||||
@@ -19,7 +19,7 @@ from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(
|
||||
est_time=120,
|
||||
suite="stage-b-kernel-benchmark-1-gpu-large",
|
||||
suite="base-b-kernel-benchmark-1-gpu-large",
|
||||
disabled="requires multi-GPU, self-skips in CI",
|
||||
)
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ from sglang.utils import is_in_ci
|
||||
|
||||
register_cuda_ci(
|
||||
est_time=120,
|
||||
suite="stage-b-kernel-benchmark-1-gpu-large",
|
||||
suite="base-b-kernel-benchmark-1-gpu-large",
|
||||
disabled="standalone diffusion NVFP4 benchmark",
|
||||
)
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ from sglang.utils import is_in_ci
|
||||
|
||||
register_cuda_ci(
|
||||
est_time=17,
|
||||
suite="stage-b-kernel-benchmark-1-gpu-large",
|
||||
suite="base-b-kernel-benchmark-1-gpu-large",
|
||||
disabled="Temporarily skipped to unblock flashinfer upgrade. Ref: https://github.com/sgl-project/sglang/actions/runs/23735552939/job/69139238979?pr=21422",
|
||||
)
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ from sglang.utils import is_in_ci
|
||||
|
||||
register_cuda_ci(
|
||||
est_time=45,
|
||||
suite="stage-b-kernel-benchmark-1-gpu-large",
|
||||
suite="base-b-kernel-benchmark-1-gpu-large",
|
||||
disabled="standalone benchmark",
|
||||
)
|
||||
|
||||
|
||||
@@ -24,7 +24,7 @@ from sglang.utils import is_in_ci
|
||||
|
||||
register_cuda_ci(
|
||||
est_time=120,
|
||||
suite="stage-b-kernel-benchmark-1-gpu-large",
|
||||
suite="base-b-kernel-benchmark-1-gpu-large",
|
||||
disabled="self-skips in CI, standalone tool",
|
||||
)
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ from sglang.jit_kernel.benchmark.utils import (
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=13, suite="stage-b-kernel-benchmark-1-gpu-large")
|
||||
register_cuda_ci(est_time=13, suite="base-b-kernel-benchmark-1-gpu-large")
|
||||
|
||||
MAX_SEQ_LEN = 131072
|
||||
ROPE_BASE = 10000.0
|
||||
|
||||
@@ -12,7 +12,7 @@ from sglang.jit_kernel.diffusion.triton.scale_shift import (
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.utils import is_in_ci
|
||||
|
||||
register_cuda_ci(est_time=13, suite="stage-b-kernel-benchmark-1-gpu-large")
|
||||
register_cuda_ci(est_time=13, suite="base-b-kernel-benchmark-1-gpu-large")
|
||||
|
||||
if is_in_ci():
|
||||
B_RANGE, S_RANGE, D_RANGE = [1], [128], [3072]
|
||||
|
||||
@@ -18,7 +18,7 @@ from sglang.jit_kernel.tests.deepseek_v4.common import (
|
||||
to_seq_extend,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=30, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=30, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=30, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
Context = Union[LegacyContext, PagedContext]
|
||||
|
||||
@@ -18,7 +18,7 @@ from sglang.jit_kernel.tests.deepseek_v4.common import (
|
||||
to_seq_extend,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=30, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=30, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=30, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
Context = Union[LegacyContext, PagedContext]
|
||||
|
||||
@@ -14,7 +14,7 @@ from sglang.srt.layers.quantization.fp8_utils import (
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=20, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=20, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=80, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
DEVICE = "cuda"
|
||||
|
||||
@@ -17,7 +17,7 @@ from sglang.srt.layers.quantization.modelopt_quant import pad_nvfp4_weight
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
# B200-only correctness coverage for diffusion NVFP4 scaled mm.
|
||||
register_cuda_ci(est_time=15, suite="stage-b-kernel-unit-1-gpu-b200")
|
||||
register_cuda_ci(est_time=15, suite="base-b-kernel-unit-1-gpu-b200")
|
||||
|
||||
DEVICE = "cuda"
|
||||
DTYPE = torch.bfloat16
|
||||
|
||||
@@ -13,7 +13,7 @@ from sglang.jit_kernel.diffusion.cutedsl.scale_residual_norm_scale_shift import
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=28, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=28, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
DEVICE = "cuda"
|
||||
|
||||
@@ -9,7 +9,7 @@ from sglang.jit_kernel.diffusion.group_norm_silu import apply_group_norm_silu
|
||||
from sglang.jit_kernel.diffusion.triton.group_norm_silu import triton_group_norm_silu
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=8, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=8, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
DEVICE = "cuda"
|
||||
|
||||
@@ -8,7 +8,7 @@ import triton
|
||||
from sglang.jit_kernel.utils import get_ci_test_range
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=44, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=44, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=176, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
DEVICE = "cuda"
|
||||
|
||||
@@ -12,7 +12,7 @@ from sglang.jit_kernel.diffusion.triton.scale_shift import (
|
||||
from sglang.jit_kernel.utils import get_ci_test_range
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=15, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=15, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
DEVICE = "cuda"
|
||||
|
||||
@@ -8,7 +8,7 @@ from sglang.jit_kernel.activation import SUPPORTED_ACTIVATIONS, run_activation
|
||||
from sglang.jit_kernel.utils import get_ci_test_range
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=20, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=20, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=30, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ import torch
|
||||
from sglang.jit_kernel.add_constant import add_constant
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=45, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=45, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=180, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ import torch
|
||||
from sglang.jit_kernel.awq_dequantize import awq_dequantize as jit_awq_dequantize
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=9, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=9, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
try:
|
||||
|
||||
@@ -11,7 +11,7 @@ from sglang.jit_kernel.awq_marlin_repack import (
|
||||
from sglang.srt.layers.quantization.utils import pack_cols, quantize_weights
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=10, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=10, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ from sglang.srt.layers.quantization.utils import pack_cols, quantize_weights
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_marlin_utils import get_weight_perm, marlin_weights
|
||||
|
||||
register_cuda_ci(est_time=10, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=10, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ import torch
|
||||
from sglang.jit_kernel.clamp_position import clamp_position_cuda
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=12, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=12, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ import triton
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=17, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=17, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
|
||||
|
||||
@@ -38,7 +38,7 @@ from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(
|
||||
est_time=300,
|
||||
suite="stage-b-kernel-unit-8-gpu-h200",
|
||||
suite="base-b-kernel-unit-8-gpu-h200",
|
||||
)
|
||||
register_cuda_ci(
|
||||
est_time=300,
|
||||
|
||||
@@ -29,7 +29,7 @@ try:
|
||||
except ImportError:
|
||||
TRITON_AVAILABLE = False
|
||||
|
||||
register_cuda_ci(est_time=5, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=5, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ from einops import rearrange, repeat
|
||||
from sglang.jit_kernel.flash_attention import flash_attn_varlen_func
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=120, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=900, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
# Skip this test on Hopper machine
|
||||
|
||||
@@ -7,7 +7,7 @@ import torch
|
||||
from sglang.jit_kernel.utils import get_ci_test_range
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=5, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=5, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ import torch
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=100, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=100, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=400, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
# =============================================================================
|
||||
|
||||
@@ -48,7 +48,7 @@ try:
|
||||
except ImportError:
|
||||
_is_fp8_fnuz = False
|
||||
|
||||
register_cuda_ci(est_time=24, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=24, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
PAGE_SIZE = 64
|
||||
|
||||
@@ -26,7 +26,7 @@ try:
|
||||
except ImportError:
|
||||
KERNELS_AVAILABLE = False
|
||||
|
||||
register_cuda_ci(est_time=6, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=6, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ from sglang.srt.layers.quantization.marlin_utils import marlin_make_workspace
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_marlin_utils import awq_marlin_quantize, marlin_quantize
|
||||
|
||||
register_cuda_ci(est_time=13, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=13, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
MNK_FACTORS = [
|
||||
|
||||
@@ -13,7 +13,7 @@ from sglang.srt.layers.quantization.utils import (
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_marlin_utils import get_weight_perm, marlin_weights
|
||||
|
||||
register_cuda_ci(est_time=16, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=16, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
MARLIN_K_CHUNKS = [128]
|
||||
|
||||
@@ -9,7 +9,7 @@ from sglang.jit_kernel.utils import get_ci_test_range
|
||||
from sglang.srt.layers.moe.topk import biased_grouped_topk_impl
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=30, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=30, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ from sglang.jit_kernel.hadamard import (
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=128, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=128, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=512, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
# Exact M×N Hadamard matrices (±1 entries) copied from
|
||||
|
||||
@@ -13,7 +13,7 @@ from sglang.srt.mem_cache.memory_pool_host import (
|
||||
from sglang.srt.utils import is_cuda, is_hip, is_npu, is_xpu
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=10, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=10, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
|
||||
@@ -7,7 +7,7 @@ from sglang.jit_kernel.hisparse import load_cache_to_device_buffer_mla
|
||||
from sglang.srt.utils import is_cuda, is_hip, is_npu, is_xpu
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=10, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=10, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
|
||||
@@ -7,7 +7,7 @@ from sglang.jit_kernel.mla_kv_pack_quantize_fp8 import mla_kv_pack_quantize_fp8
|
||||
from sglang.jit_kernel.utils import get_ci_test_range
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=60, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=60, suite="base-b-kernel-unit-1-gpu-large")
|
||||
|
||||
DEVICE = "cuda"
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ import triton.language as tl
|
||||
from sglang.jit_kernel.moe_align import moe_align_block_size
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=28, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=28, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ import torch
|
||||
from sglang.jit_kernel.moe_lora_align import moe_lora_align_block_size
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=28, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=28, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ from sglang.srt.layers.moe.fused_moe_triton import moe_align_block_size
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_marlin_utils import awq_marlin_quantize, marlin_quantize
|
||||
|
||||
register_cuda_ci(est_time=10, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=10, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ from sglang.jit_kernel.mxfp8 import (
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=5, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=5, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ from sglang.jit_kernel.nvfp4 import (
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=5, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=5, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
FLOAT4_E2M1_MAX = 6.0
|
||||
|
||||
@@ -6,7 +6,7 @@ import torch
|
||||
from sglang.jit_kernel.nvfp4 import cutlass_scaled_fp4_mm, scaled_fp4_quant
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=5, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=5, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ except Exception:
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=5, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=5, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ import torch
|
||||
from sglang.jit_kernel.per_tensor_quant_fp8 import per_tensor_quant_fp8
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=16, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=16, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
try:
|
||||
|
||||
@@ -25,7 +25,7 @@ from sglang.srt.layers.quantization.fp8_kernel import (
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=16, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=16, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
configs = list(
|
||||
|
||||
@@ -10,7 +10,7 @@ import triton.language as tl
|
||||
from sglang.jit_kernel.rope import rotary_embedding
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=18, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=18, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ import triton
|
||||
from sglang.jit_kernel.utils import get_ci_test_range
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=37, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=37, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=148, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ import triton
|
||||
from sglang.jit_kernel.utils import get_ci_test_range
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=15, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=15, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ import torch
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=6, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=6, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ import torch
|
||||
from sglang.jit_kernel.resolve_future_token_ids import resolve_future_token_ids_cuda
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=9, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=9, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ import torch
|
||||
from sglang.jit_kernel.utils import get_ci_test_range
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=45, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=45, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=240, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ from sglang.jit_kernel.rmsnorm_hf import (
|
||||
from sglang.jit_kernel.utils import get_ci_test_range
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=30, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=30, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
EPS = 1e-5
|
||||
|
||||
@@ -7,7 +7,7 @@ import triton
|
||||
from sglang.jit_kernel.utils import get_ci_test_range
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=64, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=64, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=256, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
DEVICE = "cuda"
|
||||
|
||||
@@ -10,7 +10,7 @@ from sglang.jit_kernel.set_mla_kv_buffer import (
|
||||
from sglang.jit_kernel.utils import get_ci_test_range
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=30, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=30, suite="base-b-kernel-unit-1-gpu-large")
|
||||
|
||||
DEVICE = "cuda"
|
||||
CACHE_SIZE = 4096
|
||||
|
||||
@@ -8,7 +8,7 @@ from sglang.jit_kernel.kvcache import can_use_store_cache, store_cache
|
||||
from sglang.jit_kernel.utils import get_ci_test_range
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=28, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=28, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
register_amd_ci(est_time=55, suite="jit-kernel-unit-test-amd")
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ from sglang.jit_kernel.timestep_embedding import (
|
||||
from sglang.jit_kernel.utils import get_ci_test_range
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=16, suite="stage-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=16, suite="base-b-kernel-unit-1-gpu-large")
|
||||
register_cuda_ci(est_time=120, suite="nightly-kernel-1-gpu", nightly=True)
|
||||
|
||||
CORRECTNESS_BATCH_SIZES = get_ci_test_range(
|
||||
|
||||
@@ -16,7 +16,7 @@ from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(
|
||||
est_time=300,
|
||||
suite="stage-b-kernel-unit-8-gpu-h200",
|
||||
suite="base-b-kernel-unit-8-gpu-h200",
|
||||
)
|
||||
register_cuda_ci(
|
||||
est_time=300,
|
||||
|
||||
@@ -17,7 +17,7 @@ __all__ = [
|
||||
]
|
||||
|
||||
# `suite` stays in positional slot 2 for backward compat with existing
|
||||
# `register_cpu_ci(5, "stage-a-test-cpu")` style positional calls. New fields
|
||||
# `register_cpu_ci(5, "base-a-test-cpu")` style positional calls. New fields
|
||||
# (`stage`, `runner_config`) are kwarg-only.
|
||||
_PARAM_ORDER = ("est_time", "suite", "nightly", "disabled")
|
||||
_KWARG_ONLY = ("stage", "runner_config")
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
keyed by suite name. Consumed by pr-test.yml stage jobs as
|
||||
`fromJson(needs.check-changes.outputs.partitions)['<suite>']`.
|
||||
|
||||
partitions={"stage-b-test-1-gpu-small": {"size": 8, "arr": [0,...,7], "max_parallel": 2}, ...}
|
||||
partitions={"base-b-test-1-gpu-small": {"size": 8, "arr": [0,...,7], "max_parallel": 2}, ...}
|
||||
"""
|
||||
|
||||
import argparse
|
||||
@@ -34,11 +34,11 @@ HWBackend = _ci_register.HWBackend
|
||||
# pr-test-amd.yml / pr-test-npu.yml have their own dispatch.
|
||||
_TARGET_BACKENDS = {HWBackend.CUDA, HWBackend.CPU}
|
||||
|
||||
# stage-a is the critical-path entry gate; pin its fanout to smoke-coverage
|
||||
# base-a is the critical-path entry gate; pin its fanout to smoke-coverage
|
||||
# defaults instead of est_time. max_parallel = size (no throttle).
|
||||
_STAGE_A_OVERRIDES = {
|
||||
"stage-a-test-cpu": 4,
|
||||
"stage-a-test-1-gpu-small": 1,
|
||||
_BASE_A_OVERRIDES = {
|
||||
"base-a-test-cpu": 4,
|
||||
"base-a-test-1-gpu-small": 1,
|
||||
}
|
||||
|
||||
_REUSABLE_STAGE_USES = "./.github/workflows/_pr-test-stage.yml"
|
||||
@@ -47,7 +47,7 @@ _REUSABLE_STAGE_USES = "./.github/workflows/_pr-test-stage.yml"
|
||||
def load_run_timeouts(pr_test_yml_path: str) -> dict:
|
||||
"""Map `self_name -> run_timeout_minutes` from one pr-test*.yml. The input
|
||||
is required in `_pr-test-stage.yml` -- KeyError surfaces missing.
|
||||
Inline stage-a-test-cpu is skipped (uses `_STAGE_A_OVERRIDES`)."""
|
||||
Inline base-a-test-cpu is skipped (uses `_BASE_A_OVERRIDES`)."""
|
||||
with open(pr_test_yml_path) as f:
|
||||
wf = yaml.safe_load(f)
|
||||
timeouts = {}
|
||||
@@ -122,7 +122,7 @@ def compute_partitions(
|
||||
"""
|
||||
# Allowlist: stages pr-test.yml dispatches. Stress / weekly /
|
||||
# nightly-* live in test/registered/ but pr-test doesn't run them.
|
||||
dispatched_suites = set(run_timeouts) | set(_STAGE_A_OVERRIDES)
|
||||
dispatched_suites = set(run_timeouts) | set(_BASE_A_OVERRIDES)
|
||||
suite_tests = defaultdict(list)
|
||||
for t in tests:
|
||||
if t.backend not in _TARGET_BACKENDS:
|
||||
@@ -149,8 +149,8 @@ def compute_partitions(
|
||||
bias = fit.get("bias", 0.0)
|
||||
|
||||
# Each shard pays `bias` once, so size >= coeff*total / (target-bias).
|
||||
if suite in _STAGE_A_OVERRIDES:
|
||||
size = _STAGE_A_OVERRIDES[suite]
|
||||
if suite in _BASE_A_OVERRIDES:
|
||||
size = _BASE_A_OVERRIDES[suite]
|
||||
max_parallel = size
|
||||
else:
|
||||
target = per_shard_target_seconds(suite, run_timeouts)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user