minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Deprecate /rerun-stage; scrub CUDA target_stage infra (#25322)

Browse Source
This commit is contained in:
Liangsheng Yin
2026-05-15 02:18:28 -07:00
committed by GitHub
parent 1a1d69507d
commit eff736cc0f
10 changed files with 95 additions and 604 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/_pr-awareness-comment.yml
-1
View File
@@ -92,7 +92,6 @@ jobs:
const newBlock = [ const newBlock = [
outerStart, outerStart,
'---',
'### CI Awareness', '### CI Awareness',
'', '',
`Latest PR Test: ${ptStart}${newPrTest}${ptEnd}`, `Latest PR Test: ${ptStart}${newPrTest}${ptEnd}`,
.github/workflows/_pr-test-check-changes.yml
+14 -165
View File
@@ -3,18 +3,9 @@ name: Check Changes
on: on:
workflow_call: workflow_call:
inputs: inputs:
pr_head_sha:
type: string
default: ''
git_ref: git_ref:
type: string type: string
default: '' default: ''
target_stage:
type: string
default: ''
include_wheel_build:
type: boolean
default: false
run_all_tests: run_all_tests:
type: boolean type: boolean
default: false default: false
@@ -30,8 +21,6 @@ on:
value: ${{ jobs.run.outputs.main_package }} value: ${{ jobs.run.outputs.main_package }}
sgl_kernel: sgl_kernel:
value: ${{ jobs.run.outputs.sgl_kernel }} value: ${{ jobs.run.outputs.sgl_kernel }}
sgl_kernel_raw:
value: ${{ jobs.run.outputs.sgl_kernel_raw }}
jit_kernel: jit_kernel:
value: ${{ jobs.run.outputs.jit_kernel }} value: ${{ jobs.run.outputs.jit_kernel }}
multimodal_gen: multimodal_gen:
@@ -52,20 +41,10 @@ jobs:
name: check-changes name: check-changes
runs-on: ubuntu-latest runs-on: ubuntu-latest
outputs: outputs:
# Use API-based detection for target_stage mode (filter-api), otherwise use dorny/paths-filter (filter) main_package: ${{ steps.filter.outputs.main_package || steps.run-mode.outputs.run_all_tests }}
main_package: ${{ steps.filter-api.outputs.main_package || steps.filter.outputs.main_package || steps.run-mode.outputs.run_all_tests }} sgl_kernel: ${{ steps.filter.outputs.sgl_kernel }}
# sgl_kernel is forced to false when target_stage is set AND include_wheel_build is NOT set, jit_kernel: ${{ steps.filter.outputs.jit_kernel || steps.run-mode.outputs.run_all_tests }}
# since sgl-kernel-build-wheels normally skips in target_stage mode. When include_wheel_build multimodal_gen: ${{ steps.filter.outputs.multimodal_gen || steps.run-mode.outputs.run_all_tests }}
# is true, keep the real value so the wheel build runs and the target stage downloads its
# artifact (used by /rerun-stage on PRs that modify sgl-kernel/).
# This prevents CUSTOM_BUILD_SGL_KERNEL=true when the wheel artifacts aren't available.
# Note: If PR has kernel changes AND target_stage is set AND include_wheel_build is NOT set,
# the validate-target-stage step will fail.
sgl_kernel: ${{ (!inputs.target_stage || inputs.include_wheel_build) && (steps.filter-api.outputs.sgl_kernel || steps.filter.outputs.sgl_kernel) }}
# Raw sgl_kernel value before target_stage override (used for validation)
sgl_kernel_raw: ${{ steps.filter-api.outputs.sgl_kernel || steps.filter.outputs.sgl_kernel }}
jit_kernel: ${{ steps.filter-api.outputs.jit_kernel || steps.filter.outputs.jit_kernel || steps.run-mode.outputs.run_all_tests }}
multimodal_gen: ${{ steps.filter-api.outputs.multimodal_gen || steps.filter.outputs.multimodal_gen || steps.run-mode.outputs.run_all_tests }}
partitions: ${{ steps.partitions.outputs.partitions }} partitions: ${{ steps.partitions.outputs.partitions }}
partition_model_sha: ${{ steps.partition-model-sha.outputs.sha }} partition_model_sha: ${{ steps.partition-model-sha.outputs.sha }}
b200_runner: ${{ steps.set-runner.outputs.b200_runner }} b200_runner: ${{ steps.set-runner.outputs.b200_runner }}
@@ -75,7 +54,7 @@ jobs:
- name: Checkout code - name: Checkout code
uses: actions/checkout@v4 uses: actions/checkout@v4
with: with:
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} ref: ${{ inputs.git_ref || github.sha }}
- uses: ./.github/actions/check-maintenance - uses: ./.github/actions/check-maintenance
@@ -95,9 +74,7 @@ jobs:
- name: Detect file changes - name: Detect file changes
id: filter id: filter
uses: dorny/paths-filter@v3 uses: dorny/paths-filter@v3
# Only use paths-filter for pull_request events (where it works correctly) if: steps.run-mode.outputs.run_all_tests != 'true'
# For workflow_dispatch with target_stage, we use GitHub API in the next step
if: steps.run-mode.outputs.run_all_tests != 'true' && !inputs.target_stage
with: with:
filters: | filters: |
main_package: main_package:
@@ -128,93 +105,13 @@ jobs:
# see API-side detector below for rationale. # see API-side detector below for rationale.
- "sgl-kernel/**/!(*.md|THIRDPARTYNOTICES.txt|LICENSE)" - "sgl-kernel/**/!(*.md|THIRDPARTYNOTICES.txt|LICENSE)"
# For /rerun-stage (workflow_dispatch with target_stage), dorny/paths-filter doesn't work
# correctly because it falls back to "last commit" detection which breaks for merge commits.
# Instead, we use the GitHub API to compare the PR commit against main.
- name: Detect file changes via API (for target_stage)
id: filter-api
if: inputs.target_stage && inputs.pr_head_sha
env:
GH_TOKEN: ${{ github.token }}
run: |
echo "Detecting file changes via GitHub API for target_stage mode..."
echo "PR head SHA: ${{ inputs.pr_head_sha }}"
# Get the list of changed files by comparing PR commit against main
# This correctly handles merge commits by looking at the actual PR diff
CHANGED_FILES=$(gh api "repos/${{ github.repository }}/compare/main...${{ inputs.pr_head_sha }}" \
--jq '[.files[].filename] | .[]' 2>/dev/null || echo "")
if [ -z "$CHANGED_FILES" ]; then
echo "Warning: Could not fetch changed files from API, assuming no changes"
echo "sgl_kernel=false" >> $GITHUB_OUTPUT
echo "main_package=false" >> $GITHUB_OUTPUT
echo "jit_kernel=false" >> $GITHUB_OUTPUT
echo "multimodal_gen=false" >> $GITHUB_OUTPUT
exit 0
fi
echo "Changed files:"
echo "$CHANGED_FILES" | head -20
echo "..."
# Check for sgl-kernel changes
# Note: edits to .github/workflows/pr-test-sgl-kernel.yml are intentionally
# NOT considered sgl-kernel changes. That filter line used to be included
# so workflow refactors got retested, but in practice it only catches the
# workflow's *consumers* (test job definitions), not the wheel build steps
# themselves — and gating sgl_kernel=true on it forces a 20-30 min wheel
# rebuild + the stage-a-test-1-gpu-small gate for pure CI-yaml edits that
# can't actually affect kernel behavior. PRs that touch wheel-build logic
# in scripts/ci/cuda/ or sgl-kernel/ still trigger correctly.
if echo "$CHANGED_FILES" | grep -qE "^sgl-kernel/"; then
echo "sgl_kernel=true" >> $GITHUB_OUTPUT
echo "Detected sgl-kernel changes"
else
echo "sgl_kernel=false" >> $GITHUB_OUTPUT
fi
# Check for main_package changes (excluding multimodal_gen, jit_kernel/diffusion, jit_kernel/tests/diffusion, jit_kernel/benchmark/diffusion, cli)
# Note: Need to filter out multimodal_gen and diffusion-related paths before checking, not pipe grep -q output
MAIN_PKG_FILES=$(echo "$CHANGED_FILES" | grep -E "^(python/sglang/|python/pyproject\.toml|scripts/ci/cuda/|scripts/ci/utils/|test/|\.github/workflows/pr-test\.yml|\.github/workflows/pr-gate\.yml|\.github/actions/)" | grep -v -E "^(python/sglang/multimodal_gen/|python/sglang/jit_kernel/diffusion/|python/sglang/jit_kernel/tests/diffusion/|python/sglang/jit_kernel/benchmark/diffusion/|python/sglang/cli/)" || true)
if [ -n "$MAIN_PKG_FILES" ]; then
echo "main_package=true" >> $GITHUB_OUTPUT
echo "Detected main_package changes"
else
echo "main_package=false" >> $GITHUB_OUTPUT
fi
# Check for jit_kernel changes
if echo "$CHANGED_FILES" | grep -qE "^(python/sglang/jit_kernel/|python/pyproject\.toml|\.github/workflows/pr-test\.yml|\.github/workflows/pr-test-jit-kernel\.yml)"; then
echo "jit_kernel=true" >> $GITHUB_OUTPUT
echo "Detected jit_kernel changes"
else
echo "jit_kernel=false" >> $GITHUB_OUTPUT
fi
# Check for multimodal_gen changes, including diffusion-specific jit_kernel coverage
if echo "$CHANGED_FILES" | grep -qE "^(python/sglang/multimodal_gen/|python/sglang/cli/|python/sglang/jit_kernel/diffusion/|python/sglang/jit_kernel/tests/diffusion/|python/sglang/jit_kernel/benchmark/diffusion/|python/pyproject\.toml|\.github/workflows/pr-test\.yml|\.github/workflows/pr-test-multimodal-gen\.yml)"; then
echo "multimodal_gen=true" >> $GITHUB_OUTPUT
echo "Detected multimodal_gen changes"
else
echo "multimodal_gen=false" >> $GITHUB_OUTPUT
fi
- name: Determine full-parallel mode - name: Determine full-parallel mode
id: parallel-mode id: parallel-mode
env:
GH_TOKEN: ${{ github.token }}
run: | run: |
# `full=true` lifts the matrix-fanout throttle so each suite's # `full=true` lifts the matrix-fanout throttle so each suite's
# max_parallel = size. Conditions (matching the prior set-parallel # max_parallel = size. Conditions:
# step exactly):
# 1. Scheduled cron run. # 1. Scheduled cron run.
# 2. pull_request event with the `high priority` label. # 2. pull_request event with the `high priority` label.
# 3. workflow_dispatch with target_stage set (i.e. /rerun-stage)
# whose underlying PR carries `high priority`. The labels
# aren't on the dispatch payload, so look them up via API:
# try SHA -> /pulls (works for fork PRs), fall back to
# branch name -> gh pr list (works for non-fork PRs).
FULL=false FULL=false
if [[ "${{ github.event_name }}" == "schedule" ]]; then if [[ "${{ github.event_name }}" == "schedule" ]]; then
FULL=true FULL=true
@@ -222,22 +119,6 @@ jobs:
elif [[ "${{ github.event_name }}" == "pull_request" && "${{ contains(github.event.pull_request.labels.*.name, 'high priority') }}" == "true" ]]; then elif [[ "${{ github.event_name }}" == "pull_request" && "${{ contains(github.event.pull_request.labels.*.name, 'high priority') }}" == "true" ]]; then
FULL=true FULL=true
echo "high priority PR -> full parallelism" echo "high priority PR -> full parallelism"
elif [[ -n "${{ inputs.target_stage }}" ]]; then
LABELS=""
PR_HEAD_SHA="${{ inputs.pr_head_sha }}"
if [[ -n "$PR_HEAD_SHA" ]]; then
LABELS=$(gh api "repos/${{ github.repository }}/commits/${PR_HEAD_SHA}/pulls" \
--jq '.[0].labels[].name' 2>/dev/null || true)
fi
if [[ -z "$LABELS" ]]; then
LABELS=$(gh pr list --head "${{ github.ref_name }}" --repo "${{ github.repository }}" \
--json labels --jq '.[0].labels[].name' 2>/dev/null || true)
fi
echo "PR labels (via API): ${LABELS:-"(none)"}"
if echo "$LABELS" | grep -Fxq "high priority"; then
FULL=true
echo "high priority PR (via API) -> full parallelism"
fi
fi fi
echo "full=$FULL" >> "$GITHUB_OUTPUT" echo "full=$FULL" >> "$GITHUB_OUTPUT"
@@ -285,12 +166,9 @@ jobs:
- name: Set B200 runner tag - name: Set B200 runner tag
id: set-runner id: set-runner
run: | run: |
# Use kernel-build runner only when sgl_kernel changes are detected AND we're not in target_stage mode # Use kernel-build runner only when sgl_kernel changes are detected.
# (target_stage skips wheel builds, so we can't use custom kernels) sgl_kernel="${{ steps.filter.outputs.sgl_kernel }}"
# Use API-based detection (filter-api) for target_stage mode, otherwise use dorny/paths-filter (filter) if [[ "$sgl_kernel" == "true" ]]; then
sgl_kernel="${{ steps.filter-api.outputs.sgl_kernel || steps.filter.outputs.sgl_kernel }}"
target_stage="${{ inputs.target_stage }}"
if [[ "$sgl_kernel" == "true" && -z "$target_stage" ]]; then
echo "b200_runner=4-gpu-b200-kernel" >> $GITHUB_OUTPUT echo "b200_runner=4-gpu-b200-kernel" >> $GITHUB_OUTPUT
else else
echo "b200_runner=4-gpu-b200" >> $GITHUB_OUTPUT echo "b200_runner=4-gpu-b200" >> $GITHUB_OUTPUT
@@ -313,32 +191,6 @@ jobs:
echo "Filtered run, continue-on-error disabled" echo "Filtered run, continue-on-error disabled"
fi fi
- name: Validate target_stage with kernel changes
# Fail only when PR has sgl-kernel changes AND the caller didn't opt into include_wheel_build.
# include_wheel_build=true means sgl-kernel-build-wheels will run alongside the target stage
# (see the sgl_kernel output and sgl-kernel-build-wheels if-conditions above/below), so it's
# safe to proceed.
if: inputs.target_stage && !inputs.include_wheel_build && (steps.filter-api.outputs.sgl_kernel == 'true' || steps.filter.outputs.sgl_kernel == 'true')
run: |
echo "::error::Cannot use /rerun-stage when PR has sgl-kernel changes without include_wheel_build."
echo "::error::The sgl-kernel-build-wheels job is skipped in target_stage mode by default, but this PR modifies sgl-kernel/ files."
echo "::error::The slash-command handler should have set include_wheel_build=true automatically; falling back to /tag-and-rerun-ci."
echo ""
echo "ERROR: Cannot use /rerun-stage when PR has sgl-kernel changes without include_wheel_build."
echo ""
echo "This PR modifies files in sgl-kernel/, which requires building custom kernel wheels."
echo "Running the target stage without rebuilding the kernel would use the wrong (PyPI)"
echo "version of sgl-kernel instead of your changes."
echo ""
echo "The /rerun-stage handler sets include_wheel_build=true automatically when it detects"
echo "sgl-kernel/ changes on the PR. If you see this error, the handler may be outdated."
echo ""
echo "Alternatives:"
echo " /tag-and-rerun-ci - Re-run the full workflow including kernel builds"
echo " /rerun-ci - Re-run the full workflow"
echo ""
exit 1
- name: Show filter results in summary (table) - name: Show filter results in summary (table)
run: | run: |
{ {
@@ -346,13 +198,10 @@ jobs:
echo "" echo ""
echo "| Component | Changed |" echo "| Component | Changed |"
echo "|-------------------|---------|" echo "|-------------------|---------|"
echo "| main_package | ${{ steps.filter-api.outputs.main_package || steps.filter.outputs.main_package || steps.run-mode.outputs.run_all_tests }} |" echo "| main_package | ${{ steps.filter.outputs.main_package || steps.run-mode.outputs.run_all_tests }} |"
echo "| sgl_kernel (raw) | ${{ steps.filter-api.outputs.sgl_kernel || steps.filter.outputs.sgl_kernel }} |" echo "| sgl_kernel | ${{ steps.filter.outputs.sgl_kernel }} |"
echo "| sgl_kernel (used) | ${{ (!inputs.target_stage || inputs.include_wheel_build) && (steps.filter-api.outputs.sgl_kernel || steps.filter.outputs.sgl_kernel) }} |" echo "| jit_kernel | ${{ steps.filter.outputs.jit_kernel || steps.run-mode.outputs.run_all_tests }} |"
echo "| jit_kernel | ${{ steps.filter-api.outputs.jit_kernel || steps.filter.outputs.jit_kernel || steps.run-mode.outputs.run_all_tests }} |" echo "| multimodal_gen | ${{ steps.filter.outputs.multimodal_gen || steps.run-mode.outputs.run_all_tests }} |"
echo "| multimodal_gen | ${{ steps.filter-api.outputs.multimodal_gen || steps.filter.outputs.multimodal_gen || steps.run-mode.outputs.run_all_tests }} |"
echo "| target_stage | ${{ inputs.target_stage || '(none)' }} |"
echo "| detection_method | ${{ inputs.target_stage && 'GitHub API' || 'dorny/paths-filter' }} |"
echo "| b200_runner | ${{ steps.set-runner.outputs.b200_runner }} |" echo "| b200_runner | ${{ steps.set-runner.outputs.b200_runner }} |"
echo "| enable_retry | ${{ steps.set-retry.outputs.enable_retry }} |" echo "| enable_retry | ${{ steps.set-retry.outputs.enable_retry }} |"
echo "| continue_on_error | ${{ steps.set-continue-on-error.outputs.continue_on_error }} |" echo "| continue_on_error | ${{ steps.set-continue-on-error.outputs.continue_on_error }} |"
.github/workflows/_pr-test-sgl-kernel-build.yml
+1 -4
View File
@@ -21,9 +21,6 @@ on:
description: "Suffix appended to the upload-artifact name. '' for x86, '-aarch64' for arm." description: "Suffix appended to the upload-artifact name. '' for x86, '-aarch64' for arm."
type: string type: string
default: '' default: ''
pr_head_sha:
type: string
default: ''
git_ref: git_ref:
type: string type: string
default: '' default: ''
@@ -69,7 +66,7 @@ jobs:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
with: with:
submodules: "recursive" submodules: "recursive"
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} ref: ${{ inputs.git_ref || github.sha }}
- uses: ./.github/actions/check-maintenance - uses: ./.github/actions/check-maintenance
.github/workflows/_pr-test-stage.yml
+6 -14
View File
@@ -11,7 +11,7 @@ on:
workflow_call: workflow_call:
inputs: inputs:
self_name: self_name:
description: 'Caller job key; used for partitions[suite] lookup and target_stage gating.' description: 'Caller job key; used for partitions[suite] lookup.'
type: string type: string
required: true required: true
runner_config: runner_config:
@@ -27,7 +27,7 @@ on:
type: string type: string
required: true required: true
caller_inputs: caller_inputs:
description: 'toJson(inputs) from pr-test.yml. Read via fromJson(...).target_stage / pr_head_sha / git_ref / skip_stage_health_check / test_parallel_dispatch.' description: 'toJson(inputs) from pr-test.yml. Read via fromJson(...).git_ref / skip_stage_health_check / test_parallel_dispatch.'
type: string type: string
required: true required: true
partitions: partitions:
@@ -74,19 +74,11 @@ env:
jobs: jobs:
run: run:
name: ${{ inputs.self_name }} (${{ matrix.partition }}) name: ${{ inputs.self_name }} (${{ matrix.partition }})
# target_stage takes precedence; otherwise default per-commit gating # Runs on schedule / parallel-dispatch / non-failed PR with main_package or sgl_kernel changes.
# runs on schedule / parallel-dispatch / non-failed PR with main_package
# or sgl_kernel changes.
if: | if: |
always() && always() &&
( ((github.event_name == 'schedule' || fromJson(inputs.caller_inputs).test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
(fromJson(inputs.caller_inputs).target_stage == inputs.self_name) || (fromJson(inputs.check_changes).main_package == 'true' || fromJson(inputs.check_changes).sgl_kernel == 'true')
(
!fromJson(inputs.caller_inputs).target_stage &&
((github.event_name == 'schedule' || fromJson(inputs.caller_inputs).test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
(fromJson(inputs.check_changes).main_package == 'true' || fromJson(inputs.check_changes).sgl_kernel == 'true')
)
)
runs-on: ${{ inputs.runs_on }} runs-on: ${{ inputs.runs_on }}
timeout-minutes: 240 timeout-minutes: 240
env: env:
@@ -100,7 +92,7 @@ jobs:
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
with: with:
ref: ${{ fromJson(inputs.caller_inputs).pr_head_sha || fromJson(inputs.caller_inputs).git_ref || github.sha }} ref: ${{ fromJson(inputs.caller_inputs).git_ref || github.sha }}
- name: Resolve runner_config - name: Resolve runner_config
id: rc id: rc
.github/workflows/pr-test-extra.yml
+3 -28
View File
@@ -9,33 +9,16 @@ name: PR Test Extra
# Stages: extra-a (1-/2-gpu) and extra-b (4-/8-gpu) caller stubs reuse # Stages: extra-a (1-/2-gpu) and extra-b (4-/8-gpu) caller stubs reuse
# `_pr-test-stage.yml` and `_pr-test-check-changes.yml` from pr-test.yml. # `_pr-test-stage.yml` and `_pr-test-check-changes.yml` from pr-test.yml.
run-name: ${{ inputs.target_stage && (inputs.pr_head_sha && format('[{0}] {1}', inputs.target_stage, inputs.pr_head_sha) || format('[{0}]', inputs.target_stage)) || '' }}
on: on:
pull_request: pull_request:
branches: [main] branches: [main]
workflow_dispatch: workflow_dispatch:
inputs: inputs:
target_stage:
description: "Specific stage to run (optional, for quick testing)"
required: false
type: string
default: ""
force_continue_on_error: force_continue_on_error:
description: "Force continue-on-error (test scheduled CI behavior)" description: "Force continue-on-error (test scheduled CI behavior)"
required: false required: false
type: boolean type: boolean
default: false default: false
pr_head_sha:
description: "PR head SHA to checkout (for /rerun-stage on fork PRs)"
required: false
type: string
default: ""
include_wheel_build:
description: "When set with target_stage, also run sgl-kernel-build-wheels so the target stage uses the freshly-built kernel (for /rerun-stage on PRs that modify sgl-kernel/)"
required: false
type: boolean
default: false
test_parallel_dispatch: test_parallel_dispatch:
description: "Test parallel dispatch behavior (simulates scheduled run)" description: "Test parallel dispatch behavior (simulates scheduled run)"
required: false required: false
@@ -60,7 +43,7 @@ on:
default: false default: false
concurrency: concurrency:
group: pr-test-extra-${{ github.event_name }}-${{ github.head_ref || github.ref_name || 'default' }}-${{ inputs.pr_head_sha || 'current' }}-${{ inputs.target_stage || inputs.git_ref || 'all' }} group: pr-test-extra-${{ github.event_name }}-${{ github.head_ref || github.ref_name || 'default' }}-${{ inputs.git_ref || 'all' }}
cancel-in-progress: ${{ github.event_name != 'workflow_call' }} cancel-in-progress: ${{ github.event_name != 'workflow_call' }}
env: env:
@@ -76,7 +59,7 @@ permissions:
actions: write actions: write
contents: read contents: read
issues: read issues: read
pull-requests: read pull-requests: write
jobs: jobs:
# =============================================== check changes ==================================================== # =============================================== check changes ====================================================
@@ -96,10 +79,7 @@ jobs:
) )
uses: ./.github/workflows/_pr-test-check-changes.yml uses: ./.github/workflows/_pr-test-check-changes.yml
with: with:
pr_head_sha: ${{ inputs.pr_head_sha || '' }}
git_ref: ${{ inputs.git_ref || '' }} git_ref: ${{ inputs.git_ref || '' }}
target_stage: ${{ inputs.target_stage || '' }}
include_wheel_build: ${{ inputs.include_wheel_build == true }}
run_all_tests: ${{ inputs.run_all_tests == true }} run_all_tests: ${{ inputs.run_all_tests == true }}
force_continue_on_error: ${{ inputs.force_continue_on_error == true }} force_continue_on_error: ${{ inputs.force_continue_on_error == true }}
pr_test_yml: '.github/workflows/pr-test-extra.yml' pr_test_yml: '.github/workflows/pr-test-extra.yml'
@@ -111,8 +91,6 @@ jobs:
call-pr-awareness: call-pr-awareness:
needs: check-changes needs: check-changes
if: always() && github.event_name == 'pull_request' if: always() && github.event_name == 'pull_request'
permissions:
pull-requests: write
uses: ./.github/workflows/_pr-awareness-comment.yml uses: ./.github/workflows/_pr-awareness-comment.yml
with: with:
workflow_kind: pr-test-extra workflow_kind: pr-test-extra
@@ -122,15 +100,12 @@ jobs:
sgl-kernel-build-wheels: sgl-kernel-build-wheels:
needs: check-changes needs: check-changes
if: | if: |
always() &&
needs.check-changes.result == 'success' && needs.check-changes.result == 'success' &&
needs.check-changes.outputs.sgl_kernel == 'true' && needs.check-changes.outputs.sgl_kernel == 'true'
(!inputs.target_stage || inputs.include_wheel_build)
uses: ./.github/workflows/_pr-test-sgl-kernel-build.yml uses: ./.github/workflows/_pr-test-sgl-kernel-build.yml
with: with:
runs_on: x64-kernel-build-node runs_on: x64-kernel-build-node
job_display_name: Build Wheel job_display_name: Build Wheel
pr_head_sha: ${{ inputs.pr_head_sha || '' }}
git_ref: ${{ inputs.git_ref || '' }} git_ref: ${{ inputs.git_ref || '' }}
skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }} skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }}
secrets: inherit secrets: inherit
.github/workflows/pr-test-jit-kernel.yml
+8 -20
View File
@@ -12,18 +12,10 @@ on:
b200_runner: b200_runner:
required: true required: true
type: string type: string
pr_head_sha:
required: false
type: string
default: ''
git_ref: git_ref:
required: false required: false
type: string type: string
default: '' default: ''
target_stage:
required: false
type: string
default: ''
test_parallel_dispatch: test_parallel_dispatch:
required: false required: false
type: string type: string
@@ -46,14 +38,13 @@ jobs:
jit-kernel-unit-test: jit-kernel-unit-test:
if: | if: |
github.event_name != 'schedule' && github.event_name != 'schedule' &&
inputs.test_parallel_dispatch != 'true' && inputs.test_parallel_dispatch != 'true'
!inputs.target_stage
runs-on: 1-gpu-h100 runs-on: 1-gpu-h100
timeout-minutes: 240 timeout-minutes: 240
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
with: with:
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} ref: ${{ inputs.git_ref || github.sha }}
- uses: ./.github/actions/check-stage-health - uses: ./.github/actions/check-stage-health
@@ -87,14 +78,13 @@ jobs:
jit-kernel-multigpu-unit-test: jit-kernel-multigpu-unit-test:
if: | if: |
github.event_name != 'schedule' && github.event_name != 'schedule' &&
inputs.test_parallel_dispatch != 'true' && inputs.test_parallel_dispatch != 'true'
!inputs.target_stage
runs-on: 8-gpu-h200 runs-on: 8-gpu-h200
timeout-minutes: 240 timeout-minutes: 240
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
with: with:
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} ref: ${{ inputs.git_ref || github.sha }}
- uses: ./.github/actions/check-maintenance - uses: ./.github/actions/check-maintenance
@@ -126,14 +116,13 @@ jobs:
jit-kernel-benchmark-test: jit-kernel-benchmark-test:
if: | if: |
github.event_name != 'schedule' && github.event_name != 'schedule' &&
inputs.test_parallel_dispatch != 'true' && inputs.test_parallel_dispatch != 'true'
!inputs.target_stage
runs-on: 1-gpu-h100 runs-on: 1-gpu-h100
timeout-minutes: 240 timeout-minutes: 240
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
with: with:
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} ref: ${{ inputs.git_ref || github.sha }}
- uses: ./.github/actions/check-stage-health - uses: ./.github/actions/check-stage-health
@@ -167,14 +156,13 @@ jobs:
jit-kernel-b200-test: jit-kernel-b200-test:
if: | if: |
github.event_name != 'schedule' && github.event_name != 'schedule' &&
inputs.test_parallel_dispatch != 'true' && inputs.test_parallel_dispatch != 'true'
!inputs.target_stage
runs-on: ${{ inputs.b200_runner }} runs-on: ${{ inputs.b200_runner }}
timeout-minutes: 240 timeout-minutes: 240
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
with: with:
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} ref: ${{ inputs.git_ref || github.sha }}
- uses: ./.github/actions/check-stage-health - uses: ./.github/actions/check-stage-health
.github/workflows/pr-test-multimodal-gen.yml
+18 -60
View File
@@ -16,18 +16,10 @@ on:
required: false required: false
type: string type: string
default: 'false' default: 'false'
pr_head_sha:
required: false
type: string
default: ''
git_ref: git_ref:
required: false required: false
type: string type: string
default: '' default: ''
target_stage:
required: false
type: string
default: ''
test_parallel_dispatch: test_parallel_dispatch:
required: false required: false
type: string type: string
@@ -51,13 +43,7 @@ env:
jobs: jobs:
compute-diffusion-partitions: compute-diffusion-partitions:
if: | if: inputs.multimodal_gen == 'true'
(inputs.target_stage == 'multimodal-gen-test-1-gpu') ||
(inputs.target_stage == 'multimodal-gen-test-2-gpu') ||
(
!inputs.target_stage &&
inputs.multimodal_gen == 'true'
)
runs-on: ubuntu-latest runs-on: ubuntu-latest
outputs: outputs:
matrix-1gpu: ${{ steps.compute.outputs.matrix-1gpu }} matrix-1gpu: ${{ steps.compute.outputs.matrix-1gpu }}
@@ -70,7 +56,7 @@ jobs:
- name: Checkout code - name: Checkout code
uses: actions/checkout@v4 uses: actions/checkout@v4
with: with:
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} ref: ${{ inputs.git_ref || github.sha }}
- name: Set up Python - name: Set up Python
uses: actions/setup-python@v5 uses: actions/setup-python@v5
@@ -88,14 +74,8 @@ jobs:
always() && always() &&
needs.compute-diffusion-partitions.result == 'success' && needs.compute-diffusion-partitions.result == 'success' &&
needs.compute-diffusion-partitions.outputs.matrix-1gpu != '{"include":[]}' && needs.compute-diffusion-partitions.outputs.matrix-1gpu != '{"include":[]}' &&
( ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) &&
(inputs.target_stage == 'multimodal-gen-test-1-gpu') || inputs.multimodal_gen == 'true'
(
!inputs.target_stage &&
((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) &&
inputs.multimodal_gen == 'true'
)
)
runs-on: 1-gpu-h100 runs-on: 1-gpu-h100
timeout-minutes: 240 timeout-minutes: 240
strategy: strategy:
@@ -105,7 +85,7 @@ jobs:
- name: Checkout code - name: Checkout code
uses: actions/checkout@v4 uses: actions/checkout@v4
with: with:
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} ref: ${{ inputs.git_ref || github.sha }}
- uses: ./.github/actions/check-stage-health - uses: ./.github/actions/check-stage-health
@@ -167,14 +147,8 @@ jobs:
always() && always() &&
needs.compute-diffusion-partitions.result == 'success' && needs.compute-diffusion-partitions.result == 'success' &&
needs.compute-diffusion-partitions.outputs.matrix-2gpu != '{"include":[]}' && needs.compute-diffusion-partitions.outputs.matrix-2gpu != '{"include":[]}' &&
( ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) &&
(inputs.target_stage == 'multimodal-gen-test-2-gpu') || inputs.multimodal_gen == 'true'
(
!inputs.target_stage &&
((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) &&
inputs.multimodal_gen == 'true'
)
)
runs-on: 2-gpu-h100 runs-on: 2-gpu-h100
timeout-minutes: 240 timeout-minutes: 240
strategy: strategy:
@@ -184,7 +158,7 @@ jobs:
- name: Checkout code - name: Checkout code
uses: actions/checkout@v4 uses: actions/checkout@v4
with: with:
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} ref: ${{ inputs.git_ref || github.sha }}
- uses: ./.github/actions/check-stage-health - uses: ./.github/actions/check-stage-health
@@ -243,23 +217,15 @@ jobs:
multimodal-gen-component-accuracy: multimodal-gen-component-accuracy:
if: | if: |
( ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) &&
inputs.target_stage == 'multimodal-gen-component-accuracy' || inputs.multimodal_gen == 'true'
inputs.target_stage == 'multimodal-gen-component-accuracy-1-gpu' ||
inputs.target_stage == 'multimodal-gen-component-accuracy-2-gpu'
) ||
(
!inputs.target_stage &&
((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) &&
inputs.multimodal_gen == 'true'
)
runs-on: 2-gpu-h100 runs-on: 2-gpu-h100
timeout-minutes: 240 timeout-minutes: 240
steps: steps:
- name: Checkout code - name: Checkout code
uses: actions/checkout@v4 uses: actions/checkout@v4
with: with:
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} ref: ${{ inputs.git_ref || github.sha }}
- uses: ./.github/actions/check-stage-health - uses: ./.github/actions/check-stage-health
@@ -296,19 +262,15 @@ jobs:
multimodal-gen-test-1-b200: multimodal-gen-test-1-b200:
if: | if: |
(inputs.target_stage == 'multimodal-gen-test-1-b200') || ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) &&
( inputs.multimodal_gen == 'true'
!inputs.target_stage &&
((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) &&
inputs.multimodal_gen == 'true'
)
runs-on: ${{ inputs.b200_runner }} runs-on: ${{ inputs.b200_runner }}
timeout-minutes: 240 timeout-minutes: 240
steps: steps:
- name: Checkout code - name: Checkout code
uses: actions/checkout@v4 uses: actions/checkout@v4
with: with:
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} ref: ${{ inputs.git_ref || github.sha }}
- uses: ./.github/actions/check-stage-health - uses: ./.github/actions/check-stage-health
@@ -352,19 +314,15 @@ jobs:
multimodal-gen-unit-test: multimodal-gen-unit-test:
if: | if: |
(inputs.target_stage == 'multimodal-gen-unit-test') || ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) &&
( inputs.multimodal_gen == 'true'
!inputs.target_stage &&
((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) &&
inputs.multimodal_gen == 'true'
)
runs-on: 1-gpu-h100 runs-on: 1-gpu-h100
timeout-minutes: 120 timeout-minutes: 120
steps: steps:
- name: Checkout code - name: Checkout code
uses: actions/checkout@v4 uses: actions/checkout@v4
with: with:
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} ref: ${{ inputs.git_ref || github.sha }}
- uses: ./.github/actions/check-stage-health - uses: ./.github/actions/check-stage-health
@@ -405,7 +363,7 @@ jobs:
- name: Checkout code - name: Checkout code
uses: actions/checkout@v4 uses: actions/checkout@v4
with: with:
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} ref: ${{ inputs.git_ref || github.sha }}
- name: Set up Python - name: Set up Python
uses: actions/setup-python@v5 uses: actions/setup-python@v5
.github/workflows/pr-test-sgl-kernel.yml
+3 -7
View File
@@ -9,10 +9,6 @@ on:
b200_runner: b200_runner:
required: true required: true
type: string type: string
pr_head_sha:
required: false
type: string
default: ''
git_ref: git_ref:
required: false required: false
type: string type: string
@@ -37,7 +33,7 @@ jobs:
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
with: with:
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} ref: ${{ inputs.git_ref || github.sha }}
- uses: ./.github/actions/check-stage-health - uses: ./.github/actions/check-stage-health
@@ -72,7 +68,7 @@ jobs:
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
with: with:
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} ref: ${{ inputs.git_ref || github.sha }}
- uses: ./.github/actions/check-stage-health - uses: ./.github/actions/check-stage-health
@@ -119,7 +115,7 @@ jobs:
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
with: with:
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} ref: ${{ inputs.git_ref || github.sha }}
- uses: ./.github/actions/check-stage-health - uses: ./.github/actions/check-stage-health
.github/workflows/pr-test.yml
+13 -84
View File
@@ -1,7 +1,4 @@
name: PR Test name: PR Test
# Dynamic run-name for /rerun-stage commands to enable URL lookup
# Format: "[stage-name] sha" for fork PRs, "[stage-name]" for non-fork, default for normal runs
run-name: ${{ inputs.target_stage && (inputs.pr_head_sha && format('[{0}] {1}', inputs.target_stage, inputs.pr_head_sha) || format('[{0}]', inputs.target_stage)) || '' }}
on: on:
schedule: schedule:
@@ -10,26 +7,11 @@ on:
branches: [main] branches: [main]
workflow_dispatch: workflow_dispatch:
inputs: inputs:
target_stage:
description: "Specific stage to run (optional, for quick testing)"
required: false
type: string
default: ""
force_continue_on_error: force_continue_on_error:
description: "Force continue-on-error (test scheduled CI behavior)" description: "Force continue-on-error (test scheduled CI behavior)"
required: false required: false
type: boolean type: boolean
default: false default: false
pr_head_sha:
description: "PR head SHA to checkout (for /rerun-stage on fork PRs)"
required: false
type: string
default: ""
include_wheel_build:
description: "When set with target_stage, also run sgl-kernel-build-wheels so the target stage uses the freshly-built kernel (for /rerun-stage on PRs that modify sgl-kernel/)"
required: false
type: boolean
default: false
test_parallel_dispatch: test_parallel_dispatch:
description: "Test parallel dispatch behavior (simulates scheduled run)" description: "Test parallel dispatch behavior (simulates scheduled run)"
required: false required: false
@@ -59,13 +41,11 @@ on:
default: false default: false
concurrency: concurrency:
# Concurrency group structure: pr-test-{event}-{branch}-{pr_sha}-{stage} # Concurrency group structure: pr-test-{event}-{branch}-{git_ref}
# - event_name prevents scheduled runs from colliding with fork PRs whose branch is named 'main' # - event_name prevents scheduled runs from colliding with fork PRs whose branch is named 'main'
# (without it, both resolve the branch segment to 'main' and block each other) # (without it, both resolve the branch segment to 'main' and block each other)
# - github.head_ref (pull_request) or github.ref_name (workflow_dispatch) normalizes to branch name # - github.head_ref (pull_request) or github.ref_name (workflow_dispatch) normalizes to branch name
# - pr_head_sha isolates /rerun-stage from main branch runs group: pr-test-${{ github.event_name }}-${{ github.head_ref || github.ref_name || 'default' }}-${{ inputs.git_ref || 'all' }}
# - target_stage allows parallel stage dispatches to run independently
group: pr-test-${{ github.event_name }}-${{ github.head_ref || github.ref_name || 'default' }}-${{ inputs.pr_head_sha || 'current' }}-${{ inputs.target_stage || inputs.git_ref || 'all' }}
cancel-in-progress: ${{ github.event_name != 'workflow_call' }} cancel-in-progress: ${{ github.event_name != 'workflow_call' }}
env: env:
@@ -83,17 +63,14 @@ permissions:
actions: write actions: write
contents: read contents: read
issues: read issues: read
pull-requests: read pull-requests: write
jobs: jobs:
# =============================================== check changes ==================================================== # =============================================== check changes ====================================================
check-changes: check-changes:
uses: ./.github/workflows/_pr-test-check-changes.yml uses: ./.github/workflows/_pr-test-check-changes.yml
with: with:
pr_head_sha: ${{ inputs.pr_head_sha || '' }}
git_ref: ${{ inputs.git_ref || '' }} git_ref: ${{ inputs.git_ref || '' }}
target_stage: ${{ inputs.target_stage || '' }}
include_wheel_build: ${{ inputs.include_wheel_build == true }}
run_all_tests: ${{ inputs.run_all_tests == true }} run_all_tests: ${{ inputs.run_all_tests == true }}
force_continue_on_error: ${{ inputs.force_continue_on_error == true }} force_continue_on_error: ${{ inputs.force_continue_on_error == true }}
secrets: inherit secrets: inherit
@@ -103,8 +80,6 @@ jobs:
call-pr-awareness: call-pr-awareness:
needs: check-changes needs: check-changes
if: always() && github.event_name == 'pull_request' if: always() && github.event_name == 'pull_request'
permissions:
pull-requests: write
uses: ./.github/workflows/_pr-awareness-comment.yml uses: ./.github/workflows/_pr-awareness-comment.yml
with: with:
workflow_kind: pr-test workflow_kind: pr-test
@@ -130,7 +105,6 @@ jobs:
always() && always() &&
!cancelled() && !cancelled() &&
github.event_name == 'pull_request' && github.event_name == 'pull_request' &&
!inputs.target_stage &&
inputs.test_parallel_dispatch != true && inputs.test_parallel_dispatch != true &&
(needs.check-changes.outputs.main_package == 'true' || needs.check-changes.outputs.sgl_kernel == 'true') && (needs.check-changes.outputs.main_package == 'true' || needs.check-changes.outputs.sgl_kernel == 'true') &&
(needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped')
@@ -159,7 +133,6 @@ jobs:
always() && always() &&
!cancelled() && !cancelled() &&
github.event_name == 'pull_request' && github.event_name == 'pull_request' &&
!inputs.target_stage &&
inputs.test_parallel_dispatch != true && inputs.test_parallel_dispatch != true &&
(needs.check-changes.outputs.main_package == 'true' || needs.check-changes.outputs.sgl_kernel == 'true') && (needs.check-changes.outputs.main_package == 'true' || needs.check-changes.outputs.sgl_kernel == 'true') &&
(needs.wait-for-stage-a.result == 'success' || needs.wait-for-stage-a.result == 'skipped') && (needs.wait-for-stage-a.result == 'success' || needs.wait-for-stage-a.result == 'skipped') &&
@@ -188,11 +161,10 @@ jobs:
# =============================================== PR Gate ==================================================== # =============================================== PR Gate ====================================================
call-gate: call-gate:
needs: check-changes needs: check-changes
# Skip for scheduled runs (they run all tests) and when target_stage is specified # Skip for scheduled runs (they run all tests)
if: | if: |
github.event_name != 'schedule' && github.event_name != 'schedule' &&
inputs.test_parallel_dispatch != true && inputs.test_parallel_dispatch != true &&
!inputs.target_stage &&
( (
needs.check-changes.outputs.main_package == 'true' || needs.check-changes.outputs.main_package == 'true' ||
needs.check-changes.outputs.sgl_kernel == 'true' || needs.check-changes.outputs.sgl_kernel == 'true' ||
@@ -206,53 +178,35 @@ jobs:
sgl-kernel-build-wheels: sgl-kernel-build-wheels:
needs: [check-changes, call-gate] needs: [check-changes, call-gate]
# Skip for scheduled runs (they run stages independently). Runs in target_stage mode only when # Skip for scheduled runs (they run stages independently).
# include_wheel_build is true (i.e. /rerun-stage on a PR with sgl-kernel changes), so the
# target stage can download the freshly-built wheel.
#
# `always()` lets us run when call-gate is skipped (which it always is in target_stage mode by
# design). The explicit needs.<x>.result checks preserve old gating for the normal PR path.
if: | if: |
always() &&
github.event_name != 'schedule' && github.event_name != 'schedule' &&
inputs.test_parallel_dispatch != true && inputs.test_parallel_dispatch != true &&
needs.check-changes.result == 'success' && needs.check-changes.result == 'success' &&
needs.check-changes.outputs.sgl_kernel == 'true' && needs.check-changes.outputs.sgl_kernel == 'true' &&
( needs.call-gate.result == 'success'
(!inputs.target_stage && needs.call-gate.result == 'success') ||
(inputs.target_stage && inputs.include_wheel_build)
)
uses: ./.github/workflows/_pr-test-sgl-kernel-build.yml uses: ./.github/workflows/_pr-test-sgl-kernel-build.yml
with: with:
runs_on: x64-kernel-build-node runs_on: x64-kernel-build-node
job_display_name: Build Wheel job_display_name: Build Wheel
pr_head_sha: ${{ inputs.pr_head_sha || '' }}
git_ref: ${{ inputs.git_ref || '' }} git_ref: ${{ inputs.git_ref || '' }}
skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }} skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }}
secrets: inherit secrets: inherit
sgl-kernel-build-wheels-arm: sgl-kernel-build-wheels-arm:
needs: [check-changes, call-gate] needs: [check-changes, call-gate]
# Skip for scheduled runs (they run stages independently). Runs in target_stage mode only when # Skip for scheduled runs (they run stages independently).
# include_wheel_build is true (i.e. /rerun-stage on a PR with sgl-kernel changes).
#
# See sgl-kernel-build-wheels above for the always() + result-check rationale.
if: | if: |
always() &&
github.event_name != 'schedule' && github.event_name != 'schedule' &&
inputs.test_parallel_dispatch != true && inputs.test_parallel_dispatch != true &&
needs.check-changes.result == 'success' && needs.check-changes.result == 'success' &&
needs.check-changes.outputs.sgl_kernel == 'true' && needs.check-changes.outputs.sgl_kernel == 'true' &&
( needs.call-gate.result == 'success'
(!inputs.target_stage && needs.call-gate.result == 'success') ||
(inputs.target_stage && inputs.include_wheel_build)
)
uses: ./.github/workflows/_pr-test-sgl-kernel-build.yml uses: ./.github/workflows/_pr-test-sgl-kernel-build.yml
with: with:
runs_on: arm-kernel-build-node runs_on: arm-kernel-build-node
job_display_name: Build Wheel Arm job_display_name: Build Wheel Arm
arch_suffix: '-aarch64' arch_suffix: '-aarch64'
pr_head_sha: ${{ inputs.pr_head_sha || '' }}
git_ref: ${{ inputs.git_ref || '' }} git_ref: ${{ inputs.git_ref || '' }}
skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }} skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }}
secrets: inherit secrets: inherit
@@ -262,13 +216,11 @@ jobs:
if: | if: |
github.event_name != 'schedule' && github.event_name != 'schedule' &&
inputs.test_parallel_dispatch != true && inputs.test_parallel_dispatch != true &&
!inputs.target_stage &&
needs.check-changes.outputs.sgl_kernel == 'true' needs.check-changes.outputs.sgl_kernel == 'true'
uses: ./.github/workflows/pr-test-sgl-kernel.yml uses: ./.github/workflows/pr-test-sgl-kernel.yml
with: with:
sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }} sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }}
b200_runner: ${{ needs.check-changes.outputs.b200_runner }} b200_runner: ${{ needs.check-changes.outputs.b200_runner }}
pr_head_sha: ${{ inputs.pr_head_sha || '' }}
git_ref: ${{ inputs.git_ref || '' }} git_ref: ${{ inputs.git_ref || '' }}
skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }} skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }}
secrets: inherit secrets: inherit
@@ -282,16 +234,13 @@ jobs:
!failure() && !cancelled() && !failure() && !cancelled() &&
github.event_name != 'schedule' && github.event_name != 'schedule' &&
inputs.test_parallel_dispatch != true && inputs.test_parallel_dispatch != true &&
!inputs.target_stage &&
needs.check-changes.outputs.jit_kernel == 'true' needs.check-changes.outputs.jit_kernel == 'true'
uses: ./.github/workflows/pr-test-jit-kernel.yml uses: ./.github/workflows/pr-test-jit-kernel.yml
with: with:
jit_kernel: ${{ needs.check-changes.outputs.jit_kernel }} jit_kernel: ${{ needs.check-changes.outputs.jit_kernel }}
sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }} sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }}
b200_runner: ${{ needs.check-changes.outputs.b200_runner }} b200_runner: ${{ needs.check-changes.outputs.b200_runner }}
pr_head_sha: ${{ inputs.pr_head_sha || '' }}
git_ref: ${{ inputs.git_ref || '' }} git_ref: ${{ inputs.git_ref || '' }}
target_stage: ${{ inputs.target_stage || '' }}
test_parallel_dispatch: ${{ inputs.test_parallel_dispatch == true && 'true' || 'false' }} test_parallel_dispatch: ${{ inputs.test_parallel_dispatch == true && 'true' || 'false' }}
skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }} skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }}
secrets: inherit secrets: inherit
@@ -317,14 +266,8 @@ jobs:
needs: [check-changes, call-gate] needs: [check-changes, call-gate]
if: | if: |
always() && always() &&
( ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
(inputs.target_stage == 'stage-a-test-cpu') || (needs.check-changes.outputs.main_package == 'true')
(
!inputs.target_stage &&
((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
(needs.check-changes.outputs.main_package == 'true')
)
)
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 240 timeout-minutes: 240
strategy: strategy:
@@ -341,7 +284,7 @@ jobs:
- name: Checkout code - name: Checkout code
uses: actions/checkout@v4 uses: actions/checkout@v4
with: with:
ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} ref: ${{ inputs.git_ref || github.sha }}
- uses: ./.github/actions/check-stage-health - uses: ./.github/actions/check-stage-health
@@ -449,29 +392,15 @@ jobs:
if: | if: |
always() && always() &&
!cancelled() && !cancelled() &&
( ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
inputs.target_stage == 'multimodal-gen-test-1-gpu' || needs.check-changes.outputs.multimodal_gen == 'true'
inputs.target_stage == 'multimodal-gen-test-2-gpu' ||
inputs.target_stage == 'multimodal-gen-component-accuracy' ||
inputs.target_stage == 'multimodal-gen-component-accuracy-1-gpu' ||
inputs.target_stage == 'multimodal-gen-component-accuracy-2-gpu' ||
inputs.target_stage == 'multimodal-gen-test-1-b200' ||
inputs.target_stage == 'multimodal-gen-unit-test' ||
(
!inputs.target_stage &&
((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
needs.check-changes.outputs.multimodal_gen == 'true'
)
)
uses: ./.github/workflows/pr-test-multimodal-gen.yml uses: ./.github/workflows/pr-test-multimodal-gen.yml
with: with:
multimodal_gen: ${{ needs.check-changes.outputs.multimodal_gen }} multimodal_gen: ${{ needs.check-changes.outputs.multimodal_gen }}
sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }} sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }}
b200_runner: ${{ needs.check-changes.outputs.b200_runner }} b200_runner: ${{ needs.check-changes.outputs.b200_runner }}
continue_on_error: ${{ needs.check-changes.outputs.continue_on_error }} continue_on_error: ${{ needs.check-changes.outputs.continue_on_error }}
pr_head_sha: ${{ inputs.pr_head_sha || '' }}
git_ref: ${{ inputs.git_ref || '' }} git_ref: ${{ inputs.git_ref || '' }}
target_stage: ${{ inputs.target_stage || '' }}
test_parallel_dispatch: ${{ inputs.test_parallel_dispatch == true && 'true' || 'false' }} test_parallel_dispatch: ${{ inputs.test_parallel_dispatch == true && 'true' || 'false' }}
caller_needs_failure: ${{ (needs.call-gate.result == 'failure' || needs.sgl-kernel-build-wheels.result == 'failure' || needs.check-changes.result == 'failure') && 'true' || 'false' }} caller_needs_failure: ${{ (needs.call-gate.result == 'failure' || needs.sgl-kernel-build-wheels.result == 'failure' || needs.check-changes.result == 'failure') && 'true' || 'false' }}
skip_stage_health_check: ${{ inputs.skip_stage_health_check == true && 'true' || 'false' }} skip_stage_health_check: ${{ inputs.skip_stage_health_check == true && 'true' || 'false' }}
scripts/ci/utils/slash_command_handler.py
+29 -221
View File
@@ -20,7 +20,7 @@ def _check_rebase_gate(gh_repo, pr, token):
""" """
Pre-dispatch gate mirroring `.github/actions/check-maintenance/action.yml`. Pre-dispatch gate mirroring `.github/actions/check-maintenance/action.yml`.
Without this, /rerun-stage and /rerun-test would dispatch a workflow_run Without this, /rerun-test would dispatch a workflow_run
on a PR that's behind a required base, the action would catch it, and on a PR that's behind a required base, the action would catch it, and
every job in the run would fail at the gate — wasting runner time and every job in the run would fail at the gate — wasting runner time and
producing N error annotations instead of one comment. Pre-checking here producing N error annotations instead of one comment. Pre-checking here
@@ -388,221 +388,6 @@ def handle_rerun_failed_ci(gh_repo, pr, comment, user_perms, react_on_success=Tr
return False return False
def handle_rerun_stage(
gh_repo, pr, comment, user_perms, stage_name, token, react_on_success=True
):
"""
Handles the /rerun-stage <stage-name> command.
Triggers a workflow_dispatch to run only the specified stage, skipping dependencies.
Returns True if action was taken, False otherwise.
"""
if not user_perms.get("can_rerun_stage", False):
print("Permission denied: can_rerun_stage is false.")
return False
if not stage_name:
print("Error: No stage name provided")
comment.create_reaction("confused")
pr.create_issue_comment(
f"⛔ Please specify a stage name: `/rerun-stage <stage-name>`\n\n"
f"Examples: `/rerun-stage unit-test-backend-4-gpu`, `/rerun-stage accuracy-test-1-gpu`"
)
return False
print(f"Permission granted. Triggering workflow_dispatch for stage '{stage_name}'.")
# Valid NVIDIA stage names that support target_stage
nvidia_stages = [
"stage-a-test-1-gpu-small",
"stage-a-test-cpu",
"stage-b-test-1-gpu-small",
"stage-b-test-1-gpu-large",
"stage-b-test-2-gpu-large",
"stage-b-test-4-gpu-b200",
"stage-c-test-4-gpu-h100",
"stage-c-test-8-gpu-h200",
"stage-c-test-8-gpu-h20",
"stage-c-test-4-gpu-b200",
"stage-c-test-4-gpu-gb200",
"stage-c-test-dsv4-4-gpu-b200",
"stage-c-test-dsv4-8-gpu-h200",
"stage-c-test-deepep-4-gpu-h100",
"stage-c-test-deepep-8-gpu-h200",
"stage-c-test-dsv4-4-gpu-b200",
"stage-c-test-dsv4-8-gpu-h200",
"multimodal-gen-test-1-gpu",
"multimodal-gen-test-2-gpu",
"multimodal-gen-component-accuracy",
"multimodal-gen-component-accuracy-1-gpu",
"multimodal-gen-component-accuracy-2-gpu",
"multimodal-gen-test-1-b200",
]
# Valid AMD stage names that support target_stage
amd_stages = [
"sgl-kernel-unit-test-amd",
"sgl-kernel-unit-test-2-gpu-amd",
"stage-a-test-1-gpu-small-amd",
"stage-b-test-1-gpu-small-amd",
"stage-b-test-1-gpu-small-amd-nondeterministic",
"stage-b-test-1-gpu-small-amd-mi35x",
"stage-b-test-1-gpu-large-amd",
"stage-b-test-2-gpu-large-amd",
"multimodal-gen-test-1-gpu-amd",
"multimodal-gen-test-2-gpu-amd",
"stage-c-test-large-8-gpu-amd",
"stage-c-test-large-8-gpu-amd-mi35x",
]
valid_stages = nvidia_stages + amd_stages
is_amd_stage = stage_name in amd_stages
if stage_name not in valid_stages:
comment.create_reaction("confused")
pr.create_issue_comment(
f"⛔ Stage `{stage_name}` doesn't support isolated runs yet.\n\n"
f"**NVIDIA stages:**\n"
+ "\n".join(f"- `{s}`" for s in nvidia_stages)
+ "\n\n**AMD stages:**\n"
+ "\n".join(f"- `{s}`" for s in amd_stages)
+ "\n\nOther stages will be added soon. For now, use `/rerun-failed-ci` for those stages."
)
return False
allowed, gate_msg = _check_rebase_gate(gh_repo, pr, token)
if not allowed:
comment.create_reaction("confused")
pr.create_issue_comment(gate_msg)
return False
try:
# Get the appropriate workflow based on stage type
workflow_name = "PR Test (AMD)" if is_amd_stage else "PR Test"
workflows = gh_repo.get_workflows()
target_workflow = None
for wf in workflows:
if wf.name == workflow_name:
target_workflow = wf
break
if not target_workflow:
print(f"Error: {workflow_name} workflow not found")
return False
# Check if PR is from a fork by comparing repo owners
# Handle case where fork repo may have been deleted (pr.head.repo is None)
is_fork = (
pr.head.repo is None or pr.head.repo.owner.login != gh_repo.owner.login
)
print(f"PR is from fork: {is_fork}")
# If the PR modifies sgl-kernel/, the target stage would otherwise use the
# PyPI sgl-kernel wheel instead of the PR's changes (sgl-kernel-build-wheels
# skips in target_stage mode by default). Set include_wheel_build=true so the
# workflow runs sgl-kernel-build-wheels alongside the target stage; the target
# stage waits for the build via its needs list.
kernel_changes = has_sgl_kernel_changes(pr)
if kernel_changes:
print(
"PR modifies sgl-kernel/ - setting include_wheel_build=true so the "
"target stage gets the freshly-built wheel instead of the PyPI one."
)
# pr_head_sha is used for fork PRs (passed to workflow and used for URL lookup)
pr_head_sha = None
if is_fork:
# For fork PRs: dispatch on main and pass SHA as input
# This is needed because fork branch names don't exist in the main repo
ref = "main"
pr_head_sha = pr.head.sha
print(
f"Triggering {workflow_name} workflow on ref: {ref}, PR head SHA: {pr_head_sha}"
)
inputs = {
"target_stage": stage_name,
"pr_head_sha": pr_head_sha,
}
else:
# For non-fork PRs: dispatch on the PR branch directly
# This allows testing workflow changes before merge
ref = pr.head.ref
print(f"Triggering {workflow_name} workflow on branch: {ref}")
inputs = {"target_stage": stage_name}
# For NVIDIA stages, honor the sgl-kernel / include_wheel_build flow. AMD is
# a separate workflow that doesn't share the same wheel-build pipeline.
if kernel_changes and not is_amd_stage:
inputs["include_wheel_build"] = "true"
# include_wheel_build relies on filter-api detecting kernel changes, which
# requires pr_head_sha. Ensure it's set even for non-fork PRs, and keep
# the local pr_head_sha in sync so find_workflow_run_url builds the
# expected display_title with the SHA suffix (the workflow's run-name
# includes the SHA whenever inputs.pr_head_sha is set).
if not is_fork:
inputs["pr_head_sha"] = pr.head.sha
pr_head_sha = pr.head.sha
# Record dispatch time before triggering
dispatch_time = time.time()
# Use requests directly as PyGithub's create_dispatch only accepts HTTP 204
dispatch_url = f"https://api.github.com/repos/{gh_repo.full_name}/actions/workflows/{target_workflow.id}/dispatches"
dispatch_resp = requests.post(
dispatch_url,
json={"ref": ref, "inputs": inputs},
headers={
"Authorization": f"Bearer {token}",
"Accept": "application/vnd.github+json",
},
)
success = dispatch_resp.status_code in (200, 204)
if not success:
print(f"Dispatch failed: {dispatch_resp.status_code} {dispatch_resp.text}")
if success:
print(f"Successfully triggered workflow for stage '{stage_name}'")
if react_on_success:
comment.create_reaction("+1")
run_url = find_workflow_run_url(
gh_repo,
target_workflow.id,
ref,
stage_name,
token,
dispatch_time,
pr_head_sha=pr_head_sha,
max_wait=30,
)
if run_url:
pr.create_issue_comment(
f"🚀 Triggered `{stage_name}` to run independently"
f" (skipping dependencies)."
f" [View workflow run]({run_url})"
)
else:
pr.create_issue_comment(
f"🚀 Triggered `{stage_name}` to run independently"
f" (skipping dependencies).\n"
f"⚠️ Could not retrieve workflow run URL. "
f"Check the [Actions tab](https://github.com/{gh_repo.full_name}/actions) for progress."
)
return True
else:
print("Failed to trigger workflow_dispatch")
return False
except Exception as e:
print(f"Error triggering workflow_dispatch: {e}")
comment.create_reaction("confused")
pr.create_issue_comment(
f"⛔ Failed to trigger workflow: {str(e)}\n\n"
f"Please check the logs or contact maintainers."
)
return False
CUDA_SUITE_TO_RUNNER = { CUDA_SUITE_TO_RUNNER = {
# PR test suites # PR test suites
"stage-a-test-1-gpu-small": "1-gpu-5090", "stage-a-test-1-gpu-small": "1-gpu-5090",
@@ -1259,7 +1044,7 @@ def main():
# PR authors can always rerun failed CI and rerun individual UTs on their own PRs, # PR authors can always rerun failed CI and rerun individual UTs on their own PRs,
# even if they are not listed in CI_PERMISSIONS.json. # even if they are not listed in CI_PERMISSIONS.json.
# Note: /tag-run-ci-label and /rerun-stage still require CI_PERMISSIONS.json. # Note: /tag-run-ci-label still requires CI_PERMISSIONS.json.
# Note: /rerun-test is blocked entirely for fork PRs in handle_rerun_test() itself. # Note: /rerun-test is blocked entirely for fork PRs in handle_rerun_test() itself.
if pr.user.login == user_login: if pr.user.login == user_login:
if user_perms is None: if user_perms is None:
@@ -1313,10 +1098,33 @@ def main():
print("Combined command finished, but no actions were taken.") print("Combined command finished, but no actions were taken.")
elif first_line.startswith("/rerun-stage"): elif first_line.startswith("/rerun-stage"):
# Extract stage name from command # /rerun-stage is deprecated. Stage-level granularity is too coarse to map to
parts = first_line.split(maxsplit=1) # a specific feature, and a stage rerun re-pays the cost of all unrelated tests
stage_name = parts[1].strip() if len(parts) > 1 else None # in that stage. Use /rerun-test for selective UT runs, or /rerun-failed-ci /
handle_rerun_stage(repo, pr, comment, user_perms, stage_name, token) # `run-ci` / `run-ci-extra` labels for a full rerun.
print("/rerun-stage is deprecated; posting deprecation notice.")
comment.create_reaction("-1")
pr.create_issue_comment(
"⚠️ **`/rerun-stage` has been deprecated.**\n\n"
"Stage granularity is too coarse — a stage usually doesn't map to one "
"feature, so rerunning a stage re-pays the cost of unrelated tests. "
"If you don't know which exact test files to rerun, you shouldn't be "
"using `/rerun-stage` or `/rerun-test` in the first place.\n\n"
"**Use one of these instead:**\n"
"- **Selective tests** (you know exactly which files to rerun):\n"
" ```\n"
" /rerun-test test_foo.py test_bar.py\n"
" ```\n"
"- **Rerun only failed jobs**:\n"
" ```\n"
" /rerun-failed-ci\n"
" ```\n"
"- **Full CI rerun** (with extra coverage): add the `run-ci` or "
"`run-ci-extra` label and push a new commit (or use `/tag-and-rerun-ci`).\n\n"
"**AMD CI**: stage-level dispatch is still available via "
"Actions UI → *PR Test (AMD)* / *PR Test ROCm 7.2 (AMD)* → "
"*Run workflow* → pick a stage from the dropdown."
)
elif first_line.startswith("/rerun-group"): elif first_line.startswith("/rerun-group"):
group_names = first_line.split()[1:] group_names = first_line.split()[1:]