Add AMD stage support to /rerun-stage command and fix related bugs (#14463)
This commit is contained in:
@@ -20,6 +20,12 @@ on:
|
|||||||
- "sgl-kernel/**"
|
- "sgl-kernel/**"
|
||||||
- ".github/workflows/pr-test-amd.yml"
|
- ".github/workflows/pr-test-amd.yml"
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
target_stage:
|
||||||
|
description: "Specific stage to run (optional, for quick testing)"
|
||||||
|
required: false
|
||||||
|
type: string
|
||||||
|
default: ""
|
||||||
|
|
||||||
concurrency:
|
concurrency:
|
||||||
group: pr-test-amd-${{ github.ref }}
|
group: pr-test-amd-${{ github.ref }}
|
||||||
@@ -54,7 +60,15 @@ jobs:
|
|||||||
# =============================================== sgl-kernel ====================================================
|
# =============================================== sgl-kernel ====================================================
|
||||||
sgl-kernel-unit-test-amd:
|
sgl-kernel-unit-test-amd:
|
||||||
needs: [check-changes]
|
needs: [check-changes]
|
||||||
if: needs.check-changes.outputs.sgl_kernel == 'true'
|
if: |
|
||||||
|
always() &&
|
||||||
|
(
|
||||||
|
(inputs.target_stage == 'sgl-kernel-unit-test-amd') ||
|
||||||
|
(
|
||||||
|
!inputs.target_stage &&
|
||||||
|
needs.check-changes.outputs.sgl_kernel == 'true'
|
||||||
|
)
|
||||||
|
)
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
@@ -90,8 +104,16 @@ jobs:
|
|||||||
|
|
||||||
stage-a-test-1-amd:
|
stage-a-test-1-amd:
|
||||||
needs: [check-changes]
|
needs: [check-changes]
|
||||||
if: always() && !failure() && !cancelled() &&
|
if: |
|
||||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
always() &&
|
||||||
|
(
|
||||||
|
(inputs.target_stage == 'stage-a-test-1-amd') ||
|
||||||
|
(
|
||||||
|
!inputs.target_stage &&
|
||||||
|
(!failure() && !cancelled()) &&
|
||||||
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||||
|
)
|
||||||
|
)
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
@@ -120,8 +142,16 @@ jobs:
|
|||||||
|
|
||||||
unit-test-backend-1-gpu-amd:
|
unit-test-backend-1-gpu-amd:
|
||||||
needs: [check-changes, stage-a-test-1-amd]
|
needs: [check-changes, stage-a-test-1-amd]
|
||||||
if: always() && !failure() && !cancelled() &&
|
if: |
|
||||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
always() &&
|
||||||
|
(
|
||||||
|
(inputs.target_stage == 'unit-test-backend-1-gpu-amd') ||
|
||||||
|
(
|
||||||
|
!inputs.target_stage &&
|
||||||
|
(!failure() && !cancelled()) &&
|
||||||
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||||
|
)
|
||||||
|
)
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
@@ -150,8 +180,16 @@ jobs:
|
|||||||
|
|
||||||
unit-test-backend-2-gpu-amd:
|
unit-test-backend-2-gpu-amd:
|
||||||
needs: [check-changes, stage-a-test-1-amd]
|
needs: [check-changes, stage-a-test-1-amd]
|
||||||
if: always() && !failure() && !cancelled() &&
|
if: |
|
||||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
always() &&
|
||||||
|
(
|
||||||
|
(inputs.target_stage == 'unit-test-backend-2-gpu-amd') ||
|
||||||
|
(
|
||||||
|
!inputs.target_stage &&
|
||||||
|
(!failure() && !cancelled()) &&
|
||||||
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||||
|
)
|
||||||
|
)
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
@@ -181,8 +219,17 @@ jobs:
|
|||||||
unit-test-backend-8-gpu-amd:
|
unit-test-backend-8-gpu-amd:
|
||||||
needs: [check-changes, unit-test-backend-2-gpu-amd]
|
needs: [check-changes, unit-test-backend-2-gpu-amd]
|
||||||
# Temporarily disabled - uncomment when ready to re-enable
|
# Temporarily disabled - uncomment when ready to re-enable
|
||||||
if: false && always() && !failure() && !cancelled() &&
|
if: |
|
||||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
always() &&
|
||||||
|
(
|
||||||
|
(inputs.target_stage == 'unit-test-backend-8-gpu-amd') ||
|
||||||
|
(
|
||||||
|
false &&
|
||||||
|
!inputs.target_stage &&
|
||||||
|
(!failure() && !cancelled()) &&
|
||||||
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||||
|
)
|
||||||
|
)
|
||||||
env:
|
env:
|
||||||
RUNNER_LABELS: linux-mi300-gpu-8
|
RUNNER_LABELS: linux-mi300-gpu-8
|
||||||
strategy:
|
strategy:
|
||||||
@@ -219,8 +266,16 @@ jobs:
|
|||||||
|
|
||||||
performance-test-1-gpu-part-1-amd:
|
performance-test-1-gpu-part-1-amd:
|
||||||
needs: [check-changes, stage-a-test-1-amd]
|
needs: [check-changes, stage-a-test-1-amd]
|
||||||
if: always() && !failure() && !cancelled() &&
|
if: |
|
||||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
always() &&
|
||||||
|
(
|
||||||
|
(inputs.target_stage == 'performance-test-1-gpu-part-1-amd') ||
|
||||||
|
(
|
||||||
|
!inputs.target_stage &&
|
||||||
|
(!failure() && !cancelled()) &&
|
||||||
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||||
|
)
|
||||||
|
)
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
@@ -264,8 +319,16 @@ jobs:
|
|||||||
|
|
||||||
performance-test-1-gpu-part-2-amd:
|
performance-test-1-gpu-part-2-amd:
|
||||||
needs: [check-changes, stage-a-test-1-amd]
|
needs: [check-changes, stage-a-test-1-amd]
|
||||||
if: always() && !failure() && !cancelled() &&
|
if: |
|
||||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
always() &&
|
||||||
|
(
|
||||||
|
(inputs.target_stage == 'performance-test-1-gpu-part-2-amd') ||
|
||||||
|
(
|
||||||
|
!inputs.target_stage &&
|
||||||
|
(!failure() && !cancelled()) &&
|
||||||
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||||
|
)
|
||||||
|
)
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
@@ -303,8 +366,16 @@ jobs:
|
|||||||
|
|
||||||
performance-test-2-gpu-amd:
|
performance-test-2-gpu-amd:
|
||||||
needs: [check-changes, unit-test-backend-2-gpu-amd]
|
needs: [check-changes, unit-test-backend-2-gpu-amd]
|
||||||
if: always() && !failure() && !cancelled() &&
|
if: |
|
||||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
always() &&
|
||||||
|
(
|
||||||
|
(inputs.target_stage == 'performance-test-2-gpu-amd') ||
|
||||||
|
(
|
||||||
|
!inputs.target_stage &&
|
||||||
|
(!failure() && !cancelled()) &&
|
||||||
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||||
|
)
|
||||||
|
)
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
@@ -352,8 +423,16 @@ jobs:
|
|||||||
|
|
||||||
accuracy-test-1-gpu-amd:
|
accuracy-test-1-gpu-amd:
|
||||||
needs: [check-changes, stage-a-test-1-amd]
|
needs: [check-changes, stage-a-test-1-amd]
|
||||||
if: always() && !failure() && !cancelled() &&
|
if: |
|
||||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
always() &&
|
||||||
|
(
|
||||||
|
(inputs.target_stage == 'accuracy-test-1-gpu-amd') ||
|
||||||
|
(
|
||||||
|
!inputs.target_stage &&
|
||||||
|
(!failure() && !cancelled()) &&
|
||||||
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||||
|
)
|
||||||
|
)
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
@@ -383,8 +462,16 @@ jobs:
|
|||||||
|
|
||||||
accuracy-test-2-gpu-amd:
|
accuracy-test-2-gpu-amd:
|
||||||
needs: [check-changes, accuracy-test-1-gpu-amd]
|
needs: [check-changes, accuracy-test-1-gpu-amd]
|
||||||
if: always() && !failure() && !cancelled() &&
|
if: |
|
||||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
always() &&
|
||||||
|
(
|
||||||
|
(inputs.target_stage == 'accuracy-test-2-gpu-amd') ||
|
||||||
|
(
|
||||||
|
!inputs.target_stage &&
|
||||||
|
(!failure() && !cancelled()) &&
|
||||||
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||||
|
)
|
||||||
|
)
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
|
|||||||
+133
-76
@@ -343,11 +343,14 @@ jobs:
|
|||||||
stage-a-test-1:
|
stage-a-test-1:
|
||||||
needs: [check-changes, call-gate, sgl-kernel-build-wheels]
|
needs: [check-changes, call-gate, sgl-kernel-build-wheels]
|
||||||
if: |
|
if: |
|
||||||
(inputs.target_stage == 'stage-a-test-1') ||
|
always() &&
|
||||||
(
|
(
|
||||||
always() &&
|
(inputs.target_stage == 'stage-a-test-1') ||
|
||||||
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
(
|
||||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
!inputs.target_stage &&
|
||||||
|
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
||||||
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||||
|
)
|
||||||
)
|
)
|
||||||
runs-on: 1-gpu-runner
|
runs-on: 1-gpu-runner
|
||||||
env:
|
env:
|
||||||
@@ -380,11 +383,14 @@ jobs:
|
|||||||
multimodal-gen-test-1-gpu:
|
multimodal-gen-test-1-gpu:
|
||||||
needs: [check-changes, call-gate, sgl-kernel-build-wheels]
|
needs: [check-changes, call-gate, sgl-kernel-build-wheels]
|
||||||
if: |
|
if: |
|
||||||
(inputs.target_stage == 'multimodal-gen-test-1-gpu') ||
|
always() &&
|
||||||
(
|
(
|
||||||
always() &&
|
(inputs.target_stage == 'multimodal-gen-test-1-gpu') ||
|
||||||
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
(
|
||||||
needs.check-changes.outputs.multimodal_gen == 'true'
|
!inputs.target_stage &&
|
||||||
|
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
||||||
|
needs.check-changes.outputs.multimodal_gen == 'true'
|
||||||
|
)
|
||||||
)
|
)
|
||||||
runs-on: 1-gpu-runner
|
runs-on: 1-gpu-runner
|
||||||
strategy:
|
strategy:
|
||||||
@@ -424,11 +430,14 @@ jobs:
|
|||||||
multimodal-gen-test-2-gpu:
|
multimodal-gen-test-2-gpu:
|
||||||
needs: [check-changes, call-gate, sgl-kernel-build-wheels]
|
needs: [check-changes, call-gate, sgl-kernel-build-wheels]
|
||||||
if: |
|
if: |
|
||||||
(inputs.target_stage == 'multimodal-gen-test-2-gpu') ||
|
always() &&
|
||||||
(
|
(
|
||||||
always() &&
|
(inputs.target_stage == 'multimodal-gen-test-2-gpu') ||
|
||||||
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
(
|
||||||
needs.check-changes.outputs.multimodal_gen == 'true'
|
!inputs.target_stage &&
|
||||||
|
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
||||||
|
needs.check-changes.outputs.multimodal_gen == 'true'
|
||||||
|
)
|
||||||
)
|
)
|
||||||
runs-on: 2-gpu-runner
|
runs-on: 2-gpu-runner
|
||||||
strategy:
|
strategy:
|
||||||
@@ -468,11 +477,14 @@ jobs:
|
|||||||
quantization-test:
|
quantization-test:
|
||||||
needs: [check-changes, call-gate, stage-a-test-1]
|
needs: [check-changes, call-gate, stage-a-test-1]
|
||||||
if: |
|
if: |
|
||||||
(inputs.target_stage == 'quantization-test') ||
|
always() &&
|
||||||
(
|
(
|
||||||
always() &&
|
(inputs.target_stage == 'quantization-test') ||
|
||||||
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
(
|
||||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
!inputs.target_stage &&
|
||||||
|
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
||||||
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||||
|
)
|
||||||
)
|
)
|
||||||
runs-on: 1-gpu-runner
|
runs-on: 1-gpu-runner
|
||||||
steps:
|
steps:
|
||||||
@@ -500,11 +512,14 @@ jobs:
|
|||||||
unit-test-backend-1-gpu:
|
unit-test-backend-1-gpu:
|
||||||
needs: [check-changes, call-gate, stage-a-test-1]
|
needs: [check-changes, call-gate, stage-a-test-1]
|
||||||
if: |
|
if: |
|
||||||
(inputs.target_stage == 'unit-test-backend-1-gpu') ||
|
always() &&
|
||||||
(
|
(
|
||||||
always() &&
|
(inputs.target_stage == 'unit-test-backend-1-gpu') ||
|
||||||
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
(
|
||||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
!inputs.target_stage &&
|
||||||
|
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
||||||
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||||
|
)
|
||||||
)
|
)
|
||||||
runs-on: 1-gpu-runner
|
runs-on: 1-gpu-runner
|
||||||
env:
|
env:
|
||||||
@@ -539,11 +554,14 @@ jobs:
|
|||||||
unit-test-backend-2-gpu:
|
unit-test-backend-2-gpu:
|
||||||
needs: [check-changes, call-gate, unit-test-backend-1-gpu]
|
needs: [check-changes, call-gate, unit-test-backend-1-gpu]
|
||||||
if: |
|
if: |
|
||||||
(inputs.target_stage == 'unit-test-backend-2-gpu') ||
|
always() &&
|
||||||
(
|
(
|
||||||
always() &&
|
(inputs.target_stage == 'unit-test-backend-2-gpu') ||
|
||||||
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
(
|
||||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
!inputs.target_stage &&
|
||||||
|
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
||||||
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||||
|
)
|
||||||
)
|
)
|
||||||
runs-on: 2-gpu-runner
|
runs-on: 2-gpu-runner
|
||||||
env:
|
env:
|
||||||
@@ -577,11 +595,14 @@ jobs:
|
|||||||
unit-test-backend-4-gpu:
|
unit-test-backend-4-gpu:
|
||||||
needs: [check-changes, call-gate, unit-test-backend-2-gpu]
|
needs: [check-changes, call-gate, unit-test-backend-2-gpu]
|
||||||
if: |
|
if: |
|
||||||
(inputs.target_stage == 'unit-test-backend-4-gpu') ||
|
always() &&
|
||||||
(
|
(
|
||||||
always() &&
|
(inputs.target_stage == 'unit-test-backend-4-gpu') ||
|
||||||
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
(
|
||||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
!inputs.target_stage &&
|
||||||
|
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
||||||
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||||
|
)
|
||||||
)
|
)
|
||||||
runs-on: 4-gpu-h100
|
runs-on: 4-gpu-h100
|
||||||
env:
|
env:
|
||||||
@@ -615,11 +636,14 @@ jobs:
|
|||||||
unit-test-backend-8-gpu-h200:
|
unit-test-backend-8-gpu-h200:
|
||||||
needs: [check-changes, call-gate, unit-test-backend-2-gpu]
|
needs: [check-changes, call-gate, unit-test-backend-2-gpu]
|
||||||
if: |
|
if: |
|
||||||
(inputs.target_stage == 'unit-test-backend-8-gpu-h200') ||
|
always() &&
|
||||||
(
|
(
|
||||||
always() &&
|
(inputs.target_stage == 'unit-test-backend-8-gpu-h200') ||
|
||||||
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
(
|
||||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
!inputs.target_stage &&
|
||||||
|
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
||||||
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||||
|
)
|
||||||
)
|
)
|
||||||
runs-on: 8-gpu-h200
|
runs-on: 8-gpu-h200
|
||||||
env:
|
env:
|
||||||
@@ -653,11 +677,14 @@ jobs:
|
|||||||
unit-test-backend-8-gpu-h20:
|
unit-test-backend-8-gpu-h20:
|
||||||
needs: [check-changes, call-gate, unit-test-backend-2-gpu]
|
needs: [check-changes, call-gate, unit-test-backend-2-gpu]
|
||||||
if: |
|
if: |
|
||||||
(inputs.target_stage == 'unit-test-backend-8-gpu-h20') ||
|
always() &&
|
||||||
(
|
(
|
||||||
always() &&
|
(inputs.target_stage == 'unit-test-backend-8-gpu-h20') ||
|
||||||
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
(
|
||||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
!inputs.target_stage &&
|
||||||
|
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
||||||
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||||
|
)
|
||||||
)
|
)
|
||||||
runs-on: 8-gpu-h20
|
runs-on: 8-gpu-h20
|
||||||
env:
|
env:
|
||||||
@@ -692,11 +719,14 @@ jobs:
|
|||||||
performance-test-1-gpu-part-1:
|
performance-test-1-gpu-part-1:
|
||||||
needs: [check-changes, call-gate, stage-a-test-1]
|
needs: [check-changes, call-gate, stage-a-test-1]
|
||||||
if: |
|
if: |
|
||||||
(inputs.target_stage == 'performance-test-1-gpu-part-1') ||
|
always() &&
|
||||||
(
|
(
|
||||||
always() &&
|
(inputs.target_stage == 'performance-test-1-gpu-part-1') ||
|
||||||
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
(
|
||||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
!inputs.target_stage &&
|
||||||
|
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
||||||
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||||
|
)
|
||||||
)
|
)
|
||||||
runs-on: 1-gpu-runner
|
runs-on: 1-gpu-runner
|
||||||
env:
|
env:
|
||||||
@@ -758,11 +788,14 @@ jobs:
|
|||||||
performance-test-1-gpu-part-2:
|
performance-test-1-gpu-part-2:
|
||||||
needs: [check-changes, call-gate, stage-a-test-1]
|
needs: [check-changes, call-gate, stage-a-test-1]
|
||||||
if: |
|
if: |
|
||||||
(inputs.target_stage == 'performance-test-1-gpu-part-2') ||
|
always() &&
|
||||||
(
|
(
|
||||||
always() &&
|
(inputs.target_stage == 'performance-test-1-gpu-part-2') ||
|
||||||
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
(
|
||||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
!inputs.target_stage &&
|
||||||
|
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
||||||
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||||
|
)
|
||||||
)
|
)
|
||||||
runs-on: 1-gpu-runner
|
runs-on: 1-gpu-runner
|
||||||
env:
|
env:
|
||||||
@@ -816,11 +849,14 @@ jobs:
|
|||||||
performance-test-1-gpu-part-3:
|
performance-test-1-gpu-part-3:
|
||||||
needs: [check-changes, call-gate, stage-a-test-1]
|
needs: [check-changes, call-gate, stage-a-test-1]
|
||||||
if: |
|
if: |
|
||||||
(inputs.target_stage == 'performance-test-1-gpu-part-3') ||
|
always() &&
|
||||||
(
|
(
|
||||||
always() &&
|
(inputs.target_stage == 'performance-test-1-gpu-part-3') ||
|
||||||
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
(
|
||||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
!inputs.target_stage &&
|
||||||
|
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
||||||
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||||
|
)
|
||||||
)
|
)
|
||||||
runs-on: 1-gpu-runner
|
runs-on: 1-gpu-runner
|
||||||
env:
|
env:
|
||||||
@@ -868,11 +904,14 @@ jobs:
|
|||||||
performance-test-2-gpu:
|
performance-test-2-gpu:
|
||||||
needs: [check-changes, call-gate, unit-test-backend-2-gpu]
|
needs: [check-changes, call-gate, unit-test-backend-2-gpu]
|
||||||
if: |
|
if: |
|
||||||
(inputs.target_stage == 'performance-test-2-gpu') ||
|
always() &&
|
||||||
(
|
(
|
||||||
always() &&
|
(inputs.target_stage == 'performance-test-2-gpu') ||
|
||||||
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
(
|
||||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
!inputs.target_stage &&
|
||||||
|
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
||||||
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||||
|
)
|
||||||
)
|
)
|
||||||
runs-on: 2-gpu-runner
|
runs-on: 2-gpu-runner
|
||||||
env:
|
env:
|
||||||
@@ -932,11 +971,14 @@ jobs:
|
|||||||
accuracy-test-1-gpu:
|
accuracy-test-1-gpu:
|
||||||
needs: [check-changes, call-gate, stage-a-test-1]
|
needs: [check-changes, call-gate, stage-a-test-1]
|
||||||
if: |
|
if: |
|
||||||
(inputs.target_stage == 'accuracy-test-1-gpu') ||
|
always() &&
|
||||||
(
|
(
|
||||||
always() &&
|
(inputs.target_stage == 'accuracy-test-1-gpu') ||
|
||||||
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
(
|
||||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
!inputs.target_stage &&
|
||||||
|
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
||||||
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||||
|
)
|
||||||
)
|
)
|
||||||
runs-on: 1-gpu-runner
|
runs-on: 1-gpu-runner
|
||||||
env:
|
env:
|
||||||
@@ -969,11 +1011,14 @@ jobs:
|
|||||||
accuracy-test-2-gpu:
|
accuracy-test-2-gpu:
|
||||||
needs: [check-changes, call-gate, accuracy-test-1-gpu]
|
needs: [check-changes, call-gate, accuracy-test-1-gpu]
|
||||||
if: |
|
if: |
|
||||||
(inputs.target_stage == 'accuracy-test-2-gpu') ||
|
always() &&
|
||||||
(
|
(
|
||||||
always() &&
|
(inputs.target_stage == 'accuracy-test-2-gpu') ||
|
||||||
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
(
|
||||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
!inputs.target_stage &&
|
||||||
|
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
||||||
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||||
|
)
|
||||||
)
|
)
|
||||||
runs-on: 2-gpu-runner
|
runs-on: 2-gpu-runner
|
||||||
env:
|
env:
|
||||||
@@ -1006,11 +1051,14 @@ jobs:
|
|||||||
unit-test-deepep-4-gpu:
|
unit-test-deepep-4-gpu:
|
||||||
needs: [check-changes, call-gate, unit-test-backend-2-gpu]
|
needs: [check-changes, call-gate, unit-test-backend-2-gpu]
|
||||||
if: |
|
if: |
|
||||||
(inputs.target_stage == 'unit-test-deepep-4-gpu') ||
|
always() &&
|
||||||
(
|
(
|
||||||
always() &&
|
(inputs.target_stage == 'unit-test-deepep-4-gpu') ||
|
||||||
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
(
|
||||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
!inputs.target_stage &&
|
||||||
|
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
||||||
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||||
|
)
|
||||||
)
|
)
|
||||||
runs-on: 4-gpu-h100
|
runs-on: 4-gpu-h100
|
||||||
env:
|
env:
|
||||||
@@ -1040,11 +1088,14 @@ jobs:
|
|||||||
unit-test-deepep-8-gpu:
|
unit-test-deepep-8-gpu:
|
||||||
needs: [check-changes, call-gate, unit-test-backend-2-gpu]
|
needs: [check-changes, call-gate, unit-test-backend-2-gpu]
|
||||||
if: |
|
if: |
|
||||||
(inputs.target_stage == 'unit-test-deepep-8-gpu') ||
|
always() &&
|
||||||
(
|
(
|
||||||
always() &&
|
(inputs.target_stage == 'unit-test-deepep-8-gpu') ||
|
||||||
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
(
|
||||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
!inputs.target_stage &&
|
||||||
|
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
||||||
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||||
|
)
|
||||||
)
|
)
|
||||||
runs-on: 8-gpu-h200
|
runs-on: 8-gpu-h200
|
||||||
env:
|
env:
|
||||||
@@ -1074,11 +1125,14 @@ jobs:
|
|||||||
unit-test-backend-4-gpu-b200:
|
unit-test-backend-4-gpu-b200:
|
||||||
needs: [check-changes, call-gate, unit-test-backend-2-gpu]
|
needs: [check-changes, call-gate, unit-test-backend-2-gpu]
|
||||||
if: |
|
if: |
|
||||||
(inputs.target_stage == 'unit-test-backend-4-gpu-b200') ||
|
always() &&
|
||||||
(
|
(
|
||||||
always() &&
|
(inputs.target_stage == 'unit-test-backend-4-gpu-b200') ||
|
||||||
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
(
|
||||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
!inputs.target_stage &&
|
||||||
|
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
||||||
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||||
|
)
|
||||||
)
|
)
|
||||||
runs-on: 4-gpu-b200
|
runs-on: 4-gpu-b200
|
||||||
env:
|
env:
|
||||||
@@ -1113,11 +1167,14 @@ jobs:
|
|||||||
unit-test-backend-4-gpu-gb200:
|
unit-test-backend-4-gpu-gb200:
|
||||||
needs: [check-changes, call-gate, unit-test-backend-2-gpu, sgl-kernel-build-wheels-arm]
|
needs: [check-changes, call-gate, unit-test-backend-2-gpu, sgl-kernel-build-wheels-arm]
|
||||||
if: |
|
if: |
|
||||||
(inputs.target_stage == 'unit-test-backend-4-gpu-gb200') ||
|
always() &&
|
||||||
(
|
(
|
||||||
always() &&
|
(inputs.target_stage == 'unit-test-backend-4-gpu-gb200') ||
|
||||||
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
(
|
||||||
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
!inputs.target_stage &&
|
||||||
|
(github.event_name == 'schedule' || (!failure() && !cancelled())) &&
|
||||||
|
((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
|
||||||
|
)
|
||||||
)
|
)
|
||||||
runs-on: 4-gpu-gb200
|
runs-on: 4-gpu-gb200
|
||||||
env:
|
env:
|
||||||
|
|||||||
@@ -132,7 +132,7 @@ def handle_rerun_stage(
|
|||||||
if not stage_name:
|
if not stage_name:
|
||||||
print("Error: No stage name provided")
|
print("Error: No stage name provided")
|
||||||
comment.create_reaction("confused")
|
comment.create_reaction("confused")
|
||||||
comment.create_comment(
|
pr.create_issue_comment(
|
||||||
f"❌ Please specify a stage name: `/rerun-stage <stage-name>`\n\n"
|
f"❌ Please specify a stage name: `/rerun-stage <stage-name>`\n\n"
|
||||||
f"Examples: `/rerun-stage unit-test-backend-4-gpu`, `/rerun-stage accuracy-test-1-gpu`"
|
f"Examples: `/rerun-stage unit-test-backend-4-gpu`, `/rerun-stage accuracy-test-1-gpu`"
|
||||||
)
|
)
|
||||||
@@ -140,8 +140,8 @@ def handle_rerun_stage(
|
|||||||
|
|
||||||
print(f"Permission granted. Triggering workflow_dispatch for stage '{stage_name}'.")
|
print(f"Permission granted. Triggering workflow_dispatch for stage '{stage_name}'.")
|
||||||
|
|
||||||
# Valid stage names that support target_stage
|
# Valid NVIDIA stage names that support target_stage
|
||||||
valid_stages = [
|
nvidia_stages = [
|
||||||
"stage-a-test-1",
|
"stage-a-test-1",
|
||||||
"multimodal-gen-test-1-gpu",
|
"multimodal-gen-test-1-gpu",
|
||||||
"multimodal-gen-test-2-gpu",
|
"multimodal-gen-test-2-gpu",
|
||||||
@@ -163,43 +163,69 @@ def handle_rerun_stage(
|
|||||||
"unit-test-backend-4-gpu-gb200",
|
"unit-test-backend-4-gpu-gb200",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
# Valid AMD stage names that support target_stage
|
||||||
|
amd_stages = [
|
||||||
|
"sgl-kernel-unit-test-amd",
|
||||||
|
"stage-a-test-1-amd",
|
||||||
|
"unit-test-backend-1-gpu-amd",
|
||||||
|
"unit-test-backend-2-gpu-amd",
|
||||||
|
"unit-test-backend-8-gpu-amd",
|
||||||
|
"performance-test-1-gpu-part-1-amd",
|
||||||
|
"performance-test-1-gpu-part-2-amd",
|
||||||
|
"performance-test-2-gpu-amd",
|
||||||
|
"accuracy-test-1-gpu-amd",
|
||||||
|
"accuracy-test-2-gpu-amd",
|
||||||
|
]
|
||||||
|
|
||||||
|
valid_stages = nvidia_stages + amd_stages
|
||||||
|
is_amd_stage = stage_name in amd_stages
|
||||||
|
|
||||||
if stage_name not in valid_stages:
|
if stage_name not in valid_stages:
|
||||||
comment.create_reaction("confused")
|
comment.create_reaction("confused")
|
||||||
comment.create_comment(
|
pr.create_issue_comment(
|
||||||
f"❌ Stage `{stage_name}` doesn't support isolated runs yet.\n\n"
|
f"❌ Stage `{stage_name}` doesn't support isolated runs yet.\n\n"
|
||||||
f"Currently supported stages:\n"
|
f"**NVIDIA stages:**\n"
|
||||||
+ "\n".join(f"- `{s}`" for s in valid_stages)
|
+ "\n".join(f"- `{s}`" for s in nvidia_stages)
|
||||||
|
+ "\n\n**AMD stages:**\n"
|
||||||
|
+ "\n".join(f"- `{s}`" for s in amd_stages)
|
||||||
+ "\n\nOther stages will be added soon. For now, use `/rerun-failed-ci` for those stages."
|
+ "\n\nOther stages will be added soon. For now, use `/rerun-failed-ci` for those stages."
|
||||||
)
|
)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Get the PR Test workflow
|
# Get the appropriate workflow based on stage type
|
||||||
|
workflow_name = "PR Test (AMD)" if is_amd_stage else "PR Test"
|
||||||
workflows = gh_repo.get_workflows()
|
workflows = gh_repo.get_workflows()
|
||||||
pr_test_workflow = None
|
target_workflow = None
|
||||||
for wf in workflows:
|
for wf in workflows:
|
||||||
if wf.name == "PR Test":
|
if wf.name == workflow_name:
|
||||||
pr_test_workflow = wf
|
target_workflow = wf
|
||||||
break
|
break
|
||||||
|
|
||||||
if not pr_test_workflow:
|
if not target_workflow:
|
||||||
print("Error: PR Test workflow not found")
|
print(f"Error: {workflow_name} workflow not found")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
# Trigger workflow_dispatch on the PR's head branch
|
# Trigger workflow_dispatch on the PR's head branch
|
||||||
ref = pr.head.ref
|
ref = pr.head.ref
|
||||||
print(f"Triggering workflow on branch: {ref}")
|
print(f"Triggering {workflow_name} workflow on branch: {ref}")
|
||||||
|
|
||||||
success = pr_test_workflow.create_dispatch(
|
# AMD workflow doesn't have version input, only target_stage
|
||||||
|
if is_amd_stage:
|
||||||
|
inputs = {"target_stage": stage_name}
|
||||||
|
else:
|
||||||
|
inputs = {"version": "release", "target_stage": stage_name}
|
||||||
|
|
||||||
|
success = target_workflow.create_dispatch(
|
||||||
ref=ref,
|
ref=ref,
|
||||||
inputs={"version": "release", "target_stage": stage_name},
|
inputs=inputs,
|
||||||
)
|
)
|
||||||
|
|
||||||
if success:
|
if success:
|
||||||
print(f"Successfully triggered workflow for stage '{stage_name}'")
|
print(f"Successfully triggered workflow for stage '{stage_name}'")
|
||||||
if react_on_success:
|
if react_on_success:
|
||||||
comment.create_reaction("+1")
|
comment.create_reaction("+1")
|
||||||
comment.create_comment(
|
pr.create_issue_comment(
|
||||||
f"✅ Triggered `{stage_name}` to run independently (skipping dependencies).\n\n"
|
f"✅ Triggered `{stage_name}` to run independently (skipping dependencies).\n\n"
|
||||||
f"Check the [Actions tab](https://github.com/{gh_repo.full_name}/actions) for progress."
|
f"Check the [Actions tab](https://github.com/{gh_repo.full_name}/actions) for progress."
|
||||||
)
|
)
|
||||||
@@ -211,7 +237,7 @@ def handle_rerun_stage(
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"Error triggering workflow_dispatch: {e}")
|
print(f"Error triggering workflow_dispatch: {e}")
|
||||||
comment.create_reaction("confused")
|
comment.create_reaction("confused")
|
||||||
comment.create_comment(
|
pr.create_issue_comment(
|
||||||
f"❌ Failed to trigger workflow: {str(e)}\n\n"
|
f"❌ Failed to trigger workflow: {str(e)}\n\n"
|
||||||
f"Please check the logs or contact maintainers."
|
f"Please check the logs or contact maintainers."
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user