[CI] Re-enable GB300 CI jobs (#32719)
This commit is contained in:
Alison Shao
@@ -24,7 +24,7 @@ on:
|
|||||||
- 'nightly-test-perf-4-gpu-b200'
|
- 'nightly-test-perf-4-gpu-b200'
|
||||||
- 'nightly-test-perf-8-gpu-b200'
|
- 'nightly-test-perf-8-gpu-b200'
|
||||||
- 'nightly-test-specialized-8-gpu-b200'
|
- 'nightly-test-specialized-8-gpu-b200'
|
||||||
# - 'nightly-test-perf-4-gpu-gb300' # temporarily disabled due to runner availability
|
- 'nightly-test-perf-4-gpu-gb300'
|
||||||
- 'nightly-test-kernel-1-gpu-h100'
|
- 'nightly-test-kernel-1-gpu-h100'
|
||||||
- 'nightly-test-diffusion'
|
- 'nightly-test-diffusion'
|
||||||
- 'nightly-test-kernel-8-gpu-h200'
|
- 'nightly-test-kernel-8-gpu-h200'
|
||||||
@@ -544,69 +544,68 @@ jobs:
|
|||||||
if: failure()
|
if: failure()
|
||||||
|
|
||||||
# GB300 (Grace-Blackwell NVL4) performance tests - 4 GPU (ARM64)
|
# GB300 (Grace-Blackwell NVL4) performance tests - 4 GPU (ARM64)
|
||||||
# temporarily disabled due to runner availability
|
nightly-test-perf-4-gpu-gb300:
|
||||||
# nightly-test-perf-4-gpu-gb300:
|
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-4-gpu-gb300')
|
||||||
# if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-4-gpu-gb300')
|
name: nightly-test-perf-4-gpu-gb300 (${{ matrix.model }})
|
||||||
# name: nightly-test-perf-4-gpu-gb300 (${{ matrix.model }})
|
runs-on: 4-gpu-gb300-nightly
|
||||||
# runs-on: 4-gpu-gb300-nightly
|
strategy:
|
||||||
# strategy:
|
fail-fast: false
|
||||||
# fail-fast: false
|
matrix:
|
||||||
# matrix:
|
include:
|
||||||
# include:
|
- model: glm5-nvfp4
|
||||||
# - model: glm5-nvfp4
|
suite: nightly-4-gpu-gb300-glm5-nvfp4
|
||||||
# suite: nightly-4-gpu-gb300-glm5-nvfp4
|
- model: qwen35-fp8
|
||||||
# - model: qwen35-fp8
|
suite: nightly-4-gpu-gb300-qwen35-fp8
|
||||||
# suite: nightly-4-gpu-gb300-qwen35-fp8
|
- model: deepseek-v4-pro-fp4
|
||||||
# - model: deepseek-v4-pro-fp4
|
suite: nightly-4-gpu-gb300-deepseek-v4-pro-fp4
|
||||||
# suite: nightly-4-gpu-gb300-deepseek-v4-pro-fp4
|
- model: kimi-k25-nvfp4
|
||||||
# - model: kimi-k25-nvfp4
|
suite: nightly-4-gpu-gb300-kimi-k25-nvfp4
|
||||||
# suite: nightly-4-gpu-gb300-kimi-k25-nvfp4
|
steps:
|
||||||
# steps:
|
- name: Checkout code
|
||||||
# - name: Checkout code
|
uses: actions/checkout@v4
|
||||||
# uses: actions/checkout@v4
|
with:
|
||||||
# with:
|
ref: ${{ inputs.ref || github.ref }}
|
||||||
# ref: ${{ inputs.ref || github.ref }}
|
|
||||||
#
|
- uses: ./.github/actions/check-maintenance
|
||||||
# - uses: ./.github/actions/check-maintenance
|
|
||||||
#
|
- name: Install dependencies
|
||||||
# - name: Install dependencies
|
env:
|
||||||
# env:
|
GRACE_BLACKWELL: "1"
|
||||||
# GRACE_BLACKWELL: "1"
|
run: |
|
||||||
# run: |
|
bash scripts/ci/cuda/ci_install_deepep.sh
|
||||||
# bash scripts/ci/cuda/ci_install_deepep.sh
|
|
||||||
#
|
- name: Run test
|
||||||
# - name: Run test
|
timeout-minutes: 600
|
||||||
# timeout-minutes: 600
|
env:
|
||||||
# env:
|
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
|
||||||
# TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
|
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
|
||||||
# PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
|
GPU_CONFIG: "4-gpu-gb300"
|
||||||
# GPU_CONFIG: "4-gpu-gb300"
|
run: |
|
||||||
# run: |
|
cd test
|
||||||
# cd test
|
python3 run_suite.py --hw cuda --suite ${{ matrix.suite }} --nightly --continue-on-error --timeout-per-file 7200
|
||||||
# python3 run_suite.py --hw cuda --suite ${{ matrix.suite }} --nightly --continue-on-error --timeout-per-file 7200
|
|
||||||
#
|
- name: Publish traces to storage repo
|
||||||
# - name: Publish traces to storage repo
|
if: always()
|
||||||
# if: always()
|
continue-on-error: true
|
||||||
# continue-on-error: true
|
env:
|
||||||
# env:
|
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
|
||||||
# GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
|
GITHUB_RUN_ID: ${{ github.run_id }}
|
||||||
# GITHUB_RUN_ID: ${{ github.run_id }}
|
GITHUB_RUN_NUMBER: ${{ github.run_number }}
|
||||||
# GITHUB_RUN_NUMBER: ${{ github.run_number }}
|
run: |
|
||||||
# run: |
|
TRACE_ARGS=""
|
||||||
# TRACE_ARGS=""
|
for dir in test/performance_profiles_*/; do
|
||||||
# for dir in test/performance_profiles_*/; do
|
[ -d "$dir" ] && TRACE_ARGS="$TRACE_ARGS --traces-dir $dir"
|
||||||
# [ -d "$dir" ] && TRACE_ARGS="$TRACE_ARGS --traces-dir $dir"
|
done
|
||||||
# done
|
if [ -n "$TRACE_ARGS" ]; then
|
||||||
# if [ -n "$TRACE_ARGS" ]; then
|
python3 scripts/ci/utils/publish_traces.py $TRACE_ARGS
|
||||||
# python3 scripts/ci/utils/publish_traces.py $TRACE_ARGS
|
find test/performance_profiles_*/ -name '*.json.gz' -delete
|
||||||
# find test/performance_profiles_*/ -name '*.json.gz' -delete
|
else
|
||||||
# else
|
echo "No trace directories found, skipping publish"
|
||||||
# echo "No trace directories found, skipping publish"
|
fi
|
||||||
# fi
|
|
||||||
#
|
- uses: ./.github/actions/upload-cuda-coredumps
|
||||||
# - uses: ./.github/actions/upload-cuda-coredumps
|
if: failure()
|
||||||
# if: failure()
|
|
||||||
#
|
|
||||||
# Specialized B200 tests - 8 GPU, for specific backends and configs
|
# Specialized B200 tests - 8 GPU, for specific backends and configs
|
||||||
nightly-test-specialized-8-gpu-b200:
|
nightly-test-specialized-8-gpu-b200:
|
||||||
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-8-gpu-b200' || inputs.job_filter == 'nightly-test-specialized-8-gpu-b200')
|
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-8-gpu-b200' || inputs.job_filter == 'nightly-test-specialized-8-gpu-b200')
|
||||||
@@ -792,7 +791,7 @@ jobs:
|
|||||||
- nightly-test-vlm-perf-2-gpu-h100
|
- nightly-test-vlm-perf-2-gpu-h100
|
||||||
- nightly-test-perf-4-gpu-b200
|
- nightly-test-perf-4-gpu-b200
|
||||||
- nightly-test-specialized-8-gpu-b200
|
- nightly-test-specialized-8-gpu-b200
|
||||||
# - nightly-test-perf-4-gpu-gb300 # temporarily disabled due to runner availability
|
- nightly-test-perf-4-gpu-gb300
|
||||||
- nightly-test-diffusion
|
- nightly-test-diffusion
|
||||||
- nightly-test-precision-8-gpu-h200
|
- nightly-test-precision-8-gpu-h200
|
||||||
- consolidate-metrics
|
- consolidate-metrics
|
||||||
|
|||||||
@@ -520,20 +520,19 @@ jobs:
|
|||||||
timeout_per_file: '1800'
|
timeout_per_file: '1800'
|
||||||
secrets: inherit
|
secrets: inherit
|
||||||
|
|
||||||
# temporarily disabled due to runner availability
|
base-c-test-4-gpu-gb300:
|
||||||
# base-c-test-4-gpu-gb300:
|
needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels]
|
||||||
# needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels]
|
if: ${{ !failure() && !cancelled() }}
|
||||||
# if: ${{ !failure() && !cancelled() }}
|
uses: ./.github/workflows/_pr-test-stage.yml
|
||||||
# uses: ./.github/workflows/_pr-test-stage.yml
|
with:
|
||||||
# with:
|
self_name: base-c-test-4-gpu-gb300
|
||||||
# self_name: base-c-test-4-gpu-gb300
|
runner_config: 4-gpu-gb300
|
||||||
# runner_config: 4-gpu-gb300
|
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
||||||
# check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
caller_inputs: ${{ toJson(inputs) }}
|
||||||
# caller_inputs: ${{ toJson(inputs) }}
|
partitions: ${{ needs.check-changes.outputs.partitions }}
|
||||||
# partitions: ${{ needs.check-changes.outputs.partitions }}
|
run_timeout_minutes: '30'
|
||||||
# run_timeout_minutes: '30'
|
timeout_per_file: '1800'
|
||||||
# timeout_per_file: '1800'
|
secrets: inherit
|
||||||
# secrets: inherit
|
|
||||||
|
|
||||||
pr-test-finish:
|
pr-test-finish:
|
||||||
needs:
|
needs:
|
||||||
@@ -565,7 +564,7 @@ jobs:
|
|||||||
base-c-test-deepep-4-gpu-b200,
|
base-c-test-deepep-4-gpu-b200,
|
||||||
base-c-test-deepep-8-gpu-h200,
|
base-c-test-deepep-8-gpu-h200,
|
||||||
base-c-test-4-gpu-b200,
|
base-c-test-4-gpu-b200,
|
||||||
# base-c-test-4-gpu-gb300, # temporarily disabled due to runner availability
|
base-c-test-4-gpu-gb300,
|
||||||
]
|
]
|
||||||
if: always()
|
if: always()
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
|||||||
@@ -242,10 +242,9 @@ jobs:
|
|||||||
# - arch_label: sm120
|
# - arch_label: sm120
|
||||||
# runner: 1-gpu-5090
|
# runner: 1-gpu-5090
|
||||||
# wheel_arch: x86_64
|
# wheel_arch: x86_64
|
||||||
# temporarily disabled due to runner availability
|
- arch_label: sm100-aarch64
|
||||||
# - arch_label: sm100-aarch64
|
runner: 4-gpu-gb300
|
||||||
# runner: 4-gpu-gb300
|
wheel_arch: aarch64
|
||||||
# wheel_arch: aarch64
|
|
||||||
runs-on: ${{ matrix.runner }}
|
runs-on: ${{ matrix.runner }}
|
||||||
timeout-minutes: 120
|
timeout-minutes: 120
|
||||||
steps:
|
steps:
|
||||||
|
|||||||
Reference in New Issue
Block a user