[CI] Temporarily disable GB300 jobs (runner availability) (#31764)
This commit is contained in:
Alison Shao
@@ -24,7 +24,7 @@ on:
|
||||
- 'nightly-test-perf-4-gpu-b200'
|
||||
- 'nightly-test-perf-8-gpu-b200'
|
||||
- 'nightly-test-specialized-8-gpu-b200'
|
||||
- 'nightly-test-perf-4-gpu-gb300'
|
||||
# - 'nightly-test-perf-4-gpu-gb300' # temporarily disabled due to runner availability
|
||||
- 'nightly-test-kernel-1-gpu-h100'
|
||||
- 'nightly-test-diffusion'
|
||||
- 'nightly-test-kernel-8-gpu-h200'
|
||||
@@ -544,68 +544,69 @@ jobs:
|
||||
if: failure()
|
||||
|
||||
# GB300 (Grace-Blackwell NVL4) performance tests - 4 GPU (ARM64)
|
||||
nightly-test-perf-4-gpu-gb300:
|
||||
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-4-gpu-gb300')
|
||||
name: nightly-test-perf-4-gpu-gb300 (${{ matrix.model }})
|
||||
runs-on: 4-gpu-gb300-nightly
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- model: glm5-nvfp4
|
||||
suite: nightly-4-gpu-gb300-glm5-nvfp4
|
||||
- model: qwen35-fp8
|
||||
suite: nightly-4-gpu-gb300-qwen35-fp8
|
||||
- model: deepseek-v4-pro-fp4
|
||||
suite: nightly-4-gpu-gb300-deepseek-v4-pro-fp4
|
||||
- model: kimi-k25-nvfp4
|
||||
suite: nightly-4-gpu-gb300-kimi-k25-nvfp4
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
|
||||
- uses: ./.github/actions/check-maintenance
|
||||
|
||||
- name: Install dependencies
|
||||
env:
|
||||
GRACE_BLACKWELL: "1"
|
||||
run: |
|
||||
bash scripts/ci/cuda/ci_install_deepep.sh
|
||||
|
||||
- name: Run test
|
||||
timeout-minutes: 600
|
||||
env:
|
||||
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
|
||||
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
|
||||
GPU_CONFIG: "4-gpu-gb300"
|
||||
run: |
|
||||
cd test
|
||||
python3 run_suite.py --hw cuda --suite ${{ matrix.suite }} --nightly --continue-on-error --timeout-per-file 7200
|
||||
|
||||
- name: Publish traces to storage repo
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
|
||||
GITHUB_RUN_ID: ${{ github.run_id }}
|
||||
GITHUB_RUN_NUMBER: ${{ github.run_number }}
|
||||
run: |
|
||||
TRACE_ARGS=""
|
||||
for dir in test/performance_profiles_*/; do
|
||||
[ -d "$dir" ] && TRACE_ARGS="$TRACE_ARGS --traces-dir $dir"
|
||||
done
|
||||
if [ -n "$TRACE_ARGS" ]; then
|
||||
python3 scripts/ci/utils/publish_traces.py $TRACE_ARGS
|
||||
find test/performance_profiles_*/ -name '*.json.gz' -delete
|
||||
else
|
||||
echo "No trace directories found, skipping publish"
|
||||
fi
|
||||
|
||||
- uses: ./.github/actions/upload-cuda-coredumps
|
||||
if: failure()
|
||||
|
||||
# temporarily disabled due to runner availability
|
||||
# nightly-test-perf-4-gpu-gb300:
|
||||
# if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-4-gpu-gb300')
|
||||
# name: nightly-test-perf-4-gpu-gb300 (${{ matrix.model }})
|
||||
# runs-on: 4-gpu-gb300-nightly
|
||||
# strategy:
|
||||
# fail-fast: false
|
||||
# matrix:
|
||||
# include:
|
||||
# - model: glm5-nvfp4
|
||||
# suite: nightly-4-gpu-gb300-glm5-nvfp4
|
||||
# - model: qwen35-fp8
|
||||
# suite: nightly-4-gpu-gb300-qwen35-fp8
|
||||
# - model: deepseek-v4-pro-fp4
|
||||
# suite: nightly-4-gpu-gb300-deepseek-v4-pro-fp4
|
||||
# - model: kimi-k25-nvfp4
|
||||
# suite: nightly-4-gpu-gb300-kimi-k25-nvfp4
|
||||
# steps:
|
||||
# - name: Checkout code
|
||||
# uses: actions/checkout@v4
|
||||
# with:
|
||||
# ref: ${{ inputs.ref || github.ref }}
|
||||
#
|
||||
# - uses: ./.github/actions/check-maintenance
|
||||
#
|
||||
# - name: Install dependencies
|
||||
# env:
|
||||
# GRACE_BLACKWELL: "1"
|
||||
# run: |
|
||||
# bash scripts/ci/cuda/ci_install_deepep.sh
|
||||
#
|
||||
# - name: Run test
|
||||
# timeout-minutes: 600
|
||||
# env:
|
||||
# TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
|
||||
# PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
|
||||
# GPU_CONFIG: "4-gpu-gb300"
|
||||
# run: |
|
||||
# cd test
|
||||
# python3 run_suite.py --hw cuda --suite ${{ matrix.suite }} --nightly --continue-on-error --timeout-per-file 7200
|
||||
#
|
||||
# - name: Publish traces to storage repo
|
||||
# if: always()
|
||||
# continue-on-error: true
|
||||
# env:
|
||||
# GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
|
||||
# GITHUB_RUN_ID: ${{ github.run_id }}
|
||||
# GITHUB_RUN_NUMBER: ${{ github.run_number }}
|
||||
# run: |
|
||||
# TRACE_ARGS=""
|
||||
# for dir in test/performance_profiles_*/; do
|
||||
# [ -d "$dir" ] && TRACE_ARGS="$TRACE_ARGS --traces-dir $dir"
|
||||
# done
|
||||
# if [ -n "$TRACE_ARGS" ]; then
|
||||
# python3 scripts/ci/utils/publish_traces.py $TRACE_ARGS
|
||||
# find test/performance_profiles_*/ -name '*.json.gz' -delete
|
||||
# else
|
||||
# echo "No trace directories found, skipping publish"
|
||||
# fi
|
||||
#
|
||||
# - uses: ./.github/actions/upload-cuda-coredumps
|
||||
# if: failure()
|
||||
#
|
||||
# Specialized B200 tests - 8 GPU, for specific backends and configs
|
||||
nightly-test-specialized-8-gpu-b200:
|
||||
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-8-gpu-b200' || inputs.job_filter == 'nightly-test-specialized-8-gpu-b200')
|
||||
@@ -791,7 +792,7 @@ jobs:
|
||||
- nightly-test-vlm-perf-2-gpu-h100
|
||||
- nightly-test-perf-4-gpu-b200
|
||||
- nightly-test-specialized-8-gpu-b200
|
||||
- nightly-test-perf-4-gpu-gb300
|
||||
# - nightly-test-perf-4-gpu-gb300 # temporarily disabled due to runner availability
|
||||
- nightly-test-diffusion
|
||||
- nightly-test-precision-8-gpu-h200
|
||||
- consolidate-metrics
|
||||
|
||||
@@ -520,19 +520,20 @@ jobs:
|
||||
timeout_per_file: '1800'
|
||||
secrets: inherit
|
||||
|
||||
base-c-test-4-gpu-gb300:
|
||||
needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/_pr-test-stage.yml
|
||||
with:
|
||||
self_name: base-c-test-4-gpu-gb300
|
||||
runner_config: 4-gpu-gb300
|
||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
||||
caller_inputs: ${{ toJson(inputs) }}
|
||||
partitions: ${{ needs.check-changes.outputs.partitions }}
|
||||
run_timeout_minutes: '30'
|
||||
timeout_per_file: '1800'
|
||||
secrets: inherit
|
||||
# temporarily disabled due to runner availability
|
||||
# base-c-test-4-gpu-gb300:
|
||||
# needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels]
|
||||
# if: ${{ !failure() && !cancelled() }}
|
||||
# uses: ./.github/workflows/_pr-test-stage.yml
|
||||
# with:
|
||||
# self_name: base-c-test-4-gpu-gb300
|
||||
# runner_config: 4-gpu-gb300
|
||||
# check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
||||
# caller_inputs: ${{ toJson(inputs) }}
|
||||
# partitions: ${{ needs.check-changes.outputs.partitions }}
|
||||
# run_timeout_minutes: '30'
|
||||
# timeout_per_file: '1800'
|
||||
# secrets: inherit
|
||||
|
||||
pr-test-finish:
|
||||
needs:
|
||||
@@ -564,7 +565,7 @@ jobs:
|
||||
base-c-test-deepep-4-gpu-b200,
|
||||
base-c-test-deepep-8-gpu-h200,
|
||||
base-c-test-4-gpu-b200,
|
||||
base-c-test-4-gpu-gb300,
|
||||
# base-c-test-4-gpu-gb300, # temporarily disabled due to runner availability
|
||||
]
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -238,9 +238,10 @@ jobs:
|
||||
- arch_label: sm100
|
||||
runner: 8-gpu-b200
|
||||
wheel_arch: x86_64
|
||||
- arch_label: sm100-aarch64
|
||||
runner: 4-gpu-gb300
|
||||
wheel_arch: aarch64
|
||||
# temporarily disabled due to runner availability
|
||||
# - arch_label: sm100-aarch64
|
||||
# runner: 4-gpu-gb300
|
||||
# wheel_arch: aarch64
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
|
||||
Reference in New Issue
Block a user