[CI] Temporarily disable GB300 jobs (runner availability) (#31764)
This commit is contained in:
Alison Shao
@@ -24,7 +24,7 @@ on:
|
|||||||
- 'nightly-test-perf-4-gpu-b200'
|
- 'nightly-test-perf-4-gpu-b200'
|
||||||
- 'nightly-test-perf-8-gpu-b200'
|
- 'nightly-test-perf-8-gpu-b200'
|
||||||
- 'nightly-test-specialized-8-gpu-b200'
|
- 'nightly-test-specialized-8-gpu-b200'
|
||||||
- 'nightly-test-perf-4-gpu-gb300'
|
# - 'nightly-test-perf-4-gpu-gb300' # temporarily disabled due to runner availability
|
||||||
- 'nightly-test-kernel-1-gpu-h100'
|
- 'nightly-test-kernel-1-gpu-h100'
|
||||||
- 'nightly-test-diffusion'
|
- 'nightly-test-diffusion'
|
||||||
- 'nightly-test-kernel-8-gpu-h200'
|
- 'nightly-test-kernel-8-gpu-h200'
|
||||||
@@ -544,68 +544,69 @@ jobs:
|
|||||||
if: failure()
|
if: failure()
|
||||||
|
|
||||||
# GB300 (Grace-Blackwell NVL4) performance tests - 4 GPU (ARM64)
|
# GB300 (Grace-Blackwell NVL4) performance tests - 4 GPU (ARM64)
|
||||||
nightly-test-perf-4-gpu-gb300:
|
# temporarily disabled due to runner availability
|
||||||
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-4-gpu-gb300')
|
# nightly-test-perf-4-gpu-gb300:
|
||||||
name: nightly-test-perf-4-gpu-gb300 (${{ matrix.model }})
|
# if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-4-gpu-gb300')
|
||||||
runs-on: 4-gpu-gb300-nightly
|
# name: nightly-test-perf-4-gpu-gb300 (${{ matrix.model }})
|
||||||
strategy:
|
# runs-on: 4-gpu-gb300-nightly
|
||||||
fail-fast: false
|
# strategy:
|
||||||
matrix:
|
# fail-fast: false
|
||||||
include:
|
# matrix:
|
||||||
- model: glm5-nvfp4
|
# include:
|
||||||
suite: nightly-4-gpu-gb300-glm5-nvfp4
|
# - model: glm5-nvfp4
|
||||||
- model: qwen35-fp8
|
# suite: nightly-4-gpu-gb300-glm5-nvfp4
|
||||||
suite: nightly-4-gpu-gb300-qwen35-fp8
|
# - model: qwen35-fp8
|
||||||
- model: deepseek-v4-pro-fp4
|
# suite: nightly-4-gpu-gb300-qwen35-fp8
|
||||||
suite: nightly-4-gpu-gb300-deepseek-v4-pro-fp4
|
# - model: deepseek-v4-pro-fp4
|
||||||
- model: kimi-k25-nvfp4
|
# suite: nightly-4-gpu-gb300-deepseek-v4-pro-fp4
|
||||||
suite: nightly-4-gpu-gb300-kimi-k25-nvfp4
|
# - model: kimi-k25-nvfp4
|
||||||
steps:
|
# suite: nightly-4-gpu-gb300-kimi-k25-nvfp4
|
||||||
- name: Checkout code
|
# steps:
|
||||||
uses: actions/checkout@v4
|
# - name: Checkout code
|
||||||
with:
|
# uses: actions/checkout@v4
|
||||||
ref: ${{ inputs.ref || github.ref }}
|
# with:
|
||||||
|
# ref: ${{ inputs.ref || github.ref }}
|
||||||
- uses: ./.github/actions/check-maintenance
|
#
|
||||||
|
# - uses: ./.github/actions/check-maintenance
|
||||||
- name: Install dependencies
|
#
|
||||||
env:
|
# - name: Install dependencies
|
||||||
GRACE_BLACKWELL: "1"
|
# env:
|
||||||
run: |
|
# GRACE_BLACKWELL: "1"
|
||||||
bash scripts/ci/cuda/ci_install_deepep.sh
|
# run: |
|
||||||
|
# bash scripts/ci/cuda/ci_install_deepep.sh
|
||||||
- name: Run test
|
#
|
||||||
timeout-minutes: 600
|
# - name: Run test
|
||||||
env:
|
# timeout-minutes: 600
|
||||||
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
|
# env:
|
||||||
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
|
# TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
|
||||||
GPU_CONFIG: "4-gpu-gb300"
|
# PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
|
||||||
run: |
|
# GPU_CONFIG: "4-gpu-gb300"
|
||||||
cd test
|
# run: |
|
||||||
python3 run_suite.py --hw cuda --suite ${{ matrix.suite }} --nightly --continue-on-error --timeout-per-file 7200
|
# cd test
|
||||||
|
# python3 run_suite.py --hw cuda --suite ${{ matrix.suite }} --nightly --continue-on-error --timeout-per-file 7200
|
||||||
- name: Publish traces to storage repo
|
#
|
||||||
if: always()
|
# - name: Publish traces to storage repo
|
||||||
continue-on-error: true
|
# if: always()
|
||||||
env:
|
# continue-on-error: true
|
||||||
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
|
# env:
|
||||||
GITHUB_RUN_ID: ${{ github.run_id }}
|
# GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
|
||||||
GITHUB_RUN_NUMBER: ${{ github.run_number }}
|
# GITHUB_RUN_ID: ${{ github.run_id }}
|
||||||
run: |
|
# GITHUB_RUN_NUMBER: ${{ github.run_number }}
|
||||||
TRACE_ARGS=""
|
# run: |
|
||||||
for dir in test/performance_profiles_*/; do
|
# TRACE_ARGS=""
|
||||||
[ -d "$dir" ] && TRACE_ARGS="$TRACE_ARGS --traces-dir $dir"
|
# for dir in test/performance_profiles_*/; do
|
||||||
done
|
# [ -d "$dir" ] && TRACE_ARGS="$TRACE_ARGS --traces-dir $dir"
|
||||||
if [ -n "$TRACE_ARGS" ]; then
|
# done
|
||||||
python3 scripts/ci/utils/publish_traces.py $TRACE_ARGS
|
# if [ -n "$TRACE_ARGS" ]; then
|
||||||
find test/performance_profiles_*/ -name '*.json.gz' -delete
|
# python3 scripts/ci/utils/publish_traces.py $TRACE_ARGS
|
||||||
else
|
# find test/performance_profiles_*/ -name '*.json.gz' -delete
|
||||||
echo "No trace directories found, skipping publish"
|
# else
|
||||||
fi
|
# echo "No trace directories found, skipping publish"
|
||||||
|
# fi
|
||||||
- uses: ./.github/actions/upload-cuda-coredumps
|
#
|
||||||
if: failure()
|
# - uses: ./.github/actions/upload-cuda-coredumps
|
||||||
|
# if: failure()
|
||||||
|
#
|
||||||
# Specialized B200 tests - 8 GPU, for specific backends and configs
|
# Specialized B200 tests - 8 GPU, for specific backends and configs
|
||||||
nightly-test-specialized-8-gpu-b200:
|
nightly-test-specialized-8-gpu-b200:
|
||||||
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-8-gpu-b200' || inputs.job_filter == 'nightly-test-specialized-8-gpu-b200')
|
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-8-gpu-b200' || inputs.job_filter == 'nightly-test-specialized-8-gpu-b200')
|
||||||
@@ -791,7 +792,7 @@ jobs:
|
|||||||
- nightly-test-vlm-perf-2-gpu-h100
|
- nightly-test-vlm-perf-2-gpu-h100
|
||||||
- nightly-test-perf-4-gpu-b200
|
- nightly-test-perf-4-gpu-b200
|
||||||
- nightly-test-specialized-8-gpu-b200
|
- nightly-test-specialized-8-gpu-b200
|
||||||
- nightly-test-perf-4-gpu-gb300
|
# - nightly-test-perf-4-gpu-gb300 # temporarily disabled due to runner availability
|
||||||
- nightly-test-diffusion
|
- nightly-test-diffusion
|
||||||
- nightly-test-precision-8-gpu-h200
|
- nightly-test-precision-8-gpu-h200
|
||||||
- consolidate-metrics
|
- consolidate-metrics
|
||||||
|
|||||||
@@ -520,19 +520,20 @@ jobs:
|
|||||||
timeout_per_file: '1800'
|
timeout_per_file: '1800'
|
||||||
secrets: inherit
|
secrets: inherit
|
||||||
|
|
||||||
base-c-test-4-gpu-gb300:
|
# temporarily disabled due to runner availability
|
||||||
needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels]
|
# base-c-test-4-gpu-gb300:
|
||||||
if: ${{ !failure() && !cancelled() }}
|
# needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels]
|
||||||
uses: ./.github/workflows/_pr-test-stage.yml
|
# if: ${{ !failure() && !cancelled() }}
|
||||||
with:
|
# uses: ./.github/workflows/_pr-test-stage.yml
|
||||||
self_name: base-c-test-4-gpu-gb300
|
# with:
|
||||||
runner_config: 4-gpu-gb300
|
# self_name: base-c-test-4-gpu-gb300
|
||||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
# runner_config: 4-gpu-gb300
|
||||||
caller_inputs: ${{ toJson(inputs) }}
|
# check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
||||||
partitions: ${{ needs.check-changes.outputs.partitions }}
|
# caller_inputs: ${{ toJson(inputs) }}
|
||||||
run_timeout_minutes: '30'
|
# partitions: ${{ needs.check-changes.outputs.partitions }}
|
||||||
timeout_per_file: '1800'
|
# run_timeout_minutes: '30'
|
||||||
secrets: inherit
|
# timeout_per_file: '1800'
|
||||||
|
# secrets: inherit
|
||||||
|
|
||||||
pr-test-finish:
|
pr-test-finish:
|
||||||
needs:
|
needs:
|
||||||
@@ -564,7 +565,7 @@ jobs:
|
|||||||
base-c-test-deepep-4-gpu-b200,
|
base-c-test-deepep-4-gpu-b200,
|
||||||
base-c-test-deepep-8-gpu-h200,
|
base-c-test-deepep-8-gpu-h200,
|
||||||
base-c-test-4-gpu-b200,
|
base-c-test-4-gpu-b200,
|
||||||
base-c-test-4-gpu-gb300,
|
# base-c-test-4-gpu-gb300, # temporarily disabled due to runner availability
|
||||||
]
|
]
|
||||||
if: always()
|
if: always()
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
|||||||
@@ -238,9 +238,10 @@ jobs:
|
|||||||
- arch_label: sm100
|
- arch_label: sm100
|
||||||
runner: 8-gpu-b200
|
runner: 8-gpu-b200
|
||||||
wheel_arch: x86_64
|
wheel_arch: x86_64
|
||||||
- arch_label: sm100-aarch64
|
# temporarily disabled due to runner availability
|
||||||
runner: 4-gpu-gb300
|
# - arch_label: sm100-aarch64
|
||||||
wheel_arch: aarch64
|
# runner: 4-gpu-gb300
|
||||||
|
# wheel_arch: aarch64
|
||||||
runs-on: ${{ matrix.runner }}
|
runs-on: ${{ matrix.runner }}
|
||||||
timeout-minutes: 120
|
timeout-minutes: 120
|
||||||
steps:
|
steps:
|
||||||
|
|||||||
Reference in New Issue
Block a user