From 301ea43f35b9dd569d957968218f7761fe995767 Mon Sep 17 00:00:00 2001 From: Alison Shao <54658187+alisonshao@users.noreply.github.com> Date: Fri, 31 Jul 2026 14:22:26 -0700 Subject: [PATCH] [CI] Re-enable GB300 CI jobs (#32719) --- .github/workflows/nightly-test-nvidia.yml | 129 ++++++++++----------- .github/workflows/pr-test.yml | 29 +++-- .github/workflows/release-whl-deepgemm.yml | 7 +- 3 files changed, 81 insertions(+), 84 deletions(-) diff --git a/.github/workflows/nightly-test-nvidia.yml b/.github/workflows/nightly-test-nvidia.yml index e5ec6b4fb..06952e0a1 100644 --- a/.github/workflows/nightly-test-nvidia.yml +++ b/.github/workflows/nightly-test-nvidia.yml @@ -24,7 +24,7 @@ on: - 'nightly-test-perf-4-gpu-b200' - 'nightly-test-perf-8-gpu-b200' - 'nightly-test-specialized-8-gpu-b200' -# - 'nightly-test-perf-4-gpu-gb300' # temporarily disabled due to runner availability + - 'nightly-test-perf-4-gpu-gb300' - 'nightly-test-kernel-1-gpu-h100' - 'nightly-test-diffusion' - 'nightly-test-kernel-8-gpu-h200' @@ -544,69 +544,68 @@ jobs: if: failure() # GB300 (Grace-Blackwell NVL4) performance tests - 4 GPU (ARM64) - # temporarily disabled due to runner availability -# nightly-test-perf-4-gpu-gb300: -# if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-4-gpu-gb300') -# name: nightly-test-perf-4-gpu-gb300 (${{ matrix.model }}) -# runs-on: 4-gpu-gb300-nightly -# strategy: -# fail-fast: false -# matrix: -# include: -# - model: glm5-nvfp4 -# suite: nightly-4-gpu-gb300-glm5-nvfp4 -# - model: qwen35-fp8 -# suite: nightly-4-gpu-gb300-qwen35-fp8 -# - model: deepseek-v4-pro-fp4 -# suite: nightly-4-gpu-gb300-deepseek-v4-pro-fp4 -# - model: kimi-k25-nvfp4 -# suite: nightly-4-gpu-gb300-kimi-k25-nvfp4 -# steps: -# - name: Checkout code -# uses: actions/checkout@v4 -# with: -# ref: ${{ inputs.ref || github.ref }} -# -# - uses: ./.github/actions/check-maintenance -# -# - name: Install dependencies -# env: -# GRACE_BLACKWELL: "1" -# run: | -# bash scripts/ci/cuda/ci_install_deepep.sh -# -# - name: Run test -# timeout-minutes: 600 -# env: -# TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} -# PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} -# GPU_CONFIG: "4-gpu-gb300" -# run: | -# cd test -# python3 run_suite.py --hw cuda --suite ${{ matrix.suite }} --nightly --continue-on-error --timeout-per-file 7200 -# -# - name: Publish traces to storage repo -# if: always() -# continue-on-error: true -# env: -# GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} -# GITHUB_RUN_ID: ${{ github.run_id }} -# GITHUB_RUN_NUMBER: ${{ github.run_number }} -# run: | -# TRACE_ARGS="" -# for dir in test/performance_profiles_*/; do -# [ -d "$dir" ] && TRACE_ARGS="$TRACE_ARGS --traces-dir $dir" -# done -# if [ -n "$TRACE_ARGS" ]; then -# python3 scripts/ci/utils/publish_traces.py $TRACE_ARGS -# find test/performance_profiles_*/ -name '*.json.gz' -delete -# else -# echo "No trace directories found, skipping publish" -# fi -# -# - uses: ./.github/actions/upload-cuda-coredumps -# if: failure() -# + nightly-test-perf-4-gpu-gb300: + if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-4-gpu-gb300') + name: nightly-test-perf-4-gpu-gb300 (${{ matrix.model }}) + runs-on: 4-gpu-gb300-nightly + strategy: + fail-fast: false + matrix: + include: + - model: glm5-nvfp4 + suite: nightly-4-gpu-gb300-glm5-nvfp4 + - model: qwen35-fp8 + suite: nightly-4-gpu-gb300-qwen35-fp8 + - model: deepseek-v4-pro-fp4 + suite: nightly-4-gpu-gb300-deepseek-v4-pro-fp4 + - model: kimi-k25-nvfp4 + suite: nightly-4-gpu-gb300-kimi-k25-nvfp4 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - uses: ./.github/actions/check-maintenance + + - name: Install dependencies + env: + GRACE_BLACKWELL: "1" + run: | + bash scripts/ci/cuda/ci_install_deepep.sh + + - name: Run test + timeout-minutes: 600 + env: + TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} + PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} + GPU_CONFIG: "4-gpu-gb300" + run: | + cd test + python3 run_suite.py --hw cuda --suite ${{ matrix.suite }} --nightly --continue-on-error --timeout-per-file 7200 + + - name: Publish traces to storage repo + if: always() + continue-on-error: true + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} + run: | + TRACE_ARGS="" + for dir in test/performance_profiles_*/; do + [ -d "$dir" ] && TRACE_ARGS="$TRACE_ARGS --traces-dir $dir" + done + if [ -n "$TRACE_ARGS" ]; then + python3 scripts/ci/utils/publish_traces.py $TRACE_ARGS + find test/performance_profiles_*/ -name '*.json.gz' -delete + else + echo "No trace directories found, skipping publish" + fi + + - uses: ./.github/actions/upload-cuda-coredumps + if: failure() + # Specialized B200 tests - 8 GPU, for specific backends and configs nightly-test-specialized-8-gpu-b200: if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-8-gpu-b200' || inputs.job_filter == 'nightly-test-specialized-8-gpu-b200') @@ -792,7 +791,7 @@ jobs: - nightly-test-vlm-perf-2-gpu-h100 - nightly-test-perf-4-gpu-b200 - nightly-test-specialized-8-gpu-b200 -# - nightly-test-perf-4-gpu-gb300 # temporarily disabled due to runner availability + - nightly-test-perf-4-gpu-gb300 - nightly-test-diffusion - nightly-test-precision-8-gpu-h200 - consolidate-metrics diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index f1b7e99f8..12b943488 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -520,20 +520,19 @@ jobs: timeout_per_file: '1800' secrets: inherit - # temporarily disabled due to runner availability -# base-c-test-4-gpu-gb300: -# needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels] -# if: ${{ !failure() && !cancelled() }} -# uses: ./.github/workflows/_pr-test-stage.yml -# with: -# self_name: base-c-test-4-gpu-gb300 -# runner_config: 4-gpu-gb300 -# check_changes: ${{ toJson(needs.check-changes.outputs) }} -# caller_inputs: ${{ toJson(inputs) }} -# partitions: ${{ needs.check-changes.outputs.partitions }} -# run_timeout_minutes: '30' -# timeout_per_file: '1800' -# secrets: inherit + base-c-test-4-gpu-gb300: + needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels] + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/_pr-test-stage.yml + with: + self_name: base-c-test-4-gpu-gb300 + runner_config: 4-gpu-gb300 + check_changes: ${{ toJson(needs.check-changes.outputs) }} + caller_inputs: ${{ toJson(inputs) }} + partitions: ${{ needs.check-changes.outputs.partitions }} + run_timeout_minutes: '30' + timeout_per_file: '1800' + secrets: inherit pr-test-finish: needs: @@ -565,7 +564,7 @@ jobs: base-c-test-deepep-4-gpu-b200, base-c-test-deepep-8-gpu-h200, base-c-test-4-gpu-b200, -# base-c-test-4-gpu-gb300, # temporarily disabled due to runner availability + base-c-test-4-gpu-gb300, ] if: always() runs-on: ubuntu-latest diff --git a/.github/workflows/release-whl-deepgemm.yml b/.github/workflows/release-whl-deepgemm.yml index a55519561..3d59ffc00 100644 --- a/.github/workflows/release-whl-deepgemm.yml +++ b/.github/workflows/release-whl-deepgemm.yml @@ -242,10 +242,9 @@ jobs: # - arch_label: sm120 # runner: 1-gpu-5090 # wheel_arch: x86_64 - # temporarily disabled due to runner availability -# - arch_label: sm100-aarch64 -# runner: 4-gpu-gb300 -# wheel_arch: aarch64 + - arch_label: sm100-aarch64 + runner: 4-gpu-gb300 + wheel_arch: aarch64 runs-on: ${{ matrix.runner }} timeout-minutes: 120 steps: