diff --git a/.github/workflows/nightly-test-nvidia.yml b/.github/workflows/nightly-test-nvidia.yml index 064a7b2ed..dc9ac1f2b 100644 --- a/.github/workflows/nightly-test-nvidia.yml +++ b/.github/workflows/nightly-test-nvidia.yml @@ -24,7 +24,7 @@ on: - 'nightly-test-perf-4-gpu-b200' - 'nightly-test-perf-8-gpu-b200' - 'nightly-test-specialized-8-gpu-b200' - - 'nightly-test-perf-4-gpu-gb300' +# - 'nightly-test-perf-4-gpu-gb300' # temporarily disabled due to runner availability - 'nightly-test-kernel-1-gpu-h100' - 'nightly-test-diffusion' - 'nightly-test-kernel-8-gpu-h200' @@ -544,68 +544,69 @@ jobs: if: failure() # GB300 (Grace-Blackwell NVL4) performance tests - 4 GPU (ARM64) - nightly-test-perf-4-gpu-gb300: - if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-4-gpu-gb300') - name: nightly-test-perf-4-gpu-gb300 (${{ matrix.model }}) - runs-on: 4-gpu-gb300-nightly - strategy: - fail-fast: false - matrix: - include: - - model: glm5-nvfp4 - suite: nightly-4-gpu-gb300-glm5-nvfp4 - - model: qwen35-fp8 - suite: nightly-4-gpu-gb300-qwen35-fp8 - - model: deepseek-v4-pro-fp4 - suite: nightly-4-gpu-gb300-deepseek-v4-pro-fp4 - - model: kimi-k25-nvfp4 - suite: nightly-4-gpu-gb300-kimi-k25-nvfp4 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref || github.ref }} - - - uses: ./.github/actions/check-maintenance - - - name: Install dependencies - env: - GRACE_BLACKWELL: "1" - run: | - bash scripts/ci/cuda/ci_install_deepep.sh - - - name: Run test - timeout-minutes: 600 - env: - TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} - PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} - GPU_CONFIG: "4-gpu-gb300" - run: | - cd test - python3 run_suite.py --hw cuda --suite ${{ matrix.suite }} --nightly --continue-on-error --timeout-per-file 7200 - - - name: Publish traces to storage repo - if: always() - continue-on-error: true - env: - GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} - GITHUB_RUN_ID: ${{ github.run_id }} - GITHUB_RUN_NUMBER: ${{ github.run_number }} - run: | - TRACE_ARGS="" - for dir in test/performance_profiles_*/; do - [ -d "$dir" ] && TRACE_ARGS="$TRACE_ARGS --traces-dir $dir" - done - if [ -n "$TRACE_ARGS" ]; then - python3 scripts/ci/utils/publish_traces.py $TRACE_ARGS - find test/performance_profiles_*/ -name '*.json.gz' -delete - else - echo "No trace directories found, skipping publish" - fi - - - uses: ./.github/actions/upload-cuda-coredumps - if: failure() - + # temporarily disabled due to runner availability +# nightly-test-perf-4-gpu-gb300: +# if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-4-gpu-gb300') +# name: nightly-test-perf-4-gpu-gb300 (${{ matrix.model }}) +# runs-on: 4-gpu-gb300-nightly +# strategy: +# fail-fast: false +# matrix: +# include: +# - model: glm5-nvfp4 +# suite: nightly-4-gpu-gb300-glm5-nvfp4 +# - model: qwen35-fp8 +# suite: nightly-4-gpu-gb300-qwen35-fp8 +# - model: deepseek-v4-pro-fp4 +# suite: nightly-4-gpu-gb300-deepseek-v4-pro-fp4 +# - model: kimi-k25-nvfp4 +# suite: nightly-4-gpu-gb300-kimi-k25-nvfp4 +# steps: +# - name: Checkout code +# uses: actions/checkout@v4 +# with: +# ref: ${{ inputs.ref || github.ref }} +# +# - uses: ./.github/actions/check-maintenance +# +# - name: Install dependencies +# env: +# GRACE_BLACKWELL: "1" +# run: | +# bash scripts/ci/cuda/ci_install_deepep.sh +# +# - name: Run test +# timeout-minutes: 600 +# env: +# TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} +# PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} +# GPU_CONFIG: "4-gpu-gb300" +# run: | +# cd test +# python3 run_suite.py --hw cuda --suite ${{ matrix.suite }} --nightly --continue-on-error --timeout-per-file 7200 +# +# - name: Publish traces to storage repo +# if: always() +# continue-on-error: true +# env: +# GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} +# GITHUB_RUN_ID: ${{ github.run_id }} +# GITHUB_RUN_NUMBER: ${{ github.run_number }} +# run: | +# TRACE_ARGS="" +# for dir in test/performance_profiles_*/; do +# [ -d "$dir" ] && TRACE_ARGS="$TRACE_ARGS --traces-dir $dir" +# done +# if [ -n "$TRACE_ARGS" ]; then +# python3 scripts/ci/utils/publish_traces.py $TRACE_ARGS +# find test/performance_profiles_*/ -name '*.json.gz' -delete +# else +# echo "No trace directories found, skipping publish" +# fi +# +# - uses: ./.github/actions/upload-cuda-coredumps +# if: failure() +# # Specialized B200 tests - 8 GPU, for specific backends and configs nightly-test-specialized-8-gpu-b200: if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-8-gpu-b200' || inputs.job_filter == 'nightly-test-specialized-8-gpu-b200') @@ -791,7 +792,7 @@ jobs: - nightly-test-vlm-perf-2-gpu-h100 - nightly-test-perf-4-gpu-b200 - nightly-test-specialized-8-gpu-b200 - - nightly-test-perf-4-gpu-gb300 +# - nightly-test-perf-4-gpu-gb300 # temporarily disabled due to runner availability - nightly-test-diffusion - nightly-test-precision-8-gpu-h200 - consolidate-metrics diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index d7261d566..e7ff56f62 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -520,19 +520,20 @@ jobs: timeout_per_file: '1800' secrets: inherit - base-c-test-4-gpu-gb300: - needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels] - if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/_pr-test-stage.yml - with: - self_name: base-c-test-4-gpu-gb300 - runner_config: 4-gpu-gb300 - check_changes: ${{ toJson(needs.check-changes.outputs) }} - caller_inputs: ${{ toJson(inputs) }} - partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '30' - timeout_per_file: '1800' - secrets: inherit + # temporarily disabled due to runner availability +# base-c-test-4-gpu-gb300: +# needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels] +# if: ${{ !failure() && !cancelled() }} +# uses: ./.github/workflows/_pr-test-stage.yml +# with: +# self_name: base-c-test-4-gpu-gb300 +# runner_config: 4-gpu-gb300 +# check_changes: ${{ toJson(needs.check-changes.outputs) }} +# caller_inputs: ${{ toJson(inputs) }} +# partitions: ${{ needs.check-changes.outputs.partitions }} +# run_timeout_minutes: '30' +# timeout_per_file: '1800' +# secrets: inherit pr-test-finish: needs: @@ -564,7 +565,7 @@ jobs: base-c-test-deepep-4-gpu-b200, base-c-test-deepep-8-gpu-h200, base-c-test-4-gpu-b200, - base-c-test-4-gpu-gb300, +# base-c-test-4-gpu-gb300, # temporarily disabled due to runner availability ] if: always() runs-on: ubuntu-latest diff --git a/.github/workflows/release-whl-deepgemm.yml b/.github/workflows/release-whl-deepgemm.yml index c0bddc930..6ce930240 100644 --- a/.github/workflows/release-whl-deepgemm.yml +++ b/.github/workflows/release-whl-deepgemm.yml @@ -238,9 +238,10 @@ jobs: - arch_label: sm100 runner: 8-gpu-b200 wheel_arch: x86_64 - - arch_label: sm100-aarch64 - runner: 4-gpu-gb300 - wheel_arch: aarch64 + # temporarily disabled due to runner availability +# - arch_label: sm100-aarch64 +# runner: 4-gpu-gb300 +# wheel_arch: aarch64 runs-on: ${{ matrix.runner }} timeout-minutes: 120 steps: