diff --git a/.github/workflows/nightly-test-nvidia.yml b/.github/workflows/nightly-test-nvidia.yml index 5660007de..1ec8e85f7 100644 --- a/.github/workflows/nightly-test-nvidia.yml +++ b/.github/workflows/nightly-test-nvidia.yml @@ -506,10 +506,33 @@ jobs: - name: Run test timeout-minutes: 200 + env: + TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} + PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} + GPU_CONFIG: "4-gpu-b200" run: | cd test python3 run_suite.py --hw cuda --suite nightly-4-gpu-b200 --nightly --continue-on-error --timeout-per-file 12000 + - name: Publish traces to storage repo + if: always() + continue-on-error: true + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} + run: | + TRACE_ARGS="" + for dir in test/performance_profiles_*/; do + [ -d "$dir" ] && TRACE_ARGS="$TRACE_ARGS --traces-dir $dir" + done + if [ -n "$TRACE_ARGS" ]; then + python3 scripts/ci/utils/publish_traces.py $TRACE_ARGS + find test/performance_profiles_*/ -name '*.json.gz' -delete + else + echo "No trace directories found, skipping publish" + fi + - uses: ./.github/actions/upload-cuda-coredumps if: failure() @@ -531,10 +554,33 @@ jobs: - name: Run test timeout-minutes: 600 + env: + TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} + PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} + GPU_CONFIG: "4-gpu-gb300" run: | cd test python3 run_suite.py --hw cuda --suite nightly-4-gpu-gb300 --nightly --continue-on-error --timeout-per-file 7200 + - name: Publish traces to storage repo + if: always() + continue-on-error: true + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} + run: | + TRACE_ARGS="" + for dir in test/performance_profiles_*/; do + [ -d "$dir" ] && TRACE_ARGS="$TRACE_ARGS --traces-dir $dir" + done + if [ -n "$TRACE_ARGS" ]; then + python3 scripts/ci/utils/publish_traces.py $TRACE_ARGS + find test/performance_profiles_*/ -name '*.json.gz' -delete + else + echo "No trace directories found, skipping publish" + fi + - uses: ./.github/actions/upload-cuda-coredumps if: failure()