diff --git a/.github/workflows/nightly-72-gpu-gb200.yml b/.github/workflows/nightly-72-gpu-gb200.yml index ac9e28a01..51ad81fc6 100644 --- a/.github/workflows/nightly-72-gpu-gb200.yml +++ b/.github/workflows/nightly-72-gpu-gb200.yml @@ -287,8 +287,15 @@ jobs: OSL: ${{ matrix.config.osl }} CONFIG_FILE: ${{ matrix.config.config_file }} RESULT_FILENAME: gb200-${{ matrix.config.name }} + MATRIX_CONFIG_NAME: ${{ matrix.config.name }} SQUASH_FILE: ${{ needs.prepare-image.outputs.squash_file }} NGINX_SQUASH_FILE: ${{ needs.prepare-image.outputs.nginx_squash_file }} + # S3 log-upload credentials — consumed by srt-slurm's postprocess stage + # to upload /logs after each Slurm job; prefix derived in launch_gb200.sh. + AWS_ACCESS_KEY_ID: ${{ secrets.NV_S3_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.NV_S3_SECRET_ACCESS_KEY }} + S3_BUCKET: ${{ secrets.NV_S3_BUCKET }} + S3_ENDPOINT_URL: ${{ secrets.NV_S3_ENDPOINT_URL }} steps: - name: Checkout code @@ -361,6 +368,26 @@ jobs: echo "No log tarball found, skipping analysis" fi + - name: Upload AI analysis to S3 + if: failure() + continue-on-error: true + env: + ISL: ${{ matrix.config.isl }} + OSL: ${{ matrix.config.osl }} + run: | + ANALYSIS="${{ github.workspace }}/ai_analysis.md" + [ -f "$ANALYSIS" ] || { echo "no ai_analysis.md, skipping"; exit 0; } + case "${{ github.event_name }}" in + schedule) TRIGGER=cron ;; + workflow_dispatch) TRIGGER=manual ;; + *) TRIGGER="${{ github.event_name }}" ;; + esac + fmt() { if [ $(( $1 % 1024 )) -eq 0 ]; then echo "$(( $1 / 1024 ))k"; else echo "$1"; fi; } + SEQ_LEN="$(fmt "$ISL")$(fmt "$OSL")" + KEY="${TRIGGER}/${{ github.run_id }}-${{ github.run_attempt }}/${SEQ_LEN}/${{ matrix.config.name }}/ai_analysis.md" + aws --endpoint-url "$S3_ENDPOINT_URL" s3 cp "$ANALYSIS" "s3://${S3_BUCKET}/${KEY}" + echo "uploaded to s3://${S3_BUCKET}/${KEY}" + - name: Clean up Slurm jobs on failure/cancel if: failure() || cancelled() continue-on-error: true diff --git a/scripts/ci/slurm/launch_gb200.sh b/scripts/ci/slurm/launch_gb200.sh index 2f56878dc..0c514d0b8 100755 --- a/scripts/ci/slurm/launch_gb200.sh +++ b/scripts/ci/slurm/launch_gb200.sh @@ -17,6 +17,11 @@ # SLURM_ACCOUNT - Slurm account (default: sglang) # SRT_SLURM_BRANCH - branch of srt-slurm repo to check out # GITHUB_WORKSPACE - set automatically by GitHub Actions +# MATRIX_CONFIG_NAME- matrix entry name (e.g. dsr1-fp4-1k1k-mid-curve); used in S3 prefix +# S3_BUCKET - MinIO bucket for benchmark log uploads +# S3_ENDPOINT_URL - MinIO endpoint URL (e.g. https://minio..nip.io) +# AWS_ACCESS_KEY_ID - writer access key for S3_BUCKET (via GH secrets) +# AWS_SECRET_ACCESS_KEY - writer secret key for S3_BUCKET (via GH secrets) set -euo pipefail set -x @@ -90,6 +95,31 @@ fi # --------------------------------------------------------------------------- SRTCTL_ROOT="$SRT_REPO_DIR" +: "${S3_BUCKET:?S3_BUCKET must be set}" +: "${S3_ENDPOINT_URL:?S3_ENDPOINT_URL must be set}" +: "${AWS_ACCESS_KEY_ID:?AWS_ACCESS_KEY_ID must be set}" +: "${AWS_SECRET_ACCESS_KEY:?AWS_SECRET_ACCESS_KEY must be set}" +: "${MATRIX_CONFIG_NAME:?MATRIX_CONFIG_NAME must be set}" +: "${GITHUB_RUN_ID:?GITHUB_RUN_ID must be set}" +: "${GITHUB_RUN_ATTEMPT:?GITHUB_RUN_ATTEMPT must be set}" + +# Map the GitHub trigger into a friendlier top-level prefix: cron/manual. +case "${GITHUB_EVENT_NAME:-}" in + schedule) TRIGGER=cron ;; + workflow_dispatch) TRIGGER=manual ;; + *) TRIGGER="${GITHUB_EVENT_NAME:-unknown}" ;; +esac + +# Format ISL/OSL as "1k1k" / "1k8k" / "8k1k" etc. for the S3 prefix, so logs +# group naturally by sequence-length bucket under each run. +fmt_seq_len() { + local n=$1 + if (( n % 1024 == 0 )); then echo "$((n / 1024))k"; else echo "$n"; fi +} +SEQ_LEN="$(fmt_seq_len "$ISL")$(fmt_seq_len "$OSL")" + +S3_PREFIX="${TRIGGER}/${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}/${SEQ_LEN}/${MATRIX_CONFIG_NAME}" + cat > srtslurm.yaml <// after prefix. +reporting: + s3: + bucket: "${S3_BUCKET}" + prefix: "${S3_PREFIX}" + endpoint_url: "${S3_ENDPOINT_URL}" EOF echo "--- srtslurm.yaml ---" cat srtslurm.yaml +echo "--- S3 log upload: s3://${S3_BUCKET}/${S3_PREFIX}/ ---" make setup ARCH=aarch64