[CI] Publish 4-GPU nightly profiler traces (#28738)
This commit is contained in:
@@ -506,10 +506,33 @@ jobs:
|
|||||||
|
|
||||||
- name: Run test
|
- name: Run test
|
||||||
timeout-minutes: 200
|
timeout-minutes: 200
|
||||||
|
env:
|
||||||
|
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
|
||||||
|
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
|
||||||
|
GPU_CONFIG: "4-gpu-b200"
|
||||||
run: |
|
run: |
|
||||||
cd test
|
cd test
|
||||||
python3 run_suite.py --hw cuda --suite nightly-4-gpu-b200 --nightly --continue-on-error --timeout-per-file 12000
|
python3 run_suite.py --hw cuda --suite nightly-4-gpu-b200 --nightly --continue-on-error --timeout-per-file 12000
|
||||||
|
|
||||||
|
- name: Publish traces to storage repo
|
||||||
|
if: always()
|
||||||
|
continue-on-error: true
|
||||||
|
env:
|
||||||
|
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
|
||||||
|
GITHUB_RUN_ID: ${{ github.run_id }}
|
||||||
|
GITHUB_RUN_NUMBER: ${{ github.run_number }}
|
||||||
|
run: |
|
||||||
|
TRACE_ARGS=""
|
||||||
|
for dir in test/performance_profiles_*/; do
|
||||||
|
[ -d "$dir" ] && TRACE_ARGS="$TRACE_ARGS --traces-dir $dir"
|
||||||
|
done
|
||||||
|
if [ -n "$TRACE_ARGS" ]; then
|
||||||
|
python3 scripts/ci/utils/publish_traces.py $TRACE_ARGS
|
||||||
|
find test/performance_profiles_*/ -name '*.json.gz' -delete
|
||||||
|
else
|
||||||
|
echo "No trace directories found, skipping publish"
|
||||||
|
fi
|
||||||
|
|
||||||
- uses: ./.github/actions/upload-cuda-coredumps
|
- uses: ./.github/actions/upload-cuda-coredumps
|
||||||
if: failure()
|
if: failure()
|
||||||
|
|
||||||
@@ -531,10 +554,33 @@ jobs:
|
|||||||
|
|
||||||
- name: Run test
|
- name: Run test
|
||||||
timeout-minutes: 600
|
timeout-minutes: 600
|
||||||
|
env:
|
||||||
|
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
|
||||||
|
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
|
||||||
|
GPU_CONFIG: "4-gpu-gb300"
|
||||||
run: |
|
run: |
|
||||||
cd test
|
cd test
|
||||||
python3 run_suite.py --hw cuda --suite nightly-4-gpu-gb300 --nightly --continue-on-error --timeout-per-file 7200
|
python3 run_suite.py --hw cuda --suite nightly-4-gpu-gb300 --nightly --continue-on-error --timeout-per-file 7200
|
||||||
|
|
||||||
|
- name: Publish traces to storage repo
|
||||||
|
if: always()
|
||||||
|
continue-on-error: true
|
||||||
|
env:
|
||||||
|
GITHUB_TOKEN: ${{ secrets.GH_PAT_FOR_NIGHTLY_CI_DATA }}
|
||||||
|
GITHUB_RUN_ID: ${{ github.run_id }}
|
||||||
|
GITHUB_RUN_NUMBER: ${{ github.run_number }}
|
||||||
|
run: |
|
||||||
|
TRACE_ARGS=""
|
||||||
|
for dir in test/performance_profiles_*/; do
|
||||||
|
[ -d "$dir" ] && TRACE_ARGS="$TRACE_ARGS --traces-dir $dir"
|
||||||
|
done
|
||||||
|
if [ -n "$TRACE_ARGS" ]; then
|
||||||
|
python3 scripts/ci/utils/publish_traces.py $TRACE_ARGS
|
||||||
|
find test/performance_profiles_*/ -name '*.json.gz' -delete
|
||||||
|
else
|
||||||
|
echo "No trace directories found, skipping publish"
|
||||||
|
fi
|
||||||
|
|
||||||
- uses: ./.github/actions/upload-cuda-coredumps
|
- uses: ./.github/actions/upload-cuda-coredumps
|
||||||
if: failure()
|
if: failure()
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user