ci: dynamic partition + LPT from live sglang-ci-stats model (#25263)

.github/workflows/_pr-test-check-changes.yml

@@ -34,6 +34,8 @@ on:
         value: ${{ jobs.run.outputs.multimodal_gen }}
       partitions:
         value: ${{ jobs.run.outputs.partitions }}
+      partition_model_sha:
+        value: ${{ jobs.run.outputs.partition_model_sha }}
       b200_runner:
         value: ${{ jobs.run.outputs.b200_runner }}
       enable_retry:
@@ -60,6 +62,7 @@ jobs:
       jit_kernel: ${{ steps.filter-api.outputs.jit_kernel || steps.filter.outputs.jit_kernel || steps.run-mode.outputs.run_all_tests }}
       multimodal_gen: ${{ steps.filter-api.outputs.multimodal_gen || steps.filter.outputs.multimodal_gen || steps.run-mode.outputs.run_all_tests }}
       partitions: ${{ steps.partitions.outputs.partitions }}
+      partition_model_sha: ${{ steps.partition-model-sha.outputs.sha }}
       b200_runner: ${{ steps.set-runner.outputs.b200_runner }}
       enable_retry: ${{ steps.set-retry.outputs.enable_retry }}
       continue_on_error: ${{ steps.set-continue-on-error.outputs.continue_on_error }}
@@ -233,6 +236,33 @@ jobs:
           fi
           echo "full=$FULL" >> "$GITHUB_OUTPUT"
 
+      - name: Resolve sglang-ci-stats SHA
+        id: partition-model-sha
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          # Pin all shards to one immutable commit so dispatch and every
+          # runtime LPT use the same model snapshot. Soft fail -> static.
+          SHA=$(gh api repos/sgl-project/sglang-ci-stats/commits/main --jq '.sha' 2>/dev/null || true)
+          if [[ -n "$SHA" ]]; then
+            echo "Pinned sglang-ci-stats@$SHA"
+            echo "sha=$SHA" >> "$GITHUB_OUTPUT"
+          else
+            echo "::warning::Could not resolve sglang-ci-stats SHA; using in-source est_time"
+            echo "sha=" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Fetch live partition model
+        if: steps.partition-model-sha.outputs.sha != ''
+        run: |
+          # SHA resolved -> require fetch (curl --retry). Soft fallback
+          # would risk cross-shard LPT divergence on transient curl flake.
+          rm -f /tmp/partition-model.json
+          URL="https://raw.githubusercontent.com/sgl-project/sglang-ci-stats/${{ steps.partition-model-sha.outputs.sha }}/model.json"
+          curl --fail --silent --show-error --max-time 15 --retry 3 --retry-delay 2 \
+            "$URL" -o /tmp/partition-model.json
+          echo "Fetched partition-model.json ($(wc -c < /tmp/partition-model.json) bytes)"
+
       - name: Compute partitions
         id: partitions
         run: |
@@ -243,6 +273,7 @@ jobs:
           # See scripts/ci/utils/compute_partitions.py.
           python3 scripts/ci/utils/compute_partitions.py \
             --full-parallel ${{ steps.parallel-mode.outputs.full }} \
+            --partition-model-file /tmp/partition-model.json \
             >> "$GITHUB_OUTPUT"
 
       - name: Set B200 runner tag

.github/workflows/_pr-test-stage.yml

@@ -35,9 +35,9 @@ on:
         type: string
         required: true
       run_timeout_minutes:
-        description: 'timeout-minutes for the Run test step.'
+        description: 'timeout-minutes for the Run test step. Required so compute_partitions.py can read it from pr-test.yml without a duplicated default constant.'
         type: string
-        default: '30'
+        required: true
       timeout_per_file:
         description: 'run_suite.py --timeout-per-file value (empty = unset).'
         type: string
@@ -147,6 +147,16 @@ jobs:
           [ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/env.sh" ] && source "${SGLANG_CI_VENV_PATH}/env.sh"
           python3 scripts/ci/cuda/warmup_server.py ${{ inputs.warmup_server_models }}
 
+      - name: Fetch live partition model
+        if: fromJson(inputs.check_changes).partition_model_sha != ''
+        run: |
+          # SHA resolved by check-changes -> require fetch (curl --retry)
+          # so all shards stay on the same snapshot.
+          rm -f /tmp/partition-model.json
+          URL="https://raw.githubusercontent.com/sgl-project/sglang-ci-stats/${{ fromJson(inputs.check_changes).partition_model_sha }}/model.json"
+          curl --fail --silent --show-error --max-time 15 --retry 3 --retry-delay 2 \
+            "$URL" -o /tmp/partition-model.json
+
       - name: Run test
         timeout-minutes: ${{ fromJson(inputs.run_timeout_minutes) }}
         env:
@@ -156,6 +166,7 @@ jobs:
           python3 run_suite.py --hw cuda --suite ${{ inputs.self_name }} \
             --auto-partition-id ${{ matrix.partition }} \
             --auto-partition-size ${{ fromJson(inputs.partitions)[inputs.self_name].size }} \
+            --partition-model-file /tmp/partition-model.json \
             ${{ inputs.timeout_per_file && format('--timeout-per-file {0}', inputs.timeout_per_file) || '' }} \
             $CONTINUE_ON_ERROR_FLAG
 

.github/workflows/pr-test.yml

@@ -372,6 +372,7 @@ jobs:
       check_changes: ${{ toJson(needs.check-changes.outputs) }}
       caller_inputs: ${{ toJson(inputs) }}
       partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '30'
     secrets: inherit
 
   # Runs on H100 (80GB, SM90) - tests that don't pass on 5090 (FA3, FP8, high VRAM, etc.)
@@ -386,6 +387,7 @@ jobs:
       check_changes: ${{ toJson(needs.check-changes.outputs) }}
       caller_inputs: ${{ toJson(inputs) }}
       partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '30'
       timeout_per_file: '1800'
     secrets: inherit
 
@@ -400,6 +402,7 @@ jobs:
       check_changes: ${{ toJson(needs.check-changes.outputs) }}
       caller_inputs: ${{ toJson(inputs) }}
       partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '30'
     secrets: inherit
 
   stage-b-test-4-gpu-b200:
@@ -461,6 +464,7 @@ jobs:
       check_changes: ${{ toJson(needs.check-changes.outputs) }}
       caller_inputs: ${{ toJson(inputs) }}
       partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '30'
     secrets: inherit
 
   stage-c-test-8-gpu-h200:
@@ -474,6 +478,7 @@ jobs:
       check_changes: ${{ toJson(needs.check-changes.outputs) }}
       caller_inputs: ${{ toJson(inputs) }}
       partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '30'
       # Per-model TP must match the test's launch in test/registered/ -- see
       # FALLBACK_ARGS in scripts/ci/cuda/warmup_deep_gemm.py for extra dp/ep
       # flags. Only models that actually invoke DeepGEMM kernels at runtime
@@ -494,6 +499,7 @@ jobs:
       check_changes: ${{ toJson(needs.check-changes.outputs) }}
       caller_inputs: ${{ toJson(inputs) }}
       partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '30'
     secrets: inherit
 
   stage-c-test-deepep-4-gpu-h100:
@@ -507,6 +513,7 @@ jobs:
       check_changes: ${{ toJson(needs.check-changes.outputs) }}
       caller_inputs: ${{ toJson(inputs) }}
       partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '30'
       warmup_deep_gemm_models: 'lmsys/sglang-ci-dsv3-test:4'
       warmup_server_models: 'lmsys/sglang-ci-dsv3-test:4'
     secrets: inherit
@@ -538,6 +545,7 @@ jobs:
       check_changes: ${{ toJson(needs.check-changes.outputs) }}
       caller_inputs: ${{ toJson(inputs) }}
       partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '30'
       timeout_per_file: '1800'
     secrets: inherit
 
@@ -552,6 +560,7 @@ jobs:
       check_changes: ${{ toJson(needs.check-changes.outputs) }}
       caller_inputs: ${{ toJson(inputs) }}
       partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '45'
       timeout_per_file: '1800'
     secrets: inherit
 
@@ -566,6 +575,7 @@ jobs:
       check_changes: ${{ toJson(needs.check-changes.outputs) }}
       caller_inputs: ${{ toJson(inputs) }}
       partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '30'
       timeout_per_file: '1800'
     secrets: inherit