[AMD][DI][CI] 1/N: MI355X disaggregation nightly benchmark (#29084)

Co-authored-by: Michael <13900043+michaelzhang-ai@users.noreply.github.com>
Co-authored-by: bingxche <bingxche@amd.com>
Co-authored-by: yctseng0211 <yctseng@amd.com>

.github/workflows/nightly-amd-mi355x-disagg.yml

@@ -36,10 +36,34 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.generate.outputs.matrix }}
+      image: ${{ steps.image.outputs.image }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Resolve latest MI35x nightly image
+        id: image
+        # Track the newest published image so the nightly actually catches
+        # regressions instead of re-testing one pinned build. We resolve on the
+        # GitHub-hosted runner (clean Docker Hub access, no rate limits) and pass
+        # the tag to the self-hosted runner. Scope is deliberately tight: only
+        # the clean main MI35x ROCm 7.2 line
+        # `lmsysorg/sglang-rocm:v*-rocm720-mi35x-YYYYMMDD` (excludes -test-,
+        # miles-, mori-, vllm-, rocm7_14 variants). On any failure we emit an
+        # empty string and the launcher falls back to the recipe's pinned
+        # `runtime.image`.
+        run: |
+          set -o pipefail
+          RESOLVED=$(curl -fsSL "https://hub.docker.com/v2/repositories/lmsysorg/sglang-rocm/tags?page_size=100&ordering=last_updated" \
+            | python3 -c "import json,re,sys;p=re.compile(r'^v[0-9][0-9A-Za-z.]*-rocm720-mi35x-([0-9]{8})\$');c=sorted((m.group(1),t['name']) for t in json.load(sys.stdin).get('results',[]) for m in [p.match(t.get('name',''))] if m);print(c[-1][1] if c else '')" || true)
+          if [ -n "$RESOLVED" ]; then
+            echo "Resolved latest MI35x image: lmsysorg/sglang-rocm:$RESOLVED"
+            echo "image=lmsysorg/sglang-rocm:$RESOLVED" >> $GITHUB_OUTPUT
+          else
+            echo "Could not resolve a latest image; launcher will use recipe default."
+            echo "image=" >> $GITHUB_OUTPUT
+          fi
+
       - name: Generate benchmark matrix
         id: generate
         env:
@@ -83,8 +107,13 @@ jobs:
       CONFIG_FILE: ${{ matrix.config.config_file }}
       RESULT_FILENAME: mi355x-${{ matrix.config.name }}
       MATRIX_CONFIG_NAME: ${{ matrix.config.name }}
-      # Local snapshot on the cluster's shared NFS (preferred over downloading).
-      MODEL_PATH: /it-share/model_coverage/models--sgl-project--DeepSeek-V4-Flash-FP8/snapshots/ae01d80c06cdfe30581edfd0e1c5449dc7ed7f17
+      # NOTE: RUNNER_NAME is a built-in default env var on every runner, so the
+      # launch + cleanup steps read it directly. Do NOT set it here from
+      # ${{ runner.name }} -- the `runner` context is not available in job-level
+      # env and makes the whole workflow file invalid.
+      # Shared-NFS HuggingFace cache dir; the launcher resolves the live snapshot
+      # via refs/main. Per-model value comes from nightly-configs.yaml model_path.
+      MODEL_PATH: ${{ matrix.config.model_path }}
 
     steps:
       - name: Checkout code
@@ -93,16 +122,26 @@ jobs:
       - name: Clean up prior Slurm jobs from this runner
         continue-on-error: true
         run: |
-          STALE_JOBS=$(squeue --me --noheader --format="%i" || true)
+          # launch_mi355x.sh names the allocation
+          #   mi355x-ci-<RUNNER_NAME>-<GITHUB_RUN_ID>-<config>
+          # Here we clear THIS runner's leftovers from a crashed previous run, so
+          # we match the RUNNER_NAME prefix (older runs have a different run id).
+          # Never a blanket `squeue --me`, which would kill a concurrent leg.
+          # %200j: squeue truncates the job name (%j) by default -- widen it so
+          # the grep sees the full name.
+          if [ -z "${RUNNER_NAME:-}" ]; then echo "RUNNER_NAME unset; skipping"; exit 0; fi
+          STALE_JOBS=$(squeue --me --noheader --format="%i %200j" | grep -F "mi355x-ci-${RUNNER_NAME}-" | awk '{print $1}' || true)
           if [ -n "$STALE_JOBS" ]; then
-            echo "Cancelling stale jobs: $STALE_JOBS"
+            echo "Cancelling stale jobs for ${RUNNER_NAME}: $STALE_JOBS"
             scancel $STALE_JOBS
           fi
 
       - name: Launch MI355X 2N 1P1D benchmark
         timeout-minutes: 180
         env:
-          IMAGE_OVERRIDE: ${{ inputs.image }}
+          # Manual dispatch input wins; otherwise use the latest image resolved
+          # by the setup job; otherwise the launcher falls back to the recipe default.
+          IMAGE_OVERRIDE: ${{ inputs.image != '' && inputs.image || needs.setup.outputs.image }}
         run: bash scripts/ci/slurm/launch_mi355x.sh
 
       - name: Process results
@@ -133,9 +172,14 @@ jobs:
         if: failure() || cancelled()
         continue-on-error: true
         run: |
-          ACTIVE_JOBS=$(squeue --me --noheader --format="%i" || true)
+          # Cancel only THIS leg's allocation -- match the full job name
+          # mi355x-ci-<RUNNER_NAME>-<GITHUB_RUN_ID>-<config>. GITHUB_RUN_ID +
+          # config are unique per matrix leg, so a concurrent leg is never hit.
+          if [ -z "${RUNNER_NAME:-}" ]; then echo "RUNNER_NAME unset; skipping"; exit 0; fi
+          JOB_TAG="mi355x-ci-${RUNNER_NAME}-${GITHUB_RUN_ID}-${MATRIX_CONFIG_NAME}"
+          ACTIVE_JOBS=$(squeue --me --noheader --format="%i %200j" | grep -F "$JOB_TAG" | awk '{print $1}' || true)
           if [ -n "$ACTIVE_JOBS" ]; then
-            echo "Cancelling jobs: $ACTIVE_JOBS"
+            echo "Cancelling jobs for ${JOB_TAG}: $ACTIVE_JOBS"
             scancel $ACTIVE_JOBS
           fi