From 09b1d10d59387659f0d6d78058575717b109ad73 Mon Sep 17 00:00:00 2001
From: Bingxu Chen <bingxche@amd.com>
Date: Tue, 21 Apr 2026 15:58:23 +0800
Subject: [PATCH] [AMD] prepare for MI300x PR runner pool: registry mirror,
 runner routing, threshold tuning (#23156)

---
 .../workflows/nightly-test-amd-rocm720.yml    |  2 +
 .github/workflows/nightly-test-amd.yml        |  2 +
 .github/workflows/pr-test-amd-rocm720.yml     |  2 +
 .github/workflows/pr-test-amd.yml             | 97 +++++++++----------
 scripts/ci/amd/amd_ci_start_container.sh      | 38 ++++----
 .../ci/amd/amd_ci_start_container_disagg.sh   | 34 +++----
 .../registered/amd/test_deepseek_v32_basic.py |  2 +-
 test/registered/amd/test_deepseek_v3_mtp.py   |  3 +-
 test/registered/amd/test_moriep_small.py      |  2 +-
 test/registered/moe/test_torch_compile_moe.py |  4 +
 .../perf/test_bench_serving_1gpu_part1.py     | 18 ++--
 .../perf/test_bench_serving_1gpu_part2.py     | 47 ++++++---
 .../sampling/test_pytorch_sampling_backend.py |  7 ++
 .../tokenizer/test_multi_tokenizer.py         |  4 +-
 14 files changed, 145 insertions(+), 117 deletions(-)

diff --git a/.github/workflows/nightly-test-amd-rocm720.yml b/.github/workflows/nightly-test-amd-rocm720.yml
index 5aae11707..a7744d845 100644
--- a/.github/workflows/nightly-test-amd-rocm720.yml
+++ b/.github/workflows/nightly-test-amd-rocm720.yml
@@ -92,6 +92,8 @@ on:
 
 env:
   AITER_COMMIT_OVERRIDE: ${{ inputs.aiter_ref }}
+  DOCKERHUB_AMD_USERNAME: ${{ secrets.DOCKERHUB_AMD_USERNAME }}
+  DOCKERHUB_AMD_TOKEN: ${{ secrets.DOCKERHUB_AMD_TOKEN }}
 
 concurrency:
   # When called via workflow_call with ref set, use a unique group per caller run to avoid
diff --git a/.github/workflows/nightly-test-amd.yml b/.github/workflows/nightly-test-amd.yml
index 724c9ae88..778c838c1 100644
--- a/.github/workflows/nightly-test-amd.yml
+++ b/.github/workflows/nightly-test-amd.yml
@@ -92,6 +92,8 @@ on:
 
 env:
   AITER_COMMIT_OVERRIDE: ${{ inputs.aiter_ref }}
+  DOCKERHUB_AMD_USERNAME: ${{ secrets.DOCKERHUB_AMD_USERNAME }}
+  DOCKERHUB_AMD_TOKEN: ${{ secrets.DOCKERHUB_AMD_TOKEN }}
 
 concurrency:
   # When called via workflow_call with ref set, use a unique group per caller run to avoid
diff --git a/.github/workflows/pr-test-amd-rocm720.yml b/.github/workflows/pr-test-amd-rocm720.yml
index 4c7a75dad..36893b5cd 100644
--- a/.github/workflows/pr-test-amd-rocm720.yml
+++ b/.github/workflows/pr-test-amd-rocm720.yml
@@ -93,6 +93,8 @@ on:
 
 env:
   AITER_COMMIT_OVERRIDE: ${{ inputs.aiter_ref }}
+  DOCKERHUB_AMD_USERNAME: ${{ secrets.DOCKERHUB_AMD_USERNAME }}
+  DOCKERHUB_AMD_TOKEN: ${{ secrets.DOCKERHUB_AMD_TOKEN }}
 
 concurrency:
   # When called via workflow_call with run_all_tests=true, use a unique group per run to
diff --git a/.github/workflows/pr-test-amd.yml b/.github/workflows/pr-test-amd.yml
index cdf64bded..0a0aab374 100644
--- a/.github/workflows/pr-test-amd.yml
+++ b/.github/workflows/pr-test-amd.yml
@@ -59,6 +59,14 @@ on:
         required: false
         type: boolean
         default: false
+      runner_arch:
+        description: 'AMD runner pool to dispatch GPU jobs to'
+        required: false
+        type: choice
+        default: mi300
+        options:
+          - mi300
+          - mi325
   workflow_call:
     inputs:
       ref:
@@ -84,6 +92,8 @@ on:
 
 env:
   AITER_COMMIT_OVERRIDE: ${{ inputs.aiter_ref }}
+  DOCKERHUB_AMD_USERNAME: ${{ secrets.DOCKERHUB_AMD_USERNAME }}
+  DOCKERHUB_AMD_TOKEN: ${{ secrets.DOCKERHUB_AMD_TOKEN }}
 
 concurrency:
   # Scheduled and run_all_tests runs get unique groups (never cancel each other).
@@ -165,6 +175,7 @@ jobs:
 
   # =============================================== sgl-kernel ====================================================
   sgl-kernel-unit-test-amd:
+    name: ${{ format('sgl-kernel-unit-test-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
     needs: [check-changes, call-gate]
     if: |
       always() && !cancelled() &&
@@ -176,11 +187,7 @@ jobs:
           needs.check-changes.outputs.sgl_kernel == 'true'
         )
       )
-    strategy:
-      fail-fast: false
-      matrix:
-        runner: [linux-mi325-1gpu-sglang]
-    runs-on: ${{matrix.runner}}
+    runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -226,6 +233,7 @@ jobs:
           exit $failures
 
   sgl-kernel-unit-test-2-gpu-amd:
+    name: ${{ format('sgl-kernel-unit-test-2-gpu-amd (linux-{0}-2gpu-sglang)', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
     needs: [check-changes, call-gate]
     if: |
       always() && !cancelled() &&
@@ -237,11 +245,7 @@ jobs:
           needs.check-changes.outputs.sgl_kernel == 'true'
         )
       )
-    strategy:
-      fail-fast: false
-      matrix:
-        runner: [linux-mi325-2gpu-sglang]
-    runs-on: ${{matrix.runner}}
+    runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -280,6 +284,7 @@ jobs:
   # =============================================== primary ====================================================
 
   stage-a-test-1-gpu-small-amd:
+    name: ${{ format('stage-a-test-1-gpu-small-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
     needs: [check-changes, call-gate]
     if: |
       always() && !cancelled() &&
@@ -291,11 +296,7 @@ jobs:
           ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
         )
       )
-    strategy:
-      fail-fast: false
-      matrix:
-        runner: [linux-mi325-1gpu-sglang]
-    runs-on: ${{matrix.runner}}
+    runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -320,6 +321,7 @@ jobs:
           bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-a-test-1-gpu-small-amd ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
 
   jit-kernel-unit-test-amd:
+    name: ${{ format('jit-kernel-unit-test-amd (linux-{0}-1gpu-sglang)', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
     needs: [check-changes, call-gate]
     if: |
       always() && !cancelled() &&
@@ -331,11 +333,7 @@ jobs:
           needs.check-changes.outputs.jit_kernel == 'true'
         )
       )
-    strategy:
-      fail-fast: false
-      matrix:
-        runner: [linux-mi325-1gpu-sglang]
-    runs-on: ${{matrix.runner}}
+    runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -386,6 +384,7 @@ jobs:
           max-wait-minutes: '240'
 
   stage-b-test-1-gpu-small-amd:
+    name: ${{ format('stage-b-test-1-gpu-small-amd (linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325'), matrix.part) }}
     needs: [check-changes, wait-for-stage-a-amd]
     if: |
       always() &&
@@ -400,9 +399,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        runner: [linux-mi325-1gpu-sglang]
         part: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
-    runs-on: ${{matrix.runner}}
+    runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -421,11 +419,12 @@ jobs:
         run: bash scripts/ci/amd/amd_ci_install_dependency.sh
 
       - name: Run test
-        timeout-minutes: 30
+        timeout-minutes: 45
         run: |
           bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 14 --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
 
   stage-b-test-1-gpu-small-amd-nondeterministic:
+    name: ${{ format('stage-b-test-1-gpu-small-amd-nondeterministic (linux-{0}-1gpu-sglang)', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
     needs: [check-changes, wait-for-stage-a-amd]
     if: |
       always() &&
@@ -437,11 +436,7 @@ jobs:
           ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true'))
         )
       )
-    strategy:
-      fail-fast: false
-      matrix:
-        runner: [linux-mi325-1gpu-sglang]
-    runs-on: ${{matrix.runner}}
+    runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -460,7 +455,7 @@ jobs:
         run: bash scripts/ci/amd/amd_ci_install_dependency.sh
 
       - name: Run test
-        timeout-minutes: 30
+        timeout-minutes: 45
         run: |
           bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-nondeterministic --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
 
@@ -504,6 +499,7 @@ jobs:
           bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-small-amd-mi35x ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
 
   stage-b-test-1-gpu-large-amd:
+    name: ${{ format('stage-b-test-1-gpu-large-amd (linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325'), matrix.part) }}
     needs: [check-changes, wait-for-stage-a-amd]
     if: |
       always() &&
@@ -518,9 +514,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        runner: [linux-mi325-1gpu-sglang]
         part: [0, 1, 2]
-    runs-on: ${{matrix.runner}}
+    runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -539,11 +534,12 @@ jobs:
         run: bash scripts/ci/amd/amd_ci_install_dependency.sh
 
       - name: Run test
-        timeout-minutes: 30
+        timeout-minutes: 45
         run: |
-          bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
+          bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-1-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 2700 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
 
   stage-b-test-2-gpu-large-amd:
+    name: ${{ format('stage-b-test-2-gpu-large-amd (linux-{0}-2gpu-sglang, {1})', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325'), matrix.part) }}
     needs: [check-changes, wait-for-stage-a-amd]
     if: |
       always() &&
@@ -558,9 +554,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        runner: [linux-mi325-2gpu-sglang]
         part: [0, 1]
-    runs-on: ${{matrix.runner}}
+    runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -579,11 +574,12 @@ jobs:
         run: bash scripts/ci/amd/amd_ci_install_dependency.sh
 
       - name: Run test
-        timeout-minutes: 30
+        timeout-minutes: 45
         run: |
-          bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-2-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
+          bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-2-gpu-large-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 --timeout-per-file 2700 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
 
   multimodal-gen-test-1-gpu-amd:
+    name: ${{ format('multimodal-gen-test-1-gpu-amd (linux-{0}-1gpu-sglang, {1})', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325'), matrix.part) }}
     needs: [check-changes, call-gate]
     if: |
       always() && !cancelled() &&
@@ -599,9 +595,8 @@ jobs:
       fail-fast: false
       max-parallel: 1  # Run one at a time to avoid eviction from resource exhaustion during AITER kernel JIT
       matrix:
-        runner: [linux-mi325-1gpu-sglang]
-        part: [0, 1, 2, 3]  # 2 partitions: 11 tests ÷ 2 = ~5-6 tests each
-    runs-on: ${{matrix.runner}}
+        part: [0, 1, 2, 3]
+    runs-on: ${{ format('linux-{0}-1gpu-sglang', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -715,6 +710,7 @@ jobs:
           free -h
 
   multimodal-gen-test-2-gpu-amd:
+    name: ${{ format('multimodal-gen-test-2-gpu-amd (linux-{0}-2gpu-sglang, {1})', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325'), matrix.part) }}
     needs: [check-changes, call-gate]
     if: |
       always() && !cancelled() &&
@@ -730,9 +726,8 @@ jobs:
       fail-fast: false
       max-parallel: 1  # Run one at a time to avoid eviction from resource exhaustion during AITER kernel JIT
       matrix:
-        runner: [linux-mi325-2gpu-sglang]
         part: [0, 1, 2]  # 3 partitions: 2 parametrized + 1 standalone (test_disagg_server.py)
-    runs-on: ${{matrix.runner}}
+    runs-on: ${{ format('linux-{0}-2gpu-sglang', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -812,7 +807,7 @@ jobs:
           docker exec ci_sglang rocm-smi --showmeminfo vram 2>/dev/null || echo "rocm-smi not available"
 
       - name: Run diffusion server tests (2-GPU)
-        timeout-minutes: 80
+        timeout-minutes: 90
         run: |
           # AMD CI: All 2-GPU tests including LoRA
           # Tests: T2V, T2I, I2V, LoRA
@@ -871,6 +866,7 @@ jobs:
           max-wait-minutes: '480'
 
   stage-c-test-4-gpu-amd:
+    name: ${{ format('stage-c-test-4-gpu-amd (linux-{0}-4gpu-sglang, {1})', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325'), matrix.part) }}
     needs: [check-changes, call-gate, wait-for-stage-b-amd]
     if: |
       always() &&
@@ -885,9 +881,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        runner: [linux-mi325-4gpu-sglang]
         part: [0]
-    runs-on: ${{matrix.runner}}
+    runs-on: ${{ format('linux-{0}-4gpu-sglang', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -906,7 +901,7 @@ jobs:
         run: bash scripts/ci/amd/amd_ci_install_dependency.sh
 
       - name: Run test
-        timeout-minutes: 60
+        timeout-minutes: 90
         run: |
           bash scripts/ci/amd/amd_ci_exec.sh \
             -e NCCL_CUMEM_ENABLE=0 \
@@ -919,7 +914,7 @@ jobs:
               --suite stage-c-test-4-gpu-amd \
               --auto-partition-id ${{ matrix.part }} \
               --auto-partition-size 1 \
-              --timeout-per-file 1800 \
+              --timeout-per-file 5400 \
               --enable-retry \
               --max-attempts 2 \
               --retry-wait-seconds 120 \
@@ -927,6 +922,7 @@ jobs:
               ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
 
   stage-c-test-large-8-gpu-amd:
+    name: ${{ format('stage-c-test-large-8-gpu-amd (linux-{0}-8gpu-sglang, {1})', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325'), matrix.part) }}
     needs: [check-changes, call-gate, wait-for-stage-b-amd]
     if: |
       always() &&
@@ -939,13 +935,12 @@ jobs:
         )
       )
     env:
-      RUNNER_LABELS: linux-mi325-8gpu-sglang
+      RUNNER_LABELS: ${{ format('linux-{0}-8gpu-sglang', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
     strategy:
       fail-fast: false
       matrix:
-        runner: [linux-mi325-8gpu-sglang]
         part: [0, 1, 2]
-    runs-on: ${{matrix.runner}}
+    runs-on: ${{ format('linux-{0}-8gpu-sglang', inputs.runner_arch || (github.event_name == 'pull_request' && 'mi300' || 'mi325')) }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -970,7 +965,7 @@ jobs:
           docker exec ci_sglang bash -c "cd /sglang-checkout && NCCL_DEBUG=INFO RCCL_DEBUG=INFO torchrun --nproc_per_node=8 scripts/ci/amd/test_rccl_multi_gpu.py"
 
       - name: Run test
-        timeout-minutes: 60
+        timeout-minutes: 120
         run: |
           bash scripts/ci/amd/amd_ci_exec.sh -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-c-test-large-8-gpu-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 3 --timeout-per-file 3600 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }}
 
diff --git a/scripts/ci/amd/amd_ci_start_container.sh b/scripts/ci/amd/amd_ci_start_container.sh
index 9cc733884..74fba8886 100755
--- a/scripts/ci/amd/amd_ci_start_container.sh
+++ b/scripts/ci/amd/amd_ci_start_container.sh
@@ -154,18 +154,15 @@ find_latest_image() {
     fi
   done
 
-  # Then try the local registry.
-  for days_back in {0..6}; do
-    image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
-    echo "Checking for image: ${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" >&2
-    if docker manifest inspect --insecure "${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" >/dev/null 2>&1; then
-      echo "Found available image: ${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" >&2
-      echo "${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}"
-      return 0
-    fi
-  done
-
-  # Finally, try the public registry.
+  # If not found locally, fall back to pulling from public registry.
+  # We intentionally do not probe ${LOCAL_DOCKER_REGISTRY} here with
+  # `docker manifest inspect --insecure` because that command runs in the
+  # runner pod's network namespace, which on every observed AMD scale set
+  # cannot reach 172.29.8.23:5000 (every probe either fast-fails with TLS
+  # reject or hits a 30s TCP timeout, multiplied across 7 daily candidates).
+  # The actual local-registry pull still happens in the call site below via
+  # `docker pull "${LOCAL_DOCKER_REGISTRY}/${IMAGE}"`, which goes through the
+  # docker daemon on the host and inherits its insecure-registries config.
   for days_back in {0..6}; do
     image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
     echo "Checking for image: rocm/sgl-dev:${image_tag}" >&2
@@ -260,14 +257,17 @@ elif [[ -n "${BUILD_FROM_DOCKERFILE}" ]]; then
 else
   # Find the latest pre-built image
   IMAGE=$(find_latest_image "${GPU_ARCH}")
-  echo "Pulling Docker image: ${IMAGE}"
-  if [[ "${IMAGE}" == "${LOCAL_DOCKER_REGISTRY}/"* ]]; then
-    # Local registry is on-LAN; no need to retry.
-    docker pull "${IMAGE}"
-    docker tag "${IMAGE}" "${IMAGE#${LOCAL_DOCKER_REGISTRY}/}"
-    IMAGE="${IMAGE#${LOCAL_DOCKER_REGISTRY}/}"
+  # Try the local docker registry first (avoids Docker Hub rate limits and is
+  # faster on the LAN); if that fails for any reason, fall back to the
+  # public registry with exponential-backoff retries. Capture stderr so the
+  # real failure reason (TLS handshake, 404, connection refused, etc.) is
+  # visible in the job log instead of being silently swallowed.
+  if local_pull_output=$(docker pull "${LOCAL_DOCKER_REGISTRY}/${IMAGE}" 2>&1); then
+    echo "Pulled from local docker registry: ${LOCAL_DOCKER_REGISTRY}/${IMAGE}"
+    docker tag "${LOCAL_DOCKER_REGISTRY}/${IMAGE}" "${IMAGE}"
   else
-    # Public registry pulls can hit rate limits; retry with backoff.
+    echo "Local docker registry pull failed; falling back to public registry: ${IMAGE}" >&2
+    printf '%s\n' "${local_pull_output}" | sed 's/^/  [local-pull] /' >&2
     retry_with_backoff 6 docker pull "${IMAGE}"
   fi
 fi
diff --git a/scripts/ci/amd/amd_ci_start_container_disagg.sh b/scripts/ci/amd/amd_ci_start_container_disagg.sh
index 852114482..e2796fb95 100755
--- a/scripts/ci/amd/amd_ci_start_container_disagg.sh
+++ b/scripts/ci/amd/amd_ci_start_container_disagg.sh
@@ -141,18 +141,11 @@ find_latest_image() {
     fi
   done
 
-  # Then try the local registry.
-  for days_back in {0..6}; do
-    image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
-    echo "Checking for image: ${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" >&2
-    if docker manifest inspect --insecure "${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" >/dev/null 2>&1; then
-      echo "Found available image: ${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" >&2
-      echo "${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}"
-      return 0
-    fi
-  done
-
-  # Finally, try the public registry.
+  # If not found locally, fall back to pulling from public registry.
+  # See amd_ci_start_container.sh for why we don't probe
+  # ${LOCAL_DOCKER_REGISTRY} with `docker manifest inspect --insecure` from
+  # the runner pod's network namespace; the actual local-registry pull
+  # happens at the call site below via the docker daemon on the host.
   for days_back in {0..6}; do
     image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
     echo "Checking for image: rocm/sgl-dev:${image_tag}" >&2
@@ -210,14 +203,17 @@ find_latest_image() {
 
 # Pull and run the latest image
 IMAGE=$(find_latest_image "${GPU_ARCH}")
-echo "Pulling Docker image: ${IMAGE}"
-if [[ "${IMAGE}" == "${LOCAL_DOCKER_REGISTRY}/"* ]]; then
-  # Local registry is on-LAN; no need to retry.
-  docker pull "${IMAGE}"
-  docker tag "${IMAGE}" "${IMAGE#${LOCAL_DOCKER_REGISTRY}/}"
-  IMAGE="${IMAGE#${LOCAL_DOCKER_REGISTRY}/}"
+# Try the local docker registry first (avoids Docker Hub rate limits and is
+# faster on the LAN); if that fails for any reason, fall back to the
+# public registry with exponential-backoff retries. Capture stderr so the
+# real failure reason (TLS handshake, 404, connection refused, etc.) is
+# visible in the job log instead of being silently swallowed.
+if local_pull_output=$(docker pull "${LOCAL_DOCKER_REGISTRY}/${IMAGE}" 2>&1); then
+  echo "Pulled from local docker registry: ${LOCAL_DOCKER_REGISTRY}/${IMAGE}"
+  docker tag "${LOCAL_DOCKER_REGISTRY}/${IMAGE}" "${IMAGE}"
 else
-  # Public registry pulls can hit rate limits; retry with backoff.
+  echo "Local docker registry pull failed; falling back to public registry: ${IMAGE}" >&2
+  printf '%s\n' "${local_pull_output}" | sed 's/^/  [local-pull] /' >&2
   retry_with_backoff 6 docker pull "${IMAGE}"
 fi
 
diff --git a/test/registered/amd/test_deepseek_v32_basic.py b/test/registered/amd/test_deepseek_v32_basic.py
index cd849897e..27d5b0fc1 100644
--- a/test/registered/amd/test_deepseek_v32_basic.py
+++ b/test/registered/amd/test_deepseek_v32_basic.py
@@ -15,7 +15,7 @@ from sglang.test.test_utils import (
     write_github_step_summary,
 )
 
-register_amd_ci(est_time=1800, suite="stage-c-test-large-8-gpu-amd")
+register_amd_ci(est_time=3600, suite="stage-c-test-large-8-gpu-amd")
 
 DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2"
 
diff --git a/test/registered/amd/test_deepseek_v3_mtp.py b/test/registered/amd/test_deepseek_v3_mtp.py
index 0a9f94090..38631562c 100644
--- a/test/registered/amd/test_deepseek_v3_mtp.py
+++ b/test/registered/amd/test_deepseek_v3_mtp.py
@@ -84,7 +84,8 @@ class TestDeepseekV3MTP(CustomTestCase):
                 f'{metrics["accuracy"]=:.3f}\n'
                 f"{avg_spec_accept_length=:.2f}\n"
             )
-            self.assertGreater(metrics["accuracy"], 0.935)
+            # relax for mi300x
+            self.assertGreaterEqual(metrics["accuracy"], 0.93)
             if is_in_amd_ci():
                 self.assertGreater(avg_spec_accept_length, 2.8)
             else:
diff --git a/test/registered/amd/test_moriep_small.py b/test/registered/amd/test_moriep_small.py
index 1ffc27576..42547cf9e 100644
--- a/test/registered/amd/test_moriep_small.py
+++ b/test/registered/amd/test_moriep_small.py
@@ -60,7 +60,7 @@ common_args = [
     "1",
     "--enable-dp-lm-head",
     "--mem-fraction-static",
-    "0.6",
+    "0.72",  # relax for mi300x
     "--chunked-prefill-size",
     "32768",
     "--max-running-requests",
diff --git a/test/registered/moe/test_torch_compile_moe.py b/test/registered/moe/test_torch_compile_moe.py
index 0da421835..2175f0dc9 100644
--- a/test/registered/moe/test_torch_compile_moe.py
+++ b/test/registered/moe/test_torch_compile_moe.py
@@ -12,6 +12,7 @@ from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
+    is_in_amd_ci,
     popen_launch_server,
 )
 
@@ -73,6 +74,9 @@ class TestTorchCompileMoe(CustomTestCase):
         throughput = max_tokens / (tok - tic)
         if is_cuda():
             self.assertGreaterEqual(throughput, 285)
+        elif is_in_amd_ci():
+            # relax for mi300x
+            self.assertGreaterEqual(throughput, 240)
         else:
             self.assertGreaterEqual(throughput, 270)
 
diff --git a/test/registered/perf/test_bench_serving_1gpu_part1.py b/test/registered/perf/test_bench_serving_1gpu_part1.py
index fffbd5525..635593307 100644
--- a/test/registered/perf/test_bench_serving_1gpu_part1.py
+++ b/test/registered/perf/test_bench_serving_1gpu_part1.py
@@ -142,9 +142,6 @@ class TestBenchServing1GPUPart1(CustomTestCase):
             self.assertLess(res["median_itl_ms"], 10)
 
     def test_online_lora_latency(self):
-        if is_in_amd_ci():
-            pass
-
         res = self._run_lora_latency_test(enable_background_task=False)
 
         if is_in_ci():
@@ -154,12 +151,13 @@ class TestBenchServing1GPUPart1(CustomTestCase):
                 f"median_ttft_ms: {res['median_ttft_ms']:.2f} ms\n"
             )
             self.assertLess(res["median_e2e_latency_ms"], 2400)
-            self.assertLess(res["median_ttft_ms"], 58)
+            # relax for mi300x (LoRA TTFT ~2x slower than mi325)
+            if is_in_amd_ci():
+                self.assertLess(res["median_ttft_ms"], 100)
+            else:
+                self.assertLess(res["median_ttft_ms"], 58)
 
     def test_online_lora_latency_with_concurrent_adapter_updates(self):
-        if is_in_amd_ci():
-            pass
-
         res = self._run_lora_latency_test(enable_background_task=True)
 
         if is_in_ci():
@@ -169,7 +167,11 @@ class TestBenchServing1GPUPart1(CustomTestCase):
                 f"median_ttft_ms: {res['median_ttft_ms']:.2f} ms\n"
             )
             self.assertLess(res["median_e2e_latency_ms"], 4000)
-            self.assertLess(res["median_ttft_ms"], 80)
+            # relax for mi300x (LoRA TTFT ~2x slower than mi325)
+            if is_in_amd_ci():
+                self.assertLess(res["median_ttft_ms"], 130)
+            else:
+                self.assertLess(res["median_ttft_ms"], 80)
 
     def _run_lora_latency_test(self, enable_background_task: bool):
         """
diff --git a/test/registered/perf/test_bench_serving_1gpu_part2.py b/test/registered/perf/test_bench_serving_1gpu_part2.py
index 9f3725302..23eb51a22 100644
--- a/test/registered/perf/test_bench_serving_1gpu_part2.py
+++ b/test/registered/perf/test_bench_serving_1gpu_part2.py
@@ -41,8 +41,9 @@ class TestBenchServing1GPUPart2(CustomTestCase):
                 f"### test_vlm_offline_throughput\n"
                 f"Output throughput: {res['output_throughput']:.2f} token/s\n"
             )
+            # relax for mi300x
             if is_in_amd_ci():
-                self.assertGreater(res["output_throughput"], 2000)
+                self.assertGreater(res["output_throughput"], 900)
             else:
                 self.assertGreater(res["output_throughput"], 2500)
 
@@ -116,12 +117,16 @@ class TestBenchServing1GPUPart2(CustomTestCase):
                 )
 
             self.assertEqual(res["successful_requests"], res["total_requests"])
-            bounds = {
-                10: (45, 50),
-                25: (50, 60),
-                50: (60, 65),
-            }
-            avg_latency_bound, p95_latency_bound = bounds.get(batch_size, (60, 65))
+            # relax for mi300x
+            if is_in_amd_ci():
+                bounds = {10: (60, 65), 25: (70, 80), 50: (80, 90)}
+                default_bounds = (90, 90)
+            else:
+                bounds = {10: (45, 50), 25: (50, 60), 50: (60, 65)}
+                default_bounds = (60, 65)
+            avg_latency_bound, p95_latency_bound = bounds.get(
+                batch_size, default_bounds
+            )
             self.assertLess(res["avg_latency_ms"], avg_latency_bound)
             self.assertLess(res["p95_latency_ms"], p95_latency_bound)
 
@@ -146,9 +151,15 @@ class TestBenchServing1GPUPart2(CustomTestCase):
             )
 
         self.assertEqual(res["successful_requests"], res["total_requests"])
-        self.assertLess(res["avg_latency_ms"], 20)
-        self.assertLess(res["p95_latency_ms"], 25)
-        self.assertGreater(res["throughput"], 60)
+        # relax for mi300x
+        if is_in_amd_ci():
+            self.assertLess(res["avg_latency_ms"], 35)
+            self.assertLess(res["p95_latency_ms"], 40)
+            self.assertGreater(res["throughput"], 30)
+        else:
+            self.assertLess(res["avg_latency_ms"], 20)
+            self.assertLess(res["p95_latency_ms"], 25)
+            self.assertGreater(res["throughput"], 60)
 
     def test_embeddings_api_batch_scaling(self):
         """Test embeddings API performance with different batch sizes"""
@@ -173,12 +184,16 @@ class TestBenchServing1GPUPart2(CustomTestCase):
                 )
 
             self.assertEqual(res["successful_requests"], res["total_requests"])
-            bounds = {
-                10: (60, 65),
-                25: (115, 120),
-                50: (190, 195),
-            }
-            avg_latency_bound, p95_latency_bound = bounds.get(batch_size, (250, 250))
+            # relax for mi300x
+            if is_in_amd_ci():
+                bounds = {10: (80, 90), 25: (140, 150), 50: (230, 240)}
+                default_bounds = (300, 300)
+            else:
+                bounds = {10: (60, 65), 25: (115, 120), 50: (190, 195)}
+                default_bounds = (250, 250)
+            avg_latency_bound, p95_latency_bound = bounds.get(
+                batch_size, default_bounds
+            )
             self.assertLess(res["avg_latency_ms"], avg_latency_bound)
             self.assertLess(res["p95_latency_ms"], p95_latency_bound)
 
diff --git a/test/registered/sampling/test_pytorch_sampling_backend.py b/test/registered/sampling/test_pytorch_sampling_backend.py
index e53cb1df5..23e10e05b 100644
--- a/test/registered/sampling/test_pytorch_sampling_backend.py
+++ b/test/registered/sampling/test_pytorch_sampling_backend.py
@@ -11,6 +11,7 @@ from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
     CustomTestCase,
+    is_in_amd_ci,
     popen_launch_server,
 )
 
@@ -47,6 +48,12 @@ class TestPyTorchSamplingBackend(CustomTestCase):
         metrics = run_eval(args)
         self.assertGreaterEqual(metrics["score"], 0.65)
 
+    @unittest.skipIf(
+        is_in_amd_ci(),
+        "Skip on MI300x: greedy decode is not bit-exact across runs on MI300x "
+        "(kernel-level numerical jitter), so the assertEqual on identical "
+        "regenerated text is flaky on this runner pool.",
+    )
     def test_greedy(self):
 
         first_text = None
diff --git a/test/registered/tokenizer/test_multi_tokenizer.py b/test/registered/tokenizer/test_multi_tokenizer.py
index 3fb763159..caa0dd2cb 100644
--- a/test/registered/tokenizer/test_multi_tokenizer.py
+++ b/test/registered/tokenizer/test_multi_tokenizer.py
@@ -10,6 +10,7 @@ from sglang.test.test_utils import (
     CustomTestCase,
     auto_config_device,
     get_benchmark_args,
+    is_in_amd_ci,
     is_in_ci,
     popen_launch_server,
     run_benchmark,
@@ -70,7 +71,8 @@ class TestMultiTokenizer(CustomTestCase, MMLUMixin):
                 f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n"
             )
             self.assertLess(res["median_e2e_latency_ms"], 11000)
-            self.assertLess(res["median_ttft_ms"], 86)
+            # relax for mi300x
+            self.assertLess(res["median_ttft_ms"], 130 if is_in_amd_ci() else 86)
             self.assertLess(res["median_itl_ms"], 10)