[AMD][CI] Fix ROCm 7.0's dead apt index fail the MORI dependency install (#35764)

Co-authored-by: Cursor Agent <cursoragent@cursor.com>
Co-authored-by: quitenode <quitenode@users.noreply.github.com>

scripts/ci/amd/amd_ci_install_dependency.sh

@@ -265,6 +265,15 @@ if docker exec ci_sglang test -d /sgl-workspace/mori; then
   fi
 
   echo "[MORI] Reinstalling MORI ${MORI_COMMIT} (MORI_GPU_ARCHS=${MORI_GPU_ARCHS})"
+  # Only the rocm724 (noble) base is missing libgrpc++-dev; 7.0 and 7.2.0 built
+  # MORI without it for months before this step existed, so skip the apt round
+  # trip there. Where it does run, neither step may be fatal: apt-get update
+  # exits 100 for a single unreachable index while still keeping every index it
+  # did fetch, which under set -e is enough to take out the dependency install
+  # on every AMD runner at once. Six external apt hosts are in play, so the
+  # guard is not specific to the rocm-osdb source that first triggered this.
+  # Retries are already configured image-wide (Acquire::Retries) and do not
+  # help against a 404.
   docker exec ci_sglang bash -c "
     set -euo pipefail
     export MORI_GPU_ARCHS='${MORI_GPU_ARCHS}'
@@ -273,8 +282,10 @@ if docker exec ci_sglang test -d /sgl-workspace/mori; then
     cd /sgl-workspace/mori
     git checkout '${MORI_COMMIT}'
     git submodule update --init --recursive
-    apt-get update
-    apt-get install -y --no-install-recommends libgrpc++-dev 2>/dev/null || true
+    if [ '${IMAGE_STAGE_SUFFIX}' = '-rocm724' ]; then
+      apt-get update || echo '[MORI] apt-get update reported errors; continuing with the indexes it did fetch'
+      apt-get install -y --no-install-recommends libgrpc++-dev || echo '[MORI] libgrpc++-dev unavailable; building MORI without it'
+    fi
     python3 setup.py develop
     python3 -c 'import os, torch; print(os.path.join(os.path.dirname(torch.__file__), \"lib\"))' > /etc/ld.so.conf.d/torch.conf
     ldconfig

test/registered/unit/tools/test_amd_ci_install_dependency.py

@@ -0,0 +1,45 @@
+"""Guard on the apt calls in scripts/ci/amd/amd_ci_install_dependency.sh.
+
+Those calls run under `set -euo pipefail`, and `apt-get update` exits 100 when
+any single index is unreachable -- even though it keeps every index it did
+fetch. An unguarded call therefore fails the whole "Install dependencies" step
+on every AMD runner at once, which is what took out ~25 of 27 jobs in
+pr-test-amd run 32399046576 when AMD's internal rocm-osdb artifactory started
+404ing on an index this repo never installs from.
+
+The packages involved are optional -- rocm.Dockerfile builds MORI without them
+-- so no apt call here may be able to abort the run.
+"""
+
+import re
+import unittest
+from pathlib import Path
+
+from sglang.test.ci.ci_register import register_cpu_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_cpu_ci(est_time=1, suite="base-a-test-cpu")
+
+INSTALL_SCRIPT = (
+    Path(__file__).resolve().parents[4] / "scripts/ci/amd/amd_ci_install_dependency.sh"
+)
+
+
+class TestAmdCiInstallDependencyApt(CustomTestCase):
+    def test_apt_calls_cannot_abort_the_dependency_install(self):
+        unguarded = [
+            line.strip()
+            for line in INSTALL_SCRIPT.read_text().splitlines()
+            if re.match(r"\s*(sudo\s+)?apt-get\b", line) and "||" not in line
+        ]
+        self.assertEqual(
+            unguarded,
+            [],
+            "an unguarded apt-get under `set -e` fails the dependency install on "
+            "every AMD runner whenever one apt source is unreachable; give it an "
+            "`|| echo ...` fallback",
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()