From 126a36a98679c1d023ad52d7b71ac7fb3754923b Mon Sep 17 00:00:00 2001
From: YC Yen-Ching Tseng <yctseng@amd.com>
Date: Mon, 11 May 2026 11:08:51 +0800
Subject: [PATCH] [AMD] VRAM cleanup step to AMD nightly test workflows
 (#24612)

---
 .../workflows/nightly-test-amd-rocm720.yml    | 111 ++++++++++++++++++
 .github/workflows/nightly-test-amd.yml        | 105 +++++++++++++++++
 2 files changed, 216 insertions(+)

diff --git a/.github/workflows/nightly-test-amd-rocm720.yml b/.github/workflows/nightly-test-amd-rocm720.yml
index 1adc2e618..c27a3ec74 100644
--- a/.github/workflows/nightly-test-amd-rocm720.yml
+++ b/.github/workflows/nightly-test-amd-rocm720.yml
@@ -117,6 +117,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -146,6 +149,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -174,6 +180,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -203,6 +212,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -233,6 +245,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -264,6 +279,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -294,6 +312,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -334,6 +355,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -377,6 +401,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -420,6 +447,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -463,6 +493,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -504,6 +537,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -545,6 +581,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -575,6 +614,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -605,6 +647,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -635,6 +680,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -679,6 +727,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -723,6 +774,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -767,6 +821,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -807,6 +864,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -836,6 +896,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -868,6 +931,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -914,6 +980,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -960,6 +1029,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -1004,6 +1076,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -1048,6 +1123,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -1092,6 +1170,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -1125,6 +1206,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -1158,6 +1242,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -1191,6 +1278,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -1224,6 +1314,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -1257,6 +1350,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -1302,6 +1398,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -1346,6 +1445,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -1392,6 +1494,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2)
         run: |
           touch github_summary.md
@@ -1462,6 +1567,9 @@ jobs:
           echo "image=rocm/sgl-dev:$TAG" >> "$GITHUB_OUTPUT"
           echo "Resolved DSv4 image: rocm/sgl-dev:$TAG"
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2 DSv4)
         run: |
           touch github_summary.md
@@ -1528,6 +1636,9 @@ jobs:
           echo "image=rocm/sgl-dev:$TAG" >> "$GITHUB_OUTPUT"
           echo "Resolved DSv4 image: rocm/sgl-dev:$TAG"
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker (ROCm 7.2 DSv4)
         run: |
           touch github_summary.md
diff --git a/.github/workflows/nightly-test-amd.yml b/.github/workflows/nightly-test-amd.yml
index 752568fa6..366ef7aed 100644
--- a/.github/workflows/nightly-test-amd.yml
+++ b/.github/workflows/nightly-test-amd.yml
@@ -115,6 +115,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -145,6 +148,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -174,6 +180,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -204,6 +213,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -235,6 +247,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -267,6 +282,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -297,6 +315,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -337,6 +358,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -380,6 +404,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -423,6 +450,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -466,6 +496,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -507,6 +540,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -548,6 +584,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -578,6 +617,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -607,6 +649,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -637,6 +682,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -681,6 +729,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -725,6 +776,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -769,6 +823,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -809,6 +866,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -841,6 +901,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -873,6 +936,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -919,6 +985,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -965,6 +1034,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -1009,6 +1081,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -1053,6 +1128,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -1097,6 +1175,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -1130,6 +1211,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -1163,6 +1247,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -1196,6 +1283,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -1229,6 +1319,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -1262,6 +1355,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -1307,6 +1403,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -1351,6 +1450,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md
@@ -1397,6 +1499,9 @@ jobs:
         with:
           ref: ${{ inputs.ref || github.sha }}
 
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
       - name: Setup docker
         run: |
           touch github_summary.md