[AMD][CI] Add the Qwen3.8 MXFP4 MI35x nightly (#35383)

.github/workflows/nightly-test-amd-rocm720.yml

@@ -84,9 +84,8 @@ on:
           - nightly-8-gpu-kimi-k26-rocm720
           # 8-GPU Kimi-K3 (MI35x only - native MXFP4 needs gfx95x)
           - nightly-8-gpu-mi35x-kimi-k3-rocm720
-          # 8-GPU Qwen3-235B (MI30x + MI35x MXFP4)
+          # 8-GPU Qwen3.8-MXFP4 (MI35x only - MXFP4 needs gfx95x)
-          - nightly-8-gpu-qwen3-235b-rocm720
+          - nightly-8-gpu-mi35x-qwen38-mxfp4-rocm720
-          - nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720
           # 8-GPU Qwen 3.5 (MI30x + MI35x)
           - nightly-8-gpu-qwen35-rocm720
           - nightly-8-gpu-mi35x-qwen35-rocm720
@@ -1754,87 +1753,6 @@ jobs:
           echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
           exit ${TEST_EXIT_CODE:-0}
 
-  # ==============================================================================
-  # 8-GPU Qwen3-235B (MI30x + MI35x MXFP4)
-  # ==============================================================================
-
-  nightly-8-gpu-qwen3-235b-rocm720:
-    strategy:
-      fail-fast: false
-      matrix:
-        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }}
-    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-qwen3-235b-rocm720,'))
-    runs-on: linux-mi300-8gpu-sglang
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ inputs.ref || github.sha }}
-
-      - name: Ensure VRAM is clear
-        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
-
-      - name: Setup docker (ROCm 7.2)
-        run: |
-          touch github_summary.md
-          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
-        env:
-          GITHUB_WORKSPACE: ${{ github.workspace }}
-          ENABLE_CACHE_HOST: "1"
-
-      - name: Install dependencies
-        run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
-
-      - name: Accuracy Test + Performance Test ROCm 7.2 (8-GPU Qwen3)
-        timeout-minutes: 120
-        run: |
-          > github_summary.md  # Clear summary file
-          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-            -e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
-            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
-            python3 run_suite.py --hw amd --suite nightly-8-gpu-qwen3-235b --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
-          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
-          exit ${TEST_EXIT_CODE:-0}
-
-  nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720:
-    strategy:
-      fail-fast: false
-      matrix:
-        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }}
-    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720,'))
-    runs-on: linux-mi35x-gpu-8
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ inputs.ref || github.sha }}
-
-      - name: Ensure VRAM is clear
-        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
-
-      - name: Setup docker (ROCm 7.2)
-        run: |
-          touch github_summary.md
-          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
-        env:
-          GITHUB_WORKSPACE: ${{ github.workspace }}
-
-      - name: Install dependencies
-        run: |
-          bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
-          # Install tabulate for run_suite.py (missing in MI35x container)
-          bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate
-
-      - name: Accuracy Test + Performance Test MI35x ROCm 7.2 (8-GPU Qwen3-235B-MXFP4)
-        timeout-minutes: 120
-        run: |
-          > github_summary.md  # Clear summary file
-          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
-            python3 run_suite.py --hw amd --suite nightly-8-gpu-mi35x-qwen3-235b-mxfp4 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
-          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
-          exit ${TEST_EXIT_CODE:-0}
-
   # ==============================================================================
   # 8-GPU Qwen 3.5 (MI30x + MI35x)
   # ==============================================================================
@@ -1980,6 +1898,63 @@ jobs:
           echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
           exit ${TEST_EXIT_CODE:-0}
 
+  # ==============================================================================
+  # 8-GPU Qwen3.8-MXFP4 (MI35x only - MXFP4 needs gfx95x)
+  # ==============================================================================
+
+  nightly-8-gpu-mi35x-qwen38-mxfp4-rocm720:
+    strategy:
+      fail-fast: false
+      matrix:
+        rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }}
+    if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-qwen38-mxfp4-rocm720,'))
+    runs-on: linux-mi35x-gpu-8
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref || github.sha }}
+
+      - name: Ensure VRAM is clear
+        run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
+
+      - name: Setup docker (ROCm 7.2)
+        run: |
+          touch github_summary.md
+          bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
+        env:
+          GITHUB_WORKSPACE: ${{ github.workspace }}
+          ENABLE_CACHE_HOST: "1"
+
+      - name: Install dependencies
+        run: |
+          bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
+          # Install tabulate for run_suite.py (missing in MI35x container)
+          bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate
+
+      # ~1.2 TB of weights against a fleet-shared cache that runs at 100% used.
+      # Run 32196787596 died 40 minutes into the download on ENOSPC with no
+      # free-space figure anywhere in its log; record one up front instead.
+      - name: Check model cache space
+        run: |
+          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout \
+            bash scripts/ci/amd/check_hf_cache_space.sh \
+            amd/Qwen3.8-2.4T-A95B-Quark-MXFP4 1250
+
+      # One step, not the usual accuracy/perf pair: both phases live in the
+      # same suite so they launch from one server config. The 2.4T MXFP4
+      # checkpoint is ~1.2 TB and is loaded once per phase, which dominates
+      # wall time well before either the eval or the benchmark starts.
+      - name: Accuracy Test + Performance Test MI35x ROCm 7.2 (8-GPU Qwen3.8-MXFP4)
+        timeout-minutes: 420
+        run: |
+          > github_summary.md  # Clear summary file
+          bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
+            -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
+            python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-qwen38-mxfp4 --nightly --timeout-per-file 21600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
+          echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
+          exit ${TEST_EXIT_CODE:-0}
+
   # ==============================================================================
   # 8-GPU GLM-5.1 (MI30x + MI35x)
   # ==============================================================================
@@ -2382,9 +2357,8 @@ jobs:
       - nightly-8-gpu-kimi-k26-rocm720
       # 8-GPU Kimi-K3 (MI35x only - native MXFP4 needs gfx95x)
       - nightly-8-gpu-mi35x-kimi-k3-rocm720
-      # 8-GPU Qwen3-235B (MI30x + MI35x MXFP4)
+      # 8-GPU Qwen3.8-MXFP4 (MI35x only - MXFP4 needs gfx95x)
-      - nightly-8-gpu-qwen3-235b-rocm720
+      - nightly-8-gpu-mi35x-qwen38-mxfp4-rocm720
-      - nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720
       # 8-GPU Qwen 3.5 (MI30x + MI35x)
       - nightly-8-gpu-qwen35-rocm720
       - nightly-8-gpu-mi35x-qwen35-rocm720

scripts/ci/amd/check_hf_cache_space.sh

@@ -0,0 +1,101 @@
+#!/bin/bash
+# Report HuggingFace cache headroom before a large checkpoint is used, and
+# clear stale download artifacts.
+#
+# Usage (inside the ci_sglang container, where /sgl-data is the cache mount):
+#   check_hf_cache_space.sh <model_repo_id> [required_gib]
+#
+# Why this exists: run 32196787596 died 40 minutes into a 1.2 TB download with
+# "OSError: [Errno 28] No space left on device", and the only way to find that
+# out was reading 8,500 lines of job log -- free space was never reported
+# anywhere. This puts the number in the log every time, before the download
+# rather than after it fails.
+#
+# What it deliberately does NOT do is free space by deleting other checkpoints.
+# /sgl-data is not a per-runner disk: it is `amdprj3-k8s-2`, a 15 TB volume
+# shared by the whole AMD fleet, and it sits at 100% used. An earlier version of
+# this script evicted least-recently-used checkpoints until it hit a free-space
+# target; on that filesystem it removed 48 of them and free space went from
+# 298 MB to 227 MB, because concurrent jobs consume anything released as fast as
+# it appears. So the eviction destroyed other jobs' caches fleet-wide, forcing
+# them to re-download, and bought nothing. A volume at capacity is an
+# infrastructure problem and a per-job script cannot fix it by deleting things
+# other jobs still need.
+#
+# Never fails the job: a full cache is not necessarily fatal (the checkpoint may
+# already be cached, which is the common case), and when it is fatal the
+# download says so itself -- now against a log that already explained why.
+
+set -uo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+MODEL_REPO_ID="${1:?model repo id, e.g. amd/Qwen3.8-2.4T-A95B-Quark-MXFP4}"
+REQUIRED_GIB="${2:-0}"
+
+HF_CACHE="${HF_HOME:-/sgl-data/hf-cache}/hub"
+# HuggingFace stores `org/name` as `models--org--name`.
+MODEL_DIR="$HF_CACHE/models--${MODEL_REPO_ID//\//--}"
+
+avail_gib() {
+    df -BG --output=avail "$1" 2>/dev/null | tail -1 | tr -dc '0-9'
+}
+
+report() {
+    echo "=== HF cache space ($1) ==="
+    df -h "$HF_CACHE" 2>/dev/null || df -h /sgl-data 2>/dev/null || true
+    echo "==========================="
+}
+
+check_hf_cache_space() {
+    if [[ ! -d "$HF_CACHE" ]]; then
+        echo "HF cache $HF_CACHE does not exist yet; nothing to report."
+        return 0
+    fi
+
+    report "before"
+
+    if [[ -d "$MODEL_DIR" ]]; then
+        echo "✓ ${MODEL_REPO_ID} is already cached at ${MODEL_DIR};" \
+             "no download needed regardless of free space."
+    else
+        echo "${MODEL_REPO_ID} is NOT cached; it must be downloaded."
+    fi
+
+    # Abandoned partial downloads are pure waste and safe to drop. This is the
+    # shared helper the CUDA runner prep already uses; it only touches
+    # *.incomplete / *.tmp older than two hours, so it cannot pull the rug from
+    # under a download running right now.
+    python3 "${SCRIPT_DIR}/../utils/cleanup_hf_cache.py" || true
+
+    report "after"
+
+    local avail
+    avail=$(avail_gib "$HF_CACHE")
+    if [[ -z "$avail" ]]; then
+        echo "WARNING: could not read free space from df."
+        return 0
+    fi
+    echo "Free space: ${avail} GiB."
+
+    if [[ -d "$MODEL_DIR" ]] || (( REQUIRED_GIB == 0 )) || (( avail >= REQUIRED_GIB )); then
+        return 0
+    fi
+
+    echo "=============================================================="
+    echo "WARNING: ${MODEL_REPO_ID} is not cached and only ${avail} GiB is"
+    echo "         free, against roughly ${REQUIRED_GIB} GiB of weights. The"
+    echo "         download will likely fail with ENOSPC partway through."
+    echo ""
+    echo "         /sgl-data is shared by the whole AMD fleet, so this is a"
+    echo "         capacity problem rather than something this job can clear:"
+    echo "         deleting other checkpoints to make room just moves the"
+    echo "         failure onto whichever job needed them next. Raising it"
+    echo "         needs the runner owners."
+    echo "=============================================================="
+    return 0
+}
+
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+    check_hf_cache_space "$@"
+fi

test/registered/amd/accuracy/mi35x/test_qwen38_mxfp4_eval_mi35x.py

@@ -0,0 +1,214 @@
+"""MI35x Qwen3.8-2.4T-A95B MXFP4 GSM8K accuracy + serving-perf test (8-GPU)
+
+Tests amd/Qwen3.8-2.4T-A95B-Quark-MXFP4, AMD's day-0 Quark quantization of
+Qwen/Qwen3.8-2.4T-A95B-FP8, on a single 8-GPU MI35x node.
+
+Qwen3.8 is a 2.4T-parameter / 95B-active hybrid MoE: 23 repeats of 3 x Gated
+DeltaNet -> MoE then 1 x Gated Attention -> MoE, 512 experts with 10 routed + 1
+shared active. It reuses the Qwen3.5 architecture -- the checkpoint reports
+``Qwen3_5MoeForCausalLM`` -- so no model code is added here. What is missing,
+and what this test supplies, is nightly evidence that the ROCm kernels behind
+that path keep producing correct tokens, and at what speed.
+
+MXFP4 rather than FP8: at 2.4T parameters FP8 is ~2.4 TB against 8 x 288 GB =
+2.30 TB per MI355X node, so the FP8 checkpoint has no single-node AMD recipe
+(the cookbook serves it as MI300X TP8 x PP2 over two nodes) and single-node
+means FP4. Only the routed experts are quantized; attention, the shared expert,
+the MoE gate and ``lm_head`` stay at source precision, which is why AMD
+measures the same 97.49 GSM8K as the FP8 baseline (100% recovery) and why this
+test gates the FP8 checkpoint's quality even though it serves the MXFP4 one.
+
+Both phases launch from one ``SERVER_ARGS``, which reproduces the recipe
+published on the AMD model card. That is the reason accuracy and perf share a
+file rather than splitting into an accuracy suite and a perf suite: the
+throughput numbers then describe the exact configuration the accuracy gate
+covers, and a flag change cannot drift one out from under the other. Two of
+those flags are load-bearing rather than restatements of a default:
+
+  * ``--page-size 1`` -- ``_page_size_default`` bumps the default to 64 on HIP
+    when the container sets SGLANG_AITER_KV_CACHE_LAYOUT=vectorized_5d, so the
+    measured geometry only holds if the page size is pinned.
+  * ``--attention-backend aiter`` -- no arg override picks a backend for
+    ``Qwen3_5MoeForCausalLM`` on ROCm, so the AITER path has to be named.
+
+Perf runs only once accuracy has passed (see ``accuracy_passed``): a server
+that decodes garbage still benchmarks fine, so publishing its throughput would
+be worse than publishing nothing.
+
+The scorer extracts the last number in the reply and the server runs with no
+``--reasoning-parser``, so a ``<think>`` block still scores: the reasoning
+stays in ``message.content`` rather than being split into ``reasoning_content``,
+which would leave ``content`` empty and score 0.
+
+MXFP4 needs gfx95x, so this is MI35x-only and ROCm 7.2-only; it does not
+register on gfx942 (MI300/MI325).
+
+Registry: nightly-amd-8-gpu-mi35x-qwen38-mxfp4 suite
+"""
+
+import os
+import unittest
+from types import SimpleNamespace
+
+import requests
+
+from sglang.srt.utils import kill_process_tree
+from sglang.test.ci.ci_register import register_amd_ci
+from sglang.test.nightly_bench_utils import generate_simple_markdown_report
+from sglang.test.nightly_utils import NightlyBenchmarkRunner
+from sglang.test.run_eval import run_eval
+from sglang.test.test_utils import (
+    DEFAULT_URL_FOR_TEST,
+    CustomTestCase,
+    _parse_int_list_env,
+    is_in_ci,
+    popen_launch_server,
+    write_github_step_summary,
+)
+
+# Register for AMD CI - Qwen3.8 MXFP4 accuracy + perf on MI35x (~4h: the 1.2 TB
+# checkpoint is loaded twice, once per phase, and dominates both)
+register_amd_ci(
+    est_time=14400, suite="nightly-amd-8-gpu-mi35x-qwen38-mxfp4", nightly=True
+)
+
+QWEN38_MXFP4_MODEL_PATH = os.environ.get(
+    "QWEN38_MXFP4_MODEL_PATH", "amd/Qwen3.8-2.4T-A95B-Quark-MXFP4"
+)
+SERVER_LAUNCH_TIMEOUT = 9000
+BENCH_TIMEOUT = 9000
+TP_SIZE = 8
+# AMD measures 0.9749 on this checkpoint. The gate sits ~5% below it, matching
+# the relative tolerance the sibling Qwen3.5 MI35x evals allow.
+ACCURACY_THRESHOLD = 0.93
+PERF_RESULT_DIR = "performance_results_qwen38_mxfp4_mi35x"
+
+# The AMD model card's serve recipe, shared by both phases.
+SERVER_ARGS = [
+    "--tp",
+    str(TP_SIZE),
+    "--attention-backend",
+    "aiter",
+    "--page-size",
+    "1",
+    "--chunked-prefill-size",
+    "16384",
+    "--mem-fraction-static",
+    "0.9",
+    "--trust-remote-code",
+    "--model-loader-extra-config",
+    '{"enable_multithread_load": true}',
+    "--watchdog-timeout",
+    "1200",
+]
+# Gates the AITER MXFP4-MoE / GEMM / norm / rope kernels. The ROCm image sets
+# it; a bare-pip host does not. popen_launch_server merges this over os.environ.
+SERVER_ENV = {"SGLANG_USE_AITER": "1"}
+
+
+class TestQwen38Mxfp4MI35x(CustomTestCase):
+    """Qwen3.8-2.4T-A95B MXFP4 accuracy + serving perf for AMD MI35x."""
+
+    # Set by the accuracy phase and read by the perf phase. unittest orders
+    # methods alphabetically, so test_a_* lands before test_b_*.
+    accuracy_passed = False
+
+    @classmethod
+    def setUpClass(cls):
+        cls.base_url = DEFAULT_URL_FOR_TEST
+        cls.num_examples = int(os.environ.get("GSM8K_NUM_QUESTIONS", "1319"))
+        cls.max_tokens = int(os.environ.get("GSM8K_MAX_NEW_TOKENS", "2048"))
+
+    def test_a_gsm8k_accuracy(self):
+        """GSM8K few-shot accuracy must clear the AMD-published gate."""
+        process = popen_launch_server(
+            QWEN38_MXFP4_MODEL_PATH,
+            self.base_url,
+            timeout=SERVER_LAUNCH_TIMEOUT,
+            other_args=SERVER_ARGS,
+            env=SERVER_ENV,
+        )
+
+        try:
+            requests.get(self.base_url + "/flush_cache")
+
+            args = SimpleNamespace(
+                base_url=self.base_url,
+                model=QWEN38_MXFP4_MODEL_PATH,
+                eval_name="gsm8k",
+                num_examples=self.num_examples,
+                num_threads=512,
+                max_tokens=self.max_tokens,
+                chat_template_kwargs={"enable_thinking": False},
+            )
+            metrics = run_eval(args)
+            acc = metrics["score"]
+
+            passed = acc >= ACCURACY_THRESHOLD
+            status = "✅ PASS" if passed else "❌ FAIL"
+            print(f"  accuracy={acc:.3f} threshold={ACCURACY_THRESHOLD} {status}")
+
+            if is_in_ci():
+                summary = "### Qwen3.8-2.4T-A95B MXFP4 GSM8K (MI35x)\n\n"
+                summary += "| Model | TP | Accuracy | Threshold | Status |\n"
+                summary += "| ----- | -- | -------- | --------- | ------ |\n"
+                summary += (
+                    f"| {QWEN38_MXFP4_MODEL_PATH} | {TP_SIZE} | {acc:.3f} | "
+                    f"{ACCURACY_THRESHOLD} | {status} |\n"
+                )
+                write_github_step_summary(summary)
+
+            type(self).accuracy_passed = passed
+            self.assertGreaterEqual(
+                acc,
+                ACCURACY_THRESHOLD,
+                f"Qwen3.8 MXFP4 accuracy {acc:.3f} below threshold {ACCURACY_THRESHOLD}",
+            )
+        finally:
+            kill_process_tree(process.pid)
+
+    def test_b_serving_perf(self):
+        """Serving benchmark for the configuration the accuracy phase gated."""
+        if not self.accuracy_passed:
+            self.skipTest(
+                "GSM8K accuracy did not pass; throughput for a server that "
+                "decodes incorrectly is not worth publishing"
+            )
+
+        # The leading 1 is repeated so generate_simple_markdown_report drops it
+        # as a warmup: bench_one_batch_server measures every batch as it comes,
+        # and batch 1 is both the first and the row a cold cache distorts most.
+        batch_sizes = _parse_int_list_env("NIGHTLY_BATCH_SIZES", "1,1,8,16,64")
+        input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "1024"))
+        output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "1024"))
+
+        runner = NightlyBenchmarkRunner(
+            PERF_RESULT_DIR, type(self).__name__, self.base_url
+        )
+        runner.setup_result_directory()
+
+        try:
+            results, success = runner.run_benchmark_for_model(
+                model_path=QWEN38_MXFP4_MODEL_PATH,
+                batch_sizes=batch_sizes,
+                input_lens=input_lens,
+                output_lens=output_lens,
+                other_args=SERVER_ARGS,
+                variant="mxfp4",
+                extra_bench_args=["--trust-remote-code"],
+                timeout=SERVER_LAUNCH_TIMEOUT,
+                env=SERVER_ENV,
+            )[:2]
+            if results:
+                runner.full_report += (
+                    generate_simple_markdown_report(results, "MI35x") + "\n"
+                )
+            self.assertTrue(
+                success, f"Perf benchmark failed for {QWEN38_MXFP4_MODEL_PATH} on MI35x"
+            )
+        finally:
+            runner.write_final_report()
+
+
+if __name__ == "__main__":
+    unittest.main()

test/run_suite.py

@@ -141,6 +141,7 @@ NIGHTLY_SUITES = {
         "nightly-amd-2-gpu-mi35x-deepseek-r1-mxfp4-tp2",
         "nightly-amd-8-gpu-mi35x-deepseek-r1-mxfp4-tp4",
         "nightly-amd-accuracy-8-gpu-mi35x-kimi-k3",
+        "nightly-amd-8-gpu-mi35x-qwen38-mxfp4",
         "nightly-amd-8-gpu-mi35x-glm52-fp8",
         "nightly-amd-4-gpu",
         "nightly-amd-8-gpu",