minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[AMD][CI] Add the Qwen3.8 MXFP4 MI35x nightly (#35383)

Browse Source
This commit is contained in:
Michael
2026-08-23 17:17:10 -07:00
committed by GitHub
parent fb6e3872e1
commit 20064623ab
4 changed files with 377 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/nightly-test-amd-rocm720.yml
+61 -87
View File
@@ -84,9 +84,8 @@ on:
- nightly-8-gpu-kimi-k26-rocm720
# 8-GPU Kimi-K3 (MI35x only - native MXFP4 needs gfx95x)
- nightly-8-gpu-mi35x-kimi-k3-rocm720
# 8-GPU Qwen3-235B (MI30x + MI35x MXFP4)
- nightly-8-gpu-qwen3-235b-rocm720
- nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720
# 8-GPU Qwen3.8-MXFP4 (MI35x only - MXFP4 needs gfx95x)
- nightly-8-gpu-mi35x-qwen38-mxfp4-rocm720
# 8-GPU Qwen 3.5 (MI30x + MI35x)
- nightly-8-gpu-qwen35-rocm720
- nightly-8-gpu-mi35x-qwen35-rocm720
@@ -1754,87 +1753,6 @@ jobs:
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
# ==============================================================================
# 8-GPU Qwen3-235B (MI30x + MI35x MXFP4)
# ==============================================================================
nightly-8-gpu-qwen3-235b-rocm720:
strategy:
fail-fast: false
matrix:
rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }}
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-qwen3-235b-rocm720,'))
runs-on: linux-mi300-8gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.sha }}
- name: Ensure VRAM is clear
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Setup docker (ROCm 7.2)
run: |
touch github_summary.md
bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
ENABLE_CACHE_HOST: "1"
- name: Install dependencies
run: bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
- name: Accuracy Test + Performance Test ROCm 7.2 (8-GPU Qwen3)
timeout-minutes: 120
run: |
> github_summary.md # Clear summary file
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e SGLANG_MOE_COPY_WEIGHT_VIEWS_BEFORE_H2D=1 \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-8-gpu-qwen3-235b --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720:
strategy:
fail-fast: false
matrix:
rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }}
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720,'))
runs-on: linux-mi35x-gpu-8
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.sha }}
- name: Ensure VRAM is clear
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Setup docker (ROCm 7.2)
run: |
touch github_summary.md
bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
- name: Install dependencies
run: |
bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
# Install tabulate for run_suite.py (missing in MI35x container)
bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate
- name: Accuracy Test + Performance Test MI35x ROCm 7.2 (8-GPU Qwen3-235B-MXFP4)
timeout-minutes: 120
run: |
> github_summary.md # Clear summary file
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-8-gpu-mi35x-qwen3-235b-mxfp4 --nightly --timeout-per-file 3600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
# ==============================================================================
# 8-GPU Qwen 3.5 (MI30x + MI35x)
# ==============================================================================
@@ -1980,6 +1898,63 @@ jobs:
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
# ==============================================================================
# 8-GPU Qwen3.8-MXFP4 (MI35x only - MXFP4 needs gfx95x)
# ==============================================================================
nightly-8-gpu-mi35x-qwen38-mxfp4-rocm720:
strategy:
fail-fast: false
matrix:
rocm_version: ${{ fromJson(inputs.rocm_version && inputs.rocm_version != 'all' && format('["{0}"]', inputs.rocm_version) || '["rocm724", "rocm720"]') }}
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-qwen38-mxfp4-rocm720,'))
runs-on: linux-mi35x-gpu-8
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.sha }}
- name: Ensure VRAM is clear
run: bash scripts/ci/amd/ensure_vram_clear.sh rocm
- name: Setup docker (ROCm 7.2)
run: |
touch github_summary.md
bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version ${{ matrix.rocm_version }}
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
ENABLE_CACHE_HOST: "1"
- name: Install dependencies
run: |
bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
# Install tabulate for run_suite.py (missing in MI35x container)
bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate
# ~1.2 TB of weights against a fleet-shared cache that runs at 100% used.
# Run 32196787596 died 40 minutes into the download on ENOSPC with no
# free-space figure anywhere in its log; record one up front instead.
- name: Check model cache space
run: |
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout \
bash scripts/ci/amd/check_hf_cache_space.sh \
amd/Qwen3.8-2.4T-A95B-Quark-MXFP4 1250
# One step, not the usual accuracy/perf pair: both phases live in the
# same suite so they launch from one server config. The 2.4T MXFP4
# checkpoint is ~1.2 TB and is loaded once per phase, which dominates
# wall time well before either the eval or the benchmark starts.
- name: Accuracy Test + Performance Test MI35x ROCm 7.2 (8-GPU Qwen3.8-MXFP4)
timeout-minutes: 420
run: |
> github_summary.md # Clear summary file
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-qwen38-mxfp4 --nightly --timeout-per-file 21600 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
# ==============================================================================
# 8-GPU GLM-5.1 (MI30x + MI35x)
# ==============================================================================
@@ -2382,9 +2357,8 @@ jobs:
- nightly-8-gpu-kimi-k26-rocm720
# 8-GPU Kimi-K3 (MI35x only - native MXFP4 needs gfx95x)
- nightly-8-gpu-mi35x-kimi-k3-rocm720
# 8-GPU Qwen3-235B (MI30x + MI35x MXFP4)
- nightly-8-gpu-qwen3-235b-rocm720
- nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720
# 8-GPU Qwen3.8-MXFP4 (MI35x only - MXFP4 needs gfx95x)
- nightly-8-gpu-mi35x-qwen38-mxfp4-rocm720
# 8-GPU Qwen 3.5 (MI30x + MI35x)
- nightly-8-gpu-qwen35-rocm720
- nightly-8-gpu-mi35x-qwen35-rocm720
scripts/ci/amd/check_hf_cache_space.sh
Executable
+101
View File
@@ -0,0 +1,101 @@
#!/bin/bash
# Report HuggingFace cache headroom before a large checkpoint is used, and
# clear stale download artifacts.
#
# Usage (inside the ci_sglang container, where /sgl-data is the cache mount):
# check_hf_cache_space.sh <model_repo_id> [required_gib]
#
# Why this exists: run 32196787596 died 40 minutes into a 1.2 TB download with
# "OSError: [Errno 28] No space left on device", and the only way to find that
# out was reading 8,500 lines of job log -- free space was never reported
# anywhere. This puts the number in the log every time, before the download
# rather than after it fails.
#
# What it deliberately does NOT do is free space by deleting other checkpoints.
# /sgl-data is not a per-runner disk: it is `amdprj3-k8s-2`, a 15 TB volume
# shared by the whole AMD fleet, and it sits at 100% used. An earlier version of
# this script evicted least-recently-used checkpoints until it hit a free-space
# target; on that filesystem it removed 48 of them and free space went from
# 298 MB to 227 MB, because concurrent jobs consume anything released as fast as
# it appears. So the eviction destroyed other jobs' caches fleet-wide, forcing
# them to re-download, and bought nothing. A volume at capacity is an
# infrastructure problem and a per-job script cannot fix it by deleting things
# other jobs still need.
#
# Never fails the job: a full cache is not necessarily fatal (the checkpoint may
# already be cached, which is the common case), and when it is fatal the
# download says so itself -- now against a log that already explained why.
set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
MODEL_REPO_ID="${1:?model repo id, e.g. amd/Qwen3.8-2.4T-A95B-Quark-MXFP4}"
REQUIRED_GIB="${2:-0}"
HF_CACHE="${HF_HOME:-/sgl-data/hf-cache}/hub"
# HuggingFace stores `org/name` as `models--org--name`.
MODEL_DIR="$HF_CACHE/models--${MODEL_REPO_ID//\//--}"
avail_gib() {
df -BG --output=avail "$1" 2>/dev/null | tail -1 | tr -dc '0-9'
}
report() {
echo "=== HF cache space ($1) ==="
df -h "$HF_CACHE" 2>/dev/null || df -h /sgl-data 2>/dev/null || true
echo "==========================="
}
check_hf_cache_space() {
if [[ ! -d "$HF_CACHE" ]]; then
echo "HF cache $HF_CACHE does not exist yet; nothing to report."
return 0
fi
report "before"
if [[ -d "$MODEL_DIR" ]]; then
echo "✓ ${MODEL_REPO_ID} is already cached at ${MODEL_DIR};" \
"no download needed regardless of free space."
else
echo "${MODEL_REPO_ID} is NOT cached; it must be downloaded."
fi
# Abandoned partial downloads are pure waste and safe to drop. This is the
# shared helper the CUDA runner prep already uses; it only touches
# *.incomplete / *.tmp older than two hours, so it cannot pull the rug from
# under a download running right now.
python3 "${SCRIPT_DIR}/../utils/cleanup_hf_cache.py" || true
report "after"
local avail
avail=$(avail_gib "$HF_CACHE")
if [[ -z "$avail" ]]; then
echo "WARNING: could not read free space from df."
return 0
fi
echo "Free space: ${avail} GiB."
if [[ -d "$MODEL_DIR" ]] || (( REQUIRED_GIB == 0 )) || (( avail >= REQUIRED_GIB )); then
return 0
fi
echo "=============================================================="
echo "WARNING: ${MODEL_REPO_ID} is not cached and only ${avail} GiB is"
echo " free, against roughly ${REQUIRED_GIB} GiB of weights. The"
echo " download will likely fail with ENOSPC partway through."
echo ""
echo " /sgl-data is shared by the whole AMD fleet, so this is a"
echo " capacity problem rather than something this job can clear:"
echo " deleting other checkpoints to make room just moves the"
echo " failure onto whichever job needed them next. Raising it"
echo " needs the runner owners."
echo "=============================================================="
return 0
}
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
check_hf_cache_space "$@"
fi
test/registered/amd/accuracy/mi35x/test_qwen38_mxfp4_eval_mi35x.py
+214
View File
@@ -0,0 +1,214 @@
"""MI35x Qwen3.8-2.4T-A95B MXFP4 GSM8K accuracy + serving-perf test (8-GPU)
Tests amd/Qwen3.8-2.4T-A95B-Quark-MXFP4, AMD's day-0 Quark quantization of
Qwen/Qwen3.8-2.4T-A95B-FP8, on a single 8-GPU MI35x node.
Qwen3.8 is a 2.4T-parameter / 95B-active hybrid MoE: 23 repeats of 3 x Gated
DeltaNet -> MoE then 1 x Gated Attention -> MoE, 512 experts with 10 routed + 1
shared active. It reuses the Qwen3.5 architecture -- the checkpoint reports
``Qwen3_5MoeForCausalLM`` -- so no model code is added here. What is missing,
and what this test supplies, is nightly evidence that the ROCm kernels behind
that path keep producing correct tokens, and at what speed.
MXFP4 rather than FP8: at 2.4T parameters FP8 is ~2.4 TB against 8 x 288 GB =
2.30 TB per MI355X node, so the FP8 checkpoint has no single-node AMD recipe
(the cookbook serves it as MI300X TP8 x PP2 over two nodes) and single-node
means FP4. Only the routed experts are quantized; attention, the shared expert,
the MoE gate and ``lm_head`` stay at source precision, which is why AMD
measures the same 97.49 GSM8K as the FP8 baseline (100% recovery) and why this
test gates the FP8 checkpoint's quality even though it serves the MXFP4 one.
Both phases launch from one ``SERVER_ARGS``, which reproduces the recipe
published on the AMD model card. That is the reason accuracy and perf share a
file rather than splitting into an accuracy suite and a perf suite: the
throughput numbers then describe the exact configuration the accuracy gate
covers, and a flag change cannot drift one out from under the other. Two of
those flags are load-bearing rather than restatements of a default:
* ``--page-size 1`` -- ``_page_size_default`` bumps the default to 64 on HIP
when the container sets SGLANG_AITER_KV_CACHE_LAYOUT=vectorized_5d, so the
measured geometry only holds if the page size is pinned.
* ``--attention-backend aiter`` -- no arg override picks a backend for
``Qwen3_5MoeForCausalLM`` on ROCm, so the AITER path has to be named.
Perf runs only once accuracy has passed (see ``accuracy_passed``): a server
that decodes garbage still benchmarks fine, so publishing its throughput would
be worse than publishing nothing.
The scorer extracts the last number in the reply and the server runs with no
``--reasoning-parser``, so a ``<think>`` block still scores: the reasoning
stays in ``message.content`` rather than being split into ``reasoning_content``,
which would leave ``content`` empty and score 0.
MXFP4 needs gfx95x, so this is MI35x-only and ROCm 7.2-only; it does not
register on gfx942 (MI300/MI325).
Registry: nightly-amd-8-gpu-mi35x-qwen38-mxfp4 suite
"""
import os
import unittest
from types import SimpleNamespace
import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.nightly_bench_utils import generate_simple_markdown_report
from sglang.test.nightly_utils import NightlyBenchmarkRunner
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
CustomTestCase,
_parse_int_list_env,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
# Register for AMD CI - Qwen3.8 MXFP4 accuracy + perf on MI35x (~4h: the 1.2 TB
# checkpoint is loaded twice, once per phase, and dominates both)
register_amd_ci(
est_time=14400, suite="nightly-amd-8-gpu-mi35x-qwen38-mxfp4", nightly=True
)
QWEN38_MXFP4_MODEL_PATH = os.environ.get(
"QWEN38_MXFP4_MODEL_PATH", "amd/Qwen3.8-2.4T-A95B-Quark-MXFP4"
)
SERVER_LAUNCH_TIMEOUT = 9000
BENCH_TIMEOUT = 9000
TP_SIZE = 8
# AMD measures 0.9749 on this checkpoint. The gate sits ~5% below it, matching
# the relative tolerance the sibling Qwen3.5 MI35x evals allow.
ACCURACY_THRESHOLD = 0.93
PERF_RESULT_DIR = "performance_results_qwen38_mxfp4_mi35x"
# The AMD model card's serve recipe, shared by both phases.
SERVER_ARGS = [
"--tp",
str(TP_SIZE),
"--attention-backend",
"aiter",
"--page-size",
"1",
"--chunked-prefill-size",
"16384",
"--mem-fraction-static",
"0.9",
"--trust-remote-code",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
"--watchdog-timeout",
"1200",
]
# Gates the AITER MXFP4-MoE / GEMM / norm / rope kernels. The ROCm image sets
# it; a bare-pip host does not. popen_launch_server merges this over os.environ.
SERVER_ENV = {"SGLANG_USE_AITER": "1"}
class TestQwen38Mxfp4MI35x(CustomTestCase):
"""Qwen3.8-2.4T-A95B MXFP4 accuracy + serving perf for AMD MI35x."""
# Set by the accuracy phase and read by the perf phase. unittest orders
# methods alphabetically, so test_a_* lands before test_b_*.
accuracy_passed = False
@classmethod
def setUpClass(cls):
cls.base_url = DEFAULT_URL_FOR_TEST
cls.num_examples = int(os.environ.get("GSM8K_NUM_QUESTIONS", "1319"))
cls.max_tokens = int(os.environ.get("GSM8K_MAX_NEW_TOKENS", "2048"))
def test_a_gsm8k_accuracy(self):
"""GSM8K few-shot accuracy must clear the AMD-published gate."""
process = popen_launch_server(
QWEN38_MXFP4_MODEL_PATH,
self.base_url,
timeout=SERVER_LAUNCH_TIMEOUT,
other_args=SERVER_ARGS,
env=SERVER_ENV,
)
try:
requests.get(self.base_url + "/flush_cache")
args = SimpleNamespace(
base_url=self.base_url,
model=QWEN38_MXFP4_MODEL_PATH,
eval_name="gsm8k",
num_examples=self.num_examples,
num_threads=512,
max_tokens=self.max_tokens,
chat_template_kwargs={"enable_thinking": False},
)
metrics = run_eval(args)
acc = metrics["score"]
passed = acc >= ACCURACY_THRESHOLD
status = "✅ PASS" if passed else "❌ FAIL"
print(f" accuracy={acc:.3f} threshold={ACCURACY_THRESHOLD} {status}")
if is_in_ci():
summary = "### Qwen3.8-2.4T-A95B MXFP4 GSM8K (MI35x)\n\n"
summary += "| Model | TP | Accuracy | Threshold | Status |\n"
summary += "| ----- | -- | -------- | --------- | ------ |\n"
summary += (
f"| {QWEN38_MXFP4_MODEL_PATH} | {TP_SIZE} | {acc:.3f} | "
f"{ACCURACY_THRESHOLD} | {status} |\n"
)
write_github_step_summary(summary)
type(self).accuracy_passed = passed
self.assertGreaterEqual(
acc,
ACCURACY_THRESHOLD,
f"Qwen3.8 MXFP4 accuracy {acc:.3f} below threshold {ACCURACY_THRESHOLD}",
)
finally:
kill_process_tree(process.pid)
def test_b_serving_perf(self):
"""Serving benchmark for the configuration the accuracy phase gated."""
if not self.accuracy_passed:
self.skipTest(
"GSM8K accuracy did not pass; throughput for a server that "
"decodes incorrectly is not worth publishing"
)
# The leading 1 is repeated so generate_simple_markdown_report drops it
# as a warmup: bench_one_batch_server measures every batch as it comes,
# and batch 1 is both the first and the row a cold cache distorts most.
batch_sizes = _parse_int_list_env("NIGHTLY_BATCH_SIZES", "1,1,8,16,64")
input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "1024"))
output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "1024"))
runner = NightlyBenchmarkRunner(
PERF_RESULT_DIR, type(self).__name__, self.base_url
)
runner.setup_result_directory()
try:
results, success = runner.run_benchmark_for_model(
model_path=QWEN38_MXFP4_MODEL_PATH,
batch_sizes=batch_sizes,
input_lens=input_lens,
output_lens=output_lens,
other_args=SERVER_ARGS,
variant="mxfp4",
extra_bench_args=["--trust-remote-code"],
timeout=SERVER_LAUNCH_TIMEOUT,
env=SERVER_ENV,
)[:2]
if results:
runner.full_report += (
generate_simple_markdown_report(results, "MI35x") + "\n"
)
self.assertTrue(
success, f"Perf benchmark failed for {QWEN38_MXFP4_MODEL_PATH} on MI35x"
)
finally:
runner.write_final_report()
if __name__ == "__main__":
unittest.main()
test/run_suite.py
+1
View File
@@ -141,6 +141,7 @@ NIGHTLY_SUITES = {
"nightly-amd-2-gpu-mi35x-deepseek-r1-mxfp4-tp2",
"nightly-amd-8-gpu-mi35x-deepseek-r1-mxfp4-tp4",
"nightly-amd-accuracy-8-gpu-mi35x-kimi-k3",
"nightly-amd-8-gpu-mi35x-qwen38-mxfp4",
"nightly-amd-8-gpu-mi35x-glm52-fp8",
"nightly-amd-4-gpu",
"nightly-amd-8-gpu",