minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[AMD] [CI] Register MI35x GSM8K nightly tests (#26478)

Browse Source 
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Bingxu Chen
2026-05-28 10:31:43 +08:00
committed by GitHub
co-authored by Cursor
parent 14c1bb2721
commit 81663cb5f1
4 changed files with 214 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/amd/accuracy/mi35x/test_glm51_mxfp4_tp2_gsm8k_mi35x.py
+34 -62
View File
@@ -1,25 +1,21 @@
"""MI355X GLM-5.1-MXFP4 TP=2 GSM8K accuracy gate.
This is a PR Test (AMD) regression test for the GLM-5.1-MXFP4 TP=2
This is a nightly AMD regression test for the GLM-5.1-MXFP4 TP=2
accuracy drop seen on MI355X/gfx950 when aiter selected a bad BF16 GEMM path.
Registry: stage-c-test-large-8-gpu-amd-mi35x suite
Registry: nightly-amd-2-gpu-mi35x-glm51-mxfp4 suite
"""
import os
import resource
import unittest
from types import SimpleNamespace
from urllib.parse import urlparse
os.environ.setdefault("HF_HOME", "/data2/models/huggingface")
os.environ.setdefault("HF_HUB_CACHE", "/data2/models/huggingface/hub")
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k_eval
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
CustomTestCase,
is_in_ci,
popen_launch_server,
write_github_step_summary,
@@ -27,19 +23,16 @@ from sglang.test.test_utils import (
register_amd_ci(
est_time=3600,
suite="stage-c-test-large-8-gpu-amd-mi35x",
suite="nightly-amd-2-gpu-mi35x-glm51-mxfp4",
nightly=True,
)
GLM51_MXFP4_MODEL_ID = "amd/GLM-5.1-MXFP4"
GLM51_MXFP4_LOCAL_PATHS = (
"/data2/models/amd-GLM-5.1-MXFP4",
"/data/huggingface/hub/amd/GLM-5.1-MXFP4",
)
SERVER_LAUNCH_TIMEOUT = 5400
GSM8K_ACCURACY_THRESHOLD = 0.92
GSM8K_INVALID_THRESHOLD = 0.02
DEFAULT_NUM_QUESTIONS = 1200
DEFAULT_PARALLEL = 1200
GSM8K_NUM_EXAMPLES = None
GSM8K_NUM_THREADS = 512
def _raise_nofile_limit() -> None:
@@ -50,26 +43,16 @@ def _raise_nofile_limit() -> None:
resource.setrlimit(resource.RLIMIT_NOFILE, (target, hard))
def _get_model_path() -> str:
env_path = os.environ.get("GLM51_MXFP4_MODEL_PATH")
if env_path:
return env_path
for path in GLM51_MXFP4_LOCAL_PATHS:
if os.path.exists(path):
return path
return GLM51_MXFP4_MODEL_ID
class TestGLM51MXFP4TP2GSM8KMI35x(unittest.TestCase):
class TestGLM51MXFP4TP2GSM8KMI35x(CustomTestCase):
@classmethod
def setUpClass(cls):
_raise_nofile_limit()
cls.model = _get_model_path()
cls.model = GLM51_MXFP4_MODEL_ID
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
model=cls.model,
base_url=cls.base_url,
timeout=5400,
timeout=SERVER_LAUNCH_TIMEOUT,
other_args=[
"--tp",
"2",
@@ -100,44 +83,33 @@ class TestGLM51MXFP4TP2GSM8KMI35x(unittest.TestCase):
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_gsm8k_accuracy(self):
num_questions = int(
os.environ.get("GLM51_MXFP4_GSM8K_NUM_QUESTIONS", DEFAULT_NUM_QUESTIONS)
)
parallel = int(os.environ.get("GLM51_MXFP4_GSM8K_PARALLEL", DEFAULT_PARALLEL))
url = urlparse(self.base_url)
def test_gsm8k(self):
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=num_questions,
max_new_tokens=512,
parallel=parallel,
host=url.hostname or "127.0.0.1",
port=url.port or 30000,
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
num_examples=GSM8K_NUM_EXAMPLES,
num_threads=GSM8K_NUM_THREADS,
max_tokens=512,
temperature=0.0,
)
metrics = run_gsm8k_eval(args)
accuracy = metrics["accuracy"]
invalid = metrics["invalid"]
summary = (
"### GLM-5.1-MXFP4 TP=2 GSM8K (MI355X)\n\n"
"| Model | TP | Questions | Accuracy | Invalid | Threshold | Status |\n"
"| ----- | -- | --------- | -------- | ------- | --------- | ------ |\n"
)
passed = (
accuracy >= GSM8K_ACCURACY_THRESHOLD and invalid <= GSM8K_INVALID_THRESHOLD
)
status = "PASS" if passed else "FAIL"
summary += (
f"| {self.model} | 2 | {num_questions} | {accuracy:.3f} | "
f"{invalid:.3f} | accuracy >= {GSM8K_ACCURACY_THRESHOLD:.2f} | {status} |\n"
)
if is_in_ci():
write_github_step_summary(summary)
metrics = run_eval(args)
print(f"{metrics=}", flush=True)
score = metrics["score"]
self.assertGreaterEqual(accuracy, GSM8K_ACCURACY_THRESHOLD)
self.assertLessEqual(invalid, GSM8K_INVALID_THRESHOLD)
if is_in_ci():
write_github_step_summary(
"### GLM-5.1-MXFP4 TP=2 GSM8K (MI355X)\n\n"
"| Model | Examples | Max Parallel | Score | Threshold | Latency |\n"
"| ----- | --------- | ------------ | ----- | --------- | ------- |\n"
f"| {self.model} | full | default ({GSM8K_NUM_THREADS}) | "
f"{score:.3f} | {GSM8K_ACCURACY_THRESHOLD:.2f} | "
f"{metrics.get('latency', 0):.1f}s |\n"
)
self.assertGreaterEqual(score, GSM8K_ACCURACY_THRESHOLD)
if __name__ == "__main__":
test/registered/amd/test_deepseek_r1_hicache_mi35x.py
+20 -25
View File
@@ -1,4 +1,4 @@
"""MI35x DeepSeek-R1-0528 FP8 HiCache PR Test (8-GPU)
"""MI35x DeepSeek-R1-0528 FP8 HiCache Nightly Test (8-GPU)
Regression guard: launches DeepSeek-R1-0528 (native FP8, MLA, aiter attention
backend) on MI35x with the full L1+L2+L3 HiCache hierarchy wired up
@@ -6,13 +6,13 @@ backend) on MI35x with the full L1+L2+L3 HiCache hierarchy wired up
GSM8K few-shot completion and asserts the accuracy still matches the
established threshold. The goal is to catch regressions where HiCache
breaks DSR1-0528 generation correctness, not to stress-test the cascade
overflow path (that lives in the nightly suite).
overflow path.
Acceptance: GSM8K (200 questions, 5-shot, completion API) score >= 0.93,
Acceptance: GSM8K (1319 questions, 5-shot, completion API) score >= 0.93,
matching ``test_deepseek_r1_eval_mi35x.py`` /
``test_deepseek_r1_eval_amd.py``.
Registry: stage-c-test-large-8-gpu-amd-mi35x (per-commit PR suite).
Registry: nightly-amd-8-gpu-mi35x-deepseek-r1-hicache suite.
"""
import os
@@ -21,12 +21,6 @@ import tempfile
import unittest
from types import SimpleNamespace
# MI35x CI runner caches HF models on a fast local volume. We only fall
# back to this if HF_HOME isn't already set by the runner (e.g. repro_ci.sh
# points HF_HOME at /sgl-data/hf-cache); never force HF_HUB_CACHE so
# huggingface_hub keeps deriving it as $HF_HOME/hub.
os.environ.setdefault("HF_HOME", "/data2/models/huggingface")
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.run_eval import run_eval
@@ -38,18 +32,21 @@ from sglang.test.test_utils import (
write_github_step_summary,
)
# ~15 min: 5-8 min weight load + ~2-3 min GSM8K + slack.
register_amd_ci(est_time=900, suite="stage-c-test-large-8-gpu-amd-mi35x")
# DSR1-0528 can spend 20+ min in weight loading on MI35x before warmup.
register_amd_ci(
est_time=5400,
suite="nightly-amd-8-gpu-mi35x-deepseek-r1-hicache",
nightly=True,
)
DEEPSEEK_R1_MODEL_PATH = "deepseek-ai/DeepSeek-R1-0528"
SERVER_LAUNCH_TIMEOUT = 1500
SERVER_LAUNCH_TIMEOUT = 3600
# Threshold matches the existing nightly AMD DSR1-0528 accuracy tests:
# test/registered/amd/accuracy/mi35x/test_deepseek_r1_eval_mi35x.py
# test/registered/amd/accuracy/mi30x/test_deepseek_r1_eval_amd.py
GSM8K_ACCURACY_THRESHOLD = 0.93
GSM8K_NUM_EXAMPLES = 200
GSM8K_NUM_SHOTS = 5
GSM8K_NUM_EXAMPLES = None
GSM8K_NUM_THREADS = 64
@@ -140,31 +137,29 @@ class TestDeepSeekR1HiCacheMI35x(CustomTestCase):
"""GSM8K few-shot completion against the HiCache-enabled DSR1-0528."""
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
num_examples=GSM8K_NUM_EXAMPLES,
num_shots=GSM8K_NUM_SHOTS,
num_threads=GSM8K_NUM_THREADS,
max_tokens=512,
temperature=0.0,
)
metrics = run_eval(args)
print(f"{metrics=}", flush=True)
score = metrics["score"]
print(f"GSM8K {metrics=}", flush=True)
if is_in_ci():
write_github_step_summary(
"### DeepSeek-R1-0528 FP8 HiCache GSM8K (MI35x)\n\n"
f"- score: `{score:.3f}` (threshold `{GSM8K_ACCURACY_THRESHOLD}`)\n"
f"- latency: `{metrics.get('latency', 0):.1f}s`\n"
"| Model | Examples | Max Parallel | Score | Threshold | Latency |\n"
"| ----- | --------- | ------------ | ----- | --------- | ------- |\n"
f"| {self.model} | full | {GSM8K_NUM_THREADS} | "
f"{score:.3f} | {GSM8K_ACCURACY_THRESHOLD:.2f} | "
f"{metrics.get('latency', 0):.1f}s |\n"
)
self.assertGreater(
score,
GSM8K_ACCURACY_THRESHOLD,
f"DSR1-0528 FP8 + HiCache GSM8K accuracy {score:.3f} "
f"below threshold {GSM8K_ACCURACY_THRESHOLD}",
)
self.assertGreaterEqual(score, GSM8K_ACCURACY_THRESHOLD)
if __name__ == "__main__":