minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[AMD] Add GLM-5.1-FP8 nightly accuracy and performance benchmarks for MI30x and MI35x (#22336)

Browse Source
This commit is contained in:
Michael
2026-04-08 22:57:43 -07:00
committed by GitHub
parent e379befbac
commit ef6bfc1197
6 changed files with 920 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/nightly-test-amd-rocm720.yml
+68 -22
View File
@@ -44,6 +44,7 @@ on:
- nightly-8-gpu-qwen3-235b-rocm720
- nightly-8-gpu-qwen35-rocm720
- nightly-8-gpu-glm5-rocm720
- nightly-8-gpu-glm51-rocm720
- nightly-8-gpu-minimax-m25-rocm720
- nightly-1-gpu-zimage-turbo-rocm720
- nightly-test-1-gpu-mi35x-rocm720
@@ -61,7 +62,7 @@ on:
- nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720
- nightly-8-gpu-mi35x-qwen35-rocm720
- nightly-8-gpu-mi35x-glm5-rocm720
- nightly-8-gpu-mi35x-glm47-fp8-rocm720
- nightly-8-gpu-mi35x-glm51-rocm720
- nightly-8-gpu-mi35x-minimax-m25-rocm720
job_filter:
description: 'Or type comma-separated job names (overrides dropdown if non-empty)'
@@ -709,6 +710,50 @@ jobs:
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
# 8-GPU GLM-5.1 (Accuracy + Performance combined) ROCm 7.2
nightly-8-gpu-glm51-rocm720:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-glm51-rocm720,'))
runs-on: linux-mi325-8gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.ref }}
- name: Setup docker (ROCm 7.2)
run: |
touch github_summary.md
bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
- name: Install dependencies
run: |
bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
bash scripts/ci/amd/amd_ci_exec.sh pip install git+https://github.com/huggingface/transformers.git@96f807a33b75
- name: Accuracy Test ROCm 7.2 (8-GPU GLM-5.1 NSA)
timeout-minutes: 120
run: |
> github_summary.md # Clear summary file
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-glm51 --nightly --timeout-per-file 3600 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
- name: Performance Test ROCm 7.2 (8-GPU GLM-5.1)
timeout-minutes: 120
continue-on-error: true
run: |
> github_summary.md # Clear summary file
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e SGLANG_USE_AITER=1 \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-glm51 --nightly --timeout-per-file 5400 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
# 8-GPU MiniMax-M2.5 (Accuracy + Performance combined) ROCm 7.2
nightly-8-gpu-minimax-m25-rocm720:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-minimax-m25-rocm720,'))
@@ -1288,7 +1333,7 @@ jobs:
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
# MI35x 8-GPU GLM-5 (Accuracy + Performance combined) ROCm 7.2
# MI35x 8-GPU GLM-5 (Accuracy only) ROCm 7.2
nightly-8-gpu-mi35x-glm5-rocm720:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm5-rocm720,'))
runs-on: linux-mi35x-gpu-8
@@ -1322,20 +1367,9 @@ jobs:
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
- name: Performance Test MI35x ROCm 7.2 (8-GPU GLM-5)
timeout-minutes: 120
continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed
run: |
> github_summary.md # Clear summary file
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-glm5 --nightly --timeout-per-file 5400 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
# MI35x 8-GPU GLM-4.7-FP8 (Accuracy) ROCm 7.2
nightly-8-gpu-mi35x-glm47-fp8-rocm720:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm47-fp8-rocm720,'))
# MI35x 8-GPU GLM-5.1 (Accuracy + Performance combined) ROCm 7.2
nightly-8-gpu-mi35x-glm51-rocm720:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm51-rocm720,'))
runs-on: linux-mi35x-gpu-8
steps:
- name: Checkout code
@@ -1352,17 +1386,28 @@ jobs:
- name: Install dependencies
run: |
bash scripts/ci/amd/amd_ci_install_dependency.sh
# Install tabulate for run_suite.py (missing in MI35x container)
bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps
bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate
bash scripts/ci/amd/amd_ci_exec.sh pip install git+https://github.com/huggingface/transformers.git@96f807a33b75
- name: Accuracy Test MI35x ROCm 7.2 (8-GPU GLM-4.7-FP8)
timeout-minutes: 120
- name: Accuracy Test MI35x ROCm 7.2 (8-GPU GLM-5.1 NSA)
timeout-minutes: 180
run: |
> github_summary.md # Clear summary file
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-glm47-fp8 --nightly --timeout-per-file 3600 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-glm51 --nightly --timeout-per-file 7200 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
- name: Performance Test MI35x ROCm 7.2 (8-GPU GLM-5.1)
timeout-minutes: 120
continue-on-error: true
run: |
> github_summary.md # Clear summary file
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-glm51 --nightly --timeout-per-file 5400 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
@@ -1469,6 +1514,7 @@ jobs:
- nightly-8-gpu-qwen3-235b-rocm720
- nightly-8-gpu-qwen35-rocm720
- nightly-8-gpu-glm5-rocm720
- nightly-8-gpu-glm51-rocm720
- nightly-8-gpu-minimax-m25-rocm720
# MI30x ROCm 7.2 Diffusion Tests
- nightly-1-gpu-zimage-turbo-rocm720
@@ -1488,7 +1534,7 @@ jobs:
- nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720
- nightly-8-gpu-mi35x-qwen35-rocm720
- nightly-8-gpu-mi35x-glm5-rocm720
- nightly-8-gpu-mi35x-glm47-fp8-rocm720
- nightly-8-gpu-mi35x-glm51-rocm720
- nightly-8-gpu-mi35x-minimax-m25-rocm720
runs-on: ubuntu-latest
steps:
.github/workflows/nightly-test-amd.yml
+88 -5
View File
@@ -44,6 +44,8 @@ on:
- nightly-8-gpu-qwen3-235b
- nightly-8-gpu-qwen35
- nightly-8-gpu-glm5
- nightly-8-gpu-glm51
- nightly-8-gpu-glm51-mxfp4
- nightly-8-gpu-minimax-m25
- nightly-1-gpu-zimage-turbo
- nightly-test-1-gpu-mi35x
@@ -61,6 +63,8 @@ on:
- nightly-8-gpu-mi35x-qwen3-235b-mxfp4
- nightly-8-gpu-mi35x-qwen35
- nightly-8-gpu-mi35x-glm5
- nightly-8-gpu-mi35x-glm51
- nightly-8-gpu-mi35x-glm51-mxfp4
- nightly-8-gpu-mi35x-minimax-m25
job_filter:
description: 'Or type comma-separated job names (overrides dropdown if non-empty)'
@@ -712,6 +716,50 @@ jobs:
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
# 8-GPU GLM-5.1 (Accuracy + Performance combined)
nightly-8-gpu-glm51:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-glm51,'))
runs-on: linux-mi325-8gpu-sglang
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.ref }}
- name: Setup docker
run: |
touch github_summary.md
bash scripts/ci/amd/amd_ci_start_container.sh
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
- name: Install dependencies
run: |
bash scripts/ci/amd/amd_ci_install_dependency.sh
bash scripts/ci/amd/amd_ci_exec.sh pip install git+https://github.com/huggingface/transformers.git@96f807a33b75
- name: Accuracy Test (8-GPU GLM-5.1 NSA)
timeout-minutes: 120
run: |
> github_summary.md # Clear summary file
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-glm51 --nightly --timeout-per-file 3600 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
- name: Performance Test (8-GPU GLM-5.1)
timeout-minutes: 120
continue-on-error: true
run: |
> github_summary.md # Clear summary file
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e SGLANG_USE_AITER=1 \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-glm51 --nightly --timeout-per-file 5400 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
# 8-GPU MiniMax-M2.5 (Accuracy + Performance combined)
nightly-8-gpu-minimax-m25:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-minimax-m25,'))
@@ -1294,7 +1342,7 @@ jobs:
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
# MI35x 8-GPU GLM-5 (Accuracy + Performance combined)
# MI35x 8-GPU GLM-5 (Accuracy only)
nightly-8-gpu-mi35x-glm5:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm5,'))
runs-on: linux-mi35x-gpu-8
@@ -1328,14 +1376,47 @@ jobs:
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
- name: Performance Test MI35x (8-GPU GLM-5)
timeout-minutes: 120
continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed
# MI35x 8-GPU GLM-5.1 (Accuracy + Performance combined)
nightly-8-gpu-mi35x-glm51:
if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm51,'))
runs-on: linux-mi35x-gpu-8
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.ref }}
- name: Setup docker
run: |
touch github_summary.md
bash scripts/ci/amd/amd_ci_start_container.sh
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
- name: Install dependencies
run: |
bash scripts/ci/amd/amd_ci_install_dependency.sh
bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate
bash scripts/ci/amd/amd_ci_exec.sh pip install git+https://github.com/huggingface/transformers.git@96f807a33b75
- name: Accuracy Test MI35x (8-GPU GLM-5.1 NSA)
timeout-minutes: 180
run: |
> github_summary.md # Clear summary file
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-glm5 --nightly --timeout-per-file 5400 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-glm51 --nightly --timeout-per-file 7200 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
- name: Performance Test MI35x (8-GPU GLM-5.1)
timeout-minutes: 120
continue-on-error: true
run: |
> github_summary.md # Clear summary file
bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \
-e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \
python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-glm51 --nightly --timeout-per-file 5400 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$?
echo "$(<github_summary.md )" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
@@ -1442,6 +1523,7 @@ jobs:
- nightly-8-gpu-qwen3-235b
- nightly-8-gpu-qwen35
- nightly-8-gpu-glm5
- nightly-8-gpu-glm51
- nightly-8-gpu-minimax-m25
# MI30x Diffusion Tests
- nightly-1-gpu-zimage-turbo
@@ -1459,6 +1541,7 @@ jobs:
- nightly-8-gpu-mi35x-qwen3-235b-mxfp4
- nightly-8-gpu-mi35x-qwen35
- nightly-8-gpu-mi35x-glm5
- nightly-8-gpu-mi35x-glm51
- nightly-8-gpu-mi35x-minimax-m25
# MI35x perf jobs excluded from check - perf failures don't block CI
# - nightly-perf-8-gpu-mi35x-deepseek-v32-basic
test/registered/amd/accuracy/mi30x/test_glm51_eval_amd.py
+238
View File
@@ -0,0 +1,238 @@
"""AMD GLM-5.1 GSM8K Completion Evaluation Test (8-GPU)
Tests GLM-5.1-FP8 with NSA attention backend using few-shot
completion benchmark on MI325/MI300X.
Registry: nightly-amd-accuracy-8-gpu-glm51 suite
"""
import ast
import os
import re
import time
import unittest
from dataclasses import dataclass, field
from typing import List, Optional, Tuple
import numpy as np
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
from sglang.utils import download_and_cache_file, read_jsonl
register_amd_ci(
est_time=3600,
suite="nightly-amd-accuracy-8-gpu-glm51",
nightly=True,
)
INVALID = -9999999
@dataclass
class ModelConfig:
model_path: str
tp_size: int = 8
accuracy_threshold: float = 0.50
other_args: List[str] = field(default_factory=list)
env_vars: dict = field(default_factory=dict)
timeout: Optional[int] = None
variant: Optional[str] = None
def get_display_name(self) -> str:
if self.variant:
return f"{self.model_path} ({self.variant})"
return self.model_path
GLM51_MODELS = [
ModelConfig(
model_path="zai-org/GLM-5.1-FP8",
tp_size=8,
accuracy_threshold=0.93,
timeout=3600,
variant="nsa",
other_args=[
"--trust-remote-code",
"--reasoning-parser",
"glm45",
"--tool-call-parser",
"glm47",
"--nsa-prefill-backend",
"tilelang",
"--nsa-decode-backend",
"tilelang",
"--chunked-prefill-size",
"131072",
"--mem-fraction-static",
"0.80",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
"--watchdog-timeout",
"1200",
],
env_vars={"SGLANG_USE_AITER": "1"},
),
]
def get_one_example(lines, i, include_answer):
ret = "Question: " + lines[i]["question"] + "\nAnswer:"
if include_answer:
ret += " " + lines[i]["answer"]
return ret
def get_few_shot_examples(lines, k):
ret = ""
for i in range(k):
ret += get_one_example(lines, i, True) + "\n\n"
return ret
def get_answer_value(answer_str):
answer_str = answer_str.replace(",", "")
numbers = re.findall(r"\d+", answer_str)
if len(numbers) < 1:
return INVALID
try:
return ast.literal_eval(numbers[-1])
except SyntaxError:
return INVALID
def run_gsm8k_benchmark(
base_url: str,
num_questions: int = 200,
num_shots: int = 5,
parallel: int = 64,
) -> Tuple[float, float, float]:
import sglang as sgl
from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint
url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
data_path = download_and_cache_file(url)
lines = list(read_jsonl(data_path))
few_shot_examples = get_few_shot_examples(lines, num_shots)
questions = []
labels = []
for i in range(len(lines[:num_questions])):
questions.append(get_one_example(lines, i, False))
labels.append(get_answer_value(lines[i]["answer"]))
assert all(l != INVALID for l in labels)
arguments = [{"question": q} for q in questions]
@sgl.function
def few_shot_gsm8k(s, question):
s += few_shot_examples + question
s += sgl.gen(
"answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"]
)
backend = RuntimeEndpoint(base_url)
sgl.set_default_backend(backend)
tic = time.perf_counter()
states = few_shot_gsm8k.run_batch(
arguments, temperature=0, num_threads=parallel, progress_bar=True
)
latency = time.perf_counter() - tic
preds = [get_answer_value(states[i]["answer"]) for i in range(len(states))]
acc = np.mean(np.array(preds) == np.array(labels))
invalid = np.mean(np.array(preds) == INVALID)
return float(acc), float(invalid), float(latency)
class TestGLM51EvalAMD(unittest.TestCase):
"""GLM-5.1 GSM8K Completion Evaluation Test for AMD MI325/MI300X."""
@classmethod
def setUpClass(cls):
cls.models = GLM51_MODELS
cls.base_url = DEFAULT_URL_FOR_TEST
cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200"))
def test_glm51_accuracy(self):
all_results = []
summary = "### GLM-5.1 Models (MI325)\n\n"
summary += "| Model | Variant | TP | Accuracy | Threshold | Status |\n"
summary += "| ----- | ------- | -- | -------- | --------- | ------ |\n"
for config in self.models:
display_name = config.get_display_name()
with self.subTest(model=display_name):
print(f"\n{'='*60}")
print(f"Testing: {display_name}")
print(f"{'='*60}")
env = os.environ.copy()
for key, value in config.env_vars.items():
env[key] = value
other_args = list(config.other_args)
other_args.extend(["--tp", str(config.tp_size)])
timeout = config.timeout or DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
try:
process = popen_launch_server(
model=config.model_path,
base_url=self.base_url,
timeout=timeout,
other_args=other_args,
env=env,
)
try:
acc, invalid, latency = run_gsm8k_benchmark(
self.base_url, num_questions=self.num_questions
)
passed = acc >= config.accuracy_threshold
status = "✅ PASS" if passed else "❌ FAIL"
print(
f" accuracy={acc:.3f} threshold={config.accuracy_threshold} {status}"
)
all_results.append(
{
"model": display_name,
"accuracy": acc,
"passed": passed,
}
)
summary += f"| {config.model_path} | {config.variant or 'N/A'} | {config.tp_size} | {acc:.3f} | {config.accuracy_threshold} | {status} |\n"
finally:
kill_process_tree(process.pid)
except Exception as e:
summary += f"| {config.model_path} | {config.variant or 'N/A'} | {config.tp_size} | N/A | {config.accuracy_threshold} | ❌ ERROR |\n"
all_results.append(
{
"model": display_name,
"accuracy": None,
"passed": False,
"error": str(e),
}
)
if is_in_ci():
write_github_step_summary(summary)
failed = [r for r in all_results if not r["passed"]]
if failed:
raise AssertionError(f"Failed models: {[r['model'] for r in failed]}")
if __name__ == "__main__":
unittest.main()
test/registered/amd/accuracy/mi35x/test_glm51_eval_mi35x.py
+242
View File
@@ -0,0 +1,242 @@
"""MI35x GLM-5.1 GSM8K Completion Evaluation Test (8-GPU)
Tests GLM-5.1-FP8 with NSA attention backend using few-shot
completion benchmark on MI35x.
Registry: nightly-amd-8-gpu-mi35x-glm51 suite
"""
import ast
import os
os.environ.setdefault("HF_HOME", "/data2/models/huggingface")
os.environ.setdefault("HF_HUB_CACHE", "/data2/models/huggingface/hub")
import re
import time
import unittest
from dataclasses import dataclass, field
from typing import List, Optional, Tuple
import numpy as np
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
is_in_ci,
popen_launch_server,
write_github_step_summary,
)
from sglang.utils import download_and_cache_file, read_jsonl
register_amd_ci(
est_time=5400,
suite="nightly-amd-8-gpu-mi35x-glm51",
nightly=True,
)
INVALID = -9999999
@dataclass
class ModelConfig:
model_path: str
tp_size: int = 8
accuracy_threshold: float = 0.50
other_args: List[str] = field(default_factory=list)
env_vars: dict = field(default_factory=dict)
timeout: Optional[int] = None
variant: Optional[str] = None
def get_display_name(self) -> str:
if self.variant:
return f"{self.model_path} ({self.variant})"
return self.model_path
MI35X_GLM51_MODELS = [
ModelConfig(
model_path="zai-org/GLM-5.1-FP8",
tp_size=8,
accuracy_threshold=0.93,
timeout=5400,
variant="nsa",
other_args=[
"--trust-remote-code",
"--reasoning-parser",
"glm45",
"--tool-call-parser",
"glm47",
"--nsa-prefill-backend",
"tilelang",
"--nsa-decode-backend",
"tilelang",
"--chunked-prefill-size",
"131072",
"--mem-fraction-static",
"0.80",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
"--watchdog-timeout",
"1200",
],
env_vars={},
),
]
def get_one_example(lines, i, include_answer):
ret = "Question: " + lines[i]["question"] + "\nAnswer:"
if include_answer:
ret += " " + lines[i]["answer"]
return ret
def get_few_shot_examples(lines, k):
ret = ""
for i in range(k):
ret += get_one_example(lines, i, True) + "\n\n"
return ret
def get_answer_value(answer_str):
answer_str = answer_str.replace(",", "")
numbers = re.findall(r"\d+", answer_str)
if len(numbers) < 1:
return INVALID
try:
return ast.literal_eval(numbers[-1])
except SyntaxError:
return INVALID
def run_gsm8k_benchmark(
base_url: str,
num_questions: int = 200,
num_shots: int = 5,
parallel: int = 64,
) -> Tuple[float, float, float]:
import sglang as sgl
from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint
url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
data_path = download_and_cache_file(url)
lines = list(read_jsonl(data_path))
few_shot_examples = get_few_shot_examples(lines, num_shots)
questions = []
labels = []
for i in range(len(lines[:num_questions])):
questions.append(get_one_example(lines, i, False))
labels.append(get_answer_value(lines[i]["answer"]))
assert all(l != INVALID for l in labels)
arguments = [{"question": q} for q in questions]
@sgl.function
def few_shot_gsm8k(s, question):
s += few_shot_examples + question
s += sgl.gen(
"answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"]
)
backend = RuntimeEndpoint(base_url)
sgl.set_default_backend(backend)
tic = time.perf_counter()
states = few_shot_gsm8k.run_batch(
arguments, temperature=0, num_threads=parallel, progress_bar=True
)
latency = time.perf_counter() - tic
preds = [get_answer_value(states[i]["answer"]) for i in range(len(states))]
acc = np.mean(np.array(preds) == np.array(labels))
invalid = np.mean(np.array(preds) == INVALID)
return float(acc), float(invalid), float(latency)
class TestGLM51EvalMI35x(unittest.TestCase):
"""GLM-5.1 GSM8K Completion Evaluation Test for AMD MI35x."""
@classmethod
def setUpClass(cls):
cls.models = MI35X_GLM51_MODELS
cls.base_url = DEFAULT_URL_FOR_TEST
cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200"))
def test_glm51_accuracy(self):
all_results = []
summary = "### GLM-5.1 Models (MI35x)\n\n"
summary += "| Model | Variant | TP | Accuracy | Threshold | Status |\n"
summary += "| ----- | ------- | -- | -------- | --------- | ------ |\n"
for config in self.models:
display_name = config.get_display_name()
with self.subTest(model=display_name):
print(f"\n{'='*60}")
print(f"Testing: {display_name}")
print(f"{'='*60}")
env = os.environ.copy()
for key, value in config.env_vars.items():
env[key] = value
other_args = list(config.other_args)
other_args.extend(["--tp", str(config.tp_size)])
timeout = config.timeout or DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
try:
process = popen_launch_server(
model=config.model_path,
base_url=self.base_url,
timeout=timeout,
other_args=other_args,
env=env,
)
try:
acc, invalid, latency = run_gsm8k_benchmark(
self.base_url, num_questions=self.num_questions
)
passed = acc >= config.accuracy_threshold
status = "✅ PASS" if passed else "❌ FAIL"
print(
f" accuracy={acc:.3f} threshold={config.accuracy_threshold} {status}"
)
all_results.append(
{
"model": display_name,
"accuracy": acc,
"passed": passed,
}
)
summary += f"| {config.model_path} | {config.variant or 'N/A'} | {config.tp_size} | {acc:.3f} | {config.accuracy_threshold} | {status} |\n"
finally:
kill_process_tree(process.pid)
except Exception as e:
summary += f"| {config.model_path} | {config.variant or 'N/A'} | {config.tp_size} | N/A | {config.accuracy_threshold} | ❌ ERROR |\n"
all_results.append(
{
"model": display_name,
"accuracy": None,
"passed": False,
"error": str(e),
}
)
if is_in_ci():
write_github_step_summary(summary)
failed = [r for r in all_results if not r["passed"]]
if failed:
raise AssertionError(f"Failed models: {[r['model'] for r in failed]}")
if __name__ == "__main__":
unittest.main()
test/registered/amd/perf/mi30x/test_glm51_perf_amd.py
+138
View File
@@ -0,0 +1,138 @@
"""Nightly performance benchmark for GLM-5.1 on MI30x.
Tests GLM-5.1-FP8 with NSA attention backend using bench_one_batch
on 8 GPUs with TP=8, FP8 KV cache.
Model path can be configured via GLM51_MODEL_PATH environment variable.
Registry: nightly-perf-8-gpu-glm51 suite
"""
import os
import unittest
from typing import List
from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.nightly_bench_utils import BenchmarkResult
from sglang.test.nightly_utils import NightlyBenchmarkRunner
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
register_amd_ci(est_time=5400, suite="nightly-perf-8-gpu-glm51", nightly=True)
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
model_header = results[0].model_path
if results[0].run_name and results[0].run_name != "default":
model_header += f" ({results[0].run_name})"
gpu_config = os.getenv("GPU_CONFIG", "MI325")
if gpu_config:
model_header += f" [{gpu_config}]"
summary = f"### {model_header}\n"
summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n"
summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n"
report_results = (
results[1:]
if len(results) > 1 and results[0].batch_size == results[1].batch_size
else results
)
for result in report_results:
itl = 1 / (result.output_throughput / result.batch_size) * 1000
summary += f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | {result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n"
return summary
GLM51_MODEL_PATH = os.environ.get("GLM51_MODEL_PATH", "zai-org/GLM-5.1-FP8")
PROFILE_DIR = "performance_profiles_glm51"
class TestNightlyGLM51Performance(unittest.TestCase):
"""Nightly performance benchmark for GLM-5.1 on MI30x.
Tests GLM-5.1-FP8 with NSA attention backend on TP=8.
"""
@classmethod
def setUpClass(cls):
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = [1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
cls.model_config = {
"name": "glm51",
"model_path": GLM51_MODEL_PATH,
"other_args": [
"--trust-remote-code",
"--reasoning-parser",
"glm45",
"--tool-call-parser",
"glm47",
"--tp",
"8",
"--nsa-prefill-backend",
"tilelang",
"--nsa-decode-backend",
"tilelang",
"--kv-cache-dtype",
"fp8_e4m3",
"--chunked-prefill-size",
"131072",
"--mem-fraction-static",
"0.85",
"--model-loader-extra-config",
'{"enable_multithread_load": true}',
"--watchdog-timeout",
"1200",
],
"env_vars": {
"SGLANG_USE_AITER": "1",
},
}
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
cls.runner.full_report = f"## {cls.__name__}\n"
def test_bench_glm51(self):
old_env = {}
for key, value in self.model_config.get("env_vars", {}).items():
old_env[key] = os.environ.get(key)
os.environ[key] = value
try:
result_tuple = self.runner.run_benchmark_for_model(
model_path=self.model_config["model_path"],
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=self.model_config["other_args"],
variant=self.model_config["name"],
extra_bench_args=["--trust-remote-code"],
enable_profile=False,
timeout=5400,
)
results = result_tuple[0]
success = result_tuple[1]
if results:
self.runner.full_report += (
generate_simple_markdown_report(results) + "\n"
)
self.assertTrue(success, f"Benchmark failed for {GLM51_MODEL_PATH}")
finally:
for key, value in old_env.items():
if value is None:
os.environ.pop(key, None)
else:
os.environ[key] = value
self.runner.write_final_report()
if __name__ == "__main__":
unittest.main()
test/registered/amd/perf/mi35x/test_glm51_perf_mi35x.py
+146
View File
@@ -0,0 +1,146 @@
"""MI35x Nightly performance benchmark for GLM-5.1.
Tests GLM-5.1-FP8 with NSA attention backend using bench_one_batch
on 8 GPUs with TP=8, FP8 KV cache.
Registry: nightly-perf-8-gpu-mi35x-glm51 suite
"""
import os
os.environ.setdefault("HF_HOME", "/data2/models/huggingface")
os.environ.setdefault("HF_HUB_CACHE", "/data2/models/huggingface/hub")
import unittest
from typing import List
from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.nightly_bench_utils import BenchmarkResult
from sglang.test.nightly_utils import NightlyBenchmarkRunner
from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env
register_amd_ci(est_time=5400, suite="nightly-perf-8-gpu-mi35x-glm51", nightly=True)
def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str:
model_header = results[0].model_path
if results[0].run_name and results[0].run_name != "default":
model_header += f" ({results[0].run_name})"
gpu_config = os.getenv("GPU_CONFIG", "MI35x")
if gpu_config:
model_header += f" [{gpu_config}]"
summary = f"### {model_header}\n"
summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n"
summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n"
report_results = (
results[1:]
if len(results) > 1 and results[0].batch_size == results[1].batch_size
else results
)
for result in report_results:
itl = 1 / (result.output_throughput / result.batch_size) * 1000
summary += f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | {result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n"
return summary
GLM51_MODEL_PATH = os.environ.get("GLM51_MODEL_PATH", "zai-org/GLM-5.1-FP8")
PROFILE_DIR = "performance_profiles_glm51_mi35x"
class TestGLM51PerfMI35x(unittest.TestCase):
"""Nightly performance benchmark for GLM-5.1 on MI35x.
Tests GLM-5.1-FP8 with NSA attention backend on TP=8.
"""
@classmethod
def setUpClass(cls):
cls.base_url = DEFAULT_URL_FOR_TEST
cls.batch_sizes = [1, 8, 16, 64]
cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
cls.model_config = {
"name": "glm51-mi35x",
"model_path": GLM51_MODEL_PATH,
"other_args": [
"--trust-remote-code",
"--reasoning-parser",
"glm45",
"--tool-call-parser",
"glm47",
"--tp",
"8",
"--nsa-prefill-backend",
"tilelang",
"--nsa-decode-backend",
"tilelang",
"--kv-cache-dtype",
"fp8_e4m3",
"--chunked-prefill-size",
"131072",
"--mem-fraction-static",
"0.85",
"--model-loader-extra-config",
'{"enable_multithread_load": true, "num_threads": 8}',
"--watchdog-timeout",
"1200",
],
"env_vars": {
"SGLANG_USE_AITER": "1",
"SGLANG_ROCM_FUSED_DECODE_MLA": "0",
"ROCM_QUICK_REDUCE_QUANTIZATION": "INT4",
"SAFETENSORS_FAST_GPU": "1",
},
}
os.environ.setdefault("SGLANG_BENCH_TIMEOUT", "3600")
cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
cls.runner.setup_profile_directory()
cls.runner.full_report = f"## {cls.__name__}\n"
def test_glm51_perf(self):
old_env = {}
for key, value in self.model_config.get("env_vars", {}).items():
old_env[key] = os.environ.get(key)
os.environ[key] = value
try:
result_tuple = self.runner.run_benchmark_for_model(
model_path=self.model_config["model_path"],
batch_sizes=self.batch_sizes,
input_lens=self.input_lens,
output_lens=self.output_lens,
other_args=self.model_config["other_args"],
variant=self.model_config["name"],
extra_bench_args=["--trust-remote-code"],
enable_profile=False,
timeout=5400,
)
results = result_tuple[0]
success = result_tuple[1]
if results:
self.runner.full_report += (
generate_simple_markdown_report(results) + "\n"
)
self.assertTrue(
success, f"Benchmark failed for {GLM51_MODEL_PATH} on MI35x"
)
finally:
for key, value in old_env.items():
if value is None:
os.environ.pop(key, None)
else:
os.environ[key] = value
self.runner.write_final_report()
if __name__ == "__main__":
unittest.main()