diff --git a/.github/workflows/nightly-test-amd-rocm720.yml b/.github/workflows/nightly-test-amd-rocm720.yml index ae763597b..8e905eaf0 100644 --- a/.github/workflows/nightly-test-amd-rocm720.yml +++ b/.github/workflows/nightly-test-amd-rocm720.yml @@ -44,6 +44,7 @@ on: - nightly-8-gpu-qwen3-235b-rocm720 - nightly-8-gpu-qwen35-rocm720 - nightly-8-gpu-glm5-rocm720 + - nightly-8-gpu-glm51-rocm720 - nightly-8-gpu-minimax-m25-rocm720 - nightly-1-gpu-zimage-turbo-rocm720 - nightly-test-1-gpu-mi35x-rocm720 @@ -61,7 +62,7 @@ on: - nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720 - nightly-8-gpu-mi35x-qwen35-rocm720 - nightly-8-gpu-mi35x-glm5-rocm720 - - nightly-8-gpu-mi35x-glm47-fp8-rocm720 + - nightly-8-gpu-mi35x-glm51-rocm720 - nightly-8-gpu-mi35x-minimax-m25-rocm720 job_filter: description: 'Or type comma-separated job names (overrides dropdown if non-empty)' @@ -709,6 +710,50 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} + # 8-GPU GLM-5.1 (Accuracy + Performance combined) ROCm 7.2 + nightly-8-gpu-glm51-rocm720: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-glm51-rocm720,')) + runs-on: linux-mi325-8gpu-sglang + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Setup docker (ROCm 7.2) + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps + bash scripts/ci/amd/amd_ci_exec.sh pip install git+https://github.com/huggingface/transformers.git@96f807a33b75 + + - name: Accuracy Test ROCm 7.2 (8-GPU GLM-5.1 NSA) + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-glm51 --nightly --timeout-per-file 3600 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test ROCm 7.2 (8-GPU GLM-5.1) + timeout-minutes: 120 + continue-on-error: true + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_USE_AITER=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-glm51 --nightly --timeout-per-file 5400 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + # 8-GPU MiniMax-M2.5 (Accuracy + Performance combined) ROCm 7.2 nightly-8-gpu-minimax-m25-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-minimax-m25-rocm720,')) @@ -1288,7 +1333,7 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # MI35x 8-GPU GLM-5 (Accuracy + Performance combined) ROCm 7.2 + # MI35x 8-GPU GLM-5 (Accuracy only) ROCm 7.2 nightly-8-gpu-mi35x-glm5-rocm720: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm5-rocm720,')) runs-on: linux-mi35x-gpu-8 @@ -1322,20 +1367,9 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - - name: Performance Test MI35x ROCm 7.2 (8-GPU GLM-5) - timeout-minutes: 120 - continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed - run: | - > github_summary.md # Clear summary file - bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ - -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-glm5 --nightly --timeout-per-file 5400 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? - echo "$(> $GITHUB_STEP_SUMMARY || true - exit ${TEST_EXIT_CODE:-0} - - # MI35x 8-GPU GLM-4.7-FP8 (Accuracy) ROCm 7.2 - nightly-8-gpu-mi35x-glm47-fp8-rocm720: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm47-fp8-rocm720,')) + # MI35x 8-GPU GLM-5.1 (Accuracy + Performance combined) ROCm 7.2 + nightly-8-gpu-mi35x-glm51-rocm720: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm51-rocm720,')) runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code @@ -1352,17 +1386,28 @@ jobs: - name: Install dependencies run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh - # Install tabulate for run_suite.py (missing in MI35x container) + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + bash scripts/ci/amd/amd_ci_exec.sh pip install git+https://github.com/huggingface/transformers.git@96f807a33b75 - - name: Accuracy Test MI35x ROCm 7.2 (8-GPU GLM-4.7-FP8) - timeout-minutes: 120 + - name: Accuracy Test MI35x ROCm 7.2 (8-GPU GLM-5.1 NSA) + timeout-minutes: 180 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-glm47-fp8 --nightly --timeout-per-file 3600 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-glm51 --nightly --timeout-per-file 7200 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test MI35x ROCm 7.2 (8-GPU GLM-5.1) + timeout-minutes: 120 + continue-on-error: true + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-glm51 --nightly --timeout-per-file 5400 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} @@ -1469,6 +1514,7 @@ jobs: - nightly-8-gpu-qwen3-235b-rocm720 - nightly-8-gpu-qwen35-rocm720 - nightly-8-gpu-glm5-rocm720 + - nightly-8-gpu-glm51-rocm720 - nightly-8-gpu-minimax-m25-rocm720 # MI30x ROCm 7.2 Diffusion Tests - nightly-1-gpu-zimage-turbo-rocm720 @@ -1488,7 +1534,7 @@ jobs: - nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720 - nightly-8-gpu-mi35x-qwen35-rocm720 - nightly-8-gpu-mi35x-glm5-rocm720 - - nightly-8-gpu-mi35x-glm47-fp8-rocm720 + - nightly-8-gpu-mi35x-glm51-rocm720 - nightly-8-gpu-mi35x-minimax-m25-rocm720 runs-on: ubuntu-latest steps: diff --git a/.github/workflows/nightly-test-amd.yml b/.github/workflows/nightly-test-amd.yml index 8495f51d5..b5f3b2f73 100644 --- a/.github/workflows/nightly-test-amd.yml +++ b/.github/workflows/nightly-test-amd.yml @@ -44,6 +44,8 @@ on: - nightly-8-gpu-qwen3-235b - nightly-8-gpu-qwen35 - nightly-8-gpu-glm5 + - nightly-8-gpu-glm51 + - nightly-8-gpu-glm51-mxfp4 - nightly-8-gpu-minimax-m25 - nightly-1-gpu-zimage-turbo - nightly-test-1-gpu-mi35x @@ -61,6 +63,8 @@ on: - nightly-8-gpu-mi35x-qwen3-235b-mxfp4 - nightly-8-gpu-mi35x-qwen35 - nightly-8-gpu-mi35x-glm5 + - nightly-8-gpu-mi35x-glm51 + - nightly-8-gpu-mi35x-glm51-mxfp4 - nightly-8-gpu-mi35x-minimax-m25 job_filter: description: 'Or type comma-separated job names (overrides dropdown if non-empty)' @@ -712,6 +716,50 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} + # 8-GPU GLM-5.1 (Accuracy + Performance combined) + nightly-8-gpu-glm51: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-glm51,')) + runs-on: linux-mi325-8gpu-sglang + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd/amd_ci_install_dependency.sh + bash scripts/ci/amd/amd_ci_exec.sh pip install git+https://github.com/huggingface/transformers.git@96f807a33b75 + + - name: Accuracy Test (8-GPU GLM-5.1 NSA) + timeout-minutes: 120 + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-glm51 --nightly --timeout-per-file 3600 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test (8-GPU GLM-5.1) + timeout-minutes: 120 + continue-on-error: true + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e SGLANG_USE_AITER=1 \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-glm51 --nightly --timeout-per-file 5400 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + # 8-GPU MiniMax-M2.5 (Accuracy + Performance combined) nightly-8-gpu-minimax-m25: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-minimax-m25,')) @@ -1294,7 +1342,7 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - # MI35x 8-GPU GLM-5 (Accuracy + Performance combined) + # MI35x 8-GPU GLM-5 (Accuracy only) nightly-8-gpu-mi35x-glm5: if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm5,')) runs-on: linux-mi35x-gpu-8 @@ -1328,14 +1376,47 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - - name: Performance Test MI35x (8-GPU GLM-5) - timeout-minutes: 120 - continue-on-error: true # Perf test failure doesn't fail the job if accuracy passed + # MI35x 8-GPU GLM-5.1 (Accuracy + Performance combined) + nightly-8-gpu-mi35x-glm51: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-glm51,')) + runs-on: linux-mi35x-gpu-8 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd/amd_ci_install_dependency.sh + bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate + bash scripts/ci/amd/amd_ci_exec.sh pip install git+https://github.com/huggingface/transformers.git@96f807a33b75 + + - name: Accuracy Test MI35x (8-GPU GLM-5.1 NSA) + timeout-minutes: 180 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-glm5 --nightly --timeout-per-file 5400 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + python3 run_suite.py --hw amd --suite nightly-amd-8-gpu-mi35x-glm51 --nightly --timeout-per-file 7200 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + echo "$(> $GITHUB_STEP_SUMMARY || true + exit ${TEST_EXIT_CODE:-0} + + - name: Performance Test MI35x (8-GPU GLM-5.1) + timeout-minutes: 120 + continue-on-error: true + run: | + > github_summary.md # Clear summary file + bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ + -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ + python3 run_suite.py --hw amd --suite nightly-perf-8-gpu-mi35x-glm51 --nightly --timeout-per-file 5400 ${{ inputs.continue_on_error && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} @@ -1442,6 +1523,7 @@ jobs: - nightly-8-gpu-qwen3-235b - nightly-8-gpu-qwen35 - nightly-8-gpu-glm5 + - nightly-8-gpu-glm51 - nightly-8-gpu-minimax-m25 # MI30x Diffusion Tests - nightly-1-gpu-zimage-turbo @@ -1459,6 +1541,7 @@ jobs: - nightly-8-gpu-mi35x-qwen3-235b-mxfp4 - nightly-8-gpu-mi35x-qwen35 - nightly-8-gpu-mi35x-glm5 + - nightly-8-gpu-mi35x-glm51 - nightly-8-gpu-mi35x-minimax-m25 # MI35x perf jobs excluded from check - perf failures don't block CI # - nightly-perf-8-gpu-mi35x-deepseek-v32-basic diff --git a/test/registered/amd/accuracy/mi30x/test_glm51_eval_amd.py b/test/registered/amd/accuracy/mi30x/test_glm51_eval_amd.py new file mode 100644 index 000000000..93a4b345a --- /dev/null +++ b/test/registered/amd/accuracy/mi30x/test_glm51_eval_amd.py @@ -0,0 +1,238 @@ +"""AMD GLM-5.1 GSM8K Completion Evaluation Test (8-GPU) + +Tests GLM-5.1-FP8 with NSA attention backend using few-shot +completion benchmark on MI325/MI300X. + +Registry: nightly-amd-accuracy-8-gpu-glm51 suite +""" + +import ast +import os +import re +import time +import unittest +from dataclasses import dataclass, field +from typing import List, Optional, Tuple + +import numpy as np + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) +from sglang.utils import download_and_cache_file, read_jsonl + +register_amd_ci( + est_time=3600, + suite="nightly-amd-accuracy-8-gpu-glm51", + nightly=True, +) + +INVALID = -9999999 + + +@dataclass +class ModelConfig: + model_path: str + tp_size: int = 8 + accuracy_threshold: float = 0.50 + other_args: List[str] = field(default_factory=list) + env_vars: dict = field(default_factory=dict) + timeout: Optional[int] = None + variant: Optional[str] = None + + def get_display_name(self) -> str: + if self.variant: + return f"{self.model_path} ({self.variant})" + return self.model_path + + +GLM51_MODELS = [ + ModelConfig( + model_path="zai-org/GLM-5.1-FP8", + tp_size=8, + accuracy_threshold=0.93, + timeout=3600, + variant="nsa", + other_args=[ + "--trust-remote-code", + "--reasoning-parser", + "glm45", + "--tool-call-parser", + "glm47", + "--nsa-prefill-backend", + "tilelang", + "--nsa-decode-backend", + "tilelang", + "--chunked-prefill-size", + "131072", + "--mem-fraction-static", + "0.80", + "--model-loader-extra-config", + '{"enable_multithread_load": true}', + "--watchdog-timeout", + "1200", + ], + env_vars={"SGLANG_USE_AITER": "1"}, + ), +] + + +def get_one_example(lines, i, include_answer): + ret = "Question: " + lines[i]["question"] + "\nAnswer:" + if include_answer: + ret += " " + lines[i]["answer"] + return ret + + +def get_few_shot_examples(lines, k): + ret = "" + for i in range(k): + ret += get_one_example(lines, i, True) + "\n\n" + return ret + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def run_gsm8k_benchmark( + base_url: str, + num_questions: int = 200, + num_shots: int = 5, + parallel: int = 64, +) -> Tuple[float, float, float]: + import sglang as sgl + from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint + + url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" + data_path = download_and_cache_file(url) + lines = list(read_jsonl(data_path)) + + few_shot_examples = get_few_shot_examples(lines, num_shots) + + questions = [] + labels = [] + for i in range(len(lines[:num_questions])): + questions.append(get_one_example(lines, i, False)) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + arguments = [{"question": q} for q in questions] + + @sgl.function + def few_shot_gsm8k(s, question): + s += few_shot_examples + question + s += sgl.gen( + "answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"] + ) + + backend = RuntimeEndpoint(base_url) + sgl.set_default_backend(backend) + + tic = time.perf_counter() + states = few_shot_gsm8k.run_batch( + arguments, temperature=0, num_threads=parallel, progress_bar=True + ) + latency = time.perf_counter() - tic + + preds = [get_answer_value(states[i]["answer"]) for i in range(len(states))] + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + + return float(acc), float(invalid), float(latency) + + +class TestGLM51EvalAMD(unittest.TestCase): + """GLM-5.1 GSM8K Completion Evaluation Test for AMD MI325/MI300X.""" + + @classmethod + def setUpClass(cls): + cls.models = GLM51_MODELS + cls.base_url = DEFAULT_URL_FOR_TEST + cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200")) + + def test_glm51_accuracy(self): + all_results = [] + summary = "### GLM-5.1 Models (MI325)\n\n" + summary += "| Model | Variant | TP | Accuracy | Threshold | Status |\n" + summary += "| ----- | ------- | -- | -------- | --------- | ------ |\n" + + for config in self.models: + display_name = config.get_display_name() + with self.subTest(model=display_name): + print(f"\n{'='*60}") + print(f"Testing: {display_name}") + print(f"{'='*60}") + + env = os.environ.copy() + for key, value in config.env_vars.items(): + env[key] = value + + other_args = list(config.other_args) + other_args.extend(["--tp", str(config.tp_size)]) + timeout = config.timeout or DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + + try: + process = popen_launch_server( + model=config.model_path, + base_url=self.base_url, + timeout=timeout, + other_args=other_args, + env=env, + ) + + try: + acc, invalid, latency = run_gsm8k_benchmark( + self.base_url, num_questions=self.num_questions + ) + passed = acc >= config.accuracy_threshold + status = "✅ PASS" if passed else "❌ FAIL" + print( + f" accuracy={acc:.3f} threshold={config.accuracy_threshold} {status}" + ) + + all_results.append( + { + "model": display_name, + "accuracy": acc, + "passed": passed, + } + ) + summary += f"| {config.model_path} | {config.variant or 'N/A'} | {config.tp_size} | {acc:.3f} | {config.accuracy_threshold} | {status} |\n" + + finally: + kill_process_tree(process.pid) + + except Exception as e: + summary += f"| {config.model_path} | {config.variant or 'N/A'} | {config.tp_size} | N/A | {config.accuracy_threshold} | ❌ ERROR |\n" + all_results.append( + { + "model": display_name, + "accuracy": None, + "passed": False, + "error": str(e), + } + ) + + if is_in_ci(): + write_github_step_summary(summary) + + failed = [r for r in all_results if not r["passed"]] + if failed: + raise AssertionError(f"Failed models: {[r['model'] for r in failed]}") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/accuracy/mi35x/test_glm51_eval_mi35x.py b/test/registered/amd/accuracy/mi35x/test_glm51_eval_mi35x.py new file mode 100644 index 000000000..3267a0f34 --- /dev/null +++ b/test/registered/amd/accuracy/mi35x/test_glm51_eval_mi35x.py @@ -0,0 +1,242 @@ +"""MI35x GLM-5.1 GSM8K Completion Evaluation Test (8-GPU) + +Tests GLM-5.1-FP8 with NSA attention backend using few-shot +completion benchmark on MI35x. + +Registry: nightly-amd-8-gpu-mi35x-glm51 suite +""" + +import ast +import os + +os.environ.setdefault("HF_HOME", "/data2/models/huggingface") +os.environ.setdefault("HF_HUB_CACHE", "/data2/models/huggingface/hub") + +import re +import time +import unittest +from dataclasses import dataclass, field +from typing import List, Optional, Tuple + +import numpy as np + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) +from sglang.utils import download_and_cache_file, read_jsonl + +register_amd_ci( + est_time=5400, + suite="nightly-amd-8-gpu-mi35x-glm51", + nightly=True, +) + +INVALID = -9999999 + + +@dataclass +class ModelConfig: + model_path: str + tp_size: int = 8 + accuracy_threshold: float = 0.50 + other_args: List[str] = field(default_factory=list) + env_vars: dict = field(default_factory=dict) + timeout: Optional[int] = None + variant: Optional[str] = None + + def get_display_name(self) -> str: + if self.variant: + return f"{self.model_path} ({self.variant})" + return self.model_path + + +MI35X_GLM51_MODELS = [ + ModelConfig( + model_path="zai-org/GLM-5.1-FP8", + tp_size=8, + accuracy_threshold=0.93, + timeout=5400, + variant="nsa", + other_args=[ + "--trust-remote-code", + "--reasoning-parser", + "glm45", + "--tool-call-parser", + "glm47", + "--nsa-prefill-backend", + "tilelang", + "--nsa-decode-backend", + "tilelang", + "--chunked-prefill-size", + "131072", + "--mem-fraction-static", + "0.80", + "--model-loader-extra-config", + '{"enable_multithread_load": true}', + "--watchdog-timeout", + "1200", + ], + env_vars={}, + ), +] + + +def get_one_example(lines, i, include_answer): + ret = "Question: " + lines[i]["question"] + "\nAnswer:" + if include_answer: + ret += " " + lines[i]["answer"] + return ret + + +def get_few_shot_examples(lines, k): + ret = "" + for i in range(k): + ret += get_one_example(lines, i, True) + "\n\n" + return ret + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def run_gsm8k_benchmark( + base_url: str, + num_questions: int = 200, + num_shots: int = 5, + parallel: int = 64, +) -> Tuple[float, float, float]: + import sglang as sgl + from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint + + url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" + data_path = download_and_cache_file(url) + lines = list(read_jsonl(data_path)) + + few_shot_examples = get_few_shot_examples(lines, num_shots) + + questions = [] + labels = [] + for i in range(len(lines[:num_questions])): + questions.append(get_one_example(lines, i, False)) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + arguments = [{"question": q} for q in questions] + + @sgl.function + def few_shot_gsm8k(s, question): + s += few_shot_examples + question + s += sgl.gen( + "answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"] + ) + + backend = RuntimeEndpoint(base_url) + sgl.set_default_backend(backend) + + tic = time.perf_counter() + states = few_shot_gsm8k.run_batch( + arguments, temperature=0, num_threads=parallel, progress_bar=True + ) + latency = time.perf_counter() - tic + + preds = [get_answer_value(states[i]["answer"]) for i in range(len(states))] + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + + return float(acc), float(invalid), float(latency) + + +class TestGLM51EvalMI35x(unittest.TestCase): + """GLM-5.1 GSM8K Completion Evaluation Test for AMD MI35x.""" + + @classmethod + def setUpClass(cls): + cls.models = MI35X_GLM51_MODELS + cls.base_url = DEFAULT_URL_FOR_TEST + cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "200")) + + def test_glm51_accuracy(self): + all_results = [] + summary = "### GLM-5.1 Models (MI35x)\n\n" + summary += "| Model | Variant | TP | Accuracy | Threshold | Status |\n" + summary += "| ----- | ------- | -- | -------- | --------- | ------ |\n" + + for config in self.models: + display_name = config.get_display_name() + with self.subTest(model=display_name): + print(f"\n{'='*60}") + print(f"Testing: {display_name}") + print(f"{'='*60}") + + env = os.environ.copy() + for key, value in config.env_vars.items(): + env[key] = value + + other_args = list(config.other_args) + other_args.extend(["--tp", str(config.tp_size)]) + timeout = config.timeout or DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + + try: + process = popen_launch_server( + model=config.model_path, + base_url=self.base_url, + timeout=timeout, + other_args=other_args, + env=env, + ) + + try: + acc, invalid, latency = run_gsm8k_benchmark( + self.base_url, num_questions=self.num_questions + ) + passed = acc >= config.accuracy_threshold + status = "✅ PASS" if passed else "❌ FAIL" + print( + f" accuracy={acc:.3f} threshold={config.accuracy_threshold} {status}" + ) + + all_results.append( + { + "model": display_name, + "accuracy": acc, + "passed": passed, + } + ) + summary += f"| {config.model_path} | {config.variant or 'N/A'} | {config.tp_size} | {acc:.3f} | {config.accuracy_threshold} | {status} |\n" + + finally: + kill_process_tree(process.pid) + + except Exception as e: + summary += f"| {config.model_path} | {config.variant or 'N/A'} | {config.tp_size} | N/A | {config.accuracy_threshold} | ❌ ERROR |\n" + all_results.append( + { + "model": display_name, + "accuracy": None, + "passed": False, + "error": str(e), + } + ) + + if is_in_ci(): + write_github_step_summary(summary) + + failed = [r for r in all_results if not r["passed"]] + if failed: + raise AssertionError(f"Failed models: {[r['model'] for r in failed]}") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/perf/mi30x/test_glm51_perf_amd.py b/test/registered/amd/perf/mi30x/test_glm51_perf_amd.py new file mode 100644 index 000000000..5b2347c9b --- /dev/null +++ b/test/registered/amd/perf/mi30x/test_glm51_perf_amd.py @@ -0,0 +1,138 @@ +"""Nightly performance benchmark for GLM-5.1 on MI30x. + +Tests GLM-5.1-FP8 with NSA attention backend using bench_one_batch +on 8 GPUs with TP=8, FP8 KV cache. + +Model path can be configured via GLM51_MODEL_PATH environment variable. + +Registry: nightly-perf-8-gpu-glm51 suite +""" + +import os +import unittest +from typing import List + +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.nightly_bench_utils import BenchmarkResult +from sglang.test.nightly_utils import NightlyBenchmarkRunner +from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env + +register_amd_ci(est_time=5400, suite="nightly-perf-8-gpu-glm51", nightly=True) + + +def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str: + model_header = results[0].model_path + if results[0].run_name and results[0].run_name != "default": + model_header += f" ({results[0].run_name})" + + gpu_config = os.getenv("GPU_CONFIG", "MI325") + if gpu_config: + model_header += f" [{gpu_config}]" + + summary = f"### {model_header}\n" + summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n" + summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n" + + report_results = ( + results[1:] + if len(results) > 1 and results[0].batch_size == results[1].batch_size + else results + ) + + for result in report_results: + itl = 1 / (result.output_throughput / result.batch_size) * 1000 + summary += f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | {result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n" + + return summary + + +GLM51_MODEL_PATH = os.environ.get("GLM51_MODEL_PATH", "zai-org/GLM-5.1-FP8") +PROFILE_DIR = "performance_profiles_glm51" + + +class TestNightlyGLM51Performance(unittest.TestCase): + """Nightly performance benchmark for GLM-5.1 on MI30x. + + Tests GLM-5.1-FP8 with NSA attention backend on TP=8. + """ + + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.batch_sizes = [1, 8, 16, 64] + cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096")) + cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) + + cls.model_config = { + "name": "glm51", + "model_path": GLM51_MODEL_PATH, + "other_args": [ + "--trust-remote-code", + "--reasoning-parser", + "glm45", + "--tool-call-parser", + "glm47", + "--tp", + "8", + "--nsa-prefill-backend", + "tilelang", + "--nsa-decode-backend", + "tilelang", + "--kv-cache-dtype", + "fp8_e4m3", + "--chunked-prefill-size", + "131072", + "--mem-fraction-static", + "0.85", + "--model-loader-extra-config", + '{"enable_multithread_load": true}', + "--watchdog-timeout", + "1200", + ], + "env_vars": { + "SGLANG_USE_AITER": "1", + }, + } + + cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) + cls.runner.setup_profile_directory() + cls.runner.full_report = f"## {cls.__name__}\n" + + def test_bench_glm51(self): + old_env = {} + for key, value in self.model_config.get("env_vars", {}).items(): + old_env[key] = os.environ.get(key) + os.environ[key] = value + + try: + result_tuple = self.runner.run_benchmark_for_model( + model_path=self.model_config["model_path"], + batch_sizes=self.batch_sizes, + input_lens=self.input_lens, + output_lens=self.output_lens, + other_args=self.model_config["other_args"], + variant=self.model_config["name"], + extra_bench_args=["--trust-remote-code"], + enable_profile=False, + timeout=5400, + ) + results = result_tuple[0] + success = result_tuple[1] + + if results: + self.runner.full_report += ( + generate_simple_markdown_report(results) + "\n" + ) + + self.assertTrue(success, f"Benchmark failed for {GLM51_MODEL_PATH}") + finally: + for key, value in old_env.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + self.runner.write_final_report() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/amd/perf/mi35x/test_glm51_perf_mi35x.py b/test/registered/amd/perf/mi35x/test_glm51_perf_mi35x.py new file mode 100644 index 000000000..e4bb32f07 --- /dev/null +++ b/test/registered/amd/perf/mi35x/test_glm51_perf_mi35x.py @@ -0,0 +1,146 @@ +"""MI35x Nightly performance benchmark for GLM-5.1. + +Tests GLM-5.1-FP8 with NSA attention backend using bench_one_batch +on 8 GPUs with TP=8, FP8 KV cache. + +Registry: nightly-perf-8-gpu-mi35x-glm51 suite +""" + +import os + +os.environ.setdefault("HF_HOME", "/data2/models/huggingface") +os.environ.setdefault("HF_HUB_CACHE", "/data2/models/huggingface/hub") + +import unittest +from typing import List + +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.nightly_bench_utils import BenchmarkResult +from sglang.test.nightly_utils import NightlyBenchmarkRunner +from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, _parse_int_list_env + +register_amd_ci(est_time=5400, suite="nightly-perf-8-gpu-mi35x-glm51", nightly=True) + + +def generate_simple_markdown_report(results: List[BenchmarkResult]) -> str: + model_header = results[0].model_path + if results[0].run_name and results[0].run_name != "default": + model_header += f" ({results[0].run_name})" + + gpu_config = os.getenv("GPU_CONFIG", "MI35x") + if gpu_config: + model_header += f" [{gpu_config}]" + + summary = f"### {model_header}\n" + summary += "| batch size | input len | latency (s) | input throughput (tok/s) | output throughput (tok/s) | ITL (ms) |\n" + summary += "| ---------- | --------- | ----------- | ------------------------ | ------------------------- | -------- |\n" + + report_results = ( + results[1:] + if len(results) > 1 and results[0].batch_size == results[1].batch_size + else results + ) + + for result in report_results: + itl = 1 / (result.output_throughput / result.batch_size) * 1000 + summary += f"| {result.batch_size} | {result.input_len} | {result.latency:.2f} | {result.input_throughput:.2f} | {result.output_throughput:.2f} | {itl:.2f} |\n" + + return summary + + +GLM51_MODEL_PATH = os.environ.get("GLM51_MODEL_PATH", "zai-org/GLM-5.1-FP8") +PROFILE_DIR = "performance_profiles_glm51_mi35x" + + +class TestGLM51PerfMI35x(unittest.TestCase): + """Nightly performance benchmark for GLM-5.1 on MI35x. + + Tests GLM-5.1-FP8 with NSA attention backend on TP=8. + """ + + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.batch_sizes = [1, 8, 16, 64] + cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096")) + cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512")) + + cls.model_config = { + "name": "glm51-mi35x", + "model_path": GLM51_MODEL_PATH, + "other_args": [ + "--trust-remote-code", + "--reasoning-parser", + "glm45", + "--tool-call-parser", + "glm47", + "--tp", + "8", + "--nsa-prefill-backend", + "tilelang", + "--nsa-decode-backend", + "tilelang", + "--kv-cache-dtype", + "fp8_e4m3", + "--chunked-prefill-size", + "131072", + "--mem-fraction-static", + "0.85", + "--model-loader-extra-config", + '{"enable_multithread_load": true, "num_threads": 8}', + "--watchdog-timeout", + "1200", + ], + "env_vars": { + "SGLANG_USE_AITER": "1", + "SGLANG_ROCM_FUSED_DECODE_MLA": "0", + "ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", + "SAFETENSORS_FAST_GPU": "1", + }, + } + + os.environ.setdefault("SGLANG_BENCH_TIMEOUT", "3600") + cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url) + cls.runner.setup_profile_directory() + cls.runner.full_report = f"## {cls.__name__}\n" + + def test_glm51_perf(self): + old_env = {} + for key, value in self.model_config.get("env_vars", {}).items(): + old_env[key] = os.environ.get(key) + os.environ[key] = value + + try: + result_tuple = self.runner.run_benchmark_for_model( + model_path=self.model_config["model_path"], + batch_sizes=self.batch_sizes, + input_lens=self.input_lens, + output_lens=self.output_lens, + other_args=self.model_config["other_args"], + variant=self.model_config["name"], + extra_bench_args=["--trust-remote-code"], + enable_profile=False, + timeout=5400, + ) + results = result_tuple[0] + success = result_tuple[1] + + if results: + self.runner.full_report += ( + generate_simple_markdown_report(results) + "\n" + ) + + self.assertTrue( + success, f"Benchmark failed for {GLM51_MODEL_PATH} on MI35x" + ) + finally: + for key, value in old_env.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + self.runner.write_final_report() + + +if __name__ == "__main__": + unittest.main()