From 816ea65058d28d4ad7695b3c950617b76d49aa2e Mon Sep 17 00:00:00 2001 From: Michael <13900043+michaelzhang-ai@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:48:29 -0700 Subject: [PATCH] [AMD] Add Kimi-K3 8-GPU MI35x nightly accuracy CI (#32568) Co-authored-by: Cursor Agent Co-authored-by: Michael --- .../workflows/nightly-test-amd-rocm720.yml | 31 ++-- .../accuracy/mi35x/test_kimi_k3_eval_mi35x.py | 163 ++++++++++++++++++ test/run_suite.py | 1 + 3 files changed, 185 insertions(+), 10 deletions(-) create mode 100644 test/registered/amd/accuracy/mi35x/test_kimi_k3_eval_mi35x.py diff --git a/.github/workflows/nightly-test-amd-rocm720.yml b/.github/workflows/nightly-test-amd-rocm720.yml index 3a81f23bd..124fbf805 100644 --- a/.github/workflows/nightly-test-amd-rocm720.yml +++ b/.github/workflows/nightly-test-amd-rocm720.yml @@ -71,9 +71,10 @@ on: - nightly-8-gpu-mi35x-deepseek-v4-pro-rocm720 - nightly-8-gpu-mi35x-deepseek-v4-pro-mtp-rocm720 - nightly-8-gpu-mi35x-deepseek-v4-pro-dspark-rocm720 - # 8-GPU Kimi-K2.6 (MI30x + MI35x) + # 8-GPU Kimi-K2.6 (MI30x) - nightly-8-gpu-kimi-k26-rocm720 - - nightly-8-gpu-mi35x-kimi-k26-rocm720 + # 8-GPU Kimi-K3 (MI35x only - native MXFP4 needs gfx95x) + - nightly-8-gpu-mi35x-kimi-k3-rocm720 # 8-GPU Qwen3-235B (MI30x + MI35x MXFP4) - nightly-8-gpu-qwen3-235b-rocm720 - nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720 @@ -1498,7 +1499,10 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true # ============================================================================== - # 8-GPU Kimi-K2.6 (MI30x + MI35x) + # 8-GPU Kimi-K2.6 (MI30x) + # + # Kept on MI30x: Kimi-K3 needs gfx95x for its native MXFP4 weights, so it + # cannot take over here the way it does on MI35x. # ============================================================================== nightly-8-gpu-kimi-k26-rocm720: @@ -1534,8 +1538,12 @@ jobs: echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} - nightly-8-gpu-mi35x-kimi-k26-rocm720: - if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-kimi-k26-rocm720,')) + # ============================================================================== + # 8-GPU Kimi-K3 (MI35x only - native MXFP4 needs gfx95x) + # ============================================================================== + + nightly-8-gpu-mi35x-kimi-k3-rocm720: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && (!(inputs.job_filter || inputs.job_select) || (inputs.job_filter || inputs.job_select) == 'all' || contains(format(',{0},', inputs.job_filter || inputs.job_select), ',nightly-8-gpu-mi35x-kimi-k3-rocm720,')) runs-on: linux-mi35x-gpu-8 steps: - name: Checkout code @@ -1560,13 +1568,15 @@ jobs: # Install tabulate for run_suite.py (missing in MI35x container) bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - - name: Accuracy Test MI35x ROCm 7.2 (8-GPU Kimi-K2.6) - timeout-minutes: 180 + # The 2.8T MXFP4 checkpoint is ~1.56 TB, so weight download and load + # dominate wall time well before the eval itself starts. + - name: Accuracy Test MI35x ROCm 7.2 (8-GPU Kimi-K3) + timeout-minutes: 300 run: | > github_summary.md # Clear summary file bash scripts/ci/amd/amd_ci_exec.sh -w /sglang-checkout/test \ -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" \ - python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-kimi-k26 --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? + python3 run_suite.py --hw amd --suite nightly-amd-accuracy-8-gpu-mi35x-kimi-k3 --nightly --timeout-per-file 14400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} @@ -2137,9 +2147,10 @@ jobs: - nightly-8-gpu-mi35x-deepseek-v4-pro-rocm720 - nightly-8-gpu-mi35x-deepseek-v4-pro-mtp-rocm720 - nightly-8-gpu-mi35x-deepseek-v4-pro-dspark-rocm720 - # 8-GPU Kimi-K2.6 (MI30x + MI35x) + # 8-GPU Kimi-K2.6 (MI30x) - nightly-8-gpu-kimi-k26-rocm720 - - nightly-8-gpu-mi35x-kimi-k26-rocm720 + # 8-GPU Kimi-K3 (MI35x only - native MXFP4 needs gfx95x) + - nightly-8-gpu-mi35x-kimi-k3-rocm720 # 8-GPU Qwen3-235B (MI30x + MI35x MXFP4) - nightly-8-gpu-qwen3-235b-rocm720 - nightly-8-gpu-mi35x-qwen3-235b-mxfp4-rocm720 diff --git a/test/registered/amd/accuracy/mi35x/test_kimi_k3_eval_mi35x.py b/test/registered/amd/accuracy/mi35x/test_kimi_k3_eval_mi35x.py new file mode 100644 index 000000000..9a3b43f93 --- /dev/null +++ b/test/registered/amd/accuracy/mi35x/test_kimi_k3_eval_mi35x.py @@ -0,0 +1,163 @@ +"""MI35x Kimi-K3 GSM8K Completion Evaluation Test (8-GPU) + +Tests moonshotai/Kimi-K3 with GSM8K few-shot benchmark on MI35x. + +Server arguments follow the Day-0 recipe in the AMD tracking issue +(sgl-project/sglang#32548) for the non-speculative config: TP8 with the +Triton attention backend, the AITER FlyDSL A8W4 SiTU MoE path, and the radix +cache disabled. + +That issue reports throughput on MI355 TP8 but no accuracy, and the cookbook +cell for this topology is still published as `verified: false`. So the recipe +is known to run at speed; what is missing, and what this test supplies, is +evidence that it produces correct tokens. + +First green run on 8xMI355X scored 0.956 with 0.2% unparsable, against the +0.92 threshold, in about 74 minutes end to end. + +K3's native MXFP4 weights need gfx95x, so this runs on MI35x only -- mxfp4 +does not register on gfx942 (MI300/MI325). The 2.8T checkpoint is ~1.5 TB in +MXFP4, roughly 192 GB of the 288 GB on each of the 8 GPUs, which is why +`--mem-fraction-static` stays at the cookbook's 0.85 and concurrency is +capped rather than left unbounded. + +The eval uses the few-shot *completion* harness that every sibling Kimi MI35x +test uses (K2, K2.5, K2.6). It deliberately bypasses the chat template: K3 +has thinking permanently enabled and routes its answer through +`reasoning_content` on the chat path, which would leave `message.content` +empty and score 0. Scoring raw completions keeps this test measuring what it +is meant to measure -- whether the ROCm kernels produce correct tokens. + +Registry: nightly-amd-accuracy-8-gpu-mi35x-kimi-k3 suite +""" + +import os +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +# Register for AMD CI - Kimi K3 accuracy test on MI35x (~150 min: the 1.5 TB +# MXFP4 checkpoint dominates startup, then 1319 GSM8K questions at TP8) +register_amd_ci( + est_time=9000, suite="nightly-amd-accuracy-8-gpu-mi35x-kimi-k3", nightly=True +) + +KIMI_K3_MODEL_PATH = os.environ.get("KIMI_K3_MODEL_PATH", "moonshotai/Kimi-K3") +SERVER_LAUNCH_TIMEOUT = 9000 +ACCURACY_THRESHOLD = 0.92 +TP_SIZE = 8 +# The one deviation from the tracking-issue recipe, which leaves concurrency +# unbounded and captures decode graphs to 256. Those runs drove bounded +# concurrency (<= 32); this eval submits every question at once, so it needs an +# explicit ceiling to stay inside the ~53 GB per GPU left after weights. +# --cuda-graph-max-bs-decode is kept equal to it, since capture across K3's 93 +# attention + 92 MoE layers is expensive and capturing above the concurrency +# ceiling buys nothing. +MAX_RUNNING_REQUESTS = 64 + + +class TestKimiK3EvalMI35x(CustomTestCase): + """Kimi-K3 GSM8K Completion Evaluation Test for AMD MI35x.""" + + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.num_questions = int(os.environ.get("GSM8K_NUM_QUESTIONS", "1319")) + cls.max_new_tokens = int(os.environ.get("GSM8K_MAX_NEW_TOKENS", "512")) + + def test_kimi_k3_gsm8k_accuracy(self): + """Test Kimi-K3 with GSM8K few-shot completion benchmark.""" + other_args = [ + "--tp", + str(TP_SIZE), + "--attention-backend", + "triton", + "--dtype", + "bfloat16", + "--mem-fraction-static", + "0.85", + "--disable-radix-cache", + "--cuda-graph-max-bs-decode", + str(MAX_RUNNING_REQUESTS), + "--max-running-requests", + str(MAX_RUNNING_REQUESTS), + "--reasoning-parser", + "kimi_k3", + "--tool-call-parser", + "kimi_k3", + "--trust-remote-code", + "--model-loader-extra-config", + '{"enable_multithread_load": true}', + "--watchdog-timeout", + "1200", + ] + env = os.environ.copy() + # AITER supplies K3's MoE on ROCm. SGLANG_AITER_K3_OPT is read in + # mxfp4.py and models/kimi_k3.py but is gated behind SGLANG_USE_AITER, + # so both must be set. AITER_SITUV2_A8W4 selects the W4A8 SiTU expert + # kernels; it additionally requires the MoE activation to be "situ", + # which K3 supplies from its own config. AITER_FLYDSL_FORCE is consumed + # by AITER itself rather than sglang -- it will not grep to anything + # in-tree. + env["SGLANG_USE_AITER"] = "1" + env["SGLANG_AITER_K3_OPT"] = "1" + env["AITER_FLYDSL_FORCE"] = "1" + env["AITER_SITUV2_A8W4"] = "1" + + process = popen_launch_server( + KIMI_K3_MODEL_PATH, + self.base_url, + timeout=SERVER_LAUNCH_TIMEOUT, + other_args=other_args, + env=env, + ) + + try: + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=8, + data_path=None, + num_questions=self.num_questions, + parallel=self.num_questions, + max_new_tokens=self.max_new_tokens, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + acc = metrics["accuracy"] + + passed = acc >= ACCURACY_THRESHOLD + status = "✅ PASS" if passed else "❌ FAIL" + print(f" accuracy={acc:.3f} threshold={ACCURACY_THRESHOLD} {status}") + + if is_in_ci(): + summary = "### Kimi-K3 Model (MI35x)\n\n" + summary += "| Model | TP | Accuracy | Threshold | Status |\n" + summary += "| ----- | -- | -------- | --------- | ------ |\n" + summary += f"| {KIMI_K3_MODEL_PATH} | {TP_SIZE} | {acc:.3f} | {ACCURACY_THRESHOLD} | {status} |\n" + write_github_step_summary(summary) + + self.assertGreaterEqual( + acc, + ACCURACY_THRESHOLD, + f"Kimi-K3 accuracy {acc:.3f} below threshold {ACCURACY_THRESHOLD}", + ) + finally: + kill_process_tree(process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/run_suite.py b/test/run_suite.py index e2ff0428f..dece7e03f 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -140,6 +140,7 @@ NIGHTLY_SUITES = { "nightly-amd-1-gpu-zimage-turbo", "nightly-amd-2-gpu-mi35x-deepseek-r1-mxfp4-tp2", "nightly-amd-8-gpu-mi35x-deepseek-r1-mxfp4-tp4", + "nightly-amd-accuracy-8-gpu-mi35x-kimi-k3", "nightly-amd-4-gpu", "nightly-amd-8-gpu", "nightly-amd-vlm",