[Intel XPU] Initially add nightly GSM8K accuracy tests for Llama-3.1-8B (TP=2) and Qwen3-32B (TP=4) (#28908)

Co-authored-by: Singhal, Shubham <shubham.singhal@intel.com>

python/sglang/test/xpu/simple_eval_gsm8k_xpu_mixin.py

@@ -0,0 +1,129 @@
+"""simple-evals GSM8K accuracy mixin for Intel XPU nightly tests.
+
+Mirrors the AMD/NVIDIA nightly flow (``test_gsm8k_eval_amd.py`` /
+``test_text_models_gsm8k_eval.py``): launch an SGLang server with XPU
+flags, then call ``sglang.test.run_eval`` with ``eval_name="gsm8k"`` so
+the same ``simple_eval_gsm8k.GSM8KEval`` evaluator scores every backend.
+
+Subclasses set ``model``, ``tp_size``, ``accuracy``, and may override
+``other_args`` / ``env`` / ``num_examples`` / ``num_threads``.
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+from abc import ABC
+from types import SimpleNamespace
+
+from sglang.srt.utils import kill_process_tree
+from sglang.test.run_eval import run_eval
+from sglang.test.test_utils import (
+    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
+    DEFAULT_URL_FOR_TEST,
+    popen_launch_server,
+    write_github_step_summary,
+)
+from sglang.test.xpu.test_xpu_utils import write_results_to_github_step_summary
+
+
+class SimpleEvalGSM8KXPUMixin(ABC):
+    model: str = ""
+    tp_size: int = 1
+
+    timeout_for_server_launch = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
+    other_args: list[str] = [
+        "--device",
+        "xpu",
+        "--attention-backend",
+        "intel_xpu",
+        "--dtype",
+        "bfloat16",
+        "--trust-remote-code",
+        "--disable-overlap-schedule",
+        "--disable-radix-cache",
+    ]
+    env: dict | None = None
+
+    server_cmd: str = ""
+    # 200 questions matches the limit used by the XPU 70B lm-eval YAML and
+    # fits inside run_suite's per-file timeout when num_threads=1 keeps
+    # throughput low. Subclasses on cheaper-per-token hardware (TP=1, no
+    # Level Zero wedge) can raise this or set None for the full 1319-question
+    # GSM8K test set, matching the AMD/NVIDIA nightly defaults.
+    num_examples: int | None = 200
+    # Single-stream eval: intel_xpu attention at TP>=2 wedges the Level Zero
+    # driver in ur_command_list_manager::appendUSMMemcpy on concurrent prefill.
+    # Subclasses on hardware that handles parallel prefill cleanly may bump.
+    num_threads: int = 1
+    # Short generations reduce the rate of prefill->decode->prefill handoffs,
+    # which is what trips the same Level Zero wedge on TP>=2 (observed at the
+    # default 2048; 512 matches the original few_shot_gsm8k limit and is still
+    # enough for GSM8K CoT answers).
+    max_tokens: int = 512
+
+    @classmethod
+    def setUpClass(cls):
+        cls.base_url = DEFAULT_URL_FOR_TEST
+        env = {**os.environ, **(cls.env or {})}
+        args = list(cls.other_args) + ["--tp-size", str(cls.tp_size)]
+        try:
+            cls.process = popen_launch_server(
+                cls.model,
+                cls.base_url,
+                timeout=cls.timeout_for_server_launch,
+                other_args=args,
+                env=env,
+            )
+            cls.server_cmd = subprocess.list2cmdline(cls.process.args)
+        except Exception as e:
+            write_github_step_summary(f"Failed to launch server for {cls.model}: {e}")
+            raise AssertionError(f"Test failed for {cls.model}: {e}")
+
+    @classmethod
+    def tearDownClass(cls):
+        kill_process_tree(cls.process.pid)
+
+    def test_gsm8k(self):
+        accuracy_threshold = getattr(self, "accuracy", 0.0)
+        output_throughput_threshold = getattr(self, "output_throughput", 0.0)
+
+        model_metrics = {
+            "server": self.server_cmd,
+            "client": "simple_eval_gsm8k",
+            "accuracy_threshold": getattr(self, "accuracy", "N/A"),
+            "output_throughput_threshold": getattr(self, "output_throughput", "N/A"),
+        }
+
+        try:
+            args = SimpleNamespace(
+                base_url=self.base_url,
+                model=self.model,
+                eval_name="gsm8k",
+                num_examples=self.num_examples,
+                num_threads=self.num_threads,
+                max_tokens=self.max_tokens,
+            )
+            metrics = run_eval(args)
+            model_metrics["accuracy"] = metrics["score"]
+            model_metrics["output_throughput"] = metrics.get("output_throughput")
+            model_metrics["latency"] = metrics["latency"]
+            self.assertGreaterEqual(
+                metrics["score"],
+                accuracy_threshold,
+                f'Accuracy of {self.model} is {metrics["score"]}, '
+                f"is lower than {accuracy_threshold}",
+            )
+            if "output_throughput" in metrics:
+                self.assertGreaterEqual(
+                    metrics["output_throughput"],
+                    output_throughput_threshold,
+                    f"Output throughput of {self.model} is "
+                    f'{metrics["output_throughput"]}, is lower than '
+                    f"{output_throughput_threshold}",
+                )
+        except Exception as e:
+            model_metrics["error"] = str(e)
+            self.fail(f"Test failed for {self.model}: {e}")
+        finally:
+            write_results_to_github_step_summary({self.model: model_metrics})

python/sglang/test/xpu/test_xpu_utils.py

@@ -0,0 +1,50 @@
+"""Helpers shared by XPU nightly model tests.
+
+The summary writer mirrors `python/sglang/test/ascend/test_ascend_utils.py`
+so XPU and Ascend nightly runs render the same Markdown table in
+`$GITHUB_STEP_SUMMARY`.
+"""
+
+from sglang.test.test_utils import is_in_ci, write_github_step_summary
+
+HEADER = """
+| Model | Server | Client | Output Throughput | Expected Output Throughput | Accuracy | Expected Accuracy | Status |
+| ----- | ------ | ------ | ----------------- | -------------------------- | -------- | ----------------- | ------ |
+"""
+
+_HEADER_WRITTEN = False
+
+
+def _write_header_once():
+    global _HEADER_WRITTEN
+    if not _HEADER_WRITTEN:
+        write_github_step_summary(HEADER)
+        _HEADER_WRITTEN = True
+
+
+def write_results_to_github_step_summary(results: dict):
+    if not is_in_ci():
+        return
+
+    _write_header_once()
+
+    def fmt(metrics, key, precision):
+        v = metrics.get(key, "-")
+        return f"{v:.{precision}f}" if isinstance(v, (int, float)) else v
+
+    summary = ""
+    for model, metrics in results.items():
+        accuracy = fmt(metrics, "accuracy", 4)
+        accuracy_threshold = metrics.get("accuracy_threshold", "N/A")
+        output_throughput = fmt(metrics, "output_throughput", 2)
+        output_throughput_threshold = metrics.get("output_throughput_threshold", "N/A")
+        server = metrics.get("server", "N/A")
+        client = metrics.get("client", "N/A")
+        error = metrics.get("error", "")
+        status = "PASS" if error == "" else f"FAIL: {error}"
+        summary += (
+            f"| {model} | {server} | {client} | {output_throughput} "
+            f"| {output_throughput_threshold} | {accuracy} "
+            f"| {accuracy_threshold} | {status} |\n"
+        )
+    write_github_step_summary(summary)