[NPU] Add GitHub test summary and deduplicate test code. Part 2 (#24689)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Elizaveta Martirosian <elizaveta.martirosian@gmail.com>
Co-authored-by: ronnie_zheng <zl19940307@163.com>
This commit is contained in:
Elizaveta Martirosian
2026-06-08 18:08:45 +03:00
committed by GitHub
co-authored by github-actions[bot] Elizaveta Martirosian ronnie_zheng
parent 62c505a196
commit 40030d8af8
12 changed files with 384 additions and 682 deletions
@@ -29,6 +29,7 @@ class GSM8KAscendMixin(ABC):
server_cmd = ""
gsm8k_num_shots = 5
num_questions = 200
gsm8k_parallel = 128
env = {
**os.environ,
@@ -82,13 +83,14 @@ class GSM8KAscendMixin(ABC):
data_path=None,
num_questions=self.num_questions,
max_new_tokens=512,
parallel=128,
parallel=self.gsm8k_parallel,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval(args)
model_metrics["accuracy"] = metrics["accuracy"]
model_metrics["output_throughput"] = metrics["output_throughput"]
model_metrics["latency"] = metrics["latency"]
self.assertGreaterEqual(
metrics["accuracy"],
accuracy_threshold,
@@ -58,6 +58,9 @@ DEEPSEEK_CODER_V2_LITE_WEIGHTS_PATH = os.path.join(
DEEPSEEK_CODER_1_3_B_BASE_PATH = os.path.join(
MODEL_WEIGHTS_DIR, "deepseek-ai/deepseek-coder-1.3b-base"
)
ECO_TECH_QWEN3_32B_W4A4_LAOS_WEIGHTS_PATH = os.path.join(
MODEL_WEIGHTS_DIR, "Eco-Tech/Qwen3-32B-w4a4-LAOS"
)
ERNIE_4_5_21B_A3B_PT_WEIGHTS_PATH = os.path.join(
MODEL_WEIGHTS_DIR, "baidu/ERNIE-4.5-21B-A3B-PT"
)
@@ -154,6 +157,9 @@ QWEN3_32B_W8A8_MINDIE_WEIGHTS_PATH = os.path.join(
MODEL_WEIGHTS_DIR, "aleoyang/Qwen3-32B-w8a8-MindIE"
)
QWQ_32B_W8A8_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "vllm-ascend/QWQ-32B-W8A8")
REDHATAI_QWEN2_5_0_5B_INSTRUCT_QUANTIZED_W8A8_WEIGHTS_PATH = os.path.join(
MODEL_WEIGHTS_DIR, "RedHatAI/Qwen2.5-0.5B-Instruct-quantized.w8a8"
)
SMOLLM_1_7B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "HuggingFaceTB/SmolLM-1.7B")
STABLELM_2_1_6B_WEIGHTS_PATH = os.path.join(
MODEL_WEIGHTS_DIR, "stabilityai/stablelm-2-1_6b"
@@ -565,7 +571,6 @@ def run_bench_serving(
HEADER = """
### Models
| Model | Server | Client | Output Throughput | Expected Output Throughput | Latency | Expected Latency | Accuracy | Expected Accuracy | Status |
| ----- | ------ | ------ | -------- | ------------------ | ------- | ---------------- | -------- | --------- | ------ |
"""
+5 -1
View File
@@ -7,6 +7,8 @@ from sglang.test.run_eval import run_eval
class TestMMLU:
mmlu_num_examples = 128
def test_mmlu(self):
accuracy_mmlu_threshold = getattr(self, "accuracy_mmlu", 0.00)
@@ -23,12 +25,14 @@ class TestMMLU:
base_url=self.base_url,
model=self.model,
eval_name="mmlu",
num_examples=128,
num_examples=self.mmlu_num_examples,
num_threads=32,
)
print("Starting mmlu test...")
metrics = run_eval(args)
model_metrics["accuracy"] = metrics["score"]
model_metrics["latency"] = metrics.get("latency", "-")
model_metrics["output_throughput"] = metrics.get("output_throughput", "-")
self.assertGreater(metrics["score"], accuracy_mmlu_threshold)
except Exception as e:
model_metrics["error"] = e