[NPU] Add GitHub test summary and deduplicate test code. Part 2 (#24689)
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: Elizaveta Martirosian <elizaveta.martirosian@gmail.com> Co-authored-by: ronnie_zheng <zl19940307@163.com>
This commit is contained in:
co-authored by
github-actions[bot]
Elizaveta Martirosian
ronnie_zheng
parent
62c505a196
commit
40030d8af8
@@ -29,6 +29,7 @@ class GSM8KAscendMixin(ABC):
|
||||
server_cmd = ""
|
||||
gsm8k_num_shots = 5
|
||||
num_questions = 200
|
||||
gsm8k_parallel = 128
|
||||
|
||||
env = {
|
||||
**os.environ,
|
||||
@@ -82,13 +83,14 @@ class GSM8KAscendMixin(ABC):
|
||||
data_path=None,
|
||||
num_questions=self.num_questions,
|
||||
max_new_tokens=512,
|
||||
parallel=128,
|
||||
parallel=self.gsm8k_parallel,
|
||||
host="http://127.0.0.1",
|
||||
port=int(self.base_url.split(":")[-1]),
|
||||
)
|
||||
metrics = run_eval(args)
|
||||
model_metrics["accuracy"] = metrics["accuracy"]
|
||||
model_metrics["output_throughput"] = metrics["output_throughput"]
|
||||
model_metrics["latency"] = metrics["latency"]
|
||||
self.assertGreaterEqual(
|
||||
metrics["accuracy"],
|
||||
accuracy_threshold,
|
||||
|
||||
@@ -58,6 +58,9 @@ DEEPSEEK_CODER_V2_LITE_WEIGHTS_PATH = os.path.join(
|
||||
DEEPSEEK_CODER_1_3_B_BASE_PATH = os.path.join(
|
||||
MODEL_WEIGHTS_DIR, "deepseek-ai/deepseek-coder-1.3b-base"
|
||||
)
|
||||
ECO_TECH_QWEN3_32B_W4A4_LAOS_WEIGHTS_PATH = os.path.join(
|
||||
MODEL_WEIGHTS_DIR, "Eco-Tech/Qwen3-32B-w4a4-LAOS"
|
||||
)
|
||||
ERNIE_4_5_21B_A3B_PT_WEIGHTS_PATH = os.path.join(
|
||||
MODEL_WEIGHTS_DIR, "baidu/ERNIE-4.5-21B-A3B-PT"
|
||||
)
|
||||
@@ -154,6 +157,9 @@ QWEN3_32B_W8A8_MINDIE_WEIGHTS_PATH = os.path.join(
|
||||
MODEL_WEIGHTS_DIR, "aleoyang/Qwen3-32B-w8a8-MindIE"
|
||||
)
|
||||
QWQ_32B_W8A8_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "vllm-ascend/QWQ-32B-W8A8")
|
||||
REDHATAI_QWEN2_5_0_5B_INSTRUCT_QUANTIZED_W8A8_WEIGHTS_PATH = os.path.join(
|
||||
MODEL_WEIGHTS_DIR, "RedHatAI/Qwen2.5-0.5B-Instruct-quantized.w8a8"
|
||||
)
|
||||
SMOLLM_1_7B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "HuggingFaceTB/SmolLM-1.7B")
|
||||
STABLELM_2_1_6B_WEIGHTS_PATH = os.path.join(
|
||||
MODEL_WEIGHTS_DIR, "stabilityai/stablelm-2-1_6b"
|
||||
@@ -565,7 +571,6 @@ def run_bench_serving(
|
||||
|
||||
|
||||
HEADER = """
|
||||
### Models
|
||||
| Model | Server | Client | Output Throughput | Expected Output Throughput | Latency | Expected Latency | Accuracy | Expected Accuracy | Status |
|
||||
| ----- | ------ | ------ | -------- | ------------------ | ------- | ---------------- | -------- | --------- | ------ |
|
||||
"""
|
||||
|
||||
@@ -7,6 +7,8 @@ from sglang.test.run_eval import run_eval
|
||||
|
||||
class TestMMLU:
|
||||
|
||||
mmlu_num_examples = 128
|
||||
|
||||
def test_mmlu(self):
|
||||
accuracy_mmlu_threshold = getattr(self, "accuracy_mmlu", 0.00)
|
||||
|
||||
@@ -23,12 +25,14 @@ class TestMMLU:
|
||||
base_url=self.base_url,
|
||||
model=self.model,
|
||||
eval_name="mmlu",
|
||||
num_examples=128,
|
||||
num_examples=self.mmlu_num_examples,
|
||||
num_threads=32,
|
||||
)
|
||||
print("Starting mmlu test...")
|
||||
metrics = run_eval(args)
|
||||
model_metrics["accuracy"] = metrics["score"]
|
||||
model_metrics["latency"] = metrics.get("latency", "-")
|
||||
model_metrics["output_throughput"] = metrics.get("output_throughput", "-")
|
||||
self.assertGreater(metrics["score"], accuracy_mmlu_threshold)
|
||||
except Exception as e:
|
||||
model_metrics["error"] = e
|
||||
|
||||
Reference in New Issue
Block a user