diff --git a/python/sglang/kernels/ops/attention/fla/layernorm_gated.py b/python/sglang/kernels/ops/attention/fla/layernorm_gated.py index c0de08c458..8513bc5c40 100644 --- a/python/sglang/kernels/ops/attention/fla/layernorm_gated.py +++ b/python/sglang/kernels/ops/attention/fla/layernorm_gated.py @@ -8,6 +8,7 @@ from contextlib import nullcontext from functools import lru_cache +from typing import Optional import torch import torch.nn.functional as F @@ -30,6 +31,7 @@ from sglang.srt.utils import ( is_npu, next_power_of_2, ) +from sglang.srt.utils.custom_op import register_custom_op _is_npu = is_npu() _use_cpu = is_cpu() and cpu_has_amx_support() @@ -323,7 +325,70 @@ def _layer_norm_fwd( if _is_npu: - from sgl_kernel_npu.fla.layernorm_gated import layer_norm_fwd_npu as _layer_norm_fwd + from sgl_kernel_npu.fla.layernorm_gated import ( + layer_norm_fwd_npu as _layer_norm_fwd_npu_raw, + ) + + @register_custom_op(out_shape="x", mutates_args=["out"]) + def _layer_norm_fwd_custom( + x: torch.Tensor, + weight: torch.Tensor, + bias: Optional[torch.Tensor], + eps: float, + z: Optional[torch.Tensor], + out: Optional[torch.Tensor], + group_size: Optional[int], + norm_before_gate: bool, + is_rms_norm: bool, + activation: Optional[str], + ) -> torch.Tensor: + _out, _mean, _rstd = _layer_norm_fwd_npu_raw( + x=x, + weight=weight, + bias=bias, + eps=eps, + z=z, + out=out, + group_size=group_size, + norm_before_gate=norm_before_gate, + is_rms_norm=is_rms_norm, + activation=activation, + ) + return _out + + def _layer_norm_fwd( + x, + weight, + bias, + eps, + z=None, + out=None, + group_size=None, + norm_before_gate=True, + is_rms_norm=False, + activation=None, + ): + if group_size is None: + group_size = x.shape[-1] + if out is not None: + assert out.shape == x.shape + else: + out = torch.empty_like(x) + y = _layer_norm_fwd_custom( + x=x, + weight=weight, + bias=bias, + eps=eps, + z=z, + out=out, + group_size=group_size, + norm_before_gate=norm_before_gate, + is_rms_norm=is_rms_norm, + activation=activation, + ) + # mean/rstd are unused by all callers on the NPU path (rms_norm_gated + # discards them); return None to keep the (y, mean, rstd) unpack contract. + return y, None, None def rms_norm_gated( diff --git a/python/sglang/kernels/ops/elementwise/elementwise.py b/python/sglang/kernels/ops/elementwise/elementwise.py index 85ba16c84a..8e41e1647a 100644 --- a/python/sglang/kernels/ops/elementwise/elementwise.py +++ b/python/sglang/kernels/ops/elementwise/elementwise.py @@ -3,9 +3,10 @@ import triton import triton.language as tl from sglang.kernels.jit.utils import is_arch_support_pdl -from sglang.srt.utils import is_hip +from sglang.srt.utils import is_hip, is_npu _is_hip = is_hip() +_is_npu = is_npu() rmsnorm_autotune = triton.autotune( @@ -443,6 +444,15 @@ def fused_sigmoid_mul( gate_stride_row = hidden_dim gate_stride_head = hidden_dim + if _is_npu and torch.compiler.is_compiling(): + gate_reshaped = ( + gate.view_as(attn_output) if gate.numel() == attn_output.numel() else gate + ) + if inplace: + attn_output.mul_(torch.sigmoid(gate_reshaped)) + return attn_output + return attn_output * torch.sigmoid(gate_reshaped) + out = attn_output if inplace else torch.empty_like(attn_output) block_h = 1024 if num_tokens < 1024 else 2048 grid = (num_tokens, triton.cdiv(hidden_dim, block_h)) diff --git a/python/sglang/srt/arg_groups/cuda_graph_hook.py b/python/sglang/srt/arg_groups/cuda_graph_hook.py index 1aeee2839b..f5825c7509 100644 --- a/python/sglang/srt/arg_groups/cuda_graph_hook.py +++ b/python/sglang/srt/arg_groups/cuda_graph_hook.py @@ -14,6 +14,7 @@ from sglang.srt.arg_groups.overrides import ( resolving_view, ) from sglang.srt.connector import ConnectorType +from sglang.srt.environ import envs from sglang.srt.model_executor.cuda_graph_config import ( ALLOWED_BACKENDS_PER_PHASE, Backend, @@ -193,7 +194,10 @@ def disable_tc_piecewise_cudagraph_if_incompatible(server_args: Any): "non-CUDA hardware (HIP/NPU/CPU/MPS/XPU)", lambda: ( get_platform().is_hip - or get_platform().is_npu + or ( + get_platform().is_npu + and not envs.SGLANG_NPU_ENABLE_PIECEWISE_CUDA_GRAPH.get() + ) or is_cpu() or is_mps() or get_platform().is_xpu diff --git a/python/sglang/srt/compilation/npu_piecewise_backend.py b/python/sglang/srt/compilation/npu_piecewise_backend.py index dc97bd5c3f..f33684d509 100644 --- a/python/sglang/srt/compilation/npu_piecewise_backend.py +++ b/python/sglang/srt/compilation/npu_piecewise_backend.py @@ -7,6 +7,10 @@ import torch.fx as fx from sglang.srt.compilation.compilation_config import CompilationConfig from sglang.srt.compilation.compilation_counter import compilation_counter +from sglang.srt.compilation.compile_phase import ( + get_pcg_capture_stream, + is_in_torch_compile_warmup, +) from sglang.srt.compilation.cuda_piecewise_backend import ( CUDAPiecewiseBackend, weak_ref_tensors, @@ -39,6 +43,12 @@ class NPUPiecewiseBackend(CUDAPiecewiseBackend): ) def __call__(self, *args): + if is_in_torch_compile_warmup(): + return self.compiled_graph_for_general_shape(*args) + + if len(self.sym_shape_indices) == 0: + return self.compiled_graph_for_general_shape(*args) + runtime_shape = args[self.sym_shape_indices[0]] if runtime_shape not in self.concrete_size_entries: # we don't need to do anything for this shape @@ -73,7 +83,12 @@ class NPUPiecewiseBackend(CUDAPiecewiseBackend): stack.enter_context(patch("torch.npu.empty_cache", lambda: None)) # mind-exploding: carefully manage the reference and memory. - with torch.npu.graph(npugraph, pool=self.graph_pool): + with torch.npu.graph( + npugraph, + pool=self.graph_pool, + stream=get_pcg_capture_stream(), + auto_dispatch_capture=True, + ): # `output` is managed by pytorch's cudagraph pool output = entry.runnable(*args) if self.is_last_graph: diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index e874228d27..4a28e71a5d 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -976,6 +976,7 @@ class Envs: DEEP_NORMAL_MODE_USE_INT8_QUANT = EnvBool(False) SGLANG_ZBAL_LOCAL_MEM_SIZE = EnvInt(0) SGLANG_ZBAL_BOOTSTRAP_URL = EnvStr("") + SGLANG_NPU_ENABLE_PIECEWISE_CUDA_GRAPH = EnvBool(False) # prefill cuda graph on NPU # =================================================================== # MUSA diff --git a/python/sglang/srt/hardware_backend/npu/cmo.py b/python/sglang/srt/hardware_backend/npu/cmo.py index 214097de60..2e3907bd6a 100644 --- a/python/sglang/srt/hardware_backend/npu/cmo.py +++ b/python/sglang/srt/hardware_backend/npu/cmo.py @@ -1,5 +1,9 @@ +from typing import List + import torch +from sglang.srt.utils.custom_op import register_custom_op + cmo_stream = None share_stream = None @@ -19,12 +23,10 @@ def set_cmo_stream(stream): cmo_stream = stream -def prepare_weight_cache(handle, cache, PREFETCH_MAX_SIZE=1000000000): - """ - PREFETCH_MAX_SIZE: maximum size (bytes) for each prefetch operation. - This affects the time spent in prefetch: - time ≈ PREFETCH_MAX_SIZE / system_bandwidth - """ +@register_custom_op() +def _prepare_weight_cache_list( + handle: torch.Tensor, cache: List[torch.Tensor], max_size: int +) -> None: import torch_npu stream = get_cmo_stream() @@ -33,26 +35,44 @@ def prepare_weight_cache(handle, cache, PREFETCH_MAX_SIZE=1000000000): set_cmo_stream(stream) stream.wait_stream(torch.npu.current_stream()) with torch.npu.stream(stream): - if isinstance(cache, list): - for weight in cache: - torch_npu.npu_prefetch( - weight, - handle, - PREFETCH_MAX_SIZE, - ) - else: - torch_npu.npu_prefetch( - cache, - handle, - PREFETCH_MAX_SIZE, - ) + for weight in cache: + torch_npu.npu_prefetch(weight, handle, max_size) -def wait_cmo_stream(): +@register_custom_op() +def _prepare_weight_cache_single( + handle: torch.Tensor, cache: torch.Tensor, max_size: int +) -> None: + import torch_npu + + stream = get_cmo_stream() + if stream is None: + stream = torch.npu.Stream() + set_cmo_stream(stream) + stream.wait_stream(torch.npu.current_stream()) + with torch.npu.stream(stream): + torch_npu.npu_prefetch(cache, handle, max_size) + + +def prepare_weight_cache(handle, cache, PREFETCH_MAX_SIZE=1000000000): + """ + PREFETCH_MAX_SIZE: maximum size (bytes) for each prefetch operation. + This affects the time spent in prefetch: + time ≈ PREFETCH_MAX_SIZE / system_bandwidth + """ + if isinstance(cache, list): + _prepare_weight_cache_list(handle, cache, PREFETCH_MAX_SIZE) + else: + _prepare_weight_cache_single(handle, cache, PREFETCH_MAX_SIZE) + + +@register_custom_op(out_shape=0) +def wait_cmo_stream(dummy: torch.Tensor) -> torch.Tensor: stream = get_cmo_stream() if stream is not None: cur_stream = torch.npu.current_stream() cur_stream.wait_stream(stream) + return dummy def get_share_stream(): @@ -65,14 +85,21 @@ def set_share_stream(stream): share_stream = stream -def wait_share_stream(): +@register_custom_op(out_shape=0) +def wait_share_stream(dummy: torch.Tensor) -> torch.Tensor: stream = get_share_stream() if stream is not None: cur_stream = torch.npu.current_stream() cur_stream.wait_stream(stream) + return dummy def shared_expert_on_independent_stream(hidden_states, forward_func): + import torch.compiler + + if torch.compiler.is_compiling(): + return forward_func(hidden_states) + stream = get_share_stream() if stream is None: stream = torch.npu.Stream() diff --git a/python/sglang/srt/hardware_backend/npu/moe/topk.py b/python/sglang/srt/hardware_backend/npu/moe/topk.py index 50b90aeb8a..d73cad4cba 100644 --- a/python/sglang/srt/hardware_backend/npu/moe/topk.py +++ b/python/sglang/srt/hardware_backend/npu/moe/topk.py @@ -1,7 +1,16 @@ from typing import TYPE_CHECKING, Optional import torch -from sgl_kernel_npu.norm.l1_norm import l1_norm + +from sglang.srt.utils.custom_op import register_custom_op + + +@register_custom_op(out_shape=0) +def _l1_norm(x: torch.Tensor) -> torch.Tensor: + from sgl_kernel_npu.norm.l1_norm import l1_norm + + return l1_norm(x) + from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location_dispatch import topk_ids_logical_to_physical @@ -59,7 +68,7 @@ def fused_topk_npu( ) if renormalize: - topk_weights = l1_norm( + topk_weights = _l1_norm( topk_weights if topk_config.num_fused_shared_experts == 0 else topk_weights[:, :-1] diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index 692fdafff7..3150a3637f 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -625,7 +625,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module): ) trace_sync("post_experts") if enable_dual_stream: - wait_share_stream() + final_hidden_states = wait_share_stream(final_hidden_states) elif enable_cuda_shared_overlap: torch.cuda.current_stream().wait_event(shared_event) diff --git a/python/sglang/srt/models/qwen3.py b/python/sglang/srt/models/qwen3.py index dca81ae256..32d79d9447 100644 --- a/python/sglang/srt/models/qwen3.py +++ b/python/sglang/srt/models/qwen3.py @@ -33,6 +33,7 @@ from sglang.srt.models.utils import apply_qk_norm from sglang.srt.platforms import current_platform from sglang.srt.runtime_context import get_exec, get_parallel, get_stream from sglang.srt.utils import add_prefix, get_bool_env_var, is_cuda, is_hip, is_npu +from sglang.srt.utils.custom_op import register_custom_op Qwen3Config = None @@ -57,6 +58,55 @@ if _is_npu: from sglang.srt.hardware_backend.npu.cmo import get_cmo_stream, wait_cmo_stream + def _split_qkv_rmsnorm_rope_fake_impl( + qkv: torch.Tensor, + sin: torch.Tensor, + cos: torch.Tensor, + q_size: int, + kv_size: int, + head_dim: int, + eps: float = 1e-6, + q_weight: Optional[torch.Tensor] = None, + k_weight: Optional[torch.Tensor] = None, + q_bias: Optional[torch.Tensor] = None, + k_bias: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + del sin, cos, head_dim, eps, q_weight, k_weight, q_bias, k_bias + output_shape = qkv.shape[:-1] + return ( + qkv.new_empty((*output_shape, q_size)), + qkv.new_empty((*output_shape, kv_size)), + qkv.new_empty((*output_shape, kv_size)), + ) + + @register_custom_op(fake_impl=_split_qkv_rmsnorm_rope_fake_impl) + def _split_qkv_rmsnorm_rope_custom( + qkv: torch.Tensor, + sin: torch.Tensor, + cos: torch.Tensor, + q_size: int, + kv_size: int, + head_dim: int, + eps: float = 1e-6, + q_weight: Optional[torch.Tensor] = None, + k_weight: Optional[torch.Tensor] = None, + q_bias: Optional[torch.Tensor] = None, + k_bias: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + return split_qkv_rmsnorm_rope( + qkv, + sin, + cos, + q_size, + kv_size, + head_dim, + eps=eps, + q_weight=q_weight, + k_weight=k_weight, + q_bias=q_bias, + k_bias=k_bias, + ) + class Qwen3Attention(nn.Module): def __init__( @@ -186,7 +236,7 @@ class Qwen3Attention(nn.Module): if self.attn.layer_id == self.start_layer: self.rotary_emb.get_cos_sin_with_position(positions) - q, k, v = split_qkv_rmsnorm_rope( + q, k, v = _split_qkv_rmsnorm_rope_custom( qkv, self.rotary_emb.position_sin, self.rotary_emb.position_cos, @@ -422,8 +472,8 @@ class Qwen3DecoderLayer(nn.Module): ), ) hidden_states = self.mlp(hidden_states, forward_batch=forward_batch) - if _is_npu and get_cmo_stream(): - wait_cmo_stream() + if _is_npu and get_cmo_stream() is not None: + hidden_states = wait_cmo_stream(hidden_states) hidden_states, residual = self.layer_communicator.postprocess_layer( hidden_states, residual, forward_batch ) diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index ef20cf9454..1132d19570 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -16,6 +16,7 @@ import logging import os +from contextlib import nullcontext from functools import lru_cache from typing import Iterable, Optional, Set, Tuple, Union @@ -1936,9 +1937,14 @@ class Qwen3_5ForCausalLM(nn.Module): # Pass through decoder layers for layer_idx in range(self.start_layer, self.end_layer): layer = self.layers[layer_idx] - with get_global_expert_distribution_recorder().with_current_layer( - layer_idx - ): + ctx = ( + nullcontext() + if check_cuda_graph_backend(Phase.PREFILL, Backend.TC_PIECEWISE) + else get_global_expert_distribution_recorder().with_current_layer( + layer_idx + ) + ) + with ctx: hidden_states, residual = layer( positions=positions, hidden_states=hidden_states, diff --git a/python/sglang/srt/models/qwen3_moe.py b/python/sglang/srt/models/qwen3_moe.py index 6cdc9634c4..c3597d6caf 100644 --- a/python/sglang/srt/models/qwen3_moe.py +++ b/python/sglang/srt/models/qwen3_moe.py @@ -73,6 +73,7 @@ from sglang.srt.utils import ( is_non_idle_and_non_empty, is_npu, ) +from sglang.srt.utils.custom_op import register_custom_op from sglang.srt.utils.hf_transformers_utils import get_rope_config _is_cuda = is_cuda() @@ -103,6 +104,55 @@ _is_npu = is_npu() if _is_npu: from sgl_kernel_npu.norm.split_qkv_rmsnorm_rope import split_qkv_rmsnorm_rope + def _split_qkv_rmsnorm_rope_fake_impl( + qkv: torch.Tensor, + sin: torch.Tensor, + cos: torch.Tensor, + q_size: int, + kv_size: int, + head_dim: int, + eps: float = 1e-6, + q_weight: Optional[torch.Tensor] = None, + k_weight: Optional[torch.Tensor] = None, + q_bias: Optional[torch.Tensor] = None, + k_bias: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + del sin, cos, head_dim, eps, q_weight, k_weight, q_bias, k_bias + output_shape = qkv.shape[:-1] + return ( + qkv.new_empty((*output_shape, q_size)), + qkv.new_empty((*output_shape, kv_size)), + qkv.new_empty((*output_shape, kv_size)), + ) + + @register_custom_op(fake_impl=_split_qkv_rmsnorm_rope_fake_impl) + def _split_qkv_rmsnorm_rope_custom( + qkv: torch.Tensor, + sin: torch.Tensor, + cos: torch.Tensor, + q_size: int, + kv_size: int, + head_dim: int, + eps: float = 1e-6, + q_weight: Optional[torch.Tensor] = None, + k_weight: Optional[torch.Tensor] = None, + q_bias: Optional[torch.Tensor] = None, + k_bias: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + return split_qkv_rmsnorm_rope( + qkv, + sin, + cos, + q_size, + kv_size, + head_dim, + eps=eps, + q_weight=q_weight, + k_weight=k_weight, + q_bias=q_bias, + k_bias=k_bias, + ) + def compute_yarn_parameters( config: PretrainedConfig, @@ -622,7 +672,7 @@ class Qwen3MoeAttention(nn.Module): qkv, _ = self.qkv_proj(hidden_states) if self.attn.layer_id == self.start_layer: self.rotary_emb.get_cos_sin_with_position(positions) - q, k, v = split_qkv_rmsnorm_rope( + q, k, v = _split_qkv_rmsnorm_rope_custom( qkv, self.rotary_emb.position_sin, self.rotary_emb.position_cos, diff --git a/test/registered/npu/basic_function/optimization_debug/test_npu_piecewise_graph_prefill_tp2.py b/test/registered/npu/basic_function/optimization_debug/test_npu_piecewise_graph_prefill_tp2.py new file mode 100644 index 0000000000..f1a4e7d9d2 --- /dev/null +++ b/test/registered/npu/basic_function/optimization_debug/test_npu_piecewise_graph_prefill_tp2.py @@ -0,0 +1,142 @@ +import json +import tempfile +import unittest +from urllib.parse import urlparse + +from sglang.test.ascend.e2e.test_npu_multi_node_utils import NIC_NAME +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + TestNpuPerformanceTestCaseBase, + run_bench_serving, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a3", nightly=False) + +MODEL = "Qwen/Qwen3.6-35B-A3B" +INPUT_THROUGHPUT_THRESHOLD = 8500 +MAX_CONCURRENCY = 32 +NUM_PROMPTS = 128 +INPUT_LEN = 4096 +OUTPUT_LEN = 1 +WARMUP_REQUESTS = 20 +MAX_BENCHMARK_ATTEMPTS = 2 + +ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "SGLANG_SET_CPU_AFFINITY": "1", + "HCCL_SOCKET_IFNAME": NIC_NAME, + "GLOO_SOCKET_IFNAME": NIC_NAME, + "STREAMS_PER_DEVICE": "32", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "96", + "HCCL_BUFFSIZE": "1200", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_NPU_FORWARD_NATIVE_GEMMA_RMS_NORM": "1", + "SGLANG_NPU_ENABLE_PIECEWISE_CUDA_GRAPH": "1", +} + +OTHER_ARGS = [ + "--trust-remote-code", + "--mem-fraction-static", + 0.7, + "--max-running-requests", + MAX_CONCURRENCY, + "--attention-backend", + "ascend", + "--disable-radix-cache", + "--cuda-graph-max-bs", + 32, + "--tp-size", + 2, + "--chunked-prefill-size", + 2048, + "--cuda-graph-backend-prefill", + "tc_piecewise", + "--cuda-graph-bs-prefill", + 128, + 256, + 512, + 1024, + 2048, +] + + +def load_benchmark_result(output_file): + with open(output_file, "r", encoding="utf-8") as f: + lines = [line for line in f if line.strip()] + return json.loads(lines[-1]) + + +def benchmark_passed(metrics): + return ( + metrics["completed"] == NUM_PROMPTS + and metrics["max_concurrency"] == MAX_CONCURRENCY + and metrics["input_throughput"] > INPUT_THROUGHPUT_THRESHOLD + ) + + +class TestPiecewiseGraphPrefillTp2(TestNpuPerformanceTestCaseBase): + model = MODEL + other_args = OTHER_ARGS + envs = ENVS + max_concurrency = MAX_CONCURRENCY + num_prompts = NUM_PROMPTS + input_len = INPUT_LEN + output_len = OUTPUT_LEN + warmup_requests = WARMUP_REQUESTS + seed = 1 + + def run_bench_serving_once(self, parsed_url): + with tempfile.NamedTemporaryFile(suffix=".jsonl") as output_file: + run_bench_serving( + host=parsed_url.hostname, + port=parsed_url.port, + model_path=self.model, + dataset_name=self.dataset_name, + request_rate=self.request_rate, + max_concurrency=self.max_concurrency, + num_prompts=self.num_prompts, + input_len=self.input_len, + output_len=self.output_len, + random_range_ratio=self.random_range_ratio, + warmup_requests=self.warmup_requests, + seed=self.seed, + output_file=output_file.name, + ) + return load_benchmark_result(output_file.name) + + def test_npu_piecewise_graph_prefill_tp2(self): + print(f"##=== Testing PCG serving: {MODEL} ===##") + parsed_url = urlparse(self.base_url) + metrics_list = [] + for attempt in range(MAX_BENCHMARK_ATTEMPTS): + metrics = self.run_bench_serving_once(parsed_url) + metrics_list.append(metrics) + if benchmark_passed(metrics): + break + + print( + "PCG benchmark attempt " + f"{attempt + 1} failed: completed={metrics['completed']}, " + f"max_concurrency={metrics['max_concurrency']}, " + f"input_throughput={metrics['input_throughput']}" + ) + + passed_metrics = [ + metrics for metrics in metrics_list if benchmark_passed(metrics) + ] + metrics = ( + passed_metrics[-1] + if passed_metrics + else max(metrics_list, key=lambda x: x["input_throughput"]) + ) + + self.assertEqual(metrics["completed"], self.num_prompts) + self.assertEqual(metrics["max_concurrency"], self.max_concurrency) + self.assertGreater(metrics["input_throughput"], INPUT_THROUGHPUT_THRESHOLD) + + +if __name__ == "__main__": + unittest.main()