diff --git a/python/sglang/kernels/ops/attention/fla/layernorm_gated.py b/python/sglang/kernels/ops/attention/fla/layernorm_gated.py index 8513bc5c40..c0de08c458 100644 --- a/python/sglang/kernels/ops/attention/fla/layernorm_gated.py +++ b/python/sglang/kernels/ops/attention/fla/layernorm_gated.py @@ -8,7 +8,6 @@ from contextlib import nullcontext from functools import lru_cache -from typing import Optional import torch import torch.nn.functional as F @@ -31,7 +30,6 @@ from sglang.srt.utils import ( is_npu, next_power_of_2, ) -from sglang.srt.utils.custom_op import register_custom_op _is_npu = is_npu() _use_cpu = is_cpu() and cpu_has_amx_support() @@ -325,70 +323,7 @@ def _layer_norm_fwd( if _is_npu: - from sgl_kernel_npu.fla.layernorm_gated import ( - layer_norm_fwd_npu as _layer_norm_fwd_npu_raw, - ) - - @register_custom_op(out_shape="x", mutates_args=["out"]) - def _layer_norm_fwd_custom( - x: torch.Tensor, - weight: torch.Tensor, - bias: Optional[torch.Tensor], - eps: float, - z: Optional[torch.Tensor], - out: Optional[torch.Tensor], - group_size: Optional[int], - norm_before_gate: bool, - is_rms_norm: bool, - activation: Optional[str], - ) -> torch.Tensor: - _out, _mean, _rstd = _layer_norm_fwd_npu_raw( - x=x, - weight=weight, - bias=bias, - eps=eps, - z=z, - out=out, - group_size=group_size, - norm_before_gate=norm_before_gate, - is_rms_norm=is_rms_norm, - activation=activation, - ) - return _out - - def _layer_norm_fwd( - x, - weight, - bias, - eps, - z=None, - out=None, - group_size=None, - norm_before_gate=True, - is_rms_norm=False, - activation=None, - ): - if group_size is None: - group_size = x.shape[-1] - if out is not None: - assert out.shape == x.shape - else: - out = torch.empty_like(x) - y = _layer_norm_fwd_custom( - x=x, - weight=weight, - bias=bias, - eps=eps, - z=z, - out=out, - group_size=group_size, - norm_before_gate=norm_before_gate, - is_rms_norm=is_rms_norm, - activation=activation, - ) - # mean/rstd are unused by all callers on the NPU path (rms_norm_gated - # discards them); return None to keep the (y, mean, rstd) unpack contract. - return y, None, None + from sgl_kernel_npu.fla.layernorm_gated import layer_norm_fwd_npu as _layer_norm_fwd def rms_norm_gated( diff --git a/python/sglang/kernels/ops/elementwise/elementwise.py b/python/sglang/kernels/ops/elementwise/elementwise.py index 8e41e1647a..85ba16c84a 100644 --- a/python/sglang/kernels/ops/elementwise/elementwise.py +++ b/python/sglang/kernels/ops/elementwise/elementwise.py @@ -3,10 +3,9 @@ import triton import triton.language as tl from sglang.kernels.jit.utils import is_arch_support_pdl -from sglang.srt.utils import is_hip, is_npu +from sglang.srt.utils import is_hip _is_hip = is_hip() -_is_npu = is_npu() rmsnorm_autotune = triton.autotune( @@ -444,15 +443,6 @@ def fused_sigmoid_mul( gate_stride_row = hidden_dim gate_stride_head = hidden_dim - if _is_npu and torch.compiler.is_compiling(): - gate_reshaped = ( - gate.view_as(attn_output) if gate.numel() == attn_output.numel() else gate - ) - if inplace: - attn_output.mul_(torch.sigmoid(gate_reshaped)) - return attn_output - return attn_output * torch.sigmoid(gate_reshaped) - out = attn_output if inplace else torch.empty_like(attn_output) block_h = 1024 if num_tokens < 1024 else 2048 grid = (num_tokens, triton.cdiv(hidden_dim, block_h)) diff --git a/python/sglang/srt/arg_groups/cuda_graph_hook.py b/python/sglang/srt/arg_groups/cuda_graph_hook.py index f5825c7509..1aeee2839b 100644 --- a/python/sglang/srt/arg_groups/cuda_graph_hook.py +++ b/python/sglang/srt/arg_groups/cuda_graph_hook.py @@ -14,7 +14,6 @@ from sglang.srt.arg_groups.overrides import ( resolving_view, ) from sglang.srt.connector import ConnectorType -from sglang.srt.environ import envs from sglang.srt.model_executor.cuda_graph_config import ( ALLOWED_BACKENDS_PER_PHASE, Backend, @@ -194,10 +193,7 @@ def disable_tc_piecewise_cudagraph_if_incompatible(server_args: Any): "non-CUDA hardware (HIP/NPU/CPU/MPS/XPU)", lambda: ( get_platform().is_hip - or ( - get_platform().is_npu - and not envs.SGLANG_NPU_ENABLE_PIECEWISE_CUDA_GRAPH.get() - ) + or get_platform().is_npu or is_cpu() or is_mps() or get_platform().is_xpu diff --git a/python/sglang/srt/compilation/npu_piecewise_backend.py b/python/sglang/srt/compilation/npu_piecewise_backend.py index f33684d509..dc97bd5c3f 100644 --- a/python/sglang/srt/compilation/npu_piecewise_backend.py +++ b/python/sglang/srt/compilation/npu_piecewise_backend.py @@ -7,10 +7,6 @@ import torch.fx as fx from sglang.srt.compilation.compilation_config import CompilationConfig from sglang.srt.compilation.compilation_counter import compilation_counter -from sglang.srt.compilation.compile_phase import ( - get_pcg_capture_stream, - is_in_torch_compile_warmup, -) from sglang.srt.compilation.cuda_piecewise_backend import ( CUDAPiecewiseBackend, weak_ref_tensors, @@ -43,12 +39,6 @@ class NPUPiecewiseBackend(CUDAPiecewiseBackend): ) def __call__(self, *args): - if is_in_torch_compile_warmup(): - return self.compiled_graph_for_general_shape(*args) - - if len(self.sym_shape_indices) == 0: - return self.compiled_graph_for_general_shape(*args) - runtime_shape = args[self.sym_shape_indices[0]] if runtime_shape not in self.concrete_size_entries: # we don't need to do anything for this shape @@ -83,12 +73,7 @@ class NPUPiecewiseBackend(CUDAPiecewiseBackend): stack.enter_context(patch("torch.npu.empty_cache", lambda: None)) # mind-exploding: carefully manage the reference and memory. - with torch.npu.graph( - npugraph, - pool=self.graph_pool, - stream=get_pcg_capture_stream(), - auto_dispatch_capture=True, - ): + with torch.npu.graph(npugraph, pool=self.graph_pool): # `output` is managed by pytorch's cudagraph pool output = entry.runnable(*args) if self.is_last_graph: diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 390bae1dfa..25a5cb72a9 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -980,7 +980,6 @@ class Envs: DEEP_NORMAL_MODE_USE_INT8_QUANT = EnvBool(False) SGLANG_ZBAL_LOCAL_MEM_SIZE = EnvInt(0) SGLANG_ZBAL_BOOTSTRAP_URL = EnvStr("") - SGLANG_NPU_ENABLE_PIECEWISE_CUDA_GRAPH = EnvBool(False) # prefill cuda graph on NPU # =================================================================== # MUSA diff --git a/python/sglang/srt/hardware_backend/npu/cmo.py b/python/sglang/srt/hardware_backend/npu/cmo.py index 2e3907bd6a..214097de60 100644 --- a/python/sglang/srt/hardware_backend/npu/cmo.py +++ b/python/sglang/srt/hardware_backend/npu/cmo.py @@ -1,9 +1,5 @@ -from typing import List - import torch -from sglang.srt.utils.custom_op import register_custom_op - cmo_stream = None share_stream = None @@ -23,56 +19,40 @@ def set_cmo_stream(stream): cmo_stream = stream -@register_custom_op() -def _prepare_weight_cache_list( - handle: torch.Tensor, cache: List[torch.Tensor], max_size: int -) -> None: - import torch_npu - - stream = get_cmo_stream() - if stream is None: - stream = torch.npu.Stream() - set_cmo_stream(stream) - stream.wait_stream(torch.npu.current_stream()) - with torch.npu.stream(stream): - for weight in cache: - torch_npu.npu_prefetch(weight, handle, max_size) - - -@register_custom_op() -def _prepare_weight_cache_single( - handle: torch.Tensor, cache: torch.Tensor, max_size: int -) -> None: - import torch_npu - - stream = get_cmo_stream() - if stream is None: - stream = torch.npu.Stream() - set_cmo_stream(stream) - stream.wait_stream(torch.npu.current_stream()) - with torch.npu.stream(stream): - torch_npu.npu_prefetch(cache, handle, max_size) - - def prepare_weight_cache(handle, cache, PREFETCH_MAX_SIZE=1000000000): """ PREFETCH_MAX_SIZE: maximum size (bytes) for each prefetch operation. This affects the time spent in prefetch: time ≈ PREFETCH_MAX_SIZE / system_bandwidth """ - if isinstance(cache, list): - _prepare_weight_cache_list(handle, cache, PREFETCH_MAX_SIZE) - else: - _prepare_weight_cache_single(handle, cache, PREFETCH_MAX_SIZE) + import torch_npu + + stream = get_cmo_stream() + if stream is None: + stream = torch.npu.Stream() + set_cmo_stream(stream) + stream.wait_stream(torch.npu.current_stream()) + with torch.npu.stream(stream): + if isinstance(cache, list): + for weight in cache: + torch_npu.npu_prefetch( + weight, + handle, + PREFETCH_MAX_SIZE, + ) + else: + torch_npu.npu_prefetch( + cache, + handle, + PREFETCH_MAX_SIZE, + ) -@register_custom_op(out_shape=0) -def wait_cmo_stream(dummy: torch.Tensor) -> torch.Tensor: +def wait_cmo_stream(): stream = get_cmo_stream() if stream is not None: cur_stream = torch.npu.current_stream() cur_stream.wait_stream(stream) - return dummy def get_share_stream(): @@ -85,21 +65,14 @@ def set_share_stream(stream): share_stream = stream -@register_custom_op(out_shape=0) -def wait_share_stream(dummy: torch.Tensor) -> torch.Tensor: +def wait_share_stream(): stream = get_share_stream() if stream is not None: cur_stream = torch.npu.current_stream() cur_stream.wait_stream(stream) - return dummy def shared_expert_on_independent_stream(hidden_states, forward_func): - import torch.compiler - - if torch.compiler.is_compiling(): - return forward_func(hidden_states) - stream = get_share_stream() if stream is None: stream = torch.npu.Stream() diff --git a/python/sglang/srt/hardware_backend/npu/moe/topk.py b/python/sglang/srt/hardware_backend/npu/moe/topk.py index d73cad4cba..50b90aeb8a 100644 --- a/python/sglang/srt/hardware_backend/npu/moe/topk.py +++ b/python/sglang/srt/hardware_backend/npu/moe/topk.py @@ -1,16 +1,7 @@ from typing import TYPE_CHECKING, Optional import torch - -from sglang.srt.utils.custom_op import register_custom_op - - -@register_custom_op(out_shape=0) -def _l1_norm(x: torch.Tensor) -> torch.Tensor: - from sgl_kernel_npu.norm.l1_norm import l1_norm - - return l1_norm(x) - +from sgl_kernel_npu.norm.l1_norm import l1_norm from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location_dispatch import topk_ids_logical_to_physical @@ -68,7 +59,7 @@ def fused_topk_npu( ) if renormalize: - topk_weights = _l1_norm( + topk_weights = l1_norm( topk_weights if topk_config.num_fused_shared_experts == 0 else topk_weights[:, :-1] diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index 3150a3637f..692fdafff7 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -625,7 +625,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module): ) trace_sync("post_experts") if enable_dual_stream: - final_hidden_states = wait_share_stream(final_hidden_states) + wait_share_stream() elif enable_cuda_shared_overlap: torch.cuda.current_stream().wait_event(shared_event) diff --git a/python/sglang/srt/models/qwen3.py b/python/sglang/srt/models/qwen3.py index 32d79d9447..dca81ae256 100644 --- a/python/sglang/srt/models/qwen3.py +++ b/python/sglang/srt/models/qwen3.py @@ -33,7 +33,6 @@ from sglang.srt.models.utils import apply_qk_norm from sglang.srt.platforms import current_platform from sglang.srt.runtime_context import get_exec, get_parallel, get_stream from sglang.srt.utils import add_prefix, get_bool_env_var, is_cuda, is_hip, is_npu -from sglang.srt.utils.custom_op import register_custom_op Qwen3Config = None @@ -58,55 +57,6 @@ if _is_npu: from sglang.srt.hardware_backend.npu.cmo import get_cmo_stream, wait_cmo_stream - def _split_qkv_rmsnorm_rope_fake_impl( - qkv: torch.Tensor, - sin: torch.Tensor, - cos: torch.Tensor, - q_size: int, - kv_size: int, - head_dim: int, - eps: float = 1e-6, - q_weight: Optional[torch.Tensor] = None, - k_weight: Optional[torch.Tensor] = None, - q_bias: Optional[torch.Tensor] = None, - k_bias: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - del sin, cos, head_dim, eps, q_weight, k_weight, q_bias, k_bias - output_shape = qkv.shape[:-1] - return ( - qkv.new_empty((*output_shape, q_size)), - qkv.new_empty((*output_shape, kv_size)), - qkv.new_empty((*output_shape, kv_size)), - ) - - @register_custom_op(fake_impl=_split_qkv_rmsnorm_rope_fake_impl) - def _split_qkv_rmsnorm_rope_custom( - qkv: torch.Tensor, - sin: torch.Tensor, - cos: torch.Tensor, - q_size: int, - kv_size: int, - head_dim: int, - eps: float = 1e-6, - q_weight: Optional[torch.Tensor] = None, - k_weight: Optional[torch.Tensor] = None, - q_bias: Optional[torch.Tensor] = None, - k_bias: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - return split_qkv_rmsnorm_rope( - qkv, - sin, - cos, - q_size, - kv_size, - head_dim, - eps=eps, - q_weight=q_weight, - k_weight=k_weight, - q_bias=q_bias, - k_bias=k_bias, - ) - class Qwen3Attention(nn.Module): def __init__( @@ -236,7 +186,7 @@ class Qwen3Attention(nn.Module): if self.attn.layer_id == self.start_layer: self.rotary_emb.get_cos_sin_with_position(positions) - q, k, v = _split_qkv_rmsnorm_rope_custom( + q, k, v = split_qkv_rmsnorm_rope( qkv, self.rotary_emb.position_sin, self.rotary_emb.position_cos, @@ -472,8 +422,8 @@ class Qwen3DecoderLayer(nn.Module): ), ) hidden_states = self.mlp(hidden_states, forward_batch=forward_batch) - if _is_npu and get_cmo_stream() is not None: - hidden_states = wait_cmo_stream(hidden_states) + if _is_npu and get_cmo_stream(): + wait_cmo_stream() hidden_states, residual = self.layer_communicator.postprocess_layer( hidden_states, residual, forward_batch ) diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index 1132d19570..ef20cf9454 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -16,7 +16,6 @@ import logging import os -from contextlib import nullcontext from functools import lru_cache from typing import Iterable, Optional, Set, Tuple, Union @@ -1937,14 +1936,9 @@ class Qwen3_5ForCausalLM(nn.Module): # Pass through decoder layers for layer_idx in range(self.start_layer, self.end_layer): layer = self.layers[layer_idx] - ctx = ( - nullcontext() - if check_cuda_graph_backend(Phase.PREFILL, Backend.TC_PIECEWISE) - else get_global_expert_distribution_recorder().with_current_layer( - layer_idx - ) - ) - with ctx: + with get_global_expert_distribution_recorder().with_current_layer( + layer_idx + ): hidden_states, residual = layer( positions=positions, hidden_states=hidden_states, diff --git a/python/sglang/srt/models/qwen3_moe.py b/python/sglang/srt/models/qwen3_moe.py index c3597d6caf..6cdc9634c4 100644 --- a/python/sglang/srt/models/qwen3_moe.py +++ b/python/sglang/srt/models/qwen3_moe.py @@ -73,7 +73,6 @@ from sglang.srt.utils import ( is_non_idle_and_non_empty, is_npu, ) -from sglang.srt.utils.custom_op import register_custom_op from sglang.srt.utils.hf_transformers_utils import get_rope_config _is_cuda = is_cuda() @@ -104,55 +103,6 @@ _is_npu = is_npu() if _is_npu: from sgl_kernel_npu.norm.split_qkv_rmsnorm_rope import split_qkv_rmsnorm_rope - def _split_qkv_rmsnorm_rope_fake_impl( - qkv: torch.Tensor, - sin: torch.Tensor, - cos: torch.Tensor, - q_size: int, - kv_size: int, - head_dim: int, - eps: float = 1e-6, - q_weight: Optional[torch.Tensor] = None, - k_weight: Optional[torch.Tensor] = None, - q_bias: Optional[torch.Tensor] = None, - k_bias: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - del sin, cos, head_dim, eps, q_weight, k_weight, q_bias, k_bias - output_shape = qkv.shape[:-1] - return ( - qkv.new_empty((*output_shape, q_size)), - qkv.new_empty((*output_shape, kv_size)), - qkv.new_empty((*output_shape, kv_size)), - ) - - @register_custom_op(fake_impl=_split_qkv_rmsnorm_rope_fake_impl) - def _split_qkv_rmsnorm_rope_custom( - qkv: torch.Tensor, - sin: torch.Tensor, - cos: torch.Tensor, - q_size: int, - kv_size: int, - head_dim: int, - eps: float = 1e-6, - q_weight: Optional[torch.Tensor] = None, - k_weight: Optional[torch.Tensor] = None, - q_bias: Optional[torch.Tensor] = None, - k_bias: Optional[torch.Tensor] = None, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - return split_qkv_rmsnorm_rope( - qkv, - sin, - cos, - q_size, - kv_size, - head_dim, - eps=eps, - q_weight=q_weight, - k_weight=k_weight, - q_bias=q_bias, - k_bias=k_bias, - ) - def compute_yarn_parameters( config: PretrainedConfig, @@ -672,7 +622,7 @@ class Qwen3MoeAttention(nn.Module): qkv, _ = self.qkv_proj(hidden_states) if self.attn.layer_id == self.start_layer: self.rotary_emb.get_cos_sin_with_position(positions) - q, k, v = _split_qkv_rmsnorm_rope_custom( + q, k, v = split_qkv_rmsnorm_rope( qkv, self.rotary_emb.position_sin, self.rotary_emb.position_cos, diff --git a/test/registered/npu/basic_function/optimization_debug/test_npu_piecewise_graph_prefill_tp2.py b/test/registered/npu/basic_function/optimization_debug/test_npu_piecewise_graph_prefill_tp2.py deleted file mode 100644 index f1a4e7d9d2..0000000000 --- a/test/registered/npu/basic_function/optimization_debug/test_npu_piecewise_graph_prefill_tp2.py +++ /dev/null @@ -1,142 +0,0 @@ -import json -import tempfile -import unittest -from urllib.parse import urlparse - -from sglang.test.ascend.e2e.test_npu_multi_node_utils import NIC_NAME -from sglang.test.ascend.e2e.test_npu_performance_utils import ( - TestNpuPerformanceTestCaseBase, - run_bench_serving, -) -from sglang.test.ci.ci_register import register_npu_ci - -register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a3", nightly=False) - -MODEL = "Qwen/Qwen3.6-35B-A3B" -INPUT_THROUGHPUT_THRESHOLD = 8500 -MAX_CONCURRENCY = 32 -NUM_PROMPTS = 128 -INPUT_LEN = 4096 -OUTPUT_LEN = 1 -WARMUP_REQUESTS = 20 -MAX_BENCHMARK_ATTEMPTS = 2 - -ENVS = { - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", - "SGLANG_SET_CPU_AFFINITY": "1", - "HCCL_SOCKET_IFNAME": NIC_NAME, - "GLOO_SOCKET_IFNAME": NIC_NAME, - "STREAMS_PER_DEVICE": "32", - "SGLANG_ENABLE_SPEC_V2": "1", - "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", - "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", - "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "96", - "HCCL_BUFFSIZE": "1200", - "HCCL_OP_EXPANSION_MODE": "AIV", - "SGLANG_NPU_FORWARD_NATIVE_GEMMA_RMS_NORM": "1", - "SGLANG_NPU_ENABLE_PIECEWISE_CUDA_GRAPH": "1", -} - -OTHER_ARGS = [ - "--trust-remote-code", - "--mem-fraction-static", - 0.7, - "--max-running-requests", - MAX_CONCURRENCY, - "--attention-backend", - "ascend", - "--disable-radix-cache", - "--cuda-graph-max-bs", - 32, - "--tp-size", - 2, - "--chunked-prefill-size", - 2048, - "--cuda-graph-backend-prefill", - "tc_piecewise", - "--cuda-graph-bs-prefill", - 128, - 256, - 512, - 1024, - 2048, -] - - -def load_benchmark_result(output_file): - with open(output_file, "r", encoding="utf-8") as f: - lines = [line for line in f if line.strip()] - return json.loads(lines[-1]) - - -def benchmark_passed(metrics): - return ( - metrics["completed"] == NUM_PROMPTS - and metrics["max_concurrency"] == MAX_CONCURRENCY - and metrics["input_throughput"] > INPUT_THROUGHPUT_THRESHOLD - ) - - -class TestPiecewiseGraphPrefillTp2(TestNpuPerformanceTestCaseBase): - model = MODEL - other_args = OTHER_ARGS - envs = ENVS - max_concurrency = MAX_CONCURRENCY - num_prompts = NUM_PROMPTS - input_len = INPUT_LEN - output_len = OUTPUT_LEN - warmup_requests = WARMUP_REQUESTS - seed = 1 - - def run_bench_serving_once(self, parsed_url): - with tempfile.NamedTemporaryFile(suffix=".jsonl") as output_file: - run_bench_serving( - host=parsed_url.hostname, - port=parsed_url.port, - model_path=self.model, - dataset_name=self.dataset_name, - request_rate=self.request_rate, - max_concurrency=self.max_concurrency, - num_prompts=self.num_prompts, - input_len=self.input_len, - output_len=self.output_len, - random_range_ratio=self.random_range_ratio, - warmup_requests=self.warmup_requests, - seed=self.seed, - output_file=output_file.name, - ) - return load_benchmark_result(output_file.name) - - def test_npu_piecewise_graph_prefill_tp2(self): - print(f"##=== Testing PCG serving: {MODEL} ===##") - parsed_url = urlparse(self.base_url) - metrics_list = [] - for attempt in range(MAX_BENCHMARK_ATTEMPTS): - metrics = self.run_bench_serving_once(parsed_url) - metrics_list.append(metrics) - if benchmark_passed(metrics): - break - - print( - "PCG benchmark attempt " - f"{attempt + 1} failed: completed={metrics['completed']}, " - f"max_concurrency={metrics['max_concurrency']}, " - f"input_throughput={metrics['input_throughput']}" - ) - - passed_metrics = [ - metrics for metrics in metrics_list if benchmark_passed(metrics) - ] - metrics = ( - passed_metrics[-1] - if passed_metrics - else max(metrics_list, key=lambda x: x["input_throughput"]) - ) - - self.assertEqual(metrics["completed"], self.num_prompts) - self.assertEqual(metrics["max_concurrency"], self.max_concurrency) - self.assertGreater(metrics["input_throughput"], INPUT_THROUGHPUT_THRESHOLD) - - -if __name__ == "__main__": - unittest.main()