From 321298da75e154fe33c972c6fba775c7df40cc8f Mon Sep 17 00:00:00 2001 From: Xiaoyu Zhang <35585791+BBuf@users.noreply.github.com> Date: Sat, 2 May 2026 10:12:47 +0800 Subject: [PATCH] [SKILL] Upgrade sglang profile and auto_benchmark skills (#24250) --- .../llm-serving-auto-benchmark/SKILL.md | 527 +++++++++++++ .../configs/cookbook-llm/README.md | 17 + .../cookbook-llm/deepseek-math-v2.yaml | 130 ++++ .../cookbook-llm/deepseek-r1-0528.yaml | 133 ++++ .../configs/cookbook-llm/deepseek-v3.1.yaml | 132 ++++ .../configs/cookbook-llm/deepseek-v3.2.yaml | 132 ++++ .../configs/cookbook-llm/deepseek-v3.yaml | 133 ++++ .../devstral-small-2-24b-instruct-2512.yaml | 123 +++ .../cookbook-llm/ernie-4.5-21b-a3b-pt.yaml | 117 +++ .../configs/cookbook-llm/glm-4.5.yaml | 122 +++ .../configs/cookbook-llm/glm-4.6.yaml | 135 ++++ .../configs/cookbook-llm/glm-4.7-flash.yaml | 126 +++ .../configs/cookbook-llm/glm-4.7.yaml | 130 ++++ .../configs/cookbook-llm/glm-5-fp8.yaml | 132 ++++ .../configs/cookbook-llm/glyph.yaml | 126 +++ .../configs/cookbook-llm/gpt-oss-120b.yaml | 132 ++++ .../configs/cookbook-llm/intern-s1.yaml | 135 ++++ .../cookbook-llm/kimi-k2-instruct.yaml | 133 ++++ .../configs/cookbook-llm/kimi-k2.5.yaml | 127 +++ .../kimi-linear-48b-a3b-instruct.yaml | 121 +++ .../configs/cookbook-llm/ling-2.5-1t.yaml | 134 ++++ .../configs/cookbook-llm/llada2-1-mini.yaml | 130 ++++ .../cookbook-llm/llama-3.1-70b-instruct.yaml | 124 +++ .../cookbook-llm/llama-3.3-70b-instruct.yaml | 118 +++ ...lama-4-maverick-17b-128e-instruct-fp8.yaml | 122 +++ .../llama-4-scout-17b-16e-instruct.yaml | 129 +++ .../configs/cookbook-llm/mimo-v2-flash.yaml | 133 ++++ .../configs/cookbook-llm/minimax-m2.1.yaml | 121 +++ .../configs/cookbook-llm/minimax-m2.5.yaml | 133 ++++ .../ministral-3-8b-instruct-2512.yaml | 121 +++ .../mistral-small-4-119b-2603.yaml | 124 +++ .../nemotron-3-nano-30b-a3b-bf16.yaml | 128 +++ .../nemotron-3-super-120b-a12b-bf16.yaml | 128 +++ .../configs/cookbook-llm/qwen3-235b-a22b.yaml | 132 ++++ .../qwen3-coder-480b-a35b-instruct.yaml | 131 ++++ .../cookbook-llm/qwen3-coder-next.yaml | 124 +++ .../qwen3-next-80b-a3b-instruct.yaml | 130 ++++ .../cookbook-llm/qwen35-397b-a17b-fp8.yaml | 132 ++++ .../configs/cookbook-llm/ring-2.5-1t.yaml | 124 +++ .../configs/cookbook-llm/step-3.5-flash.yaml | 133 ++++ .../references/container-runbook.md | 321 ++++++++ .../references/example-plan.yaml | 133 ++++ .../references/framework-reference.md | 113 +++ .../references/result-schema.md | 161 ++++ .../scripts/compare_benchmark_results.py | 308 ++++++++ .../scripts/validate_cookbook_configs.py | 434 +++++++++++ .../llm-torch-profiler-analysis/SKILL.md | 175 ++++- .../references/fuse-overlap-catalog.md | 44 +- .../references/vllm-torch-compile-fusions.md | 66 ++ .../scripts/analyze_llm_torch_profile.py | 60 +- .../scripts/probe_llm_server.py | 2 +- .../scripts/profile_common.py | 365 +++++++-- .../run_llm_single_model_matrix_host.sh | 274 +++++++ .../scripts/run_sglang_torch_profile_host.sh | 241 ++++++ .../run_trtllm_pytorch_profile_host.sh | 407 ++++++++++ .../scripts/run_vllm_torch_profile_host.sh | 343 ++++++++ .claude/skills/sglang-auto-benchmark/SKILL.md | 496 ------------ .../references/cookbook-llm/README.md | 76 -- .../cookbook-llm/deepseek-math-v2.yaml | 81 -- .../cookbook-llm/deepseek-r1-0528.yaml | 84 -- .../cookbook-llm/deepseek-v3.1.yaml | 83 -- .../cookbook-llm/deepseek-v3.2.yaml | 83 -- .../references/cookbook-llm/deepseek-v3.yaml | 84 -- .../devstral-small-2-24b-instruct-2512.yaml | 78 -- .../cookbook-llm/ernie-4.5-21b-a3b-pt.yaml | 76 -- .../references/cookbook-llm/glm-4.5.yaml | 74 -- .../references/cookbook-llm/glm-4.6.yaml | 83 -- .../cookbook-llm/glm-4.7-flash.yaml | 78 -- .../references/cookbook-llm/glm-4.7.yaml | 78 -- .../references/cookbook-llm/glm-5-fp8.yaml | 83 -- .../references/cookbook-llm/glyph.yaml | 81 -- .../references/cookbook-llm/gpt-oss-120b.yaml | 83 -- .../references/cookbook-llm/intern-s1.yaml | 84 -- .../cookbook-llm/kimi-k2-instruct.yaml | 83 -- .../references/cookbook-llm/kimi-k2.5.yaml | 80 -- .../kimi-linear-48b-a3b-instruct.yaml | 79 -- .../references/cookbook-llm/ling-2.5-1t.yaml | 91 --- .../cookbook-llm/llada2-1-mini.yaml | 83 -- .../cookbook-llm/llama-3.1-70b-instruct.yaml | 79 -- .../cookbook-llm/llama-3.3-70b-instruct.yaml | 73 -- ...lama-4-maverick-17b-128e-instruct-fp8.yaml | 79 -- .../llama-4-scout-17b-16e-instruct.yaml | 83 -- .../cookbook-llm/mimo-v2-flash.yaml | 91 --- .../references/cookbook-llm/minimax-m2.1.yaml | 77 -- .../references/cookbook-llm/minimax-m2.5.yaml | 83 -- .../ministral-3-8b-instruct-2512.yaml | 77 -- .../mistral-small-4-119b-2603.yaml | 79 -- .../nemotron-3-nano-30b-a3b-bf16.yaml | 81 -- .../nemotron-3-super-120b-a12b-bf16.yaml | 81 -- .../cookbook-llm/qwen3-235b-a22b.yaml | 83 -- .../qwen3-coder-480b-a35b-instruct.yaml | 82 -- .../cookbook-llm/qwen3-coder-next.yaml | 79 -- .../qwen3-next-80b-a3b-instruct.yaml | 82 -- .../cookbook-llm/qwen35-397b-a17b-fp8.yaml | 83 -- .../references/cookbook-llm/ring-2.5-1t.yaml | 79 -- .../cookbook-llm/step-3.5-flash.yaml | 83 -- .../references/qwen3-32b.yaml | 62 -- .../sglang-prod-incident-triage/SKILL.md | 291 +++++++ .../references/case-studies.md | 81 ++ .../references/decision-tree.md | 197 +++++ .../references/endpoints-and-signals.md | 218 ++++++ .../references/replay-trace-profile.md | 236 ++++++ .../scripts/incident_artifact_tool.py | 735 ++++++++++++++++++ .../scripts/replay_trusted_request_dump.py | 219 ++++++ .../skills/sglang-sota-performance/SKILL.md | 254 ++++++ 105 files changed, 10996 insertions(+), 3813 deletions(-) create mode 100644 .claude/skills/llm-serving-auto-benchmark/SKILL.md create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/README.md create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/deepseek-math-v2.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/deepseek-r1-0528.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/deepseek-v3.1.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/deepseek-v3.2.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/deepseek-v3.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/devstral-small-2-24b-instruct-2512.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/ernie-4.5-21b-a3b-pt.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/glm-4.5.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/glm-4.6.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/glm-4.7-flash.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/glm-4.7.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/glm-5-fp8.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/glyph.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/gpt-oss-120b.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/intern-s1.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/kimi-k2-instruct.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/kimi-k2.5.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/kimi-linear-48b-a3b-instruct.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/ling-2.5-1t.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/llada2-1-mini.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/llama-3.1-70b-instruct.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/llama-3.3-70b-instruct.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/llama-4-maverick-17b-128e-instruct-fp8.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/llama-4-scout-17b-16e-instruct.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/mimo-v2-flash.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/minimax-m2.1.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/minimax-m2.5.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/ministral-3-8b-instruct-2512.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/mistral-small-4-119b-2603.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/nemotron-3-nano-30b-a3b-bf16.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/nemotron-3-super-120b-a12b-bf16.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/qwen3-235b-a22b.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/qwen3-coder-480b-a35b-instruct.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/qwen3-coder-next.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/qwen3-next-80b-a3b-instruct.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/qwen35-397b-a17b-fp8.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/ring-2.5-1t.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/step-3.5-flash.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/references/container-runbook.md create mode 100644 .claude/skills/llm-serving-auto-benchmark/references/example-plan.yaml create mode 100644 .claude/skills/llm-serving-auto-benchmark/references/framework-reference.md create mode 100644 .claude/skills/llm-serving-auto-benchmark/references/result-schema.md create mode 100755 .claude/skills/llm-serving-auto-benchmark/scripts/compare_benchmark_results.py create mode 100755 .claude/skills/llm-serving-auto-benchmark/scripts/validate_cookbook_configs.py create mode 100644 .claude/skills/llm-torch-profiler-analysis/references/vllm-torch-compile-fusions.md create mode 100755 .claude/skills/llm-torch-profiler-analysis/scripts/run_llm_single_model_matrix_host.sh create mode 100755 .claude/skills/llm-torch-profiler-analysis/scripts/run_sglang_torch_profile_host.sh create mode 100755 .claude/skills/llm-torch-profiler-analysis/scripts/run_trtllm_pytorch_profile_host.sh create mode 100755 .claude/skills/llm-torch-profiler-analysis/scripts/run_vllm_torch_profile_host.sh delete mode 100644 .claude/skills/sglang-auto-benchmark/SKILL.md delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/README.md delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-math-v2.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-r1-0528.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-v3.1.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-v3.2.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-v3.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/devstral-small-2-24b-instruct-2512.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/ernie-4.5-21b-a3b-pt.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-4.5.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-4.6.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-4.7-flash.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-4.7.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-5-fp8.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/glyph.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/gpt-oss-120b.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/intern-s1.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/kimi-k2-instruct.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/kimi-k2.5.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/kimi-linear-48b-a3b-instruct.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/ling-2.5-1t.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/llada2-1-mini.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/llama-3.1-70b-instruct.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/llama-3.3-70b-instruct.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/llama-4-maverick-17b-128e-instruct-fp8.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/llama-4-scout-17b-16e-instruct.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/mimo-v2-flash.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/minimax-m2.1.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/minimax-m2.5.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/ministral-3-8b-instruct-2512.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/mistral-small-4-119b-2603.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/nemotron-3-nano-30b-a3b-bf16.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/nemotron-3-super-120b-a12b-bf16.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen3-235b-a22b.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen3-coder-480b-a35b-instruct.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen3-coder-next.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen3-next-80b-a3b-instruct.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen35-397b-a17b-fp8.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/ring-2.5-1t.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/cookbook-llm/step-3.5-flash.yaml delete mode 100644 .claude/skills/sglang-auto-benchmark/references/qwen3-32b.yaml create mode 100644 .claude/skills/sglang-prod-incident-triage/SKILL.md create mode 100644 .claude/skills/sglang-prod-incident-triage/references/case-studies.md create mode 100644 .claude/skills/sglang-prod-incident-triage/references/decision-tree.md create mode 100644 .claude/skills/sglang-prod-incident-triage/references/endpoints-and-signals.md create mode 100644 .claude/skills/sglang-prod-incident-triage/references/replay-trace-profile.md create mode 100755 .claude/skills/sglang-prod-incident-triage/scripts/incident_artifact_tool.py create mode 100755 .claude/skills/sglang-prod-incident-triage/scripts/replay_trusted_request_dump.py create mode 100644 .claude/skills/sglang-sota-performance/SKILL.md diff --git a/.claude/skills/llm-serving-auto-benchmark/SKILL.md b/.claude/skills/llm-serving-auto-benchmark/SKILL.md new file mode 100644 index 000000000..8c720e495 --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/SKILL.md @@ -0,0 +1,527 @@ +--- +name: llm-serving-auto-benchmark +description: Framework-independent LLM serving benchmark skill for comparing SGLang, vLLM, TensorRT-LLM, or another serving framework. Use when a user wants to find the best deployment command for one model across multiple serving frameworks under the same workload, GPU budget, and latency SLA. +--- + +# LLM Serving Auto Benchmark + +## Overview + +Use this skill to compare LLM serving frameworks such as SGLang, vLLM, and +TensorRT-LLM for the same model and workload. + +Use a config-driven workflow: + +- keep launch-only capacity choices in each framework's `base_server_flags` +- put the search knobs in `search_space` +- run the same dataset scenarios for every framework +- generate a bounded candidate list from `search_space`, with the baseline + candidate included first +- keep failed candidates in the result file +- pick the best SLA-passing candidate after normalizing the results + +For model-specific starting points, prefer the shipped configs in +`configs/cookbook-llm/`. They define a framework-neutral LLM serving cookbook +model set and translate each entry into framework-native SGLang, vLLM, and +TensorRT-LLM server flags. Validate those configs before a real run: + +```bash +python .claude/skills/llm-serving-auto-benchmark/scripts/validate_cookbook_configs.py \ + .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm +``` + +If you have captured target-environment `--help` files, add +`--help-dir `. That check only loads configs, verifies the +server flag names, and renders candidate commands; it does not launch model +servers. + +Prefer native tooling when it gives better coverage: + +- SGLang: `python -m sglang.auto_benchmark` when available, otherwise + `python -m sglang.bench_serving` +- vLLM: `vllm bench sweep serve` for server-parameter sweeps, otherwise + `vllm serve` plus `vllm bench serve` +- TensorRT-LLM: `trtllm-serve` for the OpenAI-compatible server plus the + TensorRT-LLM serving benchmark client or a common OpenAI-compatible benchmark + client + +TensorRT-LLM has one hard scope rule in this skill: the server backend is fixed +to `trtllm-serve serve --backend pytorch`. Do not search TensorRT-LLM backend +choice. If a request, config, or candidate asks for `trt`, an engine backend, or +any other non-PyTorch TensorRT-LLM server backend, reject that candidate as +unsupported for this skill and record the reason. This does not change the +benchmark client backend; the TensorRT-LLM benchmark client still uses +OpenAI-compatible modes such as `--backend openai` or `--backend openai-chat`. + +Only pick a winner after each requested framework has had its main serving knobs +tuned. + +The parameter lists in this skill are not a compatibility contract. They are +version-sensitive candidate knob families. Before every real run, record the +exact framework version or git commit and verify the concrete CLI flag names +with `--help` in the target environment. + +The default search style is framework-neutral: start from a mostly pure-TP +baseline, sweep a small set of high-impact runtime knobs, and cap the first +pass around 10 candidates per framework. Do not search memory fractions by +default. + +## Validation Environment + +This skill is target-agnostic. It assumes any one of the following is +available, and nothing more: + +- a local GPU host with Docker/Podman and the target framework images pulled; +- a remote GPU host reached via `ssh ` with the framework images already + running in a container there; +- a CI runner that can exec into a pre-built image for each framework. + +Do not assume a specific operator host name (`h100_sglang`, `b200_*`, +`radixark*`, `rtx5090_*`, etc.) inside this skill's own workflow. The concrete +SSH wiring, container names, workspace paths, and HF token plumbing for a given +box live in the operator-side per-host skills (for example `h100`, +`h100-sglang-diffusion`, `b200`, `rtx5090`, `radixark02`, `radixark03`); this +skill only requires that the caller can reach a shell inside a container with +`sglang`, `vllm`, or `tensorrt_llm` installed. + +Reference files are optional and version-sensitive. Treat historical flag notes +as evidence from one image, not as a compatibility guarantee for the next run. + +Additional H100 validation on `2026-05-01` used two 2-card models with a +bounded search of two SGLang memory-fraction candidates and two vLLM +memory-utilization candidates. The workload was random input `512`, output +`64`, 8 prompts, and 2 warmup requests, only to prove the search and summary +path can finish quickly. + +| Model | GPUs | Best SGLang | Best vLLM | Artifact root | +| --- | --- | --- | --- | --- | +| `Qwen/Qwen3-8B` | 2x H100, TP=2 | `sglang_mem086`, 21.64 req/s, 1385.05 output tok/s, mean TTFT 70.54 ms | `vllm_mem080`, 22.88 req/s, 1464.25 output tok/s, mean TTFT 60.56 ms | `/data/bbuf/validate/core_skill_validation_20260501/qwen3_8b/auto_benchmark` | +| `mistralai/Mistral-7B-Instruct-v0.3` | 2x H100, TP=2 | `sglang_mem080`, 24.09 req/s, 1541.92 output tok/s, mean TTFT 61.47 ms | `vllm_mem090`, 24.76 req/s, 1584.54 output tok/s, mean TTFT 58.63 ms | `/data/bbuf/validate/core_skill_validation_20260501/mistral_7b_instruct_v03/auto_benchmark` | + +## Skill Scope + +This skill is a playbook plus a config+validator toolchain, not a turn-key +orchestrator. The operator still launches servers, drives workloads, and writes +one normalized JSONL row per candidate. + +The `scripts/` directory contains exactly two tools: + +- `validate_cookbook_configs.py`: load cookbook YAML, render bounded candidate + server commands, and check flag names against captured `--help` snapshots + without launching servers. +- `compare_benchmark_results.py`: turn normalized per-candidate JSONL into the + markdown and optional CSV tables described in the Output Contract. + +Cookbook configs under `configs/cookbook-llm/` must pass the validator. The +shorter [references/example-plan.yaml](references/example-plan.yaml) is a +one-off runtime-plan skeleton and is not expected to pass as-is. Use +[references/result-schema.md](references/result-schema.md) as the single source +of truth for SLA key names. + +## Required Inputs + +Collect these before a long run: + +- model and tokenizer path, target frameworks, GPU model/count, multi-node + allowance, precision, and quantization constraints +- endpoint shape, workload source, dataset scenarios, SLA target, search budget, + and artifact output directory +- version manifest: framework package version or git commit, container/Python + environment, `--help` snapshots, and whether each search parameter was + accepted by that exact CLI + +If real production traffic is the goal, use the real request distribution. A +synthetic workload is fine for bring-up and first-pass comparison, but it is not +enough for a production choice. + +Record each scenario's input/output length distribution in the normalized +result rows. This is now part of the profiler handoff contract: if SGLang is +slower and `sglang-sota-performance` invokes `llm-torch-profiler-analysis`, +the profiler workload must reuse the slow SGLang benchmark scenario lengths +instead of falling back to its generic prefill `4090->1` and decode `1->2048` +defaults. + +## Known Gotchas + +Short list of failure modes that have bitten past validation runs. Check these +before starting a long sweep. + +- SGLang `fa3` attention backends need Hopper or newer. On A100, L40S, RTX + 5090, and older GPUs, drop `fa3` from the SGLang `search_space` and keep + `flashinfer` (or `triton` when FlashInfer is unavailable). +- SGLang `bench_serving` has two SGLang-facing backends: `--backend sglang` for + the native `/generate` endpoint and `--backend sglang-oai` for the + OpenAI-compatible endpoint. For cross-framework comparisons, prefer + `sglang-oai` so every framework is measured on the same request path. +- vLLM `--enable-dbo` only works when the target vLLM image is built with a + supported all2all backend. Keep DBO out of the default candidate list unless + the operator has verified the image. +- vLLM `--max-num-partial-prefills > 1` is model- and runtime-gated. Keep `1` + in the default pass; raise only after a preflight with the actual model. +- The historical TensorRT-LLM 1.0.0 validation image accepted + `--kv_cache_free_gpu_memory_fraction`; the older `--free_gpu_memory_fraction` + exited with a CLI error. TensorRT-LLM was refreshed to 1.2.1 stable and + 1.3.0 release candidates by 2026-04-28, so re-check the accepted flag name + via `--help` on the target image before a real run. +- The historical TensorRT-LLM 1.0.0 multi-GPU PyTorch-backend validation used + `--ipc=host`, `--ulimit memlock=-1`, `--ulimit stack=67108864`, + `--shm-size=16g`, and `NCCL_IB_DISABLE=1` (for single-node) or an equivalent + NCCL setup. Keep these as a starting point, not as a version-independent + requirement. +- The historical TensorRT-LLM 1.0.0 benchmark client took `--backend openai` or + `--backend openai-chat`; `--backend trtllm` was rejected. This is separate + from the server backend, which is pinned to `pytorch` by this skill. +- `trtllm` `benchmark_serving --dataset-name random` silently falls back to + ShareGPT sampling without `--random-ids` (or `--download-path`). +- `max_seq_len` / `max_model_len` / `context_length` candidates must cover + `max(input_len + output_len)` across every scenario, including values inside + `search_space`, not just the baseline. The validator checks this; do not + bypass it. + +## Secrets Hygiene + +- Never print `HF_TOKEN`, `HUGGINGFACE_HUB_TOKEN`, or any upstream API key into + a saved artifact. Pass them through container `-e VAR` (unquoted on the right + side so the host value is inherited) and keep them out of `server_command` + and `benchmark_command` fields written to the result JSONL. +- When a framework echoes the full argv at startup, scrub the log or redact + token-shaped substrings before uploading the artifact. + +## Fairness Rules + +Use these rules throughout the benchmark: + +- Run every framework on the same GPU type, GPU count, model weights, tokenizer, + precision, quantization policy, prompt distribution, output length target, and + sampling settings. +- Record framework version, git commit, container image, CUDA/NCCL versions, GPU + driver, visible GPU ids, launch command, and benchmark command. +- Warm the server before measuring. Restart or clear state between candidate + configurations when cache effects would bias the comparison. +- Compare steady-state fixed-QPS runs separately from burst throughput runs. +- Keep failed candidates in the final results with their failure reason. +- Report both raw throughput and SLA-passing throughput. The fastest failing + candidate is not the best deployment command. + +## Workflow + +### 1. Preflight + +Verify all requested frameworks before starting a search: + +```bash +python -m sglang.launch_server --help +python -m sglang.bench_serving --help +vllm serve --help +vllm serve --help=all +vllm bench serve --help +vllm bench serve --help=all +vllm bench sweep serve --help=all +trtllm-serve serve --help +python -m tensorrt_llm.serve.scripts.benchmark_serving --help +``` + +Use the framework-specific `--help` output in the target environment as the +source of truth. Do not keep a stale launch flag just because it appears in an +old note. + +vLLM 0.19 and newer use grouped help. Plain `vllm serve --help` only shows the +groups, so capture `--help=all` before deciding whether a search knob exists. + +Save these `--help` outputs into the run artifact directory. If a listed search +knob is missing from the current CLI, remove or translate that knob before +running the benchmark. Do not silently pass unknown flags. + +For TensorRT-LLM, also confirm that `trtllm-serve serve --help` accepts +`--backend pytorch`. If it does not, mark TensorRT-LLM unsupported in that +environment rather than falling back to a different server backend. + +For each framework, launch a minimal server, confirm `/v1/models` or the native +model-info endpoint, send one streaming request, run one tiny benchmark with at +least 5 requests, then save the launch command, benchmark command, server log, +and benchmark output. + +Before any GPU-backed smoke run, check the requested GPU ids directly with +`nvidia-smi`. If a requested GPU is already in use, stop and record that fact. +Do not silently borrow a different GPU count for a performance comparison. It is +fine to run a smaller one-GPU smoke only when the result is clearly labeled as a +flow check rather than a fair throughput comparison. + +If the target environment runs through containers, follow +[references/container-runbook.md](references/container-runbook.md) and save image +tags, pull commands, launch/benchmark logs, and cleanup commands. + +### 2. Normalize The Workload + +Use one canonical workload for all frameworks. Recommended JSONL row shape: + +```json +{"prompt": [{"role": "user", "content": "Summarize this text."}], "output_len": 256} +{"prompt": "Write a short explanation of CUDA graphs.", "output_len": 128} +``` + +Optional fields: + +```json +{ + "prompt": [{"role": "user", "content": "Use low temperature."}], + "output_len": 256, + "extra_request_body": {"temperature": 0.0, "top_p": 0.95}, + "metadata": {"source": "prod-sample"} +} +``` + +When converting user data: + +- inspect at least 3 rows before conversion +- preserve request-level sampling options in `extra_request_body` +- do not include the final assistant answer in the prompt when that answer is + the target completion +- keep multimodal or tool-call payloads only if all requested frameworks support + the chosen endpoint shape + +For synthetic bring-up, use the shipped two-scenario shape: + +```yaml +dataset: + kind: random + num_prompts: 80 + scenario_names: [chat, summarization] + input_len: [1000, 8000] + output_len: [1000, 1000] +``` + +Each aligned `input_len` / `output_len` pair is one scenario. Do not take the +cartesian product unless the user asks for that. +Name each scenario and keep the aligned pair in the artifacts. For custom +datasets, compute or record representative `input_len` and `output_len` +buckets, at least p50 and p95 when possible, so later profiler runs can match +the slow bucket rather than profiling an unrelated synthetic shape. + +Before searching any sequence-length limit, compute the largest +`input_len + output_len` in the dataset. SGLang `context_length`, vLLM +`max_model_len`, and TensorRT-LLM `max_seq_len` must be at least that value for +every candidate that is expected to run all scenarios. + +### 3. Pick A Search Tier + +Use the smallest tier that can answer the user's question: + +- Tier 1: smoke and sanity. One baseline plus a few high-impact knobs. +- Tier 2: default. A bounded sweep over the most likely server settings. +- Tier 3: exhaustive. Only when the search space is already tight and the user + accepts a long run. + +Default budget: + +- `num_prompts: 80` for the default cross-framework comparison; `num_prompts: + 20` per scenario is acceptable for a smoke/flow check and must be labeled as + such in the artifact (not as a performance result). +- `search.max_candidates_per_framework: 10` for the first useful pass +- candidate generation: baseline first, then a bounded product or ordered + candidate list from `search_space` +- at most 5 QPS search rounds unless the user asks for more +- stop early when every candidate in one framework is clearly OOM or fails the + basic health check + +Keep these in `base_server_flags` unless the user specifically wants a capacity +or memory study: + +- SGLang `mem_fraction_static` +- SGLang `schedule_policy` +- vLLM `gpu_memory_utilization` +- TensorRT-LLM `kv_cache_free_gpu_memory_fraction` + +These are real knobs, but they widen the search quickly and often turn a serving +comparison into a memory-limit study. + +### 4. Tune SGLang + +Prefer the SGLang auto-benchmark runner when the target checkout supports it: + +```bash +python -m sglang.auto_benchmark run --config /path/to/sglang.yaml +``` + +Otherwise launch the server manually and benchmark with: + +```bash +python -m sglang.bench_serving \ + --backend sglang \ + --dataset-name random \ + --random-input-len 1024 \ + --random-output-len 256 \ + --num-prompts 80 \ + --request-rate 8 \ + --output-file /path/to/sglang/results.json \ + --output-details +``` + +Version-sensitive SGLang knob families to verify: + +- `tp_size`, `pp_size`, `dp_size`, `ep_size` +- `attention_backend`, `prefill_attention_backend`, `decode_attention_backend` +- `sampling_backend` +- `max_running_requests`, `max_queued_requests` +- `chunked_prefill_size`, `prefill_max_requests`, `max_prefill_tokens` +- `max_total_tokens`, `page_size` +- CUDA graph and piecewise CUDA graph settings +- speculative or EAGLE settings only after the non-speculative baseline is tuned + +Keep `mem_fraction_static` and `schedule_policy` pinned in the default pass, +matching the shared cookbook config style. + +For quick smoke tests, it is reasonable to disable CUDA graph and piecewise CUDA +graph startup work if the goal is only to prove the framework flow. Record those +flags in the artifact. Do not carry that smoke setting into a performance winner +unless the user asked to tune eager-mode serving. + +### 5. Tune vLLM + +Use vLLM's sweep runner when available: + +```bash +vllm bench sweep serve \ + --serve-cmd 'vllm serve --port 8000' \ + --bench-cmd 'vllm bench serve --backend vllm --model --port 8000 --dataset-name random --num-prompts 80' \ + --serve-params /path/to/vllm_serve_params.json \ + --bench-params /path/to/vllm_bench_params.json \ + --output-dir /path/to/vllm_results +``` + +If sweep support is unavailable, run `vllm serve` for each candidate and measure +with `vllm bench serve`. + +Version-sensitive vLLM knob families to verify: + +- tensor, pipeline, data, decode-context, and expert parallelism +- `gpu_memory_utilization` +- `max_num_seqs` +- `max_num_batched_tokens` +- `max_model_len` +- `enable_chunked_prefill`, partial prefill limits, and DBO thresholds +- KV cache dtype and block size +- dtype and quantization settings +- CUDA graph capture sizes or eager-mode toggles when relevant +- prefix cache and speculative decoding settings only when the workload needs + those features + +vLLM should get a normal sweep, not one baseline command. See +[references/framework-reference.md](references/framework-reference.md) for +native command templates and cross-framework knob families. Confirm each flag on +the target image's `--help` before a run. + +Keep `gpu_memory_utilization` in the baseline for the default pass. Search it +only when the question is explicitly about fitting the model or trading capacity +against throughput. + +Keep DBO and all2all backend settings out of the default pass unless the target +vLLM environment is already set up for them. They are real tuning knobs, but a +candidate can fail at startup if the required all2all backend is not available. +Also preflight concurrent partial prefill before raising +`max_num_partial_prefills` above 1; some model/runtime combinations reject it at +startup. + +### 6. Tune TensorRT-LLM + +Use `trtllm-serve serve` as the server entrypoint when the target environment +supports it: + +```bash +trtllm-serve serve \ + --backend pytorch \ + --tp_size \ + --pp_size \ + --kv_cache_free_gpu_memory_fraction 0.75 \ + --host 0.0.0.0 \ + --port 8000 +``` + +Then benchmark the OpenAI-compatible endpoint with the TensorRT-LLM serving +benchmark client or with the same OpenAI-compatible client used for the other +frameworks. + +In the historical TensorRT-LLM 1.0.0 validation image, +`benchmark_serving --dataset-name random` sampled from ShareGPT unless either +`--download-path` or `--random-ids` was passed. For a fast synthetic smoke test, +pass `--random-ids`, then confirm the behavior on the target TensorRT-LLM image. + +TensorRT-LLM flag names are especially version-sensitive. In the validated +TensorRT-LLM 1.0.0 image, the KV-cache memory flag accepted by +`trtllm-serve serve` was `--kv_cache_free_gpu_memory_fraction`, not +`--free_gpu_memory_fraction`. TensorRT-LLM 1.2.1 is the latest stable GitHub +release as of 2026-04-28, with 1.3.0 release candidates also published; verify +the current flag with `trtllm-serve serve --help` before running a search on any +GPU target. + +TensorRT-LLM backend policy for this skill: + +- launch the server with `--backend pytorch` +- keep `backend: pytorch` in `base_server_flags` +- do not add `backend` to `search_space` +- reject `trt`, engine-backed serving, or any other non-PyTorch TensorRT-LLM + server backend as unsupported for this skill + +Version-sensitive TensorRT-LLM knob families to verify: + +- `tp_size`, `pp_size`, and `ep_size` +- max batch size, max sequence length, max number of tokens, and KV-cache budget +- inflight batching and scheduler options +- extra LLM API options YAML used by `trtllm-serve` with the PyTorch backend + +The `trtllm-serve serve` CLI exposes fewer direct runtime knobs than SGLang or +vLLM. Use direct flags when they exist, then use `--extra_llm_api_options` for +PyTorch-backend settings that are not top-level CLI flags. Keep unsupported +backend or engine requests in the failure table instead of translating them. + +Keep `kv_cache_free_gpu_memory_fraction` in the baseline for the default pass. +Search `max_batch_size`, `max_num_tokens`, `max_seq_len`, and validated +PyTorch-backend config options first. The server backend remains fixed to +`pytorch`. + +### 7. Normalize Results + +Write one JSONL row per candidate using the schema in +[references/result-schema.md](references/result-schema.md). Then run: + +```bash +python .claude/skills/llm-serving-auto-benchmark/scripts/compare_benchmark_results.py \ + --input /path/to/candidates.jsonl \ + --output /path/to/summary.md +``` + +Rank candidates in this order: + +1. SLA passed +2. highest request throughput or goodput +3. highest output token throughput +4. lower mean TTFT +5. lower mean TPOT/ITL +6. lower GPU count or simpler deployment if performance is close + +Keep the SLA gate itself unchanged. In the cookbook configs and normalized +result schema, TTFT SLA still uses `max_p99_ttft_ms` and TPOT SLA still uses +`max_p99_tpot_ms`; only the default cross-candidate comparison order switches +to mean TTFT and mean TPOT. + +## Output Contract + +Return a compact report with workload/SLA, hardware and framework versions, best +deployment-command tables per framework/scenario, one cross-framework comparison +table, exact launch and benchmark commands for winners, and artifact paths for +workload, raw/normalized results, CSV or markdown summary, and server logs. + +When SGLang is not the winner, include a profiler handoff note with the slow +SGLang scenario name and the exact input/output lengths or percentile bucket to +pass to `llm-torch-profiler-analysis`. + +Include failed or excluded candidates with reasons. Explain that this table is a +record of tried configs that were not selected: candidates that failed, were +skipped by policy, or completed but missed the SLA. Add caveats for synthetic +workloads, incomplete fair searches, or framework-specific parameter +substitutions. + +Use [references/framework-reference.md](references/framework-reference.md) when +you need command templates, source links, or knob-family mappings. Use +[references/example-plan.yaml](references/example-plan.yaml) as the starting +point for a full cross-framework run plan. diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/README.md b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/README.md new file mode 100644 index 000000000..cad9f7b44 --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/README.md @@ -0,0 +1,17 @@ +# Cookbook LLM Configs + +These configs define a framework-neutral LLM serving cookbook model set and translate each model into a three-framework run plan for SGLang, vLLM, and TensorRT-LLM. + +Scope: +- SGLang can preserve source-recipe `base_flags` and `search_space` where applicable; if a sequence limit is smaller than the default synthetic scenario, the config raises that limit so the shipped workload can run. +- vLLM uses framework-native `vllm serve` flags. The translation keeps the same model, tokenizer, dataset shape, GPU count, and high-impact batching/prefix-cache knobs; it does not copy SGLang-only parser or scheduler flags. +- TensorRT-LLM uses `trtllm-serve serve` with `backend: pytorch` fixed in `base_server_flags`. Backend choice is never searched. +- The two default random scenarios remain aligned pairs: `chat` uses `1000 -> 1000`, and `summarization` uses `8000 -> 1000`. + +Before a real run, capture the target framework `--help` output and validate the configs: + +```bash +python .claude/skills/llm-serving-auto-benchmark/scripts/validate_cookbook_configs.py .claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm +``` + +With captured help files, add `--help-dir ` to check the concrete flag names against that environment. This check only loads configs and renders candidate commands; it does not launch model servers. diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/deepseek-math-v2.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/deepseek-math-v2.yaml new file mode 100644 index 000000000..1649101f3 --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/deepseek-math-v2.yaml @@ -0,0 +1,130 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: deepseek-math-v2.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: deepseek-ai/DeepSeek-Math-V2 + tokenizer: deepseek-ai/DeepSeek-Math-V2 + precision: auto + quantization: model default +hardware: + gpu_count: 8 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: deepseek-ai/DeepSeek-Math-V2 + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/deepseek-math-v2 +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 8 + model_path: deepseek-ai/DeepSeek-Math-V2 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - flashinfer + decode_attention_backend: + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + ep_size: + - 1 + - 4 + - 8 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 8 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + search_space: + max_num_seqs: + - 32 + - 48 + - 64 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 8 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + search_space: + max_batch_size: + - 32 + - 48 + - 64 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 + ep_size: + - 1 + - 4 + - 8 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/deepseek-r1-0528.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/deepseek-r1-0528.yaml new file mode 100644 index 000000000..a7cbea9f7 --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/deepseek-r1-0528.yaml @@ -0,0 +1,133 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: deepseek-r1-0528.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: deepseek-ai/DeepSeek-R1-0528 + tokenizer: deepseek-ai/DeepSeek-R1-0528 + precision: auto + quantization: model default +hardware: + gpu_count: 8 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: deepseek-ai/DeepSeek-R1-0528 + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/deepseek-r1-0528 +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 8 + enable_symm_mem: true + model_path: deepseek-ai/DeepSeek-R1-0528 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + ep_size: + - 1 + - 4 + - 8 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 8 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + search_space: + max_num_seqs: + - 32 + - 48 + - 64 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 8 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + search_space: + max_batch_size: + - 32 + - 48 + - 64 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 + ep_size: + - 1 + - 4 + - 8 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/deepseek-v3.1.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/deepseek-v3.1.yaml new file mode 100644 index 000000000..2402e4600 --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/deepseek-v3.1.yaml @@ -0,0 +1,132 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: deepseek-v3.1.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: deepseek-ai/DeepSeek-V3.1 + tokenizer: deepseek-ai/DeepSeek-V3.1 + precision: auto + quantization: model default +hardware: + gpu_count: 8 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: deepseek-ai/DeepSeek-V3.1 + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/deepseek-v3.1 +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 8 + model_path: deepseek-ai/DeepSeek-V3.1 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + ep_size: + - 1 + - 4 + - 8 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 8 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + search_space: + max_num_seqs: + - 32 + - 48 + - 64 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 8 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + search_space: + max_batch_size: + - 32 + - 48 + - 64 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 + ep_size: + - 1 + - 4 + - 8 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/deepseek-v3.2.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/deepseek-v3.2.yaml new file mode 100644 index 000000000..51f1ef07b --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/deepseek-v3.2.yaml @@ -0,0 +1,132 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: deepseek-v3.2.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: deepseek-ai/DeepSeek-V3.2 + tokenizer: deepseek-ai/DeepSeek-V3.2 + precision: auto + quantization: model default +hardware: + gpu_count: 8 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: deepseek-ai/DeepSeek-V3.2 + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/deepseek-v3.2 +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 8 + model_path: deepseek-ai/DeepSeek-V3.2 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + ep_size: + - 1 + - 4 + - 8 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 8 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + search_space: + max_num_seqs: + - 32 + - 48 + - 64 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 8 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + search_space: + max_batch_size: + - 32 + - 48 + - 64 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 + ep_size: + - 1 + - 4 + - 8 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/deepseek-v3.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/deepseek-v3.yaml new file mode 100644 index 000000000..a32a92ce5 --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/deepseek-v3.yaml @@ -0,0 +1,133 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: deepseek-v3.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: deepseek-ai/DeepSeek-V3 + tokenizer: deepseek-ai/DeepSeek-V3 + precision: auto + quantization: model default +hardware: + gpu_count: 8 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: deepseek-ai/DeepSeek-V3 + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/deepseek-v3 +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 8 + enable_symm_mem: true + model_path: deepseek-ai/DeepSeek-V3 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + ep_size: + - 1 + - 4 + - 8 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 8 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + search_space: + max_num_seqs: + - 32 + - 48 + - 64 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 8 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + search_space: + max_batch_size: + - 32 + - 48 + - 64 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 + ep_size: + - 1 + - 4 + - 8 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/devstral-small-2-24b-instruct-2512.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/devstral-small-2-24b-instruct-2512.yaml new file mode 100644 index 000000000..4d6d6782e --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/devstral-small-2-24b-instruct-2512.yaml @@ -0,0 +1,123 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: devstral-small-2-24b-instruct-2512.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: mistralai/Devstral-Small-2-24B-Instruct-2512 + tokenizer: mistralai/Devstral-Small-2-24B-Instruct-2512 + precision: auto + quantization: model default +hardware: + gpu_count: 1 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: mistralai/Devstral-Small-2-24B-Instruct-2512 + max_concurrency: + - null + - 16 + - 32 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 16.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/devstral-small-2-24b-instruct-2512 +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + model_path: mistralai/Devstral-Small-2-24B-Instruct-2512 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 1 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + search_space: + max_num_seqs: + - 64 + - 96 + - 128 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 1 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + search_space: + max_batch_size: + - 64 + - 96 + - 128 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/ernie-4.5-21b-a3b-pt.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/ernie-4.5-21b-a3b-pt.yaml new file mode 100644 index 000000000..ab6ddbfa3 --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/ernie-4.5-21b-a3b-pt.yaml @@ -0,0 +1,117 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: ernie-4.5-21b-a3b-pt.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: baidu/ERNIE-4.5-21B-A3B-PT + tokenizer: baidu/ERNIE-4.5-21B-A3B-PT + precision: auto + quantization: model default +hardware: + gpu_count: 1 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: baidu/ERNIE-4.5-21B-A3B-PT + max_concurrency: + - null + - 16 + - 32 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 16.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/ernie-4.5-21b-a3b-pt +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + model_path: baidu/ERNIE-4.5-21B-A3B-PT + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 1 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + search_space: + max_num_seqs: + - 64 + - 96 + - 128 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 1 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + search_space: + max_batch_size: + - 64 + - 96 + - 128 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/glm-4.5.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/glm-4.5.yaml new file mode 100644 index 000000000..e5010eae5 --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/glm-4.5.yaml @@ -0,0 +1,122 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: glm-4.5.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: zai-org/GLM-4.5 + tokenizer: zai-org/GLM-4.5 + precision: auto + quantization: model default +hardware: + gpu_count: 4 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: zai-org/GLM-4.5 + max_concurrency: + - null + - 8 + - 16 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 8.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/glm-4.5 +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 4 + context_length: 9000 + model_path: zai-org/GLM-4.5 + trust_remote_code: true + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 4 + trust_remote_code: true + gpu_memory_utilization: 0.9 + max_model_len: 9000 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + search_space: + max_num_seqs: + - 64 + - 96 + - 128 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 4 + pp_size: 1 + trust_remote_code: true + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 9000 + search_space: + max_batch_size: + - 64 + - 96 + - 128 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 9000 + - 16384 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/glm-4.6.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/glm-4.6.yaml new file mode 100644 index 000000000..b32f61f97 --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/glm-4.6.yaml @@ -0,0 +1,135 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: glm-4.6.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: zai-org/GLM-4.6 + tokenizer: zai-org/GLM-4.6 + precision: auto + quantization: model default +hardware: + gpu_count: 8 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: zai-org/GLM-4.6 + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/glm-4.6 +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 8 + model_path: zai-org/GLM-4.6 + trust_remote_code: true + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + ep_size: + - 1 + - 4 + - 8 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 8 + trust_remote_code: true + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + search_space: + max_num_seqs: + - 32 + - 48 + - 64 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 8 + pp_size: 1 + trust_remote_code: true + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + search_space: + max_batch_size: + - 32 + - 48 + - 64 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 + ep_size: + - 1 + - 4 + - 8 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/glm-4.7-flash.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/glm-4.7-flash.yaml new file mode 100644 index 000000000..4e59645ee --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/glm-4.7-flash.yaml @@ -0,0 +1,126 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: glm-4.7-flash.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: zai-org/GLM-4.7-Flash + tokenizer: zai-org/GLM-4.7-Flash + precision: auto + quantization: model default +hardware: + gpu_count: 1 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: zai-org/GLM-4.7-Flash + max_concurrency: + - null + - 16 + - 32 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 16.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/glm-4.7-flash +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + model_path: zai-org/GLM-4.7-Flash + trust_remote_code: true + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 1 + trust_remote_code: true + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + search_space: + max_num_seqs: + - 64 + - 96 + - 128 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 1 + pp_size: 1 + trust_remote_code: true + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + search_space: + max_batch_size: + - 64 + - 96 + - 128 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/glm-4.7.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/glm-4.7.yaml new file mode 100644 index 000000000..0cdd86f86 --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/glm-4.7.yaml @@ -0,0 +1,130 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: glm-4.7.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: zai-org/GLM-4.7 + tokenizer: zai-org/GLM-4.7 + precision: auto + quantization: model default +hardware: + gpu_count: 4 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: zai-org/GLM-4.7 + max_concurrency: + - null + - 8 + - 16 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 8.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/glm-4.7 +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 4 + context_length: 9000 + model_path: zai-org/GLM-4.7 + trust_remote_code: true + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 + ep_size: + - 1 + - 2 + - 4 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 4 + trust_remote_code: true + gpu_memory_utilization: 0.9 + max_model_len: 9000 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + search_space: + max_num_seqs: + - 64 + - 96 + - 128 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 4 + pp_size: 1 + trust_remote_code: true + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 9000 + search_space: + max_batch_size: + - 64 + - 96 + - 128 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 9000 + - 16384 + ep_size: + - 1 + - 2 + - 4 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/glm-5-fp8.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/glm-5-fp8.yaml new file mode 100644 index 000000000..63df942fc --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/glm-5-fp8.yaml @@ -0,0 +1,132 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: glm-5-fp8.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: zai-org/GLM-5-FP8 + tokenizer: zai-org/GLM-5-FP8 + precision: auto + quantization: model default +hardware: + gpu_count: 8 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: zai-org/GLM-5-FP8 + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/glm-5-fp8 +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 8 + model_path: zai-org/GLM-5-FP8 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + ep_size: + - 1 + - 4 + - 8 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 8 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + search_space: + max_num_seqs: + - 32 + - 48 + - 64 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 8 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + search_space: + max_batch_size: + - 32 + - 48 + - 64 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 + ep_size: + - 1 + - 4 + - 8 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/glyph.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/glyph.yaml new file mode 100644 index 000000000..f9b793b9a --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/glyph.yaml @@ -0,0 +1,126 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: glyph.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: zai-org/Glyph + tokenizer: zai-org/Glyph + precision: auto + quantization: model default +hardware: + gpu_count: 4 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: zai-org/Glyph + max_concurrency: + - null + - 8 + - 16 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 8.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/glyph +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 4 + reasoning_parser: glm45 + tool_call_parser: glm45 + model_path: zai-org/Glyph + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 4 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + search_space: + max_num_seqs: + - 64 + - 96 + - 128 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 4 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + search_space: + max_batch_size: + - 64 + - 96 + - 128 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/gpt-oss-120b.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/gpt-oss-120b.yaml new file mode 100644 index 000000000..0ca920c07 --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/gpt-oss-120b.yaml @@ -0,0 +1,132 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: gpt-oss-120b.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: openai/gpt-oss-120b + tokenizer: openai/gpt-oss-120b + precision: auto + quantization: model default +hardware: + gpu_count: 8 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: openai/gpt-oss-120b + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/gpt-oss-120b +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 8 + model_path: openai/gpt-oss-120b + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + ep_size: + - 1 + - 4 + - 8 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 8 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + search_space: + max_num_seqs: + - 32 + - 48 + - 64 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 8 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + search_space: + max_batch_size: + - 32 + - 48 + - 64 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 + ep_size: + - 1 + - 4 + - 8 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/intern-s1.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/intern-s1.yaml new file mode 100644 index 000000000..ad30b4dd2 --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/intern-s1.yaml @@ -0,0 +1,135 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: intern-s1.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: internlm/Intern-S1 + tokenizer: internlm/Intern-S1 + precision: auto + quantization: model default +hardware: + gpu_count: 8 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: internlm/Intern-S1 + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/intern-s1 +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 8 + trust_remote_code: true + model_path: internlm/Intern-S1 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + ep_size: + - 1 + - 4 + - 8 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 8 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + trust_remote_code: true + search_space: + max_num_seqs: + - 32 + - 48 + - 64 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 8 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + trust_remote_code: true + search_space: + max_batch_size: + - 32 + - 48 + - 64 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 + ep_size: + - 1 + - 4 + - 8 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/kimi-k2-instruct.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/kimi-k2-instruct.yaml new file mode 100644 index 000000000..de03222c8 --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/kimi-k2-instruct.yaml @@ -0,0 +1,133 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: kimi-k2-instruct.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: moonshotai/Kimi-K2-Instruct + tokenizer: moonshotai/Kimi-K2-Instruct + precision: auto + quantization: model default +hardware: + gpu_count: 8 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: moonshotai/Kimi-K2-Instruct + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/kimi-k2-instruct +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 8 + trust_remote_code: true + model_path: moonshotai/Kimi-K2-Instruct + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + ep_size: + - 1 + - 4 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 8 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + trust_remote_code: true + search_space: + max_num_seqs: + - 32 + - 48 + - 64 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 8 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + trust_remote_code: true + search_space: + max_batch_size: + - 32 + - 48 + - 64 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 + ep_size: + - 1 + - 4 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/kimi-k2.5.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/kimi-k2.5.yaml new file mode 100644 index 000000000..51303495d --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/kimi-k2.5.yaml @@ -0,0 +1,127 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: kimi-k2.5.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: moonshotai/Kimi-K2.5 + tokenizer: moonshotai/Kimi-K2.5 + precision: auto + quantization: model default +hardware: + gpu_count: 8 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: moonshotai/Kimi-K2.5 + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/kimi-k2.5 +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 8 + trust_remote_code: true + model_path: moonshotai/Kimi-K2.5 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 8 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + trust_remote_code: true + search_space: + max_num_seqs: + - 32 + - 48 + - 64 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 8 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + trust_remote_code: true + search_space: + max_batch_size: + - 32 + - 48 + - 64 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/kimi-linear-48b-a3b-instruct.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/kimi-linear-48b-a3b-instruct.yaml new file mode 100644 index 000000000..dc2b3a1e5 --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/kimi-linear-48b-a3b-instruct.yaml @@ -0,0 +1,121 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: kimi-linear-48b-a3b-instruct.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: moonshotai/Kimi-Linear-48B-A3B-Instruct + tokenizer: moonshotai/Kimi-Linear-48B-A3B-Instruct + precision: auto + quantization: model default +hardware: + gpu_count: 4 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: moonshotai/Kimi-Linear-48B-A3B-Instruct + max_concurrency: + - null + - 8 + - 16 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 8.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/kimi-linear-48b-a3b-instruct +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 4 + trust_remote_code: true + model_path: moonshotai/Kimi-Linear-48B-A3B-Instruct + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 4 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + trust_remote_code: true + search_space: + max_num_seqs: + - 64 + - 96 + - 128 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 4 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + trust_remote_code: true + search_space: + max_batch_size: + - 64 + - 96 + - 128 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/ling-2.5-1t.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/ling-2.5-1t.yaml new file mode 100644 index 000000000..d4d076d1b --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/ling-2.5-1t.yaml @@ -0,0 +1,134 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: ling-2.5-1t.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: inclusionAI/Ling-2.5-1T + tokenizer: inclusionAI/Ling-2.5-1T + precision: auto + quantization: model default +hardware: + gpu_count: 8 + multi_node: true +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: inclusionAI/Ling-2.5-1T + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 2.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/ling-2.5-1t +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 8 + pp_size: 2 + nnodes: 2 + trust_remote_code: true + tool_call_parser: qwen + model_path: inclusionAI/Ling-2.5-1T + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + pp_size: + - 1 + - 2 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 8 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + pipeline_parallel_size: 2 + trust_remote_code: true + search_space: + max_num_seqs: + - 32 + - 48 + - 64 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 8 + pp_size: 2 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + trust_remote_code: true + search_space: + max_batch_size: + - 32 + - 48 + - 64 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/llada2-1-mini.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/llada2-1-mini.yaml new file mode 100644 index 000000000..ab6795c33 --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/llada2-1-mini.yaml @@ -0,0 +1,130 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: llada2-1-mini.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: inclusionAI/LLaDA2.1-mini + tokenizer: inclusionAI/LLaDA2.1-mini + precision: auto + quantization: model default +hardware: + gpu_count: 1 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: inclusionAI/LLaDA2.1-mini + max_concurrency: + - 1 + - 2 + - 4 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/llada2-1-mini +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 1 + dllm_algorithm: JointThreshold + trust_remote_code: true + max_running_requests: 1 + attention_backend: flashinfer + model_path: inclusionAI/LLaDA2.1-mini + mem_fraction_static: 0.77 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 1 + - 2 + - 4 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 1 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + trust_remote_code: true + search_space: + max_num_seqs: + - 1 + - 2 + - 4 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 1 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + trust_remote_code: true + search_space: + max_batch_size: + - 1 + - 2 + - 4 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/llama-3.1-70b-instruct.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/llama-3.1-70b-instruct.yaml new file mode 100644 index 000000000..cf7f88ed3 --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/llama-3.1-70b-instruct.yaml @@ -0,0 +1,124 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: llama-3.1-70b-instruct.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: meta-llama/Llama-3.1-70B-Instruct + tokenizer: meta-llama/Llama-3.1-70B-Instruct + precision: auto + quantization: model default +hardware: + gpu_count: 4 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: meta-llama/Llama-3.1-70B-Instruct + max_concurrency: + - null + - 8 + - 16 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 12.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/llama-3.1-70b-instruct +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 4 + model_path: meta-llama/Llama-3.1-70B-Instruct + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 4 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + search_space: + max_num_seqs: + - 64 + - 96 + - 128 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 4 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + search_space: + max_batch_size: + - 64 + - 96 + - 128 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/llama-3.3-70b-instruct.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/llama-3.3-70b-instruct.yaml new file mode 100644 index 000000000..a7fdc34c2 --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/llama-3.3-70b-instruct.yaml @@ -0,0 +1,118 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: llama-3.3-70b-instruct.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: meta-llama/Llama-3.3-70B-Instruct + tokenizer: meta-llama/Llama-3.3-70B-Instruct + precision: auto + quantization: model default +hardware: + gpu_count: 1 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: meta-llama/Llama-3.3-70B-Instruct + max_concurrency: + - null + - 16 + - 32 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 16.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/llama-3.3-70b-instruct +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tool_call_parser: llama3 + model_path: meta-llama/Llama-3.3-70B-Instruct + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 1 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + search_space: + max_num_seqs: + - 64 + - 96 + - 128 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 1 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + search_space: + max_batch_size: + - 64 + - 96 + - 128 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/llama-4-maverick-17b-128e-instruct-fp8.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/llama-4-maverick-17b-128e-instruct-fp8.yaml new file mode 100644 index 000000000..c97dfd6ab --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/llama-4-maverick-17b-128e-instruct-fp8.yaml @@ -0,0 +1,122 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: llama-4-maverick-17b-128e-instruct-fp8.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 + tokenizer: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 + precision: auto + quantization: model default +hardware: + gpu_count: 8 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 + max_concurrency: + - null + - 2 + - 4 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 2.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/llama-4-maverick-17b-128e-instruct-fp8 +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 8 + context_length: 1000000 + trust_remote_code: true + enable_multimodal: true + model_path: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 4 + - 8 + - 12 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 8 + gpu_memory_utilization: 0.9 + max_model_len: 1000000 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + trust_remote_code: true + search_space: + max_num_seqs: + - 4 + - 8 + - 12 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 8 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 1000000 + trust_remote_code: true + search_space: + max_batch_size: + - 4 + - 8 + - 12 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 1000000 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/llama-4-scout-17b-16e-instruct.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/llama-4-scout-17b-16e-instruct.yaml new file mode 100644 index 000000000..0b95311b2 --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/llama-4-scout-17b-16e-instruct.yaml @@ -0,0 +1,129 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: llama-4-scout-17b-16e-instruct.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: meta-llama/Llama-4-Scout-17B-16E-Instruct + tokenizer: meta-llama/Llama-4-Scout-17B-16E-Instruct + precision: bfloat16 + quantization: model default +hardware: + gpu_count: 8 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: meta-llama/Llama-4-Scout-17B-16E-Instruct + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/llama-4-scout-17b-16e-instruct +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 8 + enable_multimodal: true + context_length: 65536 + dtype: bfloat16 + trust_remote_code: true + model_path: meta-llama/Llama-4-Scout-17B-16E-Instruct + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 8 + - 16 + - 24 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 8 + gpu_memory_utilization: 0.9 + max_model_len: 65536 + dtype: bfloat16 + enable_chunked_prefill: true + kv_cache_dtype: auto + trust_remote_code: true + search_space: + max_num_seqs: + - 8 + - 16 + - 24 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 8 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 65536 + trust_remote_code: true + search_space: + max_batch_size: + - 8 + - 16 + - 24 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 65536 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/mimo-v2-flash.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/mimo-v2-flash.yaml new file mode 100644 index 000000000..f98917f09 --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/mimo-v2-flash.yaml @@ -0,0 +1,133 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: mimo-v2-flash.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: XiaomiMiMo/MiMo-V2-Flash + tokenizer: XiaomiMiMo/MiMo-V2-Flash + precision: auto + quantization: model default +hardware: + gpu_count: 8 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: XiaomiMiMo/MiMo-V2-Flash + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/mimo-v2-flash +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 8 + trust_remote_code: true + max_running_requests: 128 + chunked_prefill_size: 16384 + model_loader_extra_config: '{"enable_multithread_load": "true","num_threads": 64}' + attention_backend: fa3 + reasoning_parser: qwen3 + tool_call_parser: mimo + model_path: XiaomiMiMo/MiMo-V2-Flash + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 8 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + trust_remote_code: true + search_space: + max_num_seqs: + - 32 + - 48 + - 64 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 8 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + trust_remote_code: true + search_space: + max_batch_size: + - 32 + - 48 + - 64 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/minimax-m2.1.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/minimax-m2.1.yaml new file mode 100644 index 000000000..bbd62ea69 --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/minimax-m2.1.yaml @@ -0,0 +1,121 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: minimax-m2.1.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: MiniMaxAI/MiniMax-M2.1 + tokenizer: MiniMaxAI/MiniMax-M2.1 + precision: auto + quantization: model default +hardware: + gpu_count: 4 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: MiniMaxAI/MiniMax-M2.1 + max_concurrency: + - null + - 8 + - 16 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 8.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/minimax-m2.1 +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 4 + trust_remote_code: true + model_path: MiniMaxAI/MiniMax-M2.1 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 4 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + trust_remote_code: true + search_space: + max_num_seqs: + - 64 + - 96 + - 128 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 4 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + trust_remote_code: true + search_space: + max_batch_size: + - 64 + - 96 + - 128 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/minimax-m2.5.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/minimax-m2.5.yaml new file mode 100644 index 000000000..70c4d2b2d --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/minimax-m2.5.yaml @@ -0,0 +1,133 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: minimax-m2.5.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: MiniMaxAI/MiniMax-M2.5 + tokenizer: MiniMaxAI/MiniMax-M2.5 + precision: auto + quantization: model default +hardware: + gpu_count: 4 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: MiniMaxAI/MiniMax-M2.5 + max_concurrency: + - null + - 8 + - 16 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/minimax-m2.5 +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 4 + trust_remote_code: true + model_path: MiniMaxAI/MiniMax-M2.5 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 + ep_size: + - 1 + - 4 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 4 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + trust_remote_code: true + search_space: + max_num_seqs: + - 64 + - 96 + - 128 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 4 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + trust_remote_code: true + search_space: + max_batch_size: + - 64 + - 96 + - 128 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 + ep_size: + - 1 + - 4 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/ministral-3-8b-instruct-2512.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/ministral-3-8b-instruct-2512.yaml new file mode 100644 index 000000000..5c80f7f0a --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/ministral-3-8b-instruct-2512.yaml @@ -0,0 +1,121 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: ministral-3-8b-instruct-2512.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: mistralai/Ministral-3-8B-Instruct-2512 + tokenizer: mistralai/Ministral-3-8B-Instruct-2512 + precision: auto + quantization: model default +hardware: + gpu_count: 1 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: mistralai/Ministral-3-8B-Instruct-2512 + max_concurrency: + - null + - 16 + - 32 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 16.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/ministral-3-8b-instruct-2512 +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + trust_remote_code: true + tool_call_parser: mistral + model_path: mistralai/Ministral-3-8B-Instruct-2512 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 1 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + trust_remote_code: true + search_space: + max_num_seqs: + - 64 + - 96 + - 128 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 1 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + trust_remote_code: true + search_space: + max_batch_size: + - 64 + - 96 + - 128 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/mistral-small-4-119b-2603.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/mistral-small-4-119b-2603.yaml new file mode 100644 index 000000000..2970dca5c --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/mistral-small-4-119b-2603.yaml @@ -0,0 +1,124 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: mistral-small-4-119b-2603.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: mistralai/Mistral-Small-4-119B-2603 + tokenizer: mistralai/Mistral-Small-4-119B-2603 + precision: auto + quantization: model default +hardware: + gpu_count: 2 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: mistralai/Mistral-Small-4-119B-2603 + max_concurrency: + - null + - 8 + - 16 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 6.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/mistral-small-4-119b-2603 +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 2 + model_path: mistralai/Mistral-Small-4-119B-2603 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 2 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + search_space: + max_num_seqs: + - 64 + - 96 + - 128 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 2 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + search_space: + max_batch_size: + - 64 + - 96 + - 128 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/nemotron-3-nano-30b-a3b-bf16.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/nemotron-3-nano-30b-a3b-bf16.yaml new file mode 100644 index 000000000..bb031103e --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/nemotron-3-nano-30b-a3b-bf16.yaml @@ -0,0 +1,128 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: nemotron-3-nano-30b-a3b-bf16.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 + tokenizer: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 + precision: auto + quantization: model default +hardware: + gpu_count: 1 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 + max_concurrency: + - null + - 16 + - 32 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 16.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/nemotron-3-nano-30b-a3b-bf16 +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 1 + trust_remote_code: true + kv_cache_dtype: fp8_e4m3 + model_path: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 1 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: fp8_e4m3 + trust_remote_code: true + search_space: + max_num_seqs: + - 64 + - 96 + - 128 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 1 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + trust_remote_code: true + search_space: + max_batch_size: + - 64 + - 96 + - 128 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/nemotron-3-super-120b-a12b-bf16.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/nemotron-3-super-120b-a12b-bf16.yaml new file mode 100644 index 000000000..f9f5a03a6 --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/nemotron-3-super-120b-a12b-bf16.yaml @@ -0,0 +1,128 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: nemotron-3-super-120b-a12b-bf16.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16 + tokenizer: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16 + precision: auto + quantization: model default +hardware: + gpu_count: 4 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16 + max_concurrency: + - null + - 8 + - 16 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 6.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/nemotron-3-super-120b-a12b-bf16 +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 4 + trust_remote_code: true + kv_cache_dtype: fp8_e4m3 + model_path: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 4 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: fp8_e4m3 + trust_remote_code: true + search_space: + max_num_seqs: + - 64 + - 96 + - 128 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 4 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + trust_remote_code: true + search_space: + max_batch_size: + - 64 + - 96 + - 128 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/qwen3-235b-a22b.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/qwen3-235b-a22b.yaml new file mode 100644 index 000000000..4bc39b66e --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/qwen3-235b-a22b.yaml @@ -0,0 +1,132 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: qwen3-235b-a22b.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: Qwen/Qwen3-235B-A22B + tokenizer: Qwen/Qwen3-235B-A22B + precision: auto + quantization: model default +hardware: + gpu_count: 8 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: Qwen/Qwen3-235B-A22B + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/qwen3-235b-a22b +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 8 + model_path: Qwen/Qwen3-235B-A22B + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + ep_size: + - 1 + - 4 + - 8 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 8 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + search_space: + max_num_seqs: + - 32 + - 48 + - 64 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 8 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + search_space: + max_batch_size: + - 32 + - 48 + - 64 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 + ep_size: + - 1 + - 4 + - 8 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/qwen3-coder-480b-a35b-instruct.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/qwen3-coder-480b-a35b-instruct.yaml new file mode 100644 index 000000000..7583867f2 --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/qwen3-coder-480b-a35b-instruct.yaml @@ -0,0 +1,131 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: qwen3-coder-480b-a35b-instruct.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: Qwen/Qwen3-Coder-480B-A35B-Instruct + tokenizer: Qwen/Qwen3-Coder-480B-A35B-Instruct + precision: auto + quantization: model default +hardware: + gpu_count: 8 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: Qwen/Qwen3-Coder-480B-A35B-Instruct + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/qwen3-coder-480b-a35b-instruct +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 8 + ep_size: 2 + moe_runner_backend: triton + model_path: Qwen/Qwen3-Coder-480B-A35B-Instruct + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - flashinfer + decode_attention_backend: + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + ep_size: + - 1 + - 2 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 8 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + search_space: + max_num_seqs: + - 32 + - 48 + - 64 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 8 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + ep_size: 2 + search_space: + max_batch_size: + - 32 + - 48 + - 64 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 + ep_size: + - 1 + - 2 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/qwen3-coder-next.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/qwen3-coder-next.yaml new file mode 100644 index 000000000..8e8be77d3 --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/qwen3-coder-next.yaml @@ -0,0 +1,124 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: qwen3-coder-next.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: Qwen/Qwen3-Coder-Next + tokenizer: Qwen/Qwen3-Coder-Next + precision: auto + quantization: model default +hardware: + gpu_count: 2 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: Qwen/Qwen3-Coder-Next + max_concurrency: + - null + - 8 + - 16 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 12.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/qwen3-coder-next +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 2 + model_path: Qwen/Qwen3-Coder-Next + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 2 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + search_space: + max_num_seqs: + - 64 + - 96 + - 128 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 2 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + search_space: + max_batch_size: + - 64 + - 96 + - 128 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/qwen3-next-80b-a3b-instruct.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/qwen3-next-80b-a3b-instruct.yaml new file mode 100644 index 000000000..81c8f7d42 --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/qwen3-next-80b-a3b-instruct.yaml @@ -0,0 +1,130 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: qwen3-next-80b-a3b-instruct.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: Qwen/Qwen3-Next-80B-A3B-Instruct + tokenizer: Qwen/Qwen3-Next-80B-A3B-Instruct + precision: auto + quantization: model default +hardware: + gpu_count: 2 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: Qwen/Qwen3-Next-80B-A3B-Instruct + max_concurrency: + - null + - 8 + - 16 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 12.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/qwen3-next-80b-a3b-instruct +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 2 + model_path: Qwen/Qwen3-Next-80B-A3B-Instruct + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 + ep_size: + - 1 + - 2 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 2 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + search_space: + max_num_seqs: + - 64 + - 96 + - 128 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 2 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + search_space: + max_batch_size: + - 64 + - 96 + - 128 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 + ep_size: + - 1 + - 2 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/qwen35-397b-a17b-fp8.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/qwen35-397b-a17b-fp8.yaml new file mode 100644 index 000000000..e2d2cf369 --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/qwen35-397b-a17b-fp8.yaml @@ -0,0 +1,132 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: qwen35-397b-a17b-fp8.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: Qwen/Qwen3.5-397B-A17B-FP8 + tokenizer: Qwen/Qwen3.5-397B-A17B-FP8 + precision: auto + quantization: model default +hardware: + gpu_count: 4 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: Qwen/Qwen3.5-397B-A17B-FP8 + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/qwen35-397b-a17b-fp8 +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 4 + model_path: Qwen/Qwen3.5-397B-A17B-FP8 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + ep_size: + - 1 + - 4 + - 8 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 4 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + search_space: + max_num_seqs: + - 32 + - 48 + - 64 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 4 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + search_space: + max_batch_size: + - 32 + - 48 + - 64 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 + ep_size: + - 1 + - 4 + - 8 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/ring-2.5-1t.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/ring-2.5-1t.yaml new file mode 100644 index 000000000..5c8aa8330 --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/ring-2.5-1t.yaml @@ -0,0 +1,124 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: ring-2.5-1t.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: inclusionAI/Ring-2.5-1T + tokenizer: inclusionAI/Ring-2.5-1T + precision: auto + quantization: model default +hardware: + gpu_count: 8 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: inclusionAI/Ring-2.5-1T + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 2.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/ring-2.5-1t +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 8 + model_path: inclusionAI/Ring-2.5-1T + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 8 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + search_space: + max_num_seqs: + - 32 + - 48 + - 64 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 8 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + search_space: + max_batch_size: + - 32 + - 48 + - 64 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 diff --git a/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/step-3.5-flash.yaml b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/step-3.5-flash.yaml new file mode 100644 index 000000000..77ef1b45f --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/configs/cookbook-llm/step-3.5-flash.yaml @@ -0,0 +1,133 @@ +schema_version: 1 +source: + kind: llm_serving_cookbook + source_recipe_file: step-3.5-flash.yaml + translation: SGLang flags preserve the source recipe where applicable, with sequence limits raised when needed for the default dataset; vLLM and TensorRT-LLM use framework-native serving flags for the same model and dataset shape. +model: + name: stepfun-ai/Step-3.5-Flash + tokenizer: stepfun-ai/Step-3.5-Flash + precision: auto + quantization: model default +hardware: + gpu_count: 4 + multi_node: false +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + endpoint: /v1/completions + backend: openai-compatible + tokenizer: stepfun-ai/Step-3.5-Flash + max_concurrency: + - null + - 8 + - 16 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 8.0 + tolerance: 0.1 + sla: + max_p99_ttft_ms: 1500 + max_p99_tpot_ms: 30 + min_success_rate: 0.99 + output_dir: ./auto_benchmark_results/cookbook-llm/step-3.5-flash +search: + tier: 2 + max_candidates_per_framework: 8 + candidate_generation: baseline_first_bounded_product + resume: true +frameworks: + sglang: + enabled: true + server_command: python -m sglang.launch_server + base_server_flags: + tp_size: 4 + trust_remote_code: true + model_path: stepfun-ai/Step-3.5-Flash + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 + ep_size: + - 1 + - 4 + vllm: + enabled: true + server_command: vllm serve + config_source: framework_generic_translation + base_server_flags: + tensor_parallel_size: 4 + gpu_memory_utilization: 0.9 + max_model_len: 12288 + dtype: auto + enable_chunked_prefill: true + kv_cache_dtype: auto + trust_remote_code: true + search_space: + max_num_seqs: + - 64 + - 96 + - 128 + max_num_batched_tokens: + - 8192 + - 16384 + max_num_partial_prefills: + - 1 + max_long_partial_prefills: + - 1 + long_prefill_token_threshold: + - 0 + - 4096 + enable_prefix_caching: + - true + block_size: + - 16 + tensorrt_llm: + enabled: true + server_command: trtllm-serve serve + backend_policy: fixed_pytorch + config_source: framework_generic_translation + base_server_flags: + backend: pytorch + tp_size: 4 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + max_seq_len: 12288 + trust_remote_code: true + search_space: + max_batch_size: + - 64 + - 96 + - 128 + max_num_tokens: + - 8192 + - 16384 + max_seq_len: + - 12288 + - 16384 + ep_size: + - 1 + - 4 diff --git a/.claude/skills/llm-serving-auto-benchmark/references/container-runbook.md b/.claude/skills/llm-serving-auto-benchmark/references/container-runbook.md new file mode 100644 index 000000000..59da36c37 --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/references/container-runbook.md @@ -0,0 +1,321 @@ +# Container Runbook + +Use this runbook when the benchmark environment is container-based. It records +the exact image, command, help output, server log, benchmark log, and cleanup +step for each framework. + +This runbook is target-agnostic. Every `docker run` / `docker exec` command +works on a local box, an SSH-reachable remote GPU host, or a CI runner; the +per-host skills (for example `h100`, `b200`, `rtx5090`, `radixark02`, +`radixark03`) only add the SSH wrapper, container name, and workspace path +for a specific operator box. Substitute those values where you see +`$SGLANG_CONTAINER`, `$SGLANG_WORKSPACE`, and similar; nothing below assumes +an H100. + +## Common Setup + +Pull the images that will be used: + +```bash +docker pull lmsysorg/sglang:dev +docker pull vllm/vllm-openai:latest +docker pull nvcr.io/nvidia/tensorrt-llm/release:latest +``` + +Use quoted Docker GPU device lists: + +```bash +GPU_ARG='"device=6,7"' +docker run --gpus "$GPU_ARG" ... +``` + +The unquoted form `--gpus device=6,7` can be parsed incorrectly by Docker. + +Mount the shared Hugging Face cache and pass tokens through environment variables +when gated models are used: + +```bash +-v /data/.cache:/root/.cache \ +-e HF_TOKEN \ +-e HUGGINGFACE_HUB_TOKEN +``` + +Do not print token values into logs. + +Set the run variables once and pass them into containers that need them: + +```bash +export MODEL=TinyLlama/TinyLlama-1.1B-Chat-v1.0 +export TP=1 +export PP=1 +export PORT=8000 +export RUN_DIR=/tmp/llm-serving-auto-benchmark +mkdir -p "$RUN_DIR" +``` + +For synthetic validation, use two aligned scenarios rather than one tiny request +shape: + +```bash +# chat-like +RANDOM_INPUT_LEN=1000 +RANDOM_OUTPUT_LEN=1000 + +# summarization-like +RANDOM_INPUT_LEN=8000 +RANDOM_OUTPUT_LEN=1000 +``` + +For a fast smoke on larger models, 20 prompts per scenario is a reasonable +minimum. Do not treat that as a performance result. + +Set each framework's sequence-length limit to cover the largest scenario. For +the example above, use at least 9000 tokens for SGLang `--context-length`, vLLM +`--max-model-len`, and TensorRT-LLM `--max_seq_len`. + +Before launching a server, save the help output: + +```bash +python -m sglang.launch_server --help > artifacts/help/sglang_launch_server.txt +python -m sglang.bench_serving --help > artifacts/help/sglang_bench_serving.txt +vllm serve --help=all > artifacts/help/vllm_serve_all.txt +vllm bench serve --help=all > artifacts/help/vllm_bench_serve_all.txt +vllm bench sweep serve --help=all > artifacts/help/vllm_bench_sweep_serve_all.txt +trtllm-serve serve --help > artifacts/help/trtllm_serve.txt +python -m tensorrt_llm.serve.scripts.benchmark_serving --help \ + > artifacts/help/trtllm_benchmark_serving.txt +``` + +## SGLang + +If a prepared GPU host already has a long-running SGLang container (local or +reached via ssh; name is operator-specific), reuse it via `docker exec` +instead of creating a new container. The per-host skills — `h100`, +`h100-sglang-diffusion`, `b200`, `rtx5090`, `radixark02`, `radixark03`, +and similar — provide the concrete container name and workspace path for +that box; this runbook assumes the operator substitutes them: + +```bash +docker exec \ + -e MODEL \ + -e TP \ + -e PORT \ + "$SGLANG_CONTAINER" bash -lc " +cd \"\$SGLANG_WORKSPACE\" +python -m sglang.launch_server \\ + --model-path \"\$MODEL\" \\ + --tp-size \"\$TP\" \\ + --host 0.0.0.0 \\ + --port \"\$PORT\" +" +``` + +For a fresh container: + +```bash +docker run -d --name llmbench-sglang \ + --gpus "$GPU_ARG" \ + --network host \ + --ipc=host \ + -v /data/.cache:/root/.cache \ + -e MODEL \ + -e TP \ + -e PORT \ + -e HF_TOKEN \ + -e HUGGINGFACE_HUB_TOKEN \ + --entrypoint bash \ + lmsysorg/sglang:dev -lc ' +python -m sglang.launch_server \ + --model-path "$MODEL" \ + --tp-size "$TP" \ + --host 0.0.0.0 \ + --port "$PORT" +' +``` + +Then run either SGLang auto benchmark: + +```bash +python -m sglang.auto_benchmark run --config /path/to/sglang.yaml +``` + +or a tiny OpenAI-compatible smoke benchmark: + +```bash +python -m sglang.bench_serving \ + --backend sglang-oai \ + --host 127.0.0.1 \ + --port "$PORT" \ + --dataset-name random \ + --random-input-len 32 \ + --random-output-len 8 \ + --num-prompts 4 \ + --request-rate 1 \ + --max-concurrency 2 \ + --output-file "$RUN_DIR/sglang/results.json" \ + --output-details +``` + +## vLLM + +Server template: + +```bash +docker run -d --name llmbench-vllm \ + --gpus "$GPU_ARG" \ + --network host \ + --ipc=host \ + -v /data/.cache:/root/.cache \ + -e MODEL \ + -e TP \ + -e PORT \ + -e HF_TOKEN \ + -e HUGGINGFACE_HUB_TOKEN \ + --entrypoint bash \ + vllm/vllm-openai:latest -lc ' +vllm serve "$MODEL" \ + --host 0.0.0.0 \ + --port "$PORT" \ + --tensor-parallel-size "$TP" \ + --dtype auto \ + --gpu-memory-utilization 0.90 \ + --max-model-len 4096 \ + --max-num-seqs 64 \ + --max-num-batched-tokens 8192 \ + --enable-chunked-prefill \ + --kv-cache-dtype auto \ + --enable-prefix-caching \ + --trust-remote-code +' +``` + +Benchmark template: + +```bash +docker run --rm \ + --network host \ + -v /data/.cache:/root/.cache \ + -v "$RUN_DIR:/artifacts" \ + -e MODEL \ + -e PORT \ + --entrypoint bash \ + vllm/vllm-openai:latest -lc ' +vllm bench serve \ + --backend vllm \ + --base-url "http://127.0.0.1:$PORT" \ + --model "$MODEL" \ + --dataset-name random \ + --random-input-len 1024 \ + --random-output-len 256 \ + --num-prompts 80 \ + --request-rate 8 \ + --max-concurrency 64 \ + --save-result \ + --result-dir /artifacts/vllm \ + --result-filename results.json +' +``` + +Use `vllm bench sweep serve` when the target image supports it and the search +can be described with serve/bench parameter JSON files. + +## TensorRT-LLM + +This skill only supports the TensorRT-LLM PyTorch server backend. Keep +`--backend pytorch` in every `trtllm-serve serve` command. Do not switch the +server to `--backend trt`, an engine path, or any other backend; mark that +candidate unsupported instead. + +For single-node multi-GPU TensorRT-LLM containers, keep the IPC, ulimit, shared +memory, and NCCL settings below. In a multi-GPU PyTorch-backend validation +run (captured on an H100 host; the rule is not H100-specific), the server +entered `PyTorchConfig` but failed NCCL allreduce without these container +options; the same model and candidate list passed after adding them. Expect +the same requirement on any single-node multi-GPU target. + +Server template: + +```bash +docker run -d --name llmbench-trtllm \ + --gpus "$GPU_ARG" \ + --ipc=host \ + --ulimit memlock=-1 \ + --ulimit stack=67108864 \ + --shm-size=16g \ + --network host \ + -v /data/.cache:/root/.cache \ + -e MODEL \ + -e TP \ + -e PP \ + -e PORT \ + -e HF_TOKEN \ + -e HUGGINGFACE_HUB_TOKEN \ + -e NCCL_IB_DISABLE=1 \ + --entrypoint bash \ + nvcr.io/nvidia/tensorrt-llm/release:latest -lc ' +trtllm-serve serve "$MODEL" \ + --host 0.0.0.0 \ + --port "$PORT" \ + --backend pytorch \ + --tp_size "$TP" \ + --pp_size "$PP" \ + --max_batch_size 64 \ + --max_num_tokens 8192 \ + --max_seq_len 4096 \ + --kv_cache_free_gpu_memory_fraction 0.75 \ + --trust_remote_code +' +``` + +Benchmark template: + +```bash +docker run --rm \ + --network host \ + -v /data/.cache:/root/.cache \ + -v "$RUN_DIR:/artifacts" \ + -e MODEL \ + -e PORT \ + --entrypoint bash \ + nvcr.io/nvidia/tensorrt-llm/release:latest -lc ' +python -m tensorrt_llm.serve.scripts.benchmark_serving \ + --backend openai \ + --host 127.0.0.1 \ + --port "$PORT" \ + --endpoint /v1/completions \ + --model "$MODEL" \ + --dataset-name random \ + --random-input-len 1024 \ + --random-output-len 256 \ + --random-ids \ + --num-prompts 80 \ + --request-rate 8 \ + --max-concurrency 64 \ + --save-result \ + --result-dir /artifacts/trtllm \ + --result-filename results.json +' +``` + +For TensorRT-LLM 1.0.0, the serving benchmark client `--backend` choices are +`openai` and `openai-chat`. Do not pass `--backend trtllm`. This client flag is +separate from the server backend pinned above. + +## Cleanup + +Use unique container names per run and clean up by name: + +```bash +docker rm -f llmbench-sglang llmbench-vllm llmbench-trtllm +``` + +If a port remains bound after container cleanup, inspect it before killing +anything: + +```bash +ss -ltnp | grep ':8000' +ps -eo pid,ppid,user,etime,cmd | grep '' +``` + +Only kill raw PIDs when the command line proves they belong to the current +validation run. diff --git a/.claude/skills/llm-serving-auto-benchmark/references/example-plan.yaml b/.claude/skills/llm-serving-auto-benchmark/references/example-plan.yaml new file mode 100644 index 000000000..bc6e6c431 --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/references/example-plan.yaml @@ -0,0 +1,133 @@ +# Example run plan for the llm-serving-auto-benchmark skill. Baseline flags stay +# in base_server_flags, search knobs stay in search_space, and aligned dataset +# length pairs define scenarios. +# +# Note: this is the runtime plan shape (top-level `sla`, no `schema_version` or +# `server_command`). Cookbook configs in configs/cookbook-llm/ use the extended +# schema enforced by scripts/validate_cookbook_configs.py; do not run the +# validator against this file as-is. + +model: + name: Qwen/Qwen3-32B + tokenizer: Qwen/Qwen3-32B + precision: bf16 + quantization: none + +version_manifest: + sglang: + container_image: lmsysorg/sglang:dev + package_version: null + git_commit: null + server_help: artifacts/help/sglang_launch_server.txt + benchmark_help: artifacts/help/sglang_bench_serving.txt + vllm: + container_image: vllm/vllm-openai:latest + package_version: null + git_commit: null + server_help: artifacts/help/vllm_serve_all.txt + benchmark_help: artifacts/help/vllm_bench_serve_all.txt + sweep_help: artifacts/help/vllm_bench_sweep_serve_all.txt + tensorrt_llm: + container_image: nvcr.io/nvidia/tensorrt-llm/release:latest + package_version: null + git_commit: null + server_help: artifacts/help/trtllm_serve.txt + benchmark_help: artifacts/help/trtllm_benchmark_serving.txt + +hardware: + # Example values; replace with the actual target GPU (A100, H100, H200, + # B200, MI300, RTX 5090, etc.). gpu_model is recorded for fairness audit, + # not used as a scheduling hint. + gpu_model: NVIDIA H100 80GB HBM3 + gpu_count: 4 + multi_node: false + +dataset: + kind: random + num_prompts: 80 + scenario_names: [chat, summarization] + input_len: [1000, 8000] + output_len: [1000, 1000] + canonical_jsonl: null + +benchmark: + endpoint: /v1/chat/completions + backend: auto + request_rates: null + max_concurrency: [null, 16, 32] + qps: + lower: 1.0 + upper: 12.0 + tolerance: 0.1 + max_rounds: 5 + extra_request_body: + temperature: 0.0 + +sla: + max_p99_ttft_ms: 2000 + max_p99_tpot_ms: 80 + min_success_rate: 0.99 + +search: + tier: 2 + max_candidates_per_framework: 10 + candidate_generation: baseline_first_bounded_product + resume: true + output_dir: /bench/results/llm-serving-auto-benchmark + +frameworks: + sglang: + enabled: true + base_server_flags: + tp_size: 4 + trust_remote_code: true + mem_fraction_static: 0.82 + schedule_policy: lpm + context_length: 12288 + search_space: + # Verify these names against `python -m sglang.launch_server --help`. + prefill_attention_backend: [fa3, flashinfer] + decode_attention_backend: [fa3, flashinfer] + chunked_prefill_size: [8192, 16384] + max_running_requests: [64, 128] + + vllm: + enabled: true + base_server_flags: + tensor_parallel_size: 4 + trust_remote_code: true + gpu_memory_utilization: 0.90 + max_model_len: 12288 + dtype: auto + search_space: + # Verify these names against `vllm serve --help=all`. + max_num_seqs: [64, 128] + max_num_batched_tokens: [8192, 16384] + enable_chunked_prefill: [true] + # Raise above 1 only after the target model/runtime supports concurrent partial prefill. + max_num_partial_prefills: [1] + max_long_partial_prefills: [1] + long_prefill_token_threshold: [0, 4096] + enable_prefix_caching: [true] + kv_cache_dtype: [auto] + block_size: [16] + + tensorrt_llm: + enabled: true + backend_policy: fixed_pytorch + base_server_flags: + backend: pytorch + tp_size: 4 + pp_size: 1 + kv_cache_free_gpu_memory_fraction: 0.75 + trust_remote_code: true + search_space: + # Verify these names against `trtllm-serve serve --help`. + # Do not add backend choices here; TensorRT-LLM is fixed to the PyTorch backend. + max_batch_size: [64, 128] + max_num_tokens: [8192, 16384] + max_seq_len: [12288, 16384] + # Uncomment and point at concrete config files to sweep PyTorch-backend + # options via --extra_llm_api_options. A single [null] value contributes + # no dimension to the search. + # extra_llm_api_options: [null, /path/to/trt_llm_config_A.yaml] diff --git a/.claude/skills/llm-serving-auto-benchmark/references/framework-reference.md b/.claude/skills/llm-serving-auto-benchmark/references/framework-reference.md new file mode 100644 index 000000000..35d094a17 --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/references/framework-reference.md @@ -0,0 +1,113 @@ +# Framework Reference + +Use this file when choosing native framework commands or translating tuning +knobs across SGLang, vLLM, and TensorRT-LLM. Always verify the concrete CLI in +the target container with `--help` before a long run. + +## Native Entry Points + +| Framework | Server | Benchmark | Notes | +| --- | --- | --- | --- | +| SGLang | `python -m sglang.launch_server` | `python -m sglang.auto_benchmark` or `python -m sglang.bench_serving` | Use `auto_benchmark` when available for server-flag search. Use `bench_serving` for direct native or OpenAI-compatible endpoint checks. | +| vLLM | `vllm serve` | `vllm bench sweep serve` or `vllm bench serve` | Prefer `bench sweep serve` when sweeping server and benchmark parameter JSON files. | +| TensorRT-LLM | `trtllm-serve serve --backend pytorch` | TensorRT-LLM serving benchmark client or a common OpenAI-compatible client | This skill does not cover engine-backed serving or non-PyTorch server backends. | + +Common source docs: + +- SGLang bench serving: +- vLLM benchmark sweeps: +- vLLM `bench sweep serve`: +- TensorRT-LLM `trtllm-serve`: +- TensorRT-LLM deployment guide: + +## Command Templates + +### SGLang + +```bash +python -m sglang.launch_server \ + --model-path \ + --tp-size \ + --port 30000 + +python -m sglang.bench_serving \ + --backend sglang-oai \ + --host 127.0.0.1 \ + --port 30000 \ + --dataset-name random \ + --random-input-len 1024 \ + --random-output-len 256 \ + --num-prompts 80 \ + --request-rate 8 +``` + +Use `--backend sglang` for SGLang-native `/generate` checks. Use +`--backend sglang-oai` when comparing against vLLM or TensorRT-LLM through an +OpenAI-compatible path. + +### vLLM + +```bash +vllm serve \ + --host 0.0.0.0 \ + --port 8000 \ + --tensor-parallel-size \ + --gpu-memory-utilization 0.90 \ + --max-model-len 4096 \ + --max-num-seqs 64 \ + --max-num-batched-tokens 8192 \ + --enable-chunked-prefill + +vllm bench serve \ + --backend vllm \ + --base-url http://127.0.0.1:8000 \ + --model \ + --dataset-name random \ + --random-input-len 1024 \ + --random-output-len 256 \ + --num-prompts 80 +``` + +### TensorRT-LLM + +```bash +trtllm-serve serve \ + --backend pytorch \ + --tp_size \ + --kv_cache_free_gpu_memory_fraction 0.75 \ + --host 0.0.0.0 \ + --port 8000 +``` + +Benchmark the OpenAI-compatible endpoint with the TensorRT-LLM serving benchmark +client or the same OpenAI-compatible client used for the other frameworks. Keep +server backend choice fixed to `pytorch`. + +## Knob Family Mapping + +Do not copy flag names across frameworks. Compare knob families, then translate +to the target CLI. + +| Family | SGLang | vLLM | TensorRT-LLM | +| --- | --- | --- | --- | +| Parallelism | `--tp-size`, `--pp-size`, `--dp-size`, `--ep-size`, `--expert-parallel-size` | `--tensor-parallel-size`, `--pipeline-parallel-size`, `--data-parallel-size`, `--enable-expert-parallel` | `--tp_size`, `--pp_size`, `--ep_size`, `--gpus_per_node`, `--cluster_size` | +| Memory and KV cache | `--mem-fraction-static`, `--max-total-tokens`, `--kv-cache-dtype`, `--page-size`, `--cpu-offload-gb` | `--gpu-memory-utilization`, `--kv-cache-memory-bytes`, `--kv-cache-dtype`, `--block-size`, `--cpu-offload-gb` | `--kv_cache_free_gpu_memory_fraction`, plus `--max_num_tokens`, `--max_seq_len`, `--max_batch_size` | +| Batching and scheduler | `--max-running-requests`, `--schedule-policy`, `--chunked-prefill-size`, `--max-prefill-tokens`, `--prefill-max-requests` | `--max-num-seqs`, `--max-num-batched-tokens`, `--enable-chunked-prefill`, partial-prefill and DBO flags | `--max_batch_size`, `--max_num_tokens`, `--max_seq_len`; extra scheduler knobs may require `--extra_llm_api_options` | +| Attention/backend | `--attention-backend`, `--prefill-attention-backend`, `--decode-attention-backend`, `--sampling-backend` | `--attention-backend`, `--gdn-prefill-backend`, `--mm-encoder-attn-backend` | `--backend pytorch` is fixed; do not search backend choice | +| CUDA graph and compile | `--disable-cuda-graph`, `--cuda-graph-bs`, `--cuda-graph-max-bs`, `--disable-piecewise-cuda-graph`, `--enable-torch-compile` | `--enforce-eager`, `--compilation-config`, `--cudagraph-capture-sizes`, `--max-cudagraph-capture-size` | use direct flags or `--extra_llm_api_options`; record resolved PyTorch config from logs | +| Prefix/speculative | `--disable-radix-cache`, `--disable-chunked-prefix-cache`, speculative decoding flags | `--enable-prefix-caching`, `--speculative-config` | only use PyTorch-backend options accepted by the target image | +| Dtype, quantization, loading | `--dtype`, `--quantization`, `--load-format`, `--model-loader-extra-config`, `--trust-remote-code` | `--dtype`, `--quantization`, `--load-format`, `--model-loader-extra-config`, `--trust-remote-code`, `--hf-token` | `--trust_remote_code`, `--tokenizer`; engine build and non-PyTorch quantization flows are out of scope | + +## Version Rules + +Framework CLIs move quickly. For every real run: + +1. Record the framework package version, git commit, image tag, and help files. +2. Validate concrete flags with + `scripts/validate_cookbook_configs.py --help-dir `. +3. Move renamed or removed flags out of the run plan before benchmarking. +4. Record which frameworks were model-smoked and which only passed preflight. + +Historical validation from April 2026 used SGLang `0.5.10rc0`, vLLM `0.19.1`, +and TensorRT-LLM `1.0.0`. Treat those notes as old evidence, not as current +compatibility guarantees. diff --git a/.claude/skills/llm-serving-auto-benchmark/references/result-schema.md b/.claude/skills/llm-serving-auto-benchmark/references/result-schema.md new file mode 100644 index 000000000..8193c3ece --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/references/result-schema.md @@ -0,0 +1,161 @@ +# Result Schema + +Write one JSON object per candidate. Keep failed candidates in the same file so +the final summary explains what was tried. + +## SLA Key Convention + +One canonical naming across this skill. Config files and normalized result rows +must agree. + +| Key | Where | Type | +| --- | --- | --- | +| `max_p99_ttft_ms` | both | float, milliseconds, p99 | +| `max_p99_tpot_ms` | both | float, milliseconds, p99 | +| `min_success_rate` | both | float in [0, 1] | +| `passed` | result only | bool; recomputed after the run | + +Do not use `max_ttft_ms` or `max_tpot_ms` without the `p99_` prefix; those names +hide whether the target is a mean or a tail. Older cookbook configs used mean +latency targets by accident and have been migrated to the p99 names above. + +The config-level SLA block lives under `benchmark.sla` (cookbook configs) or at +the top level (example plan). Either location is acceptable, but the key names +must match this table. + +## JSONL Row + +The values below (`gpu_model`, `gpu_count`, file paths, numeric metrics, etc.) +are illustrative. Replace them with the actual target hardware and measured +values; this schema is not tied to H100. + +```json +{ + "framework": "sglang", + "framework_version": "0.5.0", + "framework_commit": "abcdef0", + "candidate_id": "sglang-tp8-flashinfer", + "model": "meta-llama/Llama-3.1-70B-Instruct", + "status": "ok", + "failure_reason": "", + "hardware": { + "gpu_model": "NVIDIA H100 80GB HBM3", + "gpu_count": 8, + "visible_devices": "0,1,2,3,4,5,6,7" + }, + "workload": { + "kind": "custom", + "scenario": "chat", + "dataset_path": "/bench/workload.autobench.jsonl", + "input_len": 2048, + "output_len": 512, + "input_len_p50": 1800, + "input_len_p95": 4096, + "output_len_p50": 384, + "output_len_p95": 1024, + "num_prompts": 1000, + "request_rate": 16, + "max_concurrency": 256, + "endpoint": "/v1/chat/completions" + }, + "sla": { + "max_p99_ttft_ms": 2000, + "max_p99_tpot_ms": 80, + "min_success_rate": 0.99, + "passed": true + }, + "metrics": { + "request_throughput": 15.8, + "output_token_throughput": 12500.0, + "total_token_throughput": 42000.0, + "mean_ttft_ms": 430.0, + "p99_ttft_ms": 1550.0, + "mean_tpot_ms": 26.0, + "p99_tpot_ms": 72.0, + "mean_e2e_ms": 8200.0, + "p99_e2e_ms": 19000.0, + "success_rate": 0.995 + }, + "server_command": "python -m sglang.launch_server ...", + "benchmark_command": "python -m sglang.bench_serving ...", + "validated_cli_flags": { + "server": ["tp_size", "attention_backend"], + "benchmark": ["dataset_name", "request_rate", "max_concurrency"] + }, + "artifacts": { + "server_log": "/bench/sglang/server.log", + "raw_result": "/bench/sglang/results.jsonl", + "server_help": "/bench/sglang/help_launch_server.txt", + "benchmark_help": "/bench/sglang/help_bench_serving.txt" + } +} +``` + +`input_len` and `output_len` are the representative scenario lengths used for +synthetic workloads or a named bucket. For custom production-like datasets, +also include p50/p95 buckets when available. These fields let +`sglang-sota-performance` pass the slow benchmark shape directly into +`llm-torch-profiler-analysis`: + +- prefill profile: `--prefill-input-len ` and + `--prefill-output-len 1` +- decode profile: `--decode-input-len 1` and + `--decode-output-len ` + +## Status Values + +- `ok`: benchmark finished and metrics are trustworthy +- `failed`: command failed for a known non-OOM reason +- `oom`: model or candidate exhausted GPU/host memory +- `timeout`: server or benchmark timed out +- `skipped`: intentionally not run, with a reason in `failure_reason` + +## Ranking Rule + +The default ranking is: + +1. `status == "ok"` +2. `sla.passed == true` +3. higher `metrics.request_throughput` +4. higher `metrics.output_token_throughput` +5. lower `metrics.mean_ttft_ms` +6. lower `metrics.mean_tpot_ms` +7. lower `hardware.gpu_count` + +If the user cares more about token throughput than request throughput, swap +steps 3 and 4 and state that in the final report. + +This ranking rule does not change the SLA gate. Keep `sla.max_p99_ttft_ms` and +`sla.max_p99_tpot_ms` as the tail-latency constraints; use mean TTFT and mean +TPOT only for default winner selection among rows that have already passed SLA. + +Missing metric semantics: + +- If `metrics.mean_ttft_ms` is absent from a row, the ranking script treats it + as the worst possible value, so that row falls below any candidate with a + real mean-TTFT measurement. Do not write `0` as a placeholder for "no + measurement"; leave the field out or set it to `null`. +- If `metrics.mean_tpot_ms` is absent from a row, the ranking script treats it + as the worst possible value, so that row falls below any candidate with a + real mean-TPOT measurement. Do not write `0` as a placeholder for "no + measurement"; leave the field out or set it to `null`. +- If `metrics.request_throughput` or `metrics.output_token_throughput` is + missing, the row ranks below any candidate with a real measurement in those + keys. A failed candidate that still produced partial metrics should keep the + metrics it did produce. + +## Final Report Tables + +The markdown summary must include these sections: + +1. `Best Commands By Framework`: one table per framework. Each table has one row + per workload scenario and includes the best candidate, SLA result, throughput, + latency metrics, GPU count, exact server command, and artifacts. +2. `Cross-Framework Best Comparison`: one table that compares the best SGLang, + vLLM, and TensorRT-LLM command for each scenario. Sort each scenario by the + ranking rule above so the best deployment choice is first. +3. `Failed Or SLA-Failing Candidates`: include this table when any candidate + failed, was skipped, or completed without passing SLA. This table records + tried configs that were not selected. Keep each reason concrete enough to + tell whether the candidate needs a retry, lower concurrency, a parameter fix, + or no further action. diff --git a/.claude/skills/llm-serving-auto-benchmark/scripts/compare_benchmark_results.py b/.claude/skills/llm-serving-auto-benchmark/scripts/compare_benchmark_results.py new file mode 100755 index 000000000..c7c4c21f6 --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/scripts/compare_benchmark_results.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python3 +"""Summarize normalized cross-framework benchmark JSONL results.""" + +from __future__ import annotations + +import argparse +import csv +import json +from pathlib import Path +from typing import Any + + +def _get(row: dict[str, Any], path: str, default: Any = None) -> Any: + current: Any = row + for part in path.split("."): + if not isinstance(current, dict) or part not in current: + return default + current = current[part] + return current + + +def _float(row: dict[str, Any], path: str, default: float = 0.0) -> float: + value = _get(row, path, default) + try: + return float(value) + except (TypeError, ValueError): + return default + + +def _bool(row: dict[str, Any], path: str, default: bool = False) -> bool: + value = _get(row, path, default) + if isinstance(value, bool): + return value + if isinstance(value, str): + return value.lower() in {"1", "true", "yes", "y"} + return bool(value) + + +def _mean_ttft_ms(row: dict[str, Any]) -> float: + return _float(row, "metrics.mean_ttft_ms", 1e30) + + +def _mean_tpot_ms(row: dict[str, Any]) -> float: + return _float(row, "metrics.mean_tpot_ms", 1e30) + + +def _rank_key(row: dict[str, Any]) -> tuple[Any, ...]: + return ( + _get(row, "status") == "ok", + _bool(row, "sla.passed"), + _float(row, "metrics.request_throughput"), + _float(row, "metrics.output_token_throughput"), + -_mean_ttft_ms(row), + -_mean_tpot_ms(row), + -_float(row, "hardware.gpu_count", 1e30), + ) + + +def _is_winner_candidate(row: dict[str, Any]) -> bool: + return _get(row, "status") == "ok" and _bool(row, "sla.passed") + + +def _fmt(value: Any, digits: int = 2) -> str: + if value is None: + return "" + if isinstance(value, float): + return f"{value:.{digits}f}" + return str(value) + + +def _cell(value: Any, digits: int = 2) -> str: + text = _fmt(value, digits) + return text.replace("\n", "
").replace("|", "\\|") + + +def _scenario(row: dict[str, Any]) -> str: + for path in ( + "workload.scenario", + "workload.scenario_name", + "workload.dataset_scenario", + "workload.dataset_name", + "workload.kind", + "scenario", + ): + value = _get(row, path) + if value: + return str(value) + return "default" + + +def _server_command(row: dict[str, Any]) -> str: + return str(_get(row, "server_command") or _get(row, "launch_command") or "") + + +def _artifact_summary(row: dict[str, Any]) -> str: + artifacts = _get(row, "artifacts", {}) + if not isinstance(artifacts, dict): + return "" + parts = [] + for key in ("raw_result", "server_log", "benchmark_log", "summary"): + value = artifacts.get(key) + if value: + parts.append(f"{key}: {value}") + return "
".join(parts) + + +def load_rows(path: Path) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + with path.open(encoding="utf-8") as f: + for line_no, line in enumerate(f, 1): + stripped = line.strip() + if not stripped: + continue + try: + row = json.loads(stripped) + except json.JSONDecodeError as exc: + raise SystemExit(f"{path}:{line_no}: invalid JSON: {exc}") from exc + if not isinstance(row, dict): + raise SystemExit(f"{path}:{line_no}: expected a JSON object") + rows.append(row) + return rows + + +def best_by_framework_and_scenario(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + best: dict[tuple[str, str], dict[str, Any]] = {} + for row in rows: + if not _is_winner_candidate(row): + continue + key = (str(_get(row, "framework", "unknown")), _scenario(row)) + if key not in best or _rank_key(row) > _rank_key(best[key]): + best[key] = row + return sorted( + best.values(), key=lambda row: (_scenario(row), _rank_key(row)), reverse=True + ) + + +def write_csv(path: Path, rows: list[dict[str, Any]]) -> None: + fields = [ + "framework", + "scenario", + "candidate_id", + "status", + "sla_passed", + "request_throughput", + "output_token_throughput", + "mean_ttft_ms", + "mean_tpot_ms", + "p99_ttft_ms", + "p99_tpot_ms", + "gpu_count", + "server_command", + "failure_reason", + ] + with path.open("w", encoding="utf-8", newline="") as f: + writer = csv.DictWriter(f, fieldnames=fields) + writer.writeheader() + for row in rows: + writer.writerow( + { + "framework": _get(row, "framework", ""), + "scenario": _scenario(row), + "candidate_id": _get(row, "candidate_id", ""), + "status": _get(row, "status", ""), + "sla_passed": _bool(row, "sla.passed"), + "request_throughput": _get(row, "metrics.request_throughput", ""), + "output_token_throughput": _get( + row, "metrics.output_token_throughput", "" + ), + "mean_ttft_ms": _get(row, "metrics.mean_ttft_ms", ""), + "mean_tpot_ms": _get(row, "metrics.mean_tpot_ms", ""), + "p99_ttft_ms": _get(row, "metrics.p99_ttft_ms", ""), + "p99_tpot_ms": _get(row, "metrics.p99_tpot_ms", ""), + "gpu_count": _get(row, "hardware.gpu_count", ""), + "server_command": _server_command(row), + "failure_reason": _get(row, "failure_reason", ""), + } + ) + + +def _append_best_commands_by_framework( + lines: list[str], scenario_winners: list[dict[str, Any]] +) -> None: + frameworks = sorted( + {str(_get(row, "framework", "unknown")) for row in scenario_winners} + ) + lines.extend(["## Best Commands By Framework", ""]) + for framework in frameworks: + lines.extend( + [ + f"### `{framework}`", + "", + "| Scenario | Candidate | Status | SLA | Req/s | Output tok/s | Total tok/s | Mean TTFT ms | Mean TPOT ms | Success rate | GPUs | Server command | Artifacts |", + "| --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- |", + ] + ) + rows = [row for row in scenario_winners if _get(row, "framework") == framework] + for row in sorted(rows, key=_scenario): + lines.append( + "| {scenario} | {candidate} | {status} | {sla} | {rps} | {otps} | {ttps} | {ttft} | {tpot} | {success} | {gpus} | {command} | {artifacts} |".format( + scenario=_cell(_scenario(row)), + candidate=_cell(_get(row, "candidate_id", "")), + status=_cell(_get(row, "status", "")), + sla=_cell(_bool(row, "sla.passed")), + rps=_cell(_get(row, "metrics.request_throughput")), + otps=_cell(_get(row, "metrics.output_token_throughput")), + ttps=_cell(_get(row, "metrics.total_token_throughput")), + ttft=_cell(_get(row, "metrics.mean_ttft_ms")), + tpot=_cell(_get(row, "metrics.mean_tpot_ms")), + success=_cell(_get(row, "metrics.success_rate")), + gpus=_cell(_get(row, "hardware.gpu_count")), + command=_cell(_server_command(row)), + artifacts=_cell(_artifact_summary(row)), + ) + ) + lines.append("") + + +def _append_cross_framework_table( + lines: list[str], scenario_winners: list[dict[str, Any]] +) -> None: + lines.extend( + [ + "## Cross-Framework Best Comparison", + "", + "| Scenario | Rank | Framework | Candidate | SLA | Req/s | Output tok/s | Mean TTFT ms | Mean TPOT ms | GPUs | Server command |", + "| --- | ---: | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | --- |", + ] + ) + scenario_names = sorted({_scenario(row) for row in scenario_winners}) + for scenario_name in scenario_names: + rows = [row for row in scenario_winners if _scenario(row) == scenario_name] + for rank, row in enumerate(sorted(rows, key=_rank_key, reverse=True), 1): + lines.append( + "| {scenario} | {rank} | {framework} | {candidate} | {sla} | {rps} | {otps} | {ttft} | {tpot} | {gpus} | {command} |".format( + scenario=_cell(scenario_name), + rank=rank, + framework=_cell(_get(row, "framework", "")), + candidate=_cell(_get(row, "candidate_id", "")), + sla=_cell(_bool(row, "sla.passed")), + rps=_cell(_get(row, "metrics.request_throughput")), + otps=_cell(_get(row, "metrics.output_token_throughput")), + ttft=_cell(_get(row, "metrics.mean_ttft_ms")), + tpot=_cell(_get(row, "metrics.mean_tpot_ms")), + gpus=_cell(_get(row, "hardware.gpu_count")), + command=_cell(_server_command(row)), + ) + ) + lines.append("") + + +def render_markdown(rows: list[dict[str, Any]]) -> str: + scenario_winners = best_by_framework_and_scenario(rows) + + lines = ["# Benchmark Summary", ""] + if not rows: + lines.append("No rows found.") + return "\n".join(lines) + "\n" + + _append_best_commands_by_framework(lines, scenario_winners) + _append_cross_framework_table(lines, scenario_winners) + + failed = [ + row + for row in rows + if _get(row, "status") != "ok" or not _bool(row, "sla.passed") + ] + if failed: + lines.extend( + [ + "", + "## Failed Or SLA-Failing Candidates", + "", + "This table records tried configs that were not selected. They either failed, were skipped by policy, or completed without passing the SLA.", + "", + "| Framework | Candidate | Status | SLA | Reason |", + "| --- | --- | --- | --- | --- |", + ] + ) + for row in failed: + lines.append( + "| {framework} | {candidate} | {status} | {sla} | {reason} |".format( + framework=_cell(_get(row, "framework", "")), + candidate=_cell(_get(row, "candidate_id", "")), + status=_cell(_get(row, "status", "")), + sla=_cell(_bool(row, "sla.passed")), + reason=_cell(_get(row, "failure_reason", "")), + ) + ) + return "\n".join(lines) + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--input", required=True, type=Path, help="Normalized JSONL") + parser.add_argument("--output", required=True, type=Path, help="Markdown summary") + parser.add_argument("--csv", type=Path, help="Optional CSV table") + args = parser.parse_args() + + rows = load_rows(args.input) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(render_markdown(rows), encoding="utf-8") + if args.csv: + args.csv.parent.mkdir(parents=True, exist_ok=True) + write_csv(args.csv, sorted(rows, key=_rank_key, reverse=True)) + + +if __name__ == "__main__": + main() diff --git a/.claude/skills/llm-serving-auto-benchmark/scripts/validate_cookbook_configs.py b/.claude/skills/llm-serving-auto-benchmark/scripts/validate_cookbook_configs.py new file mode 100755 index 000000000..549c446c1 --- /dev/null +++ b/.claude/skills/llm-serving-auto-benchmark/scripts/validate_cookbook_configs.py @@ -0,0 +1,434 @@ +#!/usr/bin/env python3 +"""Validate cross-framework cookbook benchmark configs. + +The validator is intentionally shallow: it proves that every config can be +loaded, translated into bounded candidate commands, and checked against the +known server flag surface. It does not launch model servers. +""" + +from __future__ import annotations + +import argparse +import itertools +import re +import shlex +from pathlib import Path +from typing import Any + +import yaml + +FRAMEWORKS = ("sglang", "vllm", "tensorrt_llm") +ALLOWED_SOURCE_KINDS = {"llm_serving_cookbook"} + +SEQUENCE_LIMIT_KEY = { + "sglang": "context_length", + "vllm": "max_model_len", + "tensorrt_llm": "max_seq_len", +} + +ALLOWED_SLA_KEYS = { + "max_p99_ttft_ms", + "max_p99_tpot_ms", + "min_success_rate", + "max_p99_e2e_ms", +} + +DEPRECATED_SLA_KEYS = { + "max_ttft_ms": "max_p99_ttft_ms", + "max_tpot_ms": "max_p99_tpot_ms", + "max_e2e_ms": "max_p99_e2e_ms", +} + +STATIC_SERVER_FLAGS = { + "sglang": { + "attention_backend", + "chunked_prefill_size", + "context_length", + "decode_attention_backend", + "dllm_algorithm", + "dtype", + "enable_multimodal", + "enable_symm_mem", + "ep_size", + "host", + "kv_cache_dtype", + "max_running_requests", + "mem_fraction_static", + "model_loader_extra_config", + "model_path", + "moe_runner_backend", + "nnodes", + "port", + "pp_size", + "prefill_attention_backend", + "reasoning_parser", + "schedule_policy", + "tool_call_parser", + "tp_size", + "trust_remote_code", + }, + "vllm": { + "block_size", + "dtype", + "enable_chunked_prefill", + "enable_prefix_caching", + "gpu_memory_utilization", + "host", + "kv_cache_dtype", + "long_prefill_token_threshold", + "max_long_partial_prefills", + "max_model_len", + "max_num_batched_tokens", + "max_num_partial_prefills", + "max_num_seqs", + "pipeline_parallel_size", + "port", + "tensor_parallel_size", + "trust_remote_code", + }, + "tensorrt_llm": { + "backend", + "ep_size", + "extra_llm_api_options", + "host", + "kv_cache_free_gpu_memory_fraction", + "max_batch_size", + "max_num_tokens", + "max_seq_len", + "port", + "pp_size", + "tp_size", + "trust_remote_code", + }, +} + +HELP_FILE_HINTS = { + "sglang": ("sglang", "launch"), + "vllm": ("vllm", "serve"), + "tensorrt_llm": ("trtllm", "serve"), +} + + +def flag_name(framework: str, key: str) -> str: + if framework in {"sglang", "vllm"}: + return "--" + key.replace("_", "-") + return "--" + key + + +def load_yaml(path: Path) -> dict[str, Any]: + with path.open(encoding="utf-8") as f: + data = yaml.safe_load(f) + if not isinstance(data, dict): + raise ValueError(f"{path}: expected a YAML mapping") + return data + + +def _as_list(value: Any) -> list[Any]: + if isinstance(value, list): + return value + return [value] + + +def _enabled(config: dict[str, Any], framework: str) -> bool: + return bool(config.get("frameworks", {}).get(framework, {}).get("enabled", False)) + + +def _max_required_sequence(dataset: dict[str, Any]) -> int: + input_len = dataset.get("input_len") + output_len = dataset.get("output_len") + if not isinstance(input_len, list) or not isinstance(output_len, list): + raise ValueError("dataset.input_len and dataset.output_len must be lists") + if len(input_len) != len(output_len): + raise ValueError("dataset.input_len and dataset.output_len must be aligned") + if not input_len: + raise ValueError("dataset.input_len and dataset.output_len must not be empty") + return max(int(i) + int(o) for i, o in zip(input_len, output_len, strict=True)) + + +def _candidate_dicts( + base_flags: dict[str, Any], + search_space: dict[str, Any], + limit: int, +) -> list[dict[str, Any]]: + candidates = [dict(base_flags)] + keys = list(search_space) + values = [_as_list(search_space[key]) for key in keys] + for combo in itertools.product(*values): + candidate = dict(base_flags) + candidate.update(dict(zip(keys, combo, strict=True))) + if candidate not in candidates: + candidates.append(candidate) + if len(candidates) >= limit: + break + return candidates + + +def _command_tokens( + framework: str, + config: dict[str, Any], + flags: dict[str, Any], +) -> list[str]: + server = config["frameworks"][framework] + command = shlex.split(server["server_command"]) + model = config["model"]["name"] + + if framework in {"vllm", "tensorrt_llm"}: + command.append(model) + + for key, value in flags.items(): + if value is None or value is False: + continue + command.append(flag_name(framework, key)) + if value is not True: + command.append(str(value)) + + return command + + +def render_command( + framework: str, config: dict[str, Any], flags: dict[str, Any] +) -> str: + return shlex.join(_command_tokens(framework, config, flags)) + + +def _extract_help_flags(text: str) -> set[str]: + return { + item.lstrip("-") for item in re.findall(r"--[A-Za-z0-9][A-Za-z0-9_-]*", text) + } + + +def load_help_flags(help_dir: Path) -> dict[str, set[str]]: + help_flags: dict[str, set[str]] = {} + for framework, hints in HELP_FILE_HINTS.items(): + matches = [] + for path in help_dir.rglob("*.txt"): + name = path.name.lower() + if all(hint in name for hint in hints): + matches.append(path) + if matches: + text = "\n".join( + path.read_text(encoding="utf-8", errors="replace") for path in matches + ) + help_flags[framework] = _extract_help_flags(text) + return help_flags + + +def _known_flag( + framework: str, + key: str, + help_flags: dict[str, set[str]] | None, +) -> bool: + static_keys = STATIC_SERVER_FLAGS[framework] + if key not in static_keys: + return False + if not help_flags or framework not in help_flags: + return True + + concrete = flag_name(framework, key).lstrip("-") + aliases = {concrete, concrete.replace("-", "_"), concrete.replace("_", "-")} + return bool(aliases & help_flags[framework]) + + +def _validate_framework( + config: dict[str, Any], + framework: str, + help_flags: dict[str, set[str]] | None, + max_candidates: int, +) -> list[str]: + errors: list[str] = [] + server = config["frameworks"].get(framework) + if not isinstance(server, dict): + return [f"missing frameworks.{framework}"] + if not server.get("enabled", False): + return [] + + base_flags = server.get("base_server_flags") + search_space = server.get("search_space") + if not isinstance(base_flags, dict): + errors.append(f"{framework}: base_server_flags must be a mapping") + base_flags = {} + if not isinstance(search_space, dict): + errors.append(f"{framework}: search_space must be a mapping") + search_space = {} + server_command_is_valid = isinstance(server.get("server_command"), str) + if not server_command_is_valid: + errors.append(f"{framework}: server_command must be a string") + + for key in set(base_flags) | set(search_space): + if not _known_flag(framework, key, help_flags): + errors.append(f"{framework}: unknown or unsupported server flag {key!r}") + + if framework == "tensorrt_llm": + if server.get("backend_policy") != "fixed_pytorch": + errors.append("tensorrt_llm: backend_policy must be fixed_pytorch") + if base_flags.get("backend") != "pytorch": + errors.append("tensorrt_llm: base backend must be pytorch") + if "backend" in search_space: + errors.append("tensorrt_llm: backend must not appear in search_space") + + candidates = _candidate_dicts(base_flags, search_space, max_candidates) + if not candidates: + errors.append(f"{framework}: no candidates generated") + can_render = server_command_is_valid and isinstance( + config.get("model", {}).get("name"), str + ) + if can_render: + for candidate in candidates: + command = render_command(framework, config, candidate) + if not command: + errors.append(f"{framework}: rendered an empty command") + + return errors + + +def validate_config( + path: Path, + help_flags: dict[str, set[str]] | None = None, +) -> list[str]: + errors: list[str] = [] + try: + config = load_yaml(path) + except Exception as exc: # noqa: BLE001 + return [str(exc)] + + if config.get("schema_version") != 1: + errors.append("schema_version must be 1") + if not isinstance(config.get("model", {}).get("name"), str): + errors.append("model.name must be set") + if config.get("source", {}).get("kind") not in ALLOWED_SOURCE_KINDS: + errors.append(f"source.kind must be one of {sorted(ALLOWED_SOURCE_KINDS)}") + + try: + required_sequence = _max_required_sequence(config["dataset"]) + except Exception as exc: # noqa: BLE001 + errors.append(str(exc)) + required_sequence = 0 + + search = config.get("search") + if not isinstance(search, dict): + errors.append("search must be a mapping") + max_candidates = 1 + else: + try: + max_candidates = int(search.get("max_candidates_per_framework", 0)) + except (TypeError, ValueError): + errors.append("search.max_candidates_per_framework must be an integer") + max_candidates = 1 + if max_candidates < 1: + errors.append("search.max_candidates_per_framework must be positive") + max_candidates = 1 + + frameworks = config.get("frameworks") + if not isinstance(frameworks, dict): + return errors + ["frameworks must be a mapping"] + + for framework in FRAMEWORKS: + errors.extend( + _validate_framework(config, framework, help_flags, max_candidates) + ) + + for framework in FRAMEWORKS: + if not _enabled(config, framework): + continue + key = SEQUENCE_LIMIT_KEY[framework] + fw = frameworks[framework] + base_flags = fw.get("base_server_flags", {}) or {} + search_space = fw.get("search_space", {}) or {} + if not isinstance(base_flags, dict) or not isinstance(search_space, dict): + continue + + try: + if framework == "sglang": + base_value = int(base_flags.get(key, required_sequence)) + else: + base_value = int(base_flags.get(key, 0)) + except (TypeError, ValueError): + errors.append(f"{framework}: base {key} is not an integer") + continue + if base_value < required_sequence: + errors.append( + f"{framework}: base {key} ({base_value}) is smaller than the largest dataset scenario ({required_sequence})" + ) + + if key in search_space: + for value in _as_list(search_space[key]): + try: + if int(value) < required_sequence: + errors.append( + f"{framework}: search_space {key} candidate {value} is smaller than the largest dataset scenario ({required_sequence})" + ) + except (TypeError, ValueError): + errors.append( + f"{framework}: search_space {key} candidate {value!r} is not an integer" + ) + + sla_block = ( + config.get("benchmark", {}).get("sla") + if isinstance(config.get("benchmark"), dict) + else None + ) + if sla_block is None: + sla_block = config.get("sla") + if isinstance(sla_block, dict): + for key in sla_block: + if key in DEPRECATED_SLA_KEYS: + errors.append( + f"sla: {key!r} is deprecated; use {DEPRECATED_SLA_KEYS[key]!r} (see references/result-schema.md)" + ) + elif key not in ALLOWED_SLA_KEYS: + errors.append( + f"sla: unknown key {key!r}; allowed keys are {sorted(ALLOWED_SLA_KEYS)}" + ) + + return errors + + +def iter_config_files(paths: list[Path]) -> list[Path]: + files: list[Path] = [] + for path in paths: + if path.is_dir(): + files.extend(sorted(path.rglob("*.yaml"))) + files.extend(sorted(path.rglob("*.yml"))) + else: + files.append(path) + return sorted(dict.fromkeys(files)) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("paths", nargs="+", type=Path) + parser.add_argument("--help-dir", type=Path) + parser.add_argument("--print-commands", action="store_true") + args = parser.parse_args() + + help_flags = load_help_flags(args.help_dir) if args.help_dir else None + failed = False + for path in iter_config_files(args.paths): + errors = validate_config(path, help_flags) + if errors: + failed = True + for error in errors: + print(f"{path}: {error}") + continue + + if args.print_commands: + config = load_yaml(path) + limit = int(config["search"].get("max_candidates_per_framework", 1)) + for framework in FRAMEWORKS: + if not _enabled(config, framework): + continue + server = config["frameworks"][framework] + candidates = _candidate_dicts( + server["base_server_flags"], + server["search_space"], + limit, + ) + print(f"# {path.name} {framework}") + print(render_command(framework, config, candidates[0])) + + if failed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/.claude/skills/llm-torch-profiler-analysis/SKILL.md b/.claude/skills/llm-torch-profiler-analysis/SKILL.md index 60f99e6bc..24fac5ddd 100644 --- a/.claude/skills/llm-torch-profiler-analysis/SKILL.md +++ b/.claude/skills/llm-torch-profiler-analysis/SKILL.md @@ -55,18 +55,39 @@ add one short note after the tables with exactly one of: | Existing trace triage | yes | yes | yes | | Single-trace live capture | yes | yes, if torch profiler is enabled on server | requires profiler control endpoints | | Two-trace mapping+formal triage | yes | yes | yes | -| Stage-aware live capture | yes | no | no | +| Stage-separated live workload | yes | yes | yes, with a writable shared trace dir or per-stage host runner | +| `--profile-by-stage` capture | yes | no | no | | `--profile-prefix` control | yes | usually ignored on HTTP profiler route | usually ignored on HTTP profiler route | For TensorRT-LLM, live capture only works when the server exposes `/start_profile` and `/stop_profile`, and when the deployment already provides a shared trace path plus the required env vars. -## Validation Notes +## Real H100 Validation -This unified workflow has been validated with a `4x H100` matrix across SGLang, -vLLM, and TensorRT-LLM. Use these model shapes as representative coverage when -refreshing or extending the skill: +The current reference run is the `4x H100` matrix captured on `2026-04-23` on +`h100_sglang` under: + +- `/data/bbuf/validate/unified_llm_profiler_skill/runs/20260423_h100_large_model_matrix_v3` + +Rendered markdown bundle: + +- `/data/bbuf/validate/unified_llm_profiler_skill/runs/20260423_h100_large_model_matrix_v3/h100_large_model_matrix_v3_bundle.md` + +Validated model directories: + +- `mixtral_8x7b_instruct` +- `qwen2_5_32b_instruct` +- `qwen3_32b` + +Each model directory contains: + +- `analysis_sglang.txt` +- `analysis_vllm.txt` +- `analysis_trtllm.txt` +- framework-specific trace roots and probe artifacts + +Validated matrix: | Model | SGLang | vLLM | TensorRT-LLM | Result | | --- | --- | --- | --- | --- | @@ -74,17 +95,49 @@ refreshing or extending the skill: | `Qwen/Qwen2.5-32B-Instruct` | `4x H100` | `4x H100` | `4x H100` | three tables rendered correctly on all three frameworks; benchmark probes returned direct, non-empty text | | `Qwen/Qwen3-32B` | `4x H100` | `4x H100` | `4x H100` | three tables rendered correctly on all three frameworks; vLLM and TensorRT-LLM chat probes often emitted `` prefixes | +Use this run as the main H100 reference. +The older `2026-04-22` single-card Qwen3 matrix is still useful for bring-up, but it is +not the default reference anymore. + +Stage-separated workload validation captured on `2026-05-01` on `h100_sglang`: + +- `/data/bbuf/validate/unified_llm_profiler_skill/runs/20260501_stage_split_validation` +- `/data/bbuf/validate/unified_llm_profiler_skill/runs/20260501_stage_split_validation_large` + +Validated models: + +| Model | GPU | Workloads | Result | +| --- | --- | --- | --- | +| `Qwen/Qwen2.5-0.5B-Instruct` | `1x H100` | prefill `4090->1`, decode `1->2048` | generated separate `prefill/*.trace.json.gz` and `decode/*.trace.json.gz`; kernel, overlap, and fuse tables rendered with separate `extend/prefill` and `decode` sections | +| `Qwen/Qwen2.5-1.5B-Instruct` | `1x H100` | prefill `4090->1`, decode `1->2048` | generated separate `prefill/*.trace.json.gz` and `decode/*.trace.json.gz`; kernel, overlap, and fuse tables rendered with separate `extend/prefill` and `decode` sections | +| `Qwen/Qwen2.5-7B-Instruct` | `1x H100` | prefill `4090->1`, decode `1->2048` | generated separate traces; prefill kernel table captured 28-layer GEMM/FA3/RMSNorm work, decode captured 5-step graph launches, and fuse rows were split by stage | +| `Qwen/Qwen2.5-14B-Instruct` | `1x H100` | prefill `4090->1`, decode `1->2048` | generated separate traces; prefill kernel table captured 48-layer GEMM/FA3/RMSNorm work, decode captured 5-step graph launches, and fuse rows were split by stage | +| `Qwen/Qwen3-8B` | `2x H100`, TP=2 | prefill `4090->1`, decode `1->2048`, warmup 10/capture 5 | generated separate prefill/decode traces and all three tables; unique probe prompts avoided prefix-cache pollution in the prefill table | +| `mistralai/Mistral-7B-Instruct-v0.3` | `2x H100`, TP=2 | prefill `4090->1`, decode `1->2048`, warmup 10/capture 5 | generated separate prefill/decode traces and all three tables; server logs showed no repeated-prompt prefix-cache shortcut during the active prefill window | + +This validation also covers the compatibility fix for older SGLang profiler +state machines: workload-separated live capture labels stages by output +directory and avoids nesting SGLang's internal `profile_by_stage` state machine +inside each workload. The helper +adds one internal scheduler guard step because SGLang increments `forward_ct` +before checking whether the profiler should stop; without that guard, a +`num_steps=1` prefill capture can stop just before the actual prefill forward. +The 2026-05-01 two-card validation artifacts for the additional models are: + +- `/data/bbuf/validate/core_skill_validation_20260501/qwen3_8b/profiler` +- `/data/bbuf/validate/core_skill_validation_20260501/mistral_7b_instruct_v03/profiler` + To render a validated run into one markdown document: ```bash python3 scripts/render_triage_markdown_bundle.py \ - --analysis-root /path/to/analysis_root \ - --output /path/to/analysis_bundle.md + --analysis-root /data/bbuf/validate/unified_llm_profiler_skill/runs/20260423_h100_large_model_matrix_v3 \ + --output /data/bbuf/validate/unified_llm_profiler_skill/runs/20260423_h100_large_model_matrix_v3/h100_large_model_matrix_v3_bundle.md ``` The bundle groups by model and keeps the three tables for each framework. -Validation notes: +H100 notes: - all three frameworks now render kernel, overlap, and fuse tables with separate `extend/prefill` and `decode` sections when the trace contains a clean stage split - SGLang live capture is validated and calls the server profiler API directly instead of shelling out to `sglang.profiler` @@ -92,7 +145,8 @@ Validation notes: - SGLang kernel-site reconstruction keeps sampling disabled in the mapping path so the optimized parser does not perturb SGLang table output; equality rechecks matched for `Mixtral-8x7B-Instruct-v0.1`, `Qwen3-32B`, and `nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8` - vLLM live capture requires `--output-dir` to match the server `torch_profiler_dir`; the validated H100 flow uses `--profiler-config {"profiler":"torch","torch_profiler_dir":"..."}` and then drives `/start_profile` and `/stop_profile` - TensorRT-LLM validation stays on `--backend pytorch`; the H100 flow writes the trace with `TLLM_TORCH_PROFILE_TRACE` and then analyzes the saved trace -- the 2026-04-22 TensorRT-LLM 1.0.0 `py_executor.py` profiler setup still needed a `with_stack=True` override for table-quality Python locations; re-check this on TensorRT-LLM 1.2.1 or any 1.3.x release-candidate image before assuming the override is still required +- the 2026-04-22 TensorRT-LLM 1.0.0 `py_executor.py` profiler setup still needed a `with_stack=True` override for table-quality Python locations, and the matrix runner generated that override under `/data/bbuf/validate/unified_llm_profiler_skill/overrides/trtllm`; re-check this on TensorRT-LLM 1.2.1 or any 1.3.x release-candidate image before assuming the override is still required +- on this host, keep all trace roots under `/data/...`, not `/home/...` ## When To Use It @@ -119,6 +173,43 @@ Handle it as a backend-selection issue, not as native-kernel profiler evidence. ## Main Flows +## Stage-Separated Live Capture Contract + +Live capture must not use one mixed prompt as the default. +By default, `analyze_llm_torch_profile.py --url ...` captures two labeled +workloads and then renders the same three tables with separate stage sections: + +- prefill: synthetic input length `4090`, output length `1` +- decode: synthetic input length `1`, output length `2048` + +Every live profiler path warms up `10` steps before arming the profiler and then +captures `5` active steps by default. Keep this warmup/active split aligned +across SGLang, vLLM, and TensorRT-LLM before comparing kernel tables. + +Use these options to override the contract when the benchmark workload is known: + +```bash +--profile-workload both \ +--warmup-steps 10 --num-steps 5 \ +--prefill-input-len 4090 --prefill-output-len 1 \ +--decode-input-len 1 --decode-output-len 2048 +``` + +Allowed `--profile-workload` values: + +- `both`: default; capture prefill and decode separately +- `prefill`: capture only the long-input / one-token workload +- `decode`: capture only the one-input / long-output workload +- `legacy`: keep the old `--probe-prompt` / `--probe-max-new-tokens` behavior + +For `sglang-sota-performance`, do not use the defaults if the slow SGLang +benchmark scenario has a known input/output distribution. +Set the profiler lengths from that slow scenario instead: prefill uses the slow +input length with output `1`, and decode uses input `1` with the slow output +length. For a mixed dataset, profile the slowest representative bucket such as +the p50 or p95 input/output pair used in the benchmark report, and record the +bucket in the artifact notes. + ### 1. Single-trace triage from an existing profile dir or trace ```bash @@ -136,14 +227,24 @@ mapping/formal pair is needed. python3 scripts/analyze_llm_torch_profile.py \ --framework sglang \ --url http://127.0.0.1:30000 \ - --output-dir /tmp/llm-profiler/sglang_profile_live \ + --output-dir /data/bbuf/validate/unified_llm_profiler_skill/runs/example/sglang_profile_live \ --num-steps 5 \ - --profile-by-stage + --warmup-steps 10 \ + --profile-by-stage \ + --profile-workload both ``` The script sends `POST /start_profile` to the SGLang server directly. -The script writes `server_args.json`, sends the probe requests after profiling is armed, -and waits longer for trace flush than the earlier implementation. +Keep `--output-dir` under `/data/...` so later analysis and docs can see the trace. +The script writes `server_args.json`, warms up with the same workload shape, +sends the active probe requests after profiling is armed, captures separate +`prefill/` and `decode/` profile roots by default, and waits longer for trace +flush than the earlier implementation. +For the default workload-separated capture, the directory name labels the stage +and the SGLang internal `profile_by_stage` mode is not used inside each +workload. This avoids mixing a one-token prefill probe with a separate decode +profile. The helper still adds one internal guard step because older SGLang +profilers check the target counter before running the next forward. ### 3. Single-trace live capture from vLLM @@ -151,7 +252,7 @@ Launch vLLM with torch profiler enabled, for example: ```bash vllm serve meta-llama/Llama-3.1-8B-Instruct \ - --profiler-config '{"profiler":"torch","torch_profiler_dir":"/tmp/llm-profiler/vllm_profile"}' + --profiler-config '{"profiler":"torch","torch_profiler_dir":"/data/bbuf/validate/unified_llm_profiler_skill/runs/example/vllm_profile"}' ``` Then run: @@ -160,14 +261,20 @@ Then run: python3 scripts/analyze_llm_torch_profile.py \ --framework vllm \ --url http://127.0.0.1:8000 \ - --output-dir /tmp/llm-profiler/vllm_profile \ + --output-dir /data/bbuf/validate/unified_llm_profiler_skill/runs/example/vllm_profile \ --num-steps 5 \ - --no-profile-by-stage + --warmup-steps 10 \ + --no-profile-by-stage \ + --profile-workload both ``` For vLLM, `--output-dir` must point to the same `torch_profiler_dir` the server uses. The current vLLM profiler config already defaults `torch_profiler_with_stack=true`, so the runner only needs to set `torch_profiler_dir`. +On `h100_sglang`, external vLLM containers should mount both: + +- `/data/.cache/huggingface:/root/.cache/huggingface` +- `/data/bbuf/validate/unified_llm_profiler_skill:/data/bbuf/validate/unified_llm_profiler_skill` ### 4. Single-trace live capture from TensorRT-LLM @@ -187,11 +294,15 @@ python3 scripts/analyze_llm_torch_profile.py \ --url http://127.0.0.1:8000 \ --output-dir /shared/path \ --num-steps 5 \ - --no-profile-by-stage + --no-profile-by-stage \ + --profile-workload both ``` If the deployment does not expose the profiler control endpoints, fall back to analyzing an existing trace instead of trying live capture. +If the TensorRT-LLM trace output is configured as one fixed file path, use +`scripts/run_trtllm_pytorch_profile_host.sh --stage prefill` and `--stage decode` +instead of direct `--profile-workload both`, so each stage gets its own trace file. On the current TensorRT-LLM mainline path, `py_executor.py` creates the torch profiler with `record_shapes=True` and `with_modules=True` but not `with_stack=True`. @@ -200,19 +311,21 @@ For table-quality validation, use the override generator: ```bash python3 scripts/make_trtllm_py_executor_override.py \ --source /path/to/original/py_executor.py \ - --output /tmp/llm-profiler/py_executor_with_stack.py + --output /data/bbuf/validate/unified_llm_profiler_skill/overrides/trtllm/py_executor_with_stack.py ``` -The validated TensorRT-LLM flow is: +The matrix runner does this automatically on H100 before TensorRT-LLM capture starts. -1. launch `trtllm-serve` with `TLLM_TORCH_PROFILE_TRACE=/shared/path/trace.json` +This is the validated TensorRT-LLM flow on `h100_sglang`: + +1. launch `trtllm-serve` with `TLLM_TORCH_PROFILE_TRACE=/data/.../trace.json` 2. run a few benchmark requests -3. analyze the emitted trace with `--input /shared/path/trace.json` +3. analyze the emitted trace with `--input /data/.../trace.json` ### 5. Two-trace triage from existing profile dirs or traces ```bash -python3 scripts/analyze_llm_torch_profile.py triage \ +python3 scripts/analyze_llm_torch_profile.py \ --mapping-input /path/to/graph_off_profile_dir \ --formal-input /path/to/graph_on_profile_dir ``` @@ -222,7 +335,7 @@ Use this when you need stronger overlap attribution and kernel-to-source mapping ### 6. Two-trace triage from running servers ```bash -python3 scripts/analyze_llm_torch_profile.py triage \ +python3 scripts/analyze_llm_torch_profile.py \ --framework sglang \ --mapping-url http://127.0.0.1:31025 \ --formal-url http://127.0.0.1:31026 \ @@ -241,7 +354,13 @@ For `vllm` or `TensorRT-LLM`, use the same shape but pass: `--profile-by-stage` is only meaningful on the SGLang live-capture path. -- On ordinary non-PD SGLang serving, it is still useful because prefill and decode usually have very different bottlenecks. +- With `--profile-workload both` / `prefill` / `decode`, workload directories + are the stage labels; the live-capture helper disables SGLang's internal + stage profiler per workload, warms up first, and captures the requested + active step count for the selected workload. +- On legacy or hand-captured SGLang serving, internal `profile_by_stage` is + still useful because prefill and decode usually have very different + bottlenecks. - On the current profile-v2 path inside SGLang, stage-based profiling is effectively the normal path. - PD-disaggregated serving adds one extra rule: prefill workers and decode workers must be profiled separately. That is stricter than ordinary `profile_by_stage`. - For `vllm` and `TensorRT-LLM`, disable it with `--no-profile-by-stage`. @@ -279,8 +398,10 @@ It exists to recover `kernel -> cpu_op -> python scope`. 1. If the user only wants a diagnosis, one trace is enough. 2. Prefer one-rank traces over merged traces whenever the profiler emitted both. 3. For a live server, let the script drive the profiler only when the framework-specific prerequisites are already met. -4. Prefer SGLang `--profile-by-stage` unless the user explicitly wants an all-stage mixed trace. -5. Create or clean the target trace directory before live capture so the profiler can write artifacts without permission surprises. +4. Prefer `--profile-workload both`; use `legacy` only when reproducing an old trace contract. +5. Prefer workload-separated SGLang capture; use internal `--profile-by-stage` + mainly for `legacy` or manually collected traces. +6. When on `h100_sglang`, create or clean the target trace directory through `docker exec sglang_bbuf ...` so the path is definitely writable under `/data`. ### Two-trace workflow @@ -312,6 +433,8 @@ Load these only when needed: - overlap labels, dependency-risk interpretation, and limits - [references/fuse-overlap-catalog.md](references/fuse-overlap-catalog.md) - mixed source-backed catalog of existing fuse and overlap patterns, including mainline rows plus PR-backed / in-flight rows +- [references/vllm-torch-compile-fusions.md](references/vllm-torch-compile-fusions.md) + - current vLLM torch.compile fusion passes and the source patterns they target - [references/overlap-catalog.md](references/overlap-catalog.md) - overlap-only lookup table across LLM, VLM, diffusion, disaggregation, HiSparse, and speculative scheduling diff --git a/.claude/skills/llm-torch-profiler-analysis/references/fuse-overlap-catalog.md b/.claude/skills/llm-torch-profiler-analysis/references/fuse-overlap-catalog.md index f4e67a926..917de36b6 100644 --- a/.claude/skills/llm-torch-profiler-analysis/references/fuse-overlap-catalog.md +++ b/.claude/skills/llm-torch-profiler-analysis/references/fuse-overlap-catalog.md @@ -28,21 +28,14 @@ overlap opportunity as novel. The catalog is grouped by reusable optimization family, not by one specific model. -Refresh note `2026-04-22`: rescanned current `sglang`, `flashinfer`, -`TensorRT-LLM`, and `vllm` mainline plus rechecked referenced PR state via the -GitHub API on `2026-04-22`. Stable current-code families such as Qwen-style -shared-expert top-k append, TensorRT-LLM Triton fused add+RMSNorm+FP8 quant, -and vLLM `merge_attn_states` attention-output quant are folded into the -mainline rows below. Closed-unmerged SGLang -[#22410](https://github.com/sgl-project/sglang/pull/22410) and FlashInfer -[#2840](https://github.com/flashinfer-ai/flashinfer/pull/2840) were removed -from the PR-backed sections. Keep FlashInfer -[#3058](https://github.com/flashinfer-ai/flashinfer/pull/3058) / -[#3079](https://github.com/flashinfer-ai/flashinfer/pull/3079) in mind because -that branch was reverted, and keep vLLM -[#40057](https://github.com/vllm-project/vllm/pull/40057) in mind when using -B200 FP4 MoE test coverage as a signal: it disables some B200 FP4 MoE layer -tests rather than proving the kernel family is absent. +Refresh note `2026-05-01`: rescanned current `sglang` and vLLM mainline, then +rechecked recent merged and open optimization PRs through the GitHub CLI/API. +The vLLM torch.compile pass inventory is now split out in +[`vllm-torch-compile-fusions.md`](vllm-torch-compile-fusions.md). Stable +current-code families remain folded into the mainline rows below. New +status-sensitive rows were added for DeepSeek-V4, GLM5 NSA / PDL, NVFP4 MoE, +torch.compile decode, vLLM DSV4, vLLM ROCm WMMA, and vLLM GPU/CPU sync-removal +work. Recheck PR state before treating an in-flight row as shipped. ## 1. LLM / SRT fused-kernel families @@ -147,12 +140,21 @@ Stable entries should be folded into the mainline family rows above. | PR `#22005` fused add + RMSNorm + per-token FP8 quant | `fused_add_rmsnorm_per_token_quant`
`per_token_quant_fp8` | `PR #22005`
`python/sglang/jit_kernel/csrc/elementwise/fused_add_rmsnorm_per_token_quant.cuh`
`python/sglang/jit_kernel/fused_add_rmsnorm_per_token_quant.py` | CUDA JIT kernel keeps normed values in registers and emits BF16 + FP8 outputs plus per-token scales | If FP8 online-quant traces show add+norm followed by per-token quant, treat this as an in-flight upstream CUDA fuse family. | | PR `#20667` Qwen3.5 fused QK norm + RoPE + KV cache write | `fused_qk_norm_rope_cache_pts_quant_shuffle`
`fused_qk_norm_mrope_3d_cache_pts_quant_shuffle`
`rotary_dim` | `PR #20667`
`python/sglang/srt/models/qwen3_5.py`
`python/sglang/srt/models/utils.py` | ROCm / AITER path fuses Q / K RMSNorm, partial or 3D RoPE, and direct KV cache write for Qwen3.5 attention | Treat split QK-norm + RoPE + cache-store on Qwen3.5 as a concrete in-flight upstream family, not a novel idea. | | PR `#22392` CUTLASS FP8 GEMM replacing nvjet | `cutlass_scaled_mm`
`fp8_scaled_mm`
`nvjet`
`cudaMemsetAsync` | `PR #22392`
`sgl-kernel/python/sgl_kernel/gemm.py`
`python/sglang/srt/layers/quantization/fp8_utils.py` | Runtime replacement swaps nvjet FP8 GEMMs for CUTLASS kernels, removing per-launch memset bubbles and extra output-copy kernels | Treat nvjet GEMM + memset bubble ladders as an in-flight SGLang linear-kernel family before calling them novel. | +| PR `#18612` NVFP4 CUTLASS MoE fused SiLU+Mul+quant | `silu_and_mul_scaled_nvfp4`
`nvfp4 expert quant`
`cutlass moe` | `PR #18612`
`python/sglang/srt/layers/moe/cutlass_w4a8_moe.py`
`python/sglang/jit_kernel/nvfp4.py` | Fuses MoE activation epilogue and NVFP4 expert quantization before the CUTLASS MoE second GEMM | Treat split SiLU+Mul then NVFP4 expert quant in CUTLASS MoE traces as an in-flight upstream SGLang family. | +| PR `#22918` FlashInfer per-token NVFP4 MoE | `per_token_nvfp4`
`trtllm_fp4_block_scale_moe`
`FlashInfer MoE` | `PR #22918`
`python/sglang/srt/layers/moe/fused_moe_triton/fused_moe.py` | Adds FlashInfer-backed per-token NVFP4 MoE execution so expert quant/dequant work can move into the fused MoE backend | Treat standalone per-token NVFP4 MoE support kernels as a candidate missing backend-selection path, not an automatically novel kernel idea. | +| PR `#22851` NSA top-k backend and FlashInfer / PyTorch top-k split | `nsa topk`
`flashinfer_topk`
`pytorch_topk`
`fast_topk_transform` | `PR #22851`
`python/sglang/srt/layers/attention/nsa_backend.py` | Makes NSA top-k backend selection explicit and aligns fused top-k transform with FlashInfer / PyTorch fallbacks | When NSA top-k dominates decode, first classify it as backend selection or fused-transform eligibility work. | +| PR `#24125` GLM5 NSA decode CatArrayBatchedCopy removal | `CatArrayBatchedCopy`
`GLM-5`
`NSA`
`TileLang decode` | `PR #24125`
`python/sglang/srt/layers/attention/nsa_backend.py` | Skips redundant cat/copy work in the GLM5 NSA TileLang decode path | Treat cat/copy bursts in GLM5 NSA decode as a concrete in-flight cleanup opportunity. | +| PR `#24007` MoE LoRA virtual experts for csgmv backend | `csgmv`
`virtual experts`
`MoE LoRA`
`fused_moe_lora` | `PR #24007`
`python/sglang/srt/layers/lora_backend.py`
`python/sglang/srt/layers/moe` | Routes MoE LoRA adapter work through virtual experts so csgmv-style kernels can batch it instead of launching fragmented adapter work | Treat MoE-LoRA tiny-kernel ladders as an in-flight batching/fusion family. | +| PR `#24150` torch.compile local decode support | `enable_torch_compile`
`local compile`
`decode compile`
`torchinductor` | `PR #24150`
`python/sglang/srt` | Extends SGLang torch.compile coverage to local decode regions, so Inductor-generated fusion may replace hand-authored tiny kernels | When decode traces show compiler-generated kernels or missing named fused kernels, check this in-flight compile path before calling the shape unsupported. | ## 7. PR-backed / in-flight kernel-overlap families | Pattern | Trace keywords | Primary code | Existing path | Skill should conclude | | --- | --- | --- | --- | --- | | PR `#21877` fused down-GEMM + combine superseding SBO | `enable_fused_grouped_gemm_combine`
`combine`
`down_gemm` | `PR #21877`
`python/sglang/srt/server_args.py`
`python/sglang/srt/layers/moe/token_dispatcher/deepep.py` | Fused combine eliminates the standalone combine window, so SBO is intentionally disabled when this path is on | If the trace discussion is about combine overlap, first classify it as this upstream fused-overlap family. | +| PR `#23965` PDL for DSV32 / GLM5 kernels | `enable_pdl`
`TRTLLM_ENABLE_PDL`
`cudaGridDependencySynchronize`
`DSV32`
`GLM5` | `PR #23965`
`python/sglang/srt/layers`
`sgl-kernel` | Enables programmatic dependent launch on selected DeepSeek / GLM kernels so dependent decode kernels can overlap launch-to-start gaps | Treat tight same-stream decode windows around DSV32 / GLM5 as an in-flight PDL overlap family. | +| PR `#21878` TTFT / TPOT torch.compile optimization | `enable_torch_compile`
`decode graph`
`piecewise cudagraph` | `PR #21878`
`python/sglang/srt` | Uses compiler and graph capture changes to shave TTFT / TPOT rather than adding one handwritten kernel | If the trace shows many small compiler-visible decode ops, compare against this compile-overlap / graph-capture family first. | +| PR `#24168` batched GPU-to-CPU sync for logprobs / embeddings | `logprobs`
`embeddings`
`GPU->CPU sync`
`batch sync` | `PR #24168`
`python/sglang/srt` | Batches per-request synchronization work that can otherwise serialize decode progress around logprob or embedding outputs | Treat per-request CPU sync stalls in logprob / embedding traces as a concrete in-flight SGLang scheduler/data-movement family. | ## 8. FlashInfer mainline fused-kernel families @@ -272,6 +274,15 @@ contain the same implementation. | PR `#37646` ROCm AITER fused allreduce + RMSNorm | `rocm_aiter_fused_allreduce_rmsnorm`
`custom_fused_ar_rms`
`RocmAiterAllReduceFusionPass` | `PR #37646`
`vllm/_aiter_ops.py`
`vllm/compilation/passes/pass_manager.py` | ROCm-specific compile-time path swaps the generic all-reduce fusion pass for an AITER fused allreduce-plus-RMSNorm kernel family | Treat ROCm TP all-reduce + RMSNorm ladders as an in-flight upstream fused-collective family first. | | PR `#36413` FlashInfer RMSNorm + FP4 quant fusion | `fuse_norm_quant`
`flashinfer`
`NVFP4`
`rmsnorm + fp4 quant` | `PR #36413`
`vllm/compilation/passes/fusion/rms_quant_fusion.py`
`vllm/docs/design/fusions.md` | FlashInfer-backed norm-plus-FP4 quant fusion extends the existing RMSNorm+quant family to NVFP4 flows | Treat split RMSNorm + FP4 quant ladders as an upstream in-flight family, not a fresh idea. | | PR `#39301` GLM5 router GEMM with PDL overlap | `TRTLLM_ENABLE_PDL`
`router_gemm`
`GLM5`
`FI AR RMS fusion` | `PR #39301`
`vllm/model_executor/layers/fused_moe/router/gate_linear.py`
`vllm/csrc/moe/dsv3_router_gemm_utils.h` | Extends the specialized router GEMM family to GLM5 hidden size and uses PDL to overlap the router launch with the preceding fused allreduce-plus-RMS block | Treat this as an in-flight upstream router-kernel plus launch-overlap family before calling it novel. | +| PR `#41455` ROCm WMMA paged prefill and split-K decode | `wmma`
`paged prefill`
`split-K decode`
`ROCm attention` | `PR #41455`
`vllm/v1/attention`
`vllm/_aiter_ops.py` | Adds ROCm WMMA attention kernels for paged prefill and split-K decode shapes | Treat split attention support kernels on AMD as an in-flight vLLM attention-kernel family before calling them novel. | +| PR `#41263` DeepSeek-V4 fused norm / router low-latency path | `DSV4`
`fuse norm router`
`low latency`
`router` | `PR #41263`
`vllm/model_executor/models/deepseek_v2.py`
`vllm/model_executor/layers/fused_moe/router` | Targets DeepSeek-V4 decode latency by fusing norm / router-adjacent work and low-latency model paths | Treat DSV4 norm-router ladders as a concrete in-flight upstream family. | +| PR `#41428` DSV4 fused indexer Q quant kernel | `DSV4`
`fused Indexer Q quant`
`indexer q`
`fp4` | `PR #41428`
`vllm/model_executor/models/deepseek_v2.py`
`vllm/csrc` | Improves the fused DeepSeek-V4 indexer Q quant kernel instead of materializing Q then quantizing separately | Treat DSV4 indexer-Q quant ladders as an in-flight upstream fused quant family. | +| PR `#41255` DeepSeek-V4 Tile kernels / `head_compute_mix_kernel` | `head_compute_mix_kernel`
`Tile kernel`
`DSV4`
`MLA` | `PR #41255`
`vllm/model_executor/models/deepseek_v2.py`
`vllm/csrc` | Adds DeepSeek-V4 Tile kernels that mix head compute work in one specialized kernel | Treat DSV4 MLA head-compute ladders as a known in-flight specialized-kernel family. | +| PR `#41441` DSV4 all-reduce plus `mhc_post` fusion | `DSV4`
`AR+mhc_post`
`allreduce`
`mhc_post` | `PR #41441`
`vllm/model_executor/models/deepseek_v2.py`
`vllm/compilation/passes/fusion` | Fuses or overlaps DSV4 all-reduce with post-MLA head-compute work | Treat all-reduce followed by `mhc_post` in DSV4 traces as an in-flight vLLM overlap/fusion family. | +| PR `#41446` AMD GatedDeltaNet FLA prefill kernels | `GatedDeltaNet`
`FLA prefill`
`AMD`
`Qwen3-Next` | `PR #41446`
`vllm/model_executor/models/qwen3_next.py`
`vllm/v1/attention` | Optimizes GatedDeltaNet / FLA prefill kernels on AMD linear-attention models | Treat split GDN prefill kernels on ROCm as an in-flight upstream family. | +| PR `#39748` dual-stream GDN input projection | `dual-stream`
`input projection`
`GatedDeltaNet`
`Qwen3.5` | `PR #39748`
`vllm/model_executor/models/qwen3_next.py` | Overlaps sibling input-projection branches for Qwen3 / Qwen3.5 GDN-style blocks | Treat serial GDN input projections as a known in-flight overlap opportunity. | +| PRs `#41433` / `#41434` / `#41429` / `#40561` GPU/CPU sync removal | `GPU->CPU sync`
`cpu sync`
`item()`
`non_blocking` | `PR #41433`
`PR #41434`
`PR #41429`
`PR #40561` | Removes or gates accidental GPU-to-CPU synchronization points and adds sync-detection coverage | Treat CPU gaps next to small GPU kernels as an upstream vLLM sync-removal family before proposing a kernel-only fix. | +| PR `#36823` vLLM IR `fused_add_rms_norm` overload | `vllm_ir`
`fused_add_rms_norm`
`maybe_inplace` | `PR #36823`
`vllm/compilation/passes/ir`
`vllm/compilation/passes/fusion/rms_quant_fusion.py` | Extends vLLM IR lowering so fused-add-RMSNorm variants remain visible to later compile-time fusions | Treat missing norm/quant compile fusion as potentially an IR-lowering visibility issue. | ## 17. Important toggles and caveats @@ -306,8 +317,11 @@ contain the same implementation. | `PassConfig.fuse_norm_quant` | `vllm/config/compilation.py` | Enables vLLM's RMSNorm(+residual add) -> FP8 / FP4 quant compile-time fusion family. | | `PassConfig.fuse_act_quant` | `vllm/config/compilation.py` | Enables vLLM's `SiLU+Mul -> quant` fusion family, plus ROCm AITER variants where applicable. | | `PassConfig.fuse_attn_quant` | `vllm/config/compilation.py` | Enables attention-epilogue quant fusion; requires the right backend / graph visibility, so split kernels may still be expected. | +| `PassConfig.fuse_mla_dual_rms_norm` | `vllm/config/compilation.py` | Enables the AITER-backed MLA paired-Q/KV RMSNorm fusion family on ROCm. | | `PassConfig.enable_qk_norm_rope_fusion` | `vllm/config/compilation.py` | Enables the compile-time QK RMSNorm + RoPE family on CUDA-like backends. | | `PassConfig.fuse_rope_kvcache` | `vllm/config/compilation.py` | Enables ROCm / AITER RoPE + KV-cache update fusion and is range-limited by token count. | +| `PassConfig.fuse_minimax_qk_norm` | `vllm/config/compilation.py` | Enables the MiniMax decode Q/K allreduce-plus-RMSNorm compile-time fusion family. | +| `PassConfig.fuse_act_padding` | `vllm/config/compilation.py` | Enables the ROCm AITER add-RMSNorm-plus-pad fusion family when AITER is available. | | `PassConfig.enable_sp` | `vllm/config/compilation.py` | Rewrites all-reduce into sequence-parallel staging; this is often a prerequisite for the overlap family, not just a pure fuse toggle. | | `PassConfig.fuse_gemm_comms` | `vllm/config/compilation.py` | Enables AsyncTP GEMM + collective overlap and auto-enables `enable_sp` when valid. | | `TRTLLM_ENABLE_PDL` | `vllm/csrc/dsv3_fused_a_gemm.cu`
`vllm/csrc/moe/dsv3_router_gemm_utils.h` | Enables programmatic dependent launch for the DSV3 specialized CUDA kernels, which can change launch grouping and trace shape for router / QKV-A paths. | diff --git a/.claude/skills/llm-torch-profiler-analysis/references/vllm-torch-compile-fusions.md b/.claude/skills/llm-torch-profiler-analysis/references/vllm-torch-compile-fusions.md new file mode 100644 index 000000000..e9f3109f5 --- /dev/null +++ b/.claude/skills/llm-torch-profiler-analysis/references/vllm-torch-compile-fusions.md @@ -0,0 +1,66 @@ +# vLLM Torch Compile Fusion Patterns + +Refresh: `2026-05-01`. +Source tree: vLLM `origin/main` at `7075df79b`. + +Use this file when the fuse-pattern table reports split kernels in a trace and +you need to decide whether the shape is already covered by vLLM's +`torch.compile` pattern matcher. Treat every row here as an upstream precedent +before calling a similar SGLang opportunity novel. + +## Pass Registration + +vLLM registers these passes from +`vllm/compilation/passes/pass_manager.py` through `PassConfig`. + +| Toggle | Pass | Target shape | +| --- | --- | --- | +| `enable_sp` | `SequenceParallelismPass` | all-reduce around residual/norm blocks becomes reduce-scatter, local work, and all-gather | +| `fuse_gemm_comms` | `AsyncTPPass` | GEMM plus reduce-scatter / all-gather overlap through symmetric-memory collectives | +| `fuse_allreduce_rms` | `AllReduceFusionPass` | all-reduce followed by RMSNorm, optional residual add, optional FP8 / NVFP4 quant | +| `fuse_minimax_qk_norm` | `MiniMaxQKNormPass` | MiniMax Q/K all-reduce plus RMSNorm decode path | +| `fuse_norm_quant` | `RMSNormQuantFusionPass` | RMSNorm or fused-add-RMSNorm followed by FP8 / FP4 quant | +| `fuse_norm_quant` + AITER | `RocmAiterRMSNormQuantFusionPass` | ROCm AITER RMSNorm / fused-add-RMSNorm followed by AITER or vLLM quant | +| `fuse_act_quant` | `ActivationQuantFusionPass` | SiLU-and-mul followed by FP8 / NVFP4 / block quant | +| `fuse_act_quant` + AITER | `RocmAiterSiluMulFp8GroupQuantFusionPass` | AITER SiLU-and-mul followed by FP8 group quant | +| `fuse_act_padding` + AITER | `RocmAiterTritonAddRMSNormPadFusionPass` | AITER fused-add-RMSNorm followed by padding into the next layout | +| `fuse_mla_dual_rms_norm` + AITER | `MLADualRMSNormFusionPass` | MLA paired Q and KV RMSNorms become `fused_mla_dual_rms_norm` | +| `fuse_rope_kvcache` | `RopeKVCacheFusionPass` | RoPE plus paged KV-cache update, after split cleanup passes | +| `fuse_attn_quant` | `AttnQuantFusionPass` | attention output followed by FP8 / NVFP4 quant | +| `fuse_attn_quant` | `MLAAttnQuantFusionPass` | MLA attention output followed by FP8 / NVFP4 / FP8 group quant | +| `enable_qk_norm_rope_fusion` | `QKNormRoPEFusionPass` | Q/K RMSNorm plus RoPE on packed QKV tensors | + +## Pattern Inventory + +| Source file | Pattern classes | Trace clue | Replacement | +| --- | --- | --- | --- | +| `fusion/allreduce_rms_fusion.py` | `AllReduceRMSNormPattern`, `AllReduceFusedAddRMSNormPattern`, `AllReduceFusedRMSNormStaticQuantFP8Pattern`, `AllReduceFusedAddRMSNormStaticQuantFP8Pattern`, `AllReduceFusedRMSNormStaticQuantNVFP4Pattern`, `AllReduceFusedAddRMSNormStaticQuantNVFP4Pattern` | TP all-reduce directly before RMSNorm, residual-add RMSNorm, or quant | `flashinfer_trtllm_fused_allreduce_norm` with FlashInfer allreduce fusion pattern codes | +| `fusion/rms_quant_fusion.py` | `RMSNormStaticQuantPattern`, `FusedAddRMSNormStaticQuantPattern`, `RMSNormDynamicQuantPattern`, `FusedAddRMSNormDynamicQuantPattern`, `RMSNormGroupQuantPattern`, `FusedAddRMSNormGroupQuantPattern` | RMSNorm or fused-add-RMSNorm followed by static FP8, dynamic per-token FP8, FP8 group quant, or NVFP4 quant | `_C.rms_norm_*_quant`, `_C.fused_add_rms_norm_*_quant`, or per-block quant custom op | +| `fusion/rocm_aiter_fusion.py` | `AiterRMSNormDynamicQuantPattern`, `AiterFusedAddRMSNormDynamicQuantPattern`, `AiterRMSFp8GroupQuantPattern`, `AiterFusedAddRMSFp8GroupQuantPattern` | AITER RMSNorm/fused-add-RMSNorm followed by AITER or vLLM FP8 quant | AITER fused RMSNorm-quant custom ops | +| `fusion/act_quant_fusion.py` | `SiluMulFp8StaticQuantPattern`, `SiluMulNvfp4QuantPattern`, `SiluMulBlockQuantPattern` | SiLU-and-mul activation output immediately quantized | fused activation-plus-quant custom op | +| `fusion/rocm_aiter_fusion.py` | `AiterSiluMulFp8GroupQuantPattern` | AITER SiLU-and-mul followed by FP8 group quant | AITER `act_mul_fused_fp8_group_quant` | +| `fusion/rocm_aiter_fusion.py` | `AddAiterRMSNormPadPattern` | AITER fused-add-RMSNorm output padded before the next op | AITER add-RMSNorm-pad op | +| `fusion/rocm_aiter_fusion.py` | `MLADualRMSNormPattern` | MLA Q branch and KV branch each run RMSNorm | `torch.ops.vllm.fused_mla_dual_rms_norm` backed by AITER fused QK RMSNorm | +| `fusion/qk_norm_rope_fusion.py` | `QkNormRopePattern` | Q/K RMSNorm, split/getitem reshapes, then RoPE | `_C.fused_qk_norm_rope` | +| `fusion/rope_kvcache_fusion.py` | `RopeReshapeKVCachePattern` | RoPE output followed by reshape/cache update | `vllm.fused_rope_and_unified_kv_cache_update` | +| `fusion/attn_quant_fusion.py` | `AttnFp8StaticQuantPattern`, `AttnNvfp4QuantPattern` | attention output followed by FP8 static quant or NVFP4 quant | backend attention op with fused output quant when supported | +| `fusion/mla_attn_quant_fusion.py` | `MLAAttnFp8StaticQuantPattern`, `MLAAttnNvfp4QuantPattern`, `MLAAttnFp8GroupQuantPattern` | MLA attention output followed by static FP8, NVFP4, or FP8 group quant | MLA attention op with fused output quant when supported | +| `fusion/minimax_qk_norm_fusion.py` | `MiniMaxQKNormPattern` | MiniMax `forward_qk`: Q/K variance all-reduce divided by TP world size, then RMS apply | `vllm.minimax_qk_norm_fused` / Lamport fused kernel | +| `fusion/sequence_parallelism.py` | `FirstAllReduceRMSNormPattern`, `MiddleAllReduceRMSNormPattern`, `FirstAllReduceRMSNormStaticFP8Pattern`, `MiddleAllReduceRMSNormStaticFP8Pattern` | all-reduce plus norm block in a full-graph TP model | sequence-parallel reduce-scatter, local norm, all-gather staging | +| `fusion/collective_fusion.py` | `GEMMReduceScatterPattern`, `AllGatherGEMMPattern`, `ScaledMMReduceScatterPattern`, `AllGatherScaledMMPattern`, `CutlassScaledMMReduceScatterPattern`, `AllGatherCutlassScaledMMPattern`, `FlashInferBMMFP8ReduceScatterPattern`, `FlashInferAllGatherBMMFP8Pattern` | matmul / scaled-mm / FlashInfer BMM adjacent to TP collectives | symmetric-memory fused matmul+reduce-scatter or all-gather+matmul | + +## Triage Rules + +- If the trace shows split norm/add/quant, compare first against + `RMSNormQuantFusionPass`, AITER variants, and `AllReduceFusionPass`. +- If the trace shows attention output followed by quant kernels, compare against + `AttnQuantFusionPass` or `MLAAttnQuantFusionPass`, not only handwritten + attention kernels. +- If the trace shows Q/K norm followed by RoPE or cache update, compare both + `QKNormRoPEFusionPass` and `RopeKVCacheFusionPass`; they are separate passes. +- If the trace is a TP decode trace with visible collectives, check whether + `enable_sp` and `fuse_gemm_comms` would transform the same region into + sequence-parallel or AsyncTP overlap. +- A missing vLLM compile fusion may be intentional when the graph range, backend + support check, dtype, token count, or AITER / FlashInfer availability does not + satisfy the pass-specific guard. diff --git a/.claude/skills/llm-torch-profiler-analysis/scripts/analyze_llm_torch_profile.py b/.claude/skills/llm-torch-profiler-analysis/scripts/analyze_llm_torch_profile.py index 6b6a78550..c432e4d30 100644 --- a/.claude/skills/llm-torch-profiler-analysis/scripts/analyze_llm_torch_profile.py +++ b/.claude/skills/llm-torch-profiler-analysis/scripts/analyze_llm_torch_profile.py @@ -11,6 +11,12 @@ from typing import Dict, List, Optional, Sequence, Tuple import triage_kernel_helpers as kernel_helpers import triage_overlap_helpers as overlap_helpers from profile_common import ( + DEFAULT_DECODE_INPUT_LEN, + DEFAULT_DECODE_OUTPUT_LEN, + DEFAULT_PREFILL_INPUT_LEN, + DEFAULT_PREFILL_OUTPUT_LEN, + DEFAULT_WARMUP_STEPS, + PROFILE_WORKLOAD_CHOICES, discover_trace_targets, framework_display_name, load_server_args, @@ -57,8 +63,8 @@ def build_triage_parser() -> argparse.ArgumentParser: default=None, help=( "Running server URL for single-trace triage. SGLang supports direct " - "capture through its profiler HTTP API. vLLM and TensorRT-LLM require " - "a server-side torch-profiler output path exposed via --output-dir." + "capture via sglang.profiler. vLLM and TensorRT-LLM require a server-side " + "torch-profiler output path exposed via --output-dir." ), ) parser.add_argument( @@ -132,7 +138,13 @@ def build_triage_parser() -> argparse.ArgumentParser: "--num-steps", type=int, default=5, - help="Profiler steps when generating traces from URLs.", + help="Active profiler steps when generating traces from URLs.", + ) + parser.add_argument( + "--warmup-steps", + type=int, + default=DEFAULT_WARMUP_STEPS, + help="Warmup steps to run before arming the profiler for URL capture.", ) parser.add_argument( "--profile-by-stage", action=argparse.BooleanOptionalAction, default=True @@ -151,11 +163,45 @@ def build_triage_parser() -> argparse.ArgumentParser: ) parser.add_argument("--probe-max-new-tokens", type=int, default=None) parser.add_argument("--probe-delay", type=float, default=0.5) + parser.add_argument( + "--profile-workload", + choices=PROFILE_WORKLOAD_CHOICES, + default="both", + help=( + "Live-capture workload shape. Default 'both' captures separate " + "prefill and decode profiles instead of one mixed request. Use " + "'legacy' to keep the old --probe-prompt behavior." + ), + ) + parser.add_argument( + "--prefill-input-len", + type=int, + default=DEFAULT_PREFILL_INPUT_LEN, + help="Synthetic input length for the prefill profile workload.", + ) + parser.add_argument( + "--prefill-output-len", + type=int, + default=DEFAULT_PREFILL_OUTPUT_LEN, + help="Output length for the prefill profile workload.", + ) + parser.add_argument( + "--decode-input-len", + type=int, + default=DEFAULT_DECODE_INPUT_LEN, + help="Synthetic input length for the decode profile workload.", + ) + parser.add_argument( + "--decode-output-len", + type=int, + default=DEFAULT_DECODE_OUTPUT_LEN, + help="Output length for the decode profile workload.", + ) parser.add_argument( "--start-step", type=int, default=None, - help="SGLang-only profiler start step when generating traces from URLs.", + help="Pass through to sglang.profiler when generating traces from URLs.", ) parser.add_argument( "--pid-substring", @@ -239,9 +285,15 @@ def resolve_profile_targets( probe_prompt=args.probe_prompt, probe_max_new_tokens=args.probe_max_new_tokens, probe_delay=args.probe_delay, + warmup_steps=args.warmup_steps, start_step=args.start_step, framework=framework, framework_hint_path=output_dir, + profile_workload=args.profile_workload, + prefill_input_len=args.prefill_input_len, + prefill_output_len=args.prefill_output_len, + decode_input_len=args.decode_input_len, + decode_output_len=args.decode_output_len, ) traces, server_args = discover_trace_targets(target_dir, all_traces=False) resolved_framework = resolve_framework( diff --git a/.claude/skills/llm-torch-profiler-analysis/scripts/probe_llm_server.py b/.claude/skills/llm-torch-profiler-analysis/scripts/probe_llm_server.py index 97008b789..ba7b65d00 100755 --- a/.claude/skills/llm-torch-profiler-analysis/scripts/probe_llm_server.py +++ b/.claude/skills/llm-torch-profiler-analysis/scripts/probe_llm_server.py @@ -15,7 +15,7 @@ from urllib import request from profile_common import extract_openai_chat_text DEFAULT_PROMPTS = [ - "用一句中文介绍上海。", + "Introduce Shanghai in one short sentence.", "What is 2+2? Answer briefly.", "Write one short haiku about GPUs.", ] diff --git a/.claude/skills/llm-torch-profiler-analysis/scripts/profile_common.py b/.claude/skills/llm-torch-profiler-analysis/scripts/profile_common.py index ee589e26c..8e4d7514a 100644 --- a/.claude/skills/llm-torch-profiler-analysis/scripts/profile_common.py +++ b/.claude/skills/llm-torch-profiler-analysis/scripts/profile_common.py @@ -5,10 +5,12 @@ from __future__ import annotations import gzip import json import re +import shutil import sys import tempfile import time from collections import Counter, defaultdict +from dataclasses import dataclass from functools import lru_cache from pathlib import Path from typing import Callable, Dict, Iterable, List, Optional, Sequence, Tuple @@ -42,6 +44,21 @@ TRACE_METADATA_NAMES = { } NON_KERNEL_TRACE_CATEGORIES = ("python_function", "cpu_op", "trace") PYTHON_SCOPE_NAME_PREFIXES = ("python/", "nn.module:") +PROFILE_WORKLOAD_CHOICES = ("legacy", "prefill", "decode", "both") +DEFAULT_PREFILL_INPUT_LEN = 4090 +DEFAULT_PREFILL_OUTPUT_LEN = 1 +DEFAULT_DECODE_INPUT_LEN = 1 +DEFAULT_DECODE_OUTPUT_LEN = 2048 +DEFAULT_WARMUP_STEPS = 10 + + +@dataclass(frozen=True) +class ProbePlan: + prompt: str + capture_max_new_tokens: int + capture_requests: int + warmup_max_new_tokens: int + warmup_requests: int @lru_cache(maxsize=65536) @@ -416,10 +433,16 @@ def resolve_framework( def parse_stage(path: Path) -> str: - name = path.name.lower() - if "-extend" in name or "-prefill" in name: + parts = [part.lower() for part in path.parts[-6:]] + name = " ".join(parts) + segment_path = "/" + "/".join(parts) + "/" + if any(marker in name for marker in ("-extend", "-prefill", "_extend", "_prefill")): return "extend" - if "-decode" in name: + if any(f"/{segment}/" in segment_path for segment in ("extend", "prefill")): + return "extend" + if any(marker in name for marker in ("-decode", "_decode")): + return "decode" + if "/decode/" in segment_path: return "decode" return "all" @@ -514,8 +537,19 @@ def discover_trace_targets( if path.is_file(): return [path], load_server_args(path) - trace_dir = newest_trace_dir(path) - traces = discover_trace_files(trace_dir, recursive=False) + direct_traces = discover_trace_files(path, recursive=False) + recursive_traces = discover_trace_files(path, recursive=True) + recursive_stages = {parse_stage(trace) for trace in recursive_traces} + if ( + not direct_traces + and recursive_traces + and any(stage != "all" for stage in recursive_stages) + ): + traces = recursive_traces + trace_dir = path + else: + trace_dir = newest_trace_dir(path) + traces = discover_trace_files(trace_dir, recursive=False) if not traces: raise FileNotFoundError(f"No trace files found under {trace_dir}") @@ -606,6 +640,109 @@ def send_probe_request( ) +def unique_probe_prompt(prompt: str, probe_index: int) -> str: + marker = f"profile_probe_{max(0, int(probe_index))}" + parts = prompt.split(maxsplit=1) + suffix = parts[1] if len(parts) == 2 else prompt + return f"{marker} {suffix}".strip() + + +def send_probe_requests( + *, + url: str, + prompt: str, + max_new_tokens: int, + request_count: int, + framework: str, + model: Optional[str] = None, + sampling_seed_offset: int = 0, +) -> None: + request_count = max(0, int(request_count)) + seed_offset = max(0, int(sampling_seed_offset)) + for request_idx in range(request_count): + probe_index = seed_offset + request_idx + send_probe_request( + url=url, + prompt=unique_probe_prompt(prompt, probe_index), + max_new_tokens=max_new_tokens, + sampling_seed=probe_index, + framework=framework, + model=model, + ) + + +def synthetic_prompt(input_len: int) -> str: + token_count = max(1, int(input_len)) + return " ".join(["profile"] * token_count) + + +def workload_probe( + stage: str, + *, + prefill_input_len: int, + prefill_output_len: int, + decode_input_len: int, + decode_output_len: int, +) -> Tuple[str, int]: + if stage == "prefill": + return synthetic_prompt(prefill_input_len), max(1, int(prefill_output_len)) + if stage == "decode": + return synthetic_prompt(decode_input_len), max(1, int(decode_output_len)) + raise ValueError(f"unknown profile workload stage: {stage}") + + +def build_probe_plan( + stage: str, + *, + prompt: str, + max_new_tokens: int, + num_steps: int, + probe_requests: int, + warmup_steps: int, +) -> ProbePlan: + active_steps = max(1, int(num_steps)) + requested_probes = max(1, int(probe_requests)) + warmup_steps = max(0, int(warmup_steps)) + max_new_tokens = max(1, int(max_new_tokens)) + + if stage == "prefill": + return ProbePlan( + prompt=prompt, + capture_max_new_tokens=max_new_tokens, + capture_requests=max(requested_probes, active_steps), + warmup_max_new_tokens=max_new_tokens, + warmup_requests=warmup_steps, + ) + if stage == "decode": + return ProbePlan( + prompt=prompt, + capture_max_new_tokens=max_new_tokens, + capture_requests=requested_probes, + warmup_max_new_tokens=max(1, warmup_steps), + warmup_requests=1 if warmup_steps else 0, + ) + return ProbePlan( + prompt=prompt, + capture_max_new_tokens=max_new_tokens, + capture_requests=requested_probes, + warmup_max_new_tokens=max_new_tokens, + warmup_requests=warmup_steps, + ) + + +def expand_profile_workload(profile_workload: str) -> List[str]: + workload = normalize_text(profile_workload).lower() + if workload not in PROFILE_WORKLOAD_CHOICES: + raise ValueError( + f"--profile-workload must be one of {', '.join(PROFILE_WORKLOAD_CHOICES)}" + ) + if workload == "both": + return ["prefill", "decode"] + if workload == "legacy": + return ["legacy"] + return [workload] + + def discover_openai_model(url: str) -> str: payload = try_get_json(url.rstrip("/") + "/v1/models", timeout=60.0) if not isinstance(payload, dict): @@ -690,35 +827,50 @@ def run_remote_profiler( url: str, output_dir: Optional[str], framework: str, - probe_requests: int, - probe_prompt: str, - probe_max_new_tokens: Optional[int], + probe_plan: ProbePlan, probe_delay: float, - num_steps: int, + stage: Optional[str] = None, ) -> Path: framework = canonicalize_framework(framework) output_path = ensure_remote_profiler_output_path(output_dir, framework) + if stage and output_path.is_file(): + raise ValueError( + "--profile-workload both requires a directory output path for " + f"{framework_display_name(framework)} so each stage trace can be labeled." + ) + before_traces = ( + set(discover_trace_files(output_path, recursive=True)) + if output_path.exists() + else set() + ) + model = discover_openai_model(url) if framework in {"vllm", "trtllm"} else None + if probe_plan.warmup_requests > 0: + send_probe_requests( + url=url, + prompt=probe_plan.prompt, + max_new_tokens=probe_plan.warmup_max_new_tokens, + request_count=probe_plan.warmup_requests, + framework=framework, + model=model, + ) + start_remote_profiler(url, framework) stop_error: Optional[BaseException] = None try: - if probe_requests > 0: - # Some profiler endpoints need a brief setup window after + if probe_plan.capture_requests > 0: + # `sglang.profiler` performs its own startup work before it reaches # POST /start_profile. A very short delay can send probes too early # and miss the profiling window entirely. time.sleep(max(5.0, probe_delay)) - effective_max_new_tokens = probe_max_new_tokens or max(64, num_steps * 8) - model = ( - discover_openai_model(url) if framework in {"vllm", "trtllm"} else None + send_probe_requests( + url=url, + prompt=probe_plan.prompt, + max_new_tokens=probe_plan.capture_max_new_tokens, + request_count=probe_plan.capture_requests, + framework=framework, + model=model, + sampling_seed_offset=probe_plan.warmup_requests, ) - for request_idx in range(probe_requests): - send_probe_request( - url=url, - prompt=probe_prompt, - max_new_tokens=effective_max_new_tokens, - sampling_seed=request_idx, - framework=framework, - model=model, - ) finally: try: stop_remote_profiler(url, framework) @@ -726,7 +878,22 @@ def run_remote_profiler( stop_error = exc if stop_error is not None: raise stop_error - return wait_for_profiler_artifact(output_path) + artifact = wait_for_profiler_artifact(output_path) + if stage and output_path.is_dir(): + after_traces = set(discover_trace_files(output_path, recursive=True)) + new_traces = sorted(after_traces - before_traces, key=lambda item: item.name) + if new_traces: + stage_dir = output_path / stage + stage_dir.mkdir(parents=True, exist_ok=True) + for trace in new_traces: + if stage_dir in trace.parents: + continue + target = stage_dir / trace.name + if target.exists(): + target = stage_dir / f"{time.time_ns()}-{trace.name}" + shutil.move(str(trace), str(target)) + return stage_dir + return artifact def run_sglang_profiler( @@ -736,9 +903,7 @@ def run_sglang_profiler( profile_by_stage: bool, merge_profiles: bool, profile_prefix: Optional[str], - probe_requests: int, - probe_prompt: str, - probe_max_new_tokens: Optional[int], + probe_plan: ProbePlan, probe_delay: float, start_step: Optional[int] = None, ) -> Path: @@ -765,6 +930,15 @@ def run_sglang_profiler( if start_step is not None: payload["start_step"] = str(start_step) + if probe_plan.warmup_requests > 0: + send_probe_requests( + url=url, + prompt=probe_plan.prompt, + max_new_tokens=probe_plan.warmup_max_new_tokens, + request_count=probe_plan.warmup_requests, + framework="sglang", + ) + req = request.Request( url.rstrip("/") + "/start_profile", data=json.dumps(payload).encode("utf-8"), @@ -773,17 +947,20 @@ def run_sglang_profiler( with request.urlopen(req, timeout=300.0): pass - if probe_requests > 0: + if probe_plan.capture_requests > 0: time.sleep(max(0.0, probe_delay)) - effective_max_new_tokens = probe_max_new_tokens or max(64, num_steps * 8) - for request_idx in range(probe_requests): - send_probe_request( - url=url, - prompt=probe_prompt, - max_new_tokens=effective_max_new_tokens, - sampling_seed=request_idx, - framework="sglang", - ) + send_probe_requests( + url=url, + prompt=probe_plan.prompt, + max_new_tokens=probe_plan.capture_max_new_tokens, + request_count=probe_plan.capture_requests, + framework="sglang", + sampling_seed_offset=probe_plan.warmup_requests, + ) + try: + stop_remote_profiler(url, "sglang") + except RuntimeError: + pass return wait_for_profiler_artifact(output_path, timeout_s=180.0) @@ -799,9 +976,15 @@ def run_profiler( probe_prompt: str, probe_max_new_tokens: Optional[int], probe_delay: float, + warmup_steps: int = DEFAULT_WARMUP_STEPS, start_step: Optional[int] = None, framework: str = "auto", framework_hint_path: Optional[str] = None, + profile_workload: str = "both", + prefill_input_len: int = DEFAULT_PREFILL_INPUT_LEN, + prefill_output_len: int = DEFAULT_PREFILL_OUTPUT_LEN, + decode_input_len: int = DEFAULT_DECODE_INPUT_LEN, + decode_output_len: int = DEFAULT_DECODE_OUTPUT_LEN, ) -> Path: resolved_framework = resolve_framework( framework, @@ -813,6 +996,58 @@ def run_profiler( ), ) if resolved_framework == "sglang": + stages = expand_profile_workload(profile_workload) + if stages != ["legacy"]: + output_root = ( + Path(output_dir).expanduser().resolve() + if output_dir + else Path(tempfile.mkdtemp(prefix="sglang-torch-profile-")) + ) + output_root.mkdir(parents=True, exist_ok=True) + for stage in stages: + prompt, max_new_tokens = workload_probe( + stage, + prefill_input_len=prefill_input_len, + prefill_output_len=prefill_output_len, + decode_input_len=decode_input_len, + decode_output_len=decode_output_len, + ) + probe_plan = build_probe_plan( + stage, + prompt=prompt, + max_new_tokens=max_new_tokens, + num_steps=num_steps, + probe_requests=probe_requests, + warmup_steps=warmup_steps, + ) + # SGLang increments `forward_ct` before checking whether the + # profiler reached its target. Ask for one extra step so the + # requested stage forward is captured instead of stopping just + # before it runs. + stage_num_steps = max(1, int(num_steps)) + 1 + run_sglang_profiler( + url=url, + output_dir=str(output_root / stage), + num_steps=stage_num_steps, + profile_by_stage=False, + merge_profiles=merge_profiles, + profile_prefix=( + f"{profile_prefix}-{stage}" if profile_prefix else stage + ), + probe_plan=probe_plan, + probe_delay=probe_delay, + start_step=start_step, + ) + return output_root + legacy_max_new_tokens = probe_max_new_tokens or max(64, num_steps * 8) + legacy_plan = build_probe_plan( + "legacy", + prompt=probe_prompt, + max_new_tokens=legacy_max_new_tokens, + num_steps=num_steps, + probe_requests=probe_requests, + warmup_steps=warmup_steps, + ) return run_sglang_profiler( url=url, output_dir=output_dir, @@ -820,9 +1055,7 @@ def run_profiler( profile_by_stage=profile_by_stage, merge_profiles=merge_profiles, profile_prefix=profile_prefix, - probe_requests=probe_requests, - probe_prompt=probe_prompt, - probe_max_new_tokens=probe_max_new_tokens, + probe_plan=legacy_plan, probe_delay=probe_delay, start_step=start_step, ) @@ -844,16 +1077,48 @@ def run_profiler( "--profile-prefix on the HTTP profiler control path.", file=sys.stderr, ) - return run_remote_profiler( - url=url, - output_dir=output_dir, - framework=resolved_framework, - probe_requests=probe_requests, - probe_prompt=probe_prompt, - probe_max_new_tokens=probe_max_new_tokens, - probe_delay=probe_delay, - num_steps=num_steps, - ) + stages = expand_profile_workload(profile_workload) + if stages == ["legacy"]: + legacy_max_new_tokens = probe_max_new_tokens or max(64, num_steps * 8) + return run_remote_profiler( + url=url, + output_dir=output_dir, + framework=resolved_framework, + probe_plan=build_probe_plan( + "legacy", + prompt=probe_prompt, + max_new_tokens=legacy_max_new_tokens, + num_steps=num_steps, + probe_requests=probe_requests, + warmup_steps=warmup_steps, + ), + probe_delay=probe_delay, + ) + output_root = ensure_remote_profiler_output_path(output_dir, resolved_framework) + for stage in stages: + prompt, max_new_tokens = workload_probe( + stage, + prefill_input_len=prefill_input_len, + prefill_output_len=prefill_output_len, + decode_input_len=decode_input_len, + decode_output_len=decode_output_len, + ) + run_remote_profiler( + url=url, + output_dir=str(output_root), + framework=resolved_framework, + probe_plan=build_probe_plan( + stage, + prompt=prompt, + max_new_tokens=max_new_tokens, + num_steps=num_steps, + probe_requests=probe_requests, + warmup_steps=warmup_steps, + ), + probe_delay=probe_delay, + stage=stage, + ) + return output_root def select_heaviest_pid( diff --git a/.claude/skills/llm-torch-profiler-analysis/scripts/run_llm_single_model_matrix_host.sh b/.claude/skills/llm-torch-profiler-analysis/scripts/run_llm_single_model_matrix_host.sh new file mode 100755 index 000000000..b80f3fe43 --- /dev/null +++ b/.claude/skills/llm-torch-profiler-analysis/scripts/run_llm_single_model_matrix_host.sh @@ -0,0 +1,274 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: + run_llm_single_model_matrix_host.sh \ + --model-id gpt_oss_20b \ + --model openai/gpt-oss-20b \ + --root /data/bbuf/validate/unified_llm_profiler_skill/runs/20260423_h100_large_model_matrix \ + --gpus 2,3,4,5 \ + --sglang-port 30098 \ + --vllm-formal-port 31098 \ + --vllm-mapping-port 31099 \ + --trt-formal-prefill-port 32098 \ + --trt-formal-decode-port 32099 \ + --trt-mapping-prefill-port 32198 \ + --trt-mapping-decode-port 32199 + +This script is intended to run on the H100 host. It: +1. captures SGLang live profiling and writes `analysis_sglang.txt` +2. captures vLLM formal + eager mapping traces and writes `analysis_vllm.txt` +3. captures TensorRT-LLM formal + graph-off mapping traces and writes `analysis_trtllm.txt` +4. stores one benchmark JSON per framework under the model run directory + +Default profiler workloads are stage-separated: + prefill: input 4090, output 1 + decode: input 1, output 2048 + +Environment: + Export `HF_TOKEN` and `HUGGINGFACE_HUB_TOKEN` before running. +EOF +} + +MODEL_ID="" +MODEL="" +ROOT="" +GPUS="" +TP_SIZE="" +SGLANG_PORT="" +VLLM_FORMAL_PORT="" +VLLM_MAPPING_PORT="" +TRT_FORMAL_PREFILL_PORT="" +TRT_FORMAL_DECODE_PORT="" +TRT_MAPPING_PREFILL_PORT="" +TRT_MAPPING_DECODE_PORT="" +SGLANG_MEM_FRACTION="0.85" +MAX_MODEL_LEN="4096" +KV_FRACTION="0.85" +SGLANG_SERVER_EXTRA="" +PROFILE_WORKLOAD="both" +PREFILL_INPUT_LEN=4090 +PREFILL_OUTPUT_LEN=1 +DECODE_INPUT_LEN=1 +DECODE_OUTPUT_LEN=2048 +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TRT_IMAGE="nvcr.io/nvidia/tensorrt-llm/release:latest" +TRT_OVERRIDE_ROOT="/data/bbuf/validate/unified_llm_profiler_skill/overrides/trtllm" +TRT_OVERRIDE_SOURCE="$TRT_OVERRIDE_ROOT/py_executor.original.py" +TRT_OVERRIDE_PATH="$TRT_OVERRIDE_ROOT/py_executor_with_stack.py" + +while [[ $# -gt 0 ]]; do + case "$1" in + --model-id) MODEL_ID="$2"; shift 2 ;; + --model) MODEL="$2"; shift 2 ;; + --root) ROOT="$2"; shift 2 ;; + --gpus) GPUS="$2"; shift 2 ;; + --tp-size) TP_SIZE="$2"; shift 2 ;; + --sglang-port) SGLANG_PORT="$2"; shift 2 ;; + --vllm-formal-port) VLLM_FORMAL_PORT="$2"; shift 2 ;; + --vllm-mapping-port) VLLM_MAPPING_PORT="$2"; shift 2 ;; + --trt-formal-prefill-port) TRT_FORMAL_PREFILL_PORT="$2"; shift 2 ;; + --trt-formal-decode-port) TRT_FORMAL_DECODE_PORT="$2"; shift 2 ;; + --trt-mapping-prefill-port) TRT_MAPPING_PREFILL_PORT="$2"; shift 2 ;; + --trt-mapping-decode-port) TRT_MAPPING_DECODE_PORT="$2"; shift 2 ;; + --sglang-mem-fraction) SGLANG_MEM_FRACTION="$2"; shift 2 ;; + --sglang-server-extra) SGLANG_SERVER_EXTRA="$2"; shift 2 ;; + --max-model-len) MAX_MODEL_LEN="$2"; shift 2 ;; + --kv-fraction) KV_FRACTION="$2"; shift 2 ;; + --profile-workload) PROFILE_WORKLOAD="$2"; shift 2 ;; + --prefill-input-len) PREFILL_INPUT_LEN="$2"; shift 2 ;; + --prefill-output-len) PREFILL_OUTPUT_LEN="$2"; shift 2 ;; + --decode-input-len) DECODE_INPUT_LEN="$2"; shift 2 ;; + --decode-output-len) DECODE_OUTPUT_LEN="$2"; shift 2 ;; + --help|-h) usage; exit 0 ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ -z "${HF_TOKEN:-}" && -z "${HUGGINGFACE_HUB_TOKEN:-}" ]]; then + echo "Set HF_TOKEN or HUGGINGFACE_HUB_TOKEN before running." >&2 + exit 2 +fi +if [[ -z "${HF_TOKEN:-}" ]]; then + HF_TOKEN="$HUGGINGFACE_HUB_TOKEN" +fi +if [[ -z "${HUGGINGFACE_HUB_TOKEN:-}" ]]; then + HUGGINGFACE_HUB_TOKEN="$HF_TOKEN" +fi + +for value in \ + MODEL_ID MODEL ROOT GPUS \ + SGLANG_PORT VLLM_FORMAL_PORT VLLM_MAPPING_PORT \ + TRT_FORMAL_PREFILL_PORT TRT_FORMAL_DECODE_PORT \ + TRT_MAPPING_PREFILL_PORT TRT_MAPPING_DECODE_PORT; do + if [[ -z "${!value}" ]]; then + echo "Missing required argument: $value" >&2 + usage >&2 + exit 2 + fi +done + +IFS=',' read -r -a GPU_LIST <<< "$GPUS" +GPU_COUNT="${#GPU_LIST[@]}" +if [[ "$GPU_COUNT" -lt 1 ]]; then + echo "Could not parse --gpus: $GPUS" >&2 + exit 2 +fi +if [[ -z "$TP_SIZE" ]]; then + TP_SIZE="$GPU_COUNT" +fi +if (( TP_SIZE < 1 || TP_SIZE > GPU_COUNT )); then + echo "--tp-size must be between 1 and the visible GPU count ($GPU_COUNT)." >&2 + exit 2 +fi + +MODEL_ROOT="$ROOT/$MODEL_ID" +SGLANG_ANALYSIS="$MODEL_ROOT/analysis_sglang.txt" +VLLM_FORMAL_DIR="$MODEL_ROOT/vllm_formal" +VLLM_MAPPING_DIR="$MODEL_ROOT/vllm_mapping" +VLLM_ANALYSIS="$MODEL_ROOT/analysis_vllm.txt" +TRT_FORMAL_DIR="$MODEL_ROOT/trtllm_formal" +TRT_MAPPING_DIR="$MODEL_ROOT/trtllm_mapping" +TRT_ANALYSIS="$MODEL_ROOT/analysis_trtllm.txt" + +docker exec sglang_bbuf bash -lc "mkdir -p '$MODEL_ROOT'" + +if [[ ! -s "$TRT_OVERRIDE_SOURCE" ]]; then + echo "[bootstrap] TensorRT-LLM py_executor source snapshot" + docker exec sglang_bbuf bash -lc "mkdir -p '$TRT_OVERRIDE_ROOT'" + docker run --rm --entrypoint cat "$TRT_IMAGE" \ + /usr/local/lib/python3.12/dist-packages/tensorrt_llm/_torch/pyexecutor/py_executor.py \ + | docker exec -i sglang_bbuf bash -lc "cat > '$TRT_OVERRIDE_SOURCE'" +fi +echo "[bootstrap] TensorRT-LLM py_executor override with with_stack=True and rank0-only trace export" +docker exec sglang_bbuf bash -lc "cd '$SCRIPT_DIR' && python3 make_trtllm_py_executor_override.py --source '$TRT_OVERRIDE_SOURCE' --output '$TRT_OVERRIDE_PATH'" + +sglang_args=( + --model "$MODEL" + --run-dir "$MODEL_ROOT" + --port "$SGLANG_PORT" + --gpus "$GPUS" + --tp-size "$TP_SIZE" + --mem-fraction "$SGLANG_MEM_FRACTION" + --profile-workload "$PROFILE_WORKLOAD" + --prefill-input-len "$PREFILL_INPUT_LEN" + --prefill-output-len "$PREFILL_OUTPUT_LEN" + --decode-input-len "$DECODE_INPUT_LEN" + --decode-output-len "$DECODE_OUTPUT_LEN" + --trust-remote-code +) +if [[ -n "$SGLANG_SERVER_EXTRA" ]]; then + sglang_args+=(--server-extra "$SGLANG_SERVER_EXTRA") +fi + +echo "[1/6] SGLang server + live triage" +HF_TOKEN="$HF_TOKEN" HUGGINGFACE_HUB_TOKEN="$HUGGINGFACE_HUB_TOKEN" \ + "$SCRIPT_DIR/run_sglang_torch_profile_host.sh" \ + "${sglang_args[@]}" + +echo "[2/6] vLLM formal" +HF_TOKEN="$HF_TOKEN" HUGGINGFACE_HUB_TOKEN="$HUGGINGFACE_HUB_TOKEN" \ + "$SCRIPT_DIR/run_vllm_torch_profile_host.sh" \ + --model "$MODEL" \ + --run-dir "$VLLM_FORMAL_DIR" \ + --port "$VLLM_FORMAL_PORT" \ + --gpus "$GPUS" \ + --tensor-parallel-size "$TP_SIZE" \ + --max-model-len "$MAX_MODEL_LEN" \ + --profile-workload "$PROFILE_WORKLOAD" \ + --prefill-input-len "$PREFILL_INPUT_LEN" \ + --prefill-output-len "$PREFILL_OUTPUT_LEN" \ + --decode-input-len "$DECODE_INPUT_LEN" \ + --decode-output-len "$DECODE_OUTPUT_LEN" \ + --trust-remote-code + +echo "[3/6] vLLM mapping" +HF_TOKEN="$HF_TOKEN" HUGGINGFACE_HUB_TOKEN="$HUGGINGFACE_HUB_TOKEN" \ + "$SCRIPT_DIR/run_vllm_torch_profile_host.sh" \ + --model "$MODEL" \ + --run-dir "$VLLM_MAPPING_DIR" \ + --port "$VLLM_MAPPING_PORT" \ + --gpus "$GPUS" \ + --tensor-parallel-size "$TP_SIZE" \ + --profiler-active-iterations 2 \ + --max-model-len "$MAX_MODEL_LEN" \ + --profile-workload "$PROFILE_WORKLOAD" \ + --prefill-input-len "$PREFILL_INPUT_LEN" \ + --prefill-output-len "$PREFILL_OUTPUT_LEN" \ + --decode-input-len "$DECODE_INPUT_LEN" \ + --decode-output-len "$DECODE_OUTPUT_LEN" \ + --trust-remote-code \ + --enforce-eager + +echo "[4/6] vLLM mapping-formal analysis" +docker exec sglang_bbuf bash -lc "cd '$SCRIPT_DIR' && python3 analyze_llm_torch_profile.py --framework vllm --mapping-input '$VLLM_MAPPING_DIR' --formal-input '$VLLM_FORMAL_DIR' > '$VLLM_ANALYSIS'" + +echo "[5/6] TensorRT-LLM formal + mapping captures" +HF_TOKEN="$HF_TOKEN" HUGGINGFACE_HUB_TOKEN="$HUGGINGFACE_HUB_TOKEN" \ + "$SCRIPT_DIR/run_trtllm_pytorch_profile_host.sh" \ + --model "$MODEL" \ + --run-dir "$TRT_FORMAL_DIR" \ + --stage prefill \ + --port "$TRT_FORMAL_PREFILL_PORT" \ + --gpus "$GPUS" \ + --tp-size "$TP_SIZE" \ + --kv-fraction "$KV_FRACTION" \ + --input-len "$PREFILL_INPUT_LEN" \ + --output-len "$PREFILL_OUTPUT_LEN" \ + --override-py-executor "$TRT_OVERRIDE_PATH" \ + --trust-remote-code +HF_TOKEN="$HF_TOKEN" HUGGINGFACE_HUB_TOKEN="$HUGGINGFACE_HUB_TOKEN" \ + "$SCRIPT_DIR/run_trtllm_pytorch_profile_host.sh" \ + --model "$MODEL" \ + --run-dir "$TRT_FORMAL_DIR" \ + --stage decode \ + --port "$TRT_FORMAL_DECODE_PORT" \ + --gpus "$GPUS" \ + --tp-size "$TP_SIZE" \ + --kv-fraction "$KV_FRACTION" \ + --input-len "$DECODE_INPUT_LEN" \ + --output-len "$DECODE_OUTPUT_LEN" \ + --override-py-executor "$TRT_OVERRIDE_PATH" \ + --trust-remote-code +HF_TOKEN="$HF_TOKEN" HUGGINGFACE_HUB_TOKEN="$HUGGINGFACE_HUB_TOKEN" \ + "$SCRIPT_DIR/run_trtllm_pytorch_profile_host.sh" \ + --model "$MODEL" \ + --run-dir "$TRT_MAPPING_DIR" \ + --stage prefill \ + --port "$TRT_MAPPING_PREFILL_PORT" \ + --gpus "$GPUS" \ + --tp-size "$TP_SIZE" \ + --kv-fraction "$KV_FRACTION" \ + --input-len "$PREFILL_INPUT_LEN" \ + --output-len "$PREFILL_OUTPUT_LEN" \ + --override-py-executor "$TRT_OVERRIDE_PATH" \ + --disable-cudagraph \ + --trust-remote-code +HF_TOKEN="$HF_TOKEN" HUGGINGFACE_HUB_TOKEN="$HUGGINGFACE_HUB_TOKEN" \ + "$SCRIPT_DIR/run_trtllm_pytorch_profile_host.sh" \ + --model "$MODEL" \ + --run-dir "$TRT_MAPPING_DIR" \ + --stage decode \ + --port "$TRT_MAPPING_DECODE_PORT" \ + --gpus "$GPUS" \ + --tp-size "$TP_SIZE" \ + --kv-fraction "$KV_FRACTION" \ + --input-len "$DECODE_INPUT_LEN" \ + --output-len "$DECODE_OUTPUT_LEN" \ + --override-py-executor "$TRT_OVERRIDE_PATH" \ + --disable-cudagraph \ + --trust-remote-code + +echo "[6/6] TensorRT-LLM mapping-formal analysis" +docker exec sglang_bbuf bash -lc "cd '$SCRIPT_DIR' && python3 analyze_llm_torch_profile.py --framework trtllm --mapping-input '$TRT_MAPPING_DIR' --formal-input '$TRT_FORMAL_DIR' > '$TRT_ANALYSIS'" + +echo "MODEL_ROOT=$MODEL_ROOT" +echo "ANALYSIS_SGLANG=$SGLANG_ANALYSIS" +echo "ANALYSIS_VLLM=$VLLM_ANALYSIS" +echo "ANALYSIS_TRTLLM=$TRT_ANALYSIS" diff --git a/.claude/skills/llm-torch-profiler-analysis/scripts/run_sglang_torch_profile_host.sh b/.claude/skills/llm-torch-profiler-analysis/scripts/run_sglang_torch_profile_host.sh new file mode 100755 index 000000000..c22af0e57 --- /dev/null +++ b/.claude/skills/llm-torch-profiler-analysis/scripts/run_sglang_torch_profile_host.sh @@ -0,0 +1,241 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: + run_sglang_torch_profile_host.sh \ + --model Qwen/Qwen3-8B \ + --run-dir /data/bbuf/validate/unified_llm_profiler_skill/runs/example_sglang \ + --port 30088 \ + --gpus 0 + + run_sglang_torch_profile_host.sh \ + --model openai/gpt-oss-20b \ + --run-dir /data/bbuf/validate/unified_llm_profiler_skill/runs/example_sglang_4gpu \ + --port 30088 \ + --gpus 2,3,4,5 \ + --tp-size 4 + +Options: + --model TEXT Model id or local path for SGLang. + --run-dir PATH Shared /data directory for logs and traces. + --port INT Server port. + --gpus TEXT CUDA_VISIBLE_DEVICES value, for example 0 or 2,3,4,5. + --gpu TEXT Alias for --gpus. + --tp-size INT Tensor parallel size. Defaults to the visible GPU count. + --trust-remote-code Pass --trust-remote-code. + --mem-fraction FLOAT SGLang static memory fraction. + --request-max-tokens INT Generation length for the probe request. + --prompt TEXT Probe prompt. + --warmup-steps INT Warmup steps before profiling. Defaults to 10. + --profile-workload TEXT legacy|prefill|decode|both. Defaults to both. + --prefill-input-len INT Synthetic prefill prompt length. Defaults to 4090. + --prefill-output-len INT Synthetic prefill output length. Defaults to 1. + --decode-input-len INT Synthetic decode prompt length. Defaults to 1. + --decode-output-len INT Synthetic decode output length. Defaults to 2048. + --repo-dir PATH SGLang repo path inside `sglang_bbuf`. + --server-extra TEXT Extra args appended to launch_server. + --help Show this message. + +Notes: + - Run this on the H100 host. It uses `docker exec sglang_bbuf`. + - The server is launched first, then the profiler capture runs with + stage-separated prefill/decode workloads and `--profile-by-stage`. + - A small benchmark summary is written after profiling. +EOF +} + +MODEL="" +RUN_DIR="" +PORT="" +GPUS="" +TP_SIZE="" +TRUST_REMOTE_CODE=0 +MEM_FRACTION=0.85 +REQUEST_MAX_TOKENS=12 +PROMPT="Explain the difference between CUDA graph mode and eager mode in two sentences." +WARMUP_STEPS=10 +PROFILE_WORKLOAD="both" +PREFILL_INPUT_LEN=4090 +PREFILL_OUTPUT_LEN=1 +DECODE_INPUT_LEN=1 +DECODE_OUTPUT_LEN=2048 +SGLANG_REPO_DIR="${SGLANG_REPO_DIR:-/data/bbuf/repos/sglang}" +SERVER_EXTRA="" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +while [[ $# -gt 0 ]]; do + case "$1" in + --model) + MODEL="$2" + shift 2 + ;; + --run-dir) + RUN_DIR="$2" + shift 2 + ;; + --port) + PORT="$2" + shift 2 + ;; + --gpu) + GPUS="$2" + shift 2 + ;; + --gpus) + GPUS="$2" + shift 2 + ;; + --tp-size) + TP_SIZE="$2" + shift 2 + ;; + --trust-remote-code) + TRUST_REMOTE_CODE=1 + shift + ;; + --mem-fraction) + MEM_FRACTION="$2" + shift 2 + ;; + --request-max-tokens) + REQUEST_MAX_TOKENS="$2" + shift 2 + ;; + --prompt) + PROMPT="$2" + shift 2 + ;; + --warmup-steps) + WARMUP_STEPS="$2" + shift 2 + ;; + --profile-workload) + PROFILE_WORKLOAD="$2" + shift 2 + ;; + --prefill-input-len) + PREFILL_INPUT_LEN="$2" + shift 2 + ;; + --prefill-output-len) + PREFILL_OUTPUT_LEN="$2" + shift 2 + ;; + --decode-input-len) + DECODE_INPUT_LEN="$2" + shift 2 + ;; + --decode-output-len) + DECODE_OUTPUT_LEN="$2" + shift 2 + ;; + --repo-dir) + SGLANG_REPO_DIR="$2" + shift 2 + ;; + --server-extra) + SERVER_EXTRA="$2" + shift 2 + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ -z "$MODEL" || -z "$RUN_DIR" || -z "$PORT" || -z "$GPUS" ]]; then + usage >&2 + exit 2 +fi + +IFS=',' read -r -a GPU_LIST <<< "$GPUS" +GPU_COUNT="${#GPU_LIST[@]}" +if [[ "$GPU_COUNT" -lt 1 ]]; then + echo "Could not parse --gpus: $GPUS" >&2 + exit 2 +fi +if [[ -z "$TP_SIZE" ]]; then + TP_SIZE="$GPU_COUNT" +fi +if (( TP_SIZE < 1 || TP_SIZE > GPU_COUNT )); then + echo "--tp-size must be between 1 and the visible GPU count ($GPU_COUNT)." >&2 + exit 2 +fi + +LOG_PATH="$RUN_DIR/sglang_server.log" +ANALYSIS_PATH="$RUN_DIR/analysis_sglang.txt" +PROFILE_ROOT="$RUN_DIR/sglang_profile_live" +BENCHMARK_PATH="$RUN_DIR/benchmark_sglang.json" +PID_PATH="$RUN_DIR/sglang_server.pid" +LAUNCH_PATTERN="[s]glang.launch_server.*--port $PORT" +SERVER_ARGS="python3 -m sglang.launch_server --model-path \"$MODEL\" --port \"$PORT\" --tp-size \"$TP_SIZE\" --mem-fraction-static \"$MEM_FRACTION\"" + +if [[ "$TRUST_REMOTE_CODE" -eq 1 ]]; then + SERVER_ARGS="$SERVER_ARGS --trust-remote-code" +fi +if [[ -n "$SERVER_EXTRA" ]]; then + SERVER_ARGS="$SERVER_ARGS $SERVER_EXTRA" +fi + +docker exec sglang_bbuf bash -lc "mkdir -p '$RUN_DIR' '$PROFILE_ROOT'" +docker exec sglang_bbuf bash -lc "pkill -f '$LAUNCH_PATTERN' >/dev/null 2>&1 || true" +docker exec sglang_bbuf bash -lc "mkdir -p '$RUN_DIR' '$PROFILE_ROOT' && cd '$SGLANG_REPO_DIR' && rm -f '$PID_PATH' && (CUDA_VISIBLE_DEVICES=$GPUS PYTHONPATH=python nohup $SERVER_ARGS > '$LOG_PATH' 2>&1 < /dev/null & echo \$! > '$PID_PATH')" + +cleanup() { + docker exec sglang_bbuf bash -lc "pkill -f '$LAUNCH_PATTERN' >/dev/null 2>&1 || true" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +ready=0 +for _ in $(seq 1 180); do + if curl -sf "http://127.0.0.1:${PORT}/v1/models" >/dev/null; then + ready=1 + break + fi + sleep 2 +done +if [[ "$ready" -ne 1 ]]; then + echo "SGLang server did not become ready on port ${PORT}. Recent logs:" >&2 + ssh_log=$(docker exec sglang_bbuf bash -lc "tail -n 120 '$LOG_PATH'" 2>/dev/null || true) + printf '%s\n' "$ssh_log" >&2 + exit 1 +fi + +python3 - < '$ANALYSIS_PATH'" +python3 "$SCRIPT_DIR/probe_llm_server.py" \ + --framework sglang \ + --url "http://127.0.0.1:${PORT}" \ + | docker exec -i sglang_bbuf bash -lc "cat > '$BENCHMARK_PATH'" >/dev/null +docker exec sglang_bbuf bash -lc "sed -n '1,240p' '$ANALYSIS_PATH'" +echo "BENCHMARK_PATH=$BENCHMARK_PATH" diff --git a/.claude/skills/llm-torch-profiler-analysis/scripts/run_trtllm_pytorch_profile_host.sh b/.claude/skills/llm-torch-profiler-analysis/scripts/run_trtllm_pytorch_profile_host.sh new file mode 100755 index 000000000..1e3b51e1d --- /dev/null +++ b/.claude/skills/llm-torch-profiler-analysis/scripts/run_trtllm_pytorch_profile_host.sh @@ -0,0 +1,407 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: + run_trtllm_pytorch_profile_host.sh \ + --model Qwen/Qwen3-8B \ + --run-dir /data/bbuf/validate/unified_llm_profiler_skill/runs/example \ + --stage prefill \ + --port 32188 \ + --gpus 0 + + run_trtllm_pytorch_profile_host.sh \ + --model openai/gpt-oss-20b \ + --run-dir /data/bbuf/validate/unified_llm_profiler_skill/runs/example_4gpu \ + --stage prefill \ + --port 32188 \ + --gpus 2,3,4,5 \ + --tp-size 4 + +Options: + --model TEXT Hugging Face model id. + --run-dir PATH Shared /data run directory for logs and traces. + --stage prefill|decode Capture window. Prefill profiles 4090->1 by + default; decode profiles 1->2048 by default. + --port INT Host port for trtllm-serve. + --gpus TEXT CUDA_VISIBLE_DEVICES value, for example 0 or 2,3,4,5. + --gpu TEXT Alias for --gpus. + --tp-size INT Tensor parallel size. Defaults to the visible GPU count. + --image TEXT Container image. + --shared-root PATH Shared validation root mounted into the container. + --hf-cache PATH Host Hugging Face cache path. + --override-py-executor PATH Optional py_executor.py override path. + --disable-cudagraph Generate/use a YAML override with cuda_graph_config: null. + --input-len INT Synthetic prompt length for this stage. + Defaults: prefill 4090, decode 1. + --request-max-tokens INT Generation length for this stage. + Defaults: prefill 1, decode 2048. + --output-len INT Alias for --request-max-tokens. + --prompt TEXT Probe prompt. Defaults to a synthetic prompt + sized by --input-len. + --warmup-steps INT Warmup steps before the profiler window. Defaults to 10. + --active-steps INT Active profiler steps to capture. Defaults to 5. + --max-seq-len INT Serve max sequence length. + --kv-fraction FLOAT KV cache free GPU memory fraction. + --container-name TEXT Override container name. + --trust-remote-code Pass --trust_remote_code to trtllm-serve. + --help Show this message. + +Environment: + HF_TOKEN or HUGGINGFACE_HUB_TOKEN must be set. + +Notes: + - Run this on the H100 host, not inside `sglang_bbuf`. + - It always pins TensorRT-LLM to `--backend pytorch`. + - The default image tag is floating; record the resolved TensorRT-LLM version + in the run manifest and pass --image for reproducible validation. + - Profiling uses `TLLM_PROFILE_START_STOP` and `TLLM_TORCH_PROFILE_TRACE`. + - For Python-location recovery, prefer a `py_executor.py` override with `with_stack=True`. + - A small benchmark summary is written after the trace is emitted. +EOF +} + +IMAGE="nvcr.io/nvidia/tensorrt-llm/release:latest" +SHARED_ROOT="/data/bbuf/validate/unified_llm_profiler_skill" +HF_CACHE="/data/.cache/huggingface" +OVERRIDE_PY_EXECUTOR="" +DISABLE_CUDAGRAPH=0 +REQUEST_MAX_TOKENS="" +INPUT_LEN="" +PROMPT="" +WARMUP_STEPS=10 +ACTIVE_STEPS=5 +MAX_SEQ_LEN=4096 +KV_FRACTION=0.85 +CONTAINER_NAME="" +TRUST_REMOTE_CODE=0 +TP_SIZE="" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +MODEL="" +RUN_DIR="" +STAGE="" +PORT="" +GPUS="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --model) + MODEL="$2" + shift 2 + ;; + --run-dir) + RUN_DIR="$2" + shift 2 + ;; + --stage) + STAGE="$2" + shift 2 + ;; + --port) + PORT="$2" + shift 2 + ;; + --gpu) + GPUS="$2" + shift 2 + ;; + --gpus) + GPUS="$2" + shift 2 + ;; + --tp-size) + TP_SIZE="$2" + shift 2 + ;; + --image) + IMAGE="$2" + shift 2 + ;; + --shared-root) + SHARED_ROOT="$2" + shift 2 + ;; + --hf-cache) + HF_CACHE="$2" + shift 2 + ;; + --override-py-executor) + OVERRIDE_PY_EXECUTOR="$2" + shift 2 + ;; + --disable-cudagraph) + DISABLE_CUDAGRAPH=1 + shift + ;; + --input-len) + INPUT_LEN="$2" + shift 2 + ;; + --request-max-tokens) + REQUEST_MAX_TOKENS="$2" + shift 2 + ;; + --output-len) + REQUEST_MAX_TOKENS="$2" + shift 2 + ;; + --prompt) + PROMPT="$2" + shift 2 + ;; + --warmup-steps) + WARMUP_STEPS="$2" + shift 2 + ;; + --active-steps) + ACTIVE_STEPS="$2" + shift 2 + ;; + --max-seq-len) + MAX_SEQ_LEN="$2" + shift 2 + ;; + --kv-fraction) + KV_FRACTION="$2" + shift 2 + ;; + --container-name) + CONTAINER_NAME="$2" + shift 2 + ;; + --trust-remote-code) + TRUST_REMOTE_CODE=1 + shift + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ -z "${HF_TOKEN:-}" && -z "${HUGGINGFACE_HUB_TOKEN:-}" ]]; then + echo "Set HF_TOKEN or HUGGINGFACE_HUB_TOKEN before running." >&2 + exit 2 +fi +if [[ -z "${HF_TOKEN:-}" ]]; then + HF_TOKEN="$HUGGINGFACE_HUB_TOKEN" +fi +if [[ -z "${HUGGINGFACE_HUB_TOKEN:-}" ]]; then + HUGGINGFACE_HUB_TOKEN="$HF_TOKEN" +fi + +if [[ -z "$MODEL" || -z "$RUN_DIR" || -z "$STAGE" || -z "$PORT" || -z "$GPUS" ]]; then + usage >&2 + exit 2 +fi + +IFS=',' read -r -a GPU_LIST <<< "$GPUS" +GPU_COUNT="${#GPU_LIST[@]}" +if [[ "$GPU_COUNT" -lt 1 ]]; then + echo "Could not parse --gpus: $GPUS" >&2 + exit 2 +fi +if [[ -z "$TP_SIZE" ]]; then + TP_SIZE="$GPU_COUNT" +fi +if (( TP_SIZE < 1 || TP_SIZE > GPU_COUNT )); then + echo "--tp-size must be between 1 and the visible GPU count ($GPU_COUNT)." >&2 + exit 2 +fi + +case "$STAGE" in + prefill) + TRACE_PATH="$RUN_DIR/trace-prefill.json" + LOG_PATH="$RUN_DIR/server-prefill.log" + BENCHMARK_PATH="$RUN_DIR/benchmark-prefill.json" + if [[ -z "$INPUT_LEN" ]]; then + INPUT_LEN=4090 + fi + if [[ -z "$REQUEST_MAX_TOKENS" ]]; then + REQUEST_MAX_TOKENS=1 + fi + ;; + decode) + TRACE_PATH="$RUN_DIR/trace-decode.json" + LOG_PATH="$RUN_DIR/server-decode.log" + BENCHMARK_PATH="$RUN_DIR/benchmark-decode.json" + if [[ -z "$INPUT_LEN" ]]; then + INPUT_LEN=1 + fi + if [[ -z "$REQUEST_MAX_TOKENS" ]]; then + REQUEST_MAX_TOKENS=2048 + fi + ;; + *) + echo "--stage must be prefill or decode." >&2 + exit 2 + ;; +esac + +if (( WARMUP_STEPS < 0 || ACTIVE_STEPS < 1 )); then + echo "--warmup-steps must be >= 0 and --active-steps must be >= 1." >&2 + exit 2 +fi + +case "$STAGE" in + prefill) + profile_start=$((WARMUP_STEPS + 1)) + ;; + decode) + profile_start=$((WARMUP_STEPS + 2)) + ;; +esac +profile_stop=$((profile_start + ACTIVE_STEPS - 1)) +PROFILE_START_STOP="${profile_start}-${profile_stop}" + +if [[ -z "$CONTAINER_NAME" ]]; then + model_slug="${MODEL##*/}" + model_slug="${model_slug//\//-}" + model_slug="${model_slug//./-}" + model_slug="${model_slug//_/-}" + model_slug="${model_slug// /-}" + gpu_slug="${GPUS//,/-}" + CONTAINER_NAME="trtllm-${model_slug}-${STAGE}-g${gpu_slug}-p${PORT}" +fi + +EXTRA_LLM_OPTIONS="" +if [[ "$DISABLE_CUDAGRAPH" -eq 1 ]]; then + EXTRA_CFG_PATH="$SHARED_ROOT/tmp/trt_no_cudagraph.yaml" + docker exec sglang_bbuf bash -lc "mkdir -p '$(dirname "$EXTRA_CFG_PATH")' && printf 'cuda_graph_config: null\n' > '$EXTRA_CFG_PATH'" + EXTRA_LLM_OPTIONS="--extra_llm_api_options $EXTRA_CFG_PATH" +fi + +docker exec sglang_bbuf bash -lc "mkdir -p '$RUN_DIR'" +docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true + +docker_args=( + run -d --rm + --name "$CONTAINER_NAME" + --gpus all + --ipc=host + --network host + --entrypoint bash + -e "CUDA_VISIBLE_DEVICES=$GPUS" + -e "HF_TOKEN=$HF_TOKEN" + -e "HUGGINGFACE_HUB_TOKEN=$HUGGINGFACE_HUB_TOKEN" + -e "TLLM_PROFILE_START_STOP=$PROFILE_START_STOP" + -e "TLLM_LLMAPI_ENABLE_NVTX=1" + -e "TLLM_TORCH_PROFILE_TRACE=$TRACE_PATH" + -e "RUN_DIR=$RUN_DIR" + -e "LOG_PATH=$LOG_PATH" + -e "MODEL_ID=$MODEL" + -e "SERVE_PORT=$PORT" + -v "$HF_CACHE:/root/.cache/huggingface" + -v "$SHARED_ROOT:$SHARED_ROOT" +) + +if [[ -n "$OVERRIDE_PY_EXECUTOR" ]]; then + docker_args+=( + -v "$OVERRIDE_PY_EXECUTOR:/usr/local/lib/python3.12/dist-packages/tensorrt_llm/_torch/pyexecutor/py_executor.py:ro" + ) +fi + +trust_remote_code_arg="" +if [[ "$TRUST_REMOTE_CODE" -eq 1 ]]; then + trust_remote_code_arg="--trust_remote_code" +fi + +container_cmd=$( + cat < "$LOG_PATH" 2>&1 +EOF +) + +docker_args+=("$IMAGE" -lc "$container_cmd") +docker "${docker_args[@]}" >/dev/null + +cleanup() { + docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +ready=0 +for _ in $(seq 1 180); do + if curl -sf "http://127.0.0.1:${PORT}/v1/models" >/dev/null; then + ready=1 + break + fi + sleep 2 +done +if [[ "$ready" -ne 1 ]]; then + echo "Server did not become ready on port ${PORT}. Recent logs:" >&2 + docker logs "$CONTAINER_NAME" 2>&1 | tail -n 120 >&2 || true + exit 1 +fi + +python3 - <&2 + exit 1 +fi + +python3 "$SCRIPT_DIR/probe_llm_server.py" \ + --framework trtllm \ + --url "http://127.0.0.1:${PORT}" \ + --model "$MODEL" \ + | docker exec -i sglang_bbuf bash -lc "cat > '$BENCHMARK_PATH'" >/dev/null + +echo "TRACE_PATH=$TRACE_PATH" +echo "LOG_PATH=$LOG_PATH" +echo "BENCHMARK_PATH=$BENCHMARK_PATH" diff --git a/.claude/skills/llm-torch-profiler-analysis/scripts/run_vllm_torch_profile_host.sh b/.claude/skills/llm-torch-profiler-analysis/scripts/run_vllm_torch_profile_host.sh new file mode 100755 index 000000000..c21d014dd --- /dev/null +++ b/.claude/skills/llm-torch-profiler-analysis/scripts/run_vllm_torch_profile_host.sh @@ -0,0 +1,343 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: + run_vllm_torch_profile_host.sh \ + --model Qwen/Qwen3-8B \ + --run-dir /data/bbuf/validate/unified_llm_profiler_skill/runs/example_vllm_formal \ + --port 31088 \ + --gpus 1 + + run_vllm_torch_profile_host.sh \ + --model openai/gpt-oss-20b \ + --run-dir /data/bbuf/validate/unified_llm_profiler_skill/runs/example_vllm_4gpu \ + --port 31088 \ + --gpus 2,3,4,5 \ + --tensor-parallel-size 4 + +Options: + --model TEXT Hugging Face model id. + --run-dir PATH Shared /data directory for logs and traces. + --port INT Host port for vllm serve. + --gpus TEXT CUDA_VISIBLE_DEVICES value, for example 1 or 2,3,4,5. + --gpu TEXT Alias for --gpus. + --image TEXT Container image. + --hf-cache PATH Host Hugging Face cache path. + --gpu-memory-util FLOAT vLLM --gpu-memory-utilization. + --max-model-len INT vLLM --max-model-len. + --tensor-parallel-size INT vLLM --tensor-parallel-size. Defaults to the visible GPU count. + --profiler-active-iterations INT + Torch-profiler active iterations. + --enforce-eager Launch vLLM with --enforce-eager for mapping traces. + --trust-remote-code Pass --trust-remote-code. + --request-max-tokens INT Generation length for the probe request. + --prompt TEXT Probe prompt. + --warmup-steps INT Warmup steps before profiling. Defaults to 10. + --profile-workload TEXT legacy|prefill|decode|both. Defaults to both. + --prefill-input-len INT Synthetic prefill prompt length. Defaults to 4090. + --prefill-output-len INT Synthetic prefill output length. Defaults to 1. + --decode-input-len INT Synthetic decode prompt length. Defaults to 1. + --decode-output-len INT Synthetic decode output length. Defaults to 2048. + --container-name TEXT Override container name. + --help Show this message. + +Environment: + HF_TOKEN or HUGGINGFACE_HUB_TOKEN must be set. + +Notes: + - Run this on the H100 host, not inside `sglang_bbuf`. + - This uses the vLLM torch-profiler flow: `--profiler-config`, then POST + `/start_profile` and `/stop_profile`. + - Default capture is two labeled profiles: prefill 4090->1 and decode 1->2048. + - Current vLLM profiler config already defaults `torch_profiler_with_stack=true`. + - A small benchmark summary is written after profiling. +EOF +} + +IMAGE="vllm/vllm-openai:latest" +HF_CACHE="/data/.cache/huggingface" +GPU_MEMORY_UTIL=0.90 +MAX_MODEL_LEN=4096 +TP_SIZE="" +ENFORCE_EAGER=0 +TRUST_REMOTE_CODE=0 +REQUEST_MAX_TOKENS=12 +PROFILER_ACTIVE_ITERATIONS=5 +PROMPT="Explain the difference between CUDA graph mode and eager mode in two sentences." +WARMUP_STEPS=10 +PROFILE_WORKLOAD="both" +PREFILL_INPUT_LEN=4090 +PREFILL_OUTPUT_LEN=1 +DECODE_INPUT_LEN=1 +DECODE_OUTPUT_LEN=2048 +CONTAINER_NAME="" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +MODEL="" +RUN_DIR="" +PORT="" +GPUS="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --model) + MODEL="$2" + shift 2 + ;; + --run-dir) + RUN_DIR="$2" + shift 2 + ;; + --port) + PORT="$2" + shift 2 + ;; + --gpu) + GPUS="$2" + shift 2 + ;; + --gpus) + GPUS="$2" + shift 2 + ;; + --image) + IMAGE="$2" + shift 2 + ;; + --hf-cache) + HF_CACHE="$2" + shift 2 + ;; + --gpu-memory-util) + GPU_MEMORY_UTIL="$2" + shift 2 + ;; + --max-model-len) + MAX_MODEL_LEN="$2" + shift 2 + ;; + --tensor-parallel-size) + TP_SIZE="$2" + shift 2 + ;; + --profiler-active-iterations) + PROFILER_ACTIVE_ITERATIONS="$2" + shift 2 + ;; + --enforce-eager) + ENFORCE_EAGER=1 + shift + ;; + --trust-remote-code) + TRUST_REMOTE_CODE=1 + shift + ;; + --request-max-tokens) + REQUEST_MAX_TOKENS="$2" + shift 2 + ;; + --prompt) + PROMPT="$2" + shift 2 + ;; + --warmup-steps) + WARMUP_STEPS="$2" + shift 2 + ;; + --profile-workload) + PROFILE_WORKLOAD="$2" + shift 2 + ;; + --prefill-input-len) + PREFILL_INPUT_LEN="$2" + shift 2 + ;; + --prefill-output-len) + PREFILL_OUTPUT_LEN="$2" + shift 2 + ;; + --decode-input-len) + DECODE_INPUT_LEN="$2" + shift 2 + ;; + --decode-output-len) + DECODE_OUTPUT_LEN="$2" + shift 2 + ;; + --container-name) + CONTAINER_NAME="$2" + shift 2 + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ -z "${HF_TOKEN:-}" && -z "${HUGGINGFACE_HUB_TOKEN:-}" ]]; then + echo "Set HF_TOKEN or HUGGINGFACE_HUB_TOKEN before running." >&2 + exit 2 +fi +if [[ -z "${HF_TOKEN:-}" ]]; then + HF_TOKEN="$HUGGINGFACE_HUB_TOKEN" +fi +if [[ -z "${HUGGINGFACE_HUB_TOKEN:-}" ]]; then + HUGGINGFACE_HUB_TOKEN="$HF_TOKEN" +fi + +if [[ -z "$MODEL" || -z "$RUN_DIR" || -z "$PORT" || -z "$GPUS" ]]; then + usage >&2 + exit 2 +fi + +IFS=',' read -r -a GPU_LIST <<< "$GPUS" +GPU_COUNT="${#GPU_LIST[@]}" +if [[ "$GPU_COUNT" -lt 1 ]]; then + echo "Could not parse --gpus: $GPUS" >&2 + exit 2 +fi +if [[ -z "$TP_SIZE" ]]; then + TP_SIZE="$GPU_COUNT" +fi +if (( TP_SIZE < 1 || TP_SIZE > GPU_COUNT )); then + echo "--tensor-parallel-size must be between 1 and the visible GPU count ($GPU_COUNT)." >&2 + exit 2 +fi +if (( PROFILER_ACTIVE_ITERATIONS < 1 )); then + echo "--profiler-active-iterations must be >= 1." >&2 + exit 2 +fi + +PROFILE_DIR="$RUN_DIR/vllm_profile" +LOG_PATH="$RUN_DIR/server.log" +ANALYSIS_PATH="$RUN_DIR/analysis_vllm_live.txt" +BENCHMARK_PATH="$RUN_DIR/benchmark_vllm.json" + +if [[ -z "$CONTAINER_NAME" ]]; then + model_slug="${MODEL##*/}" + model_slug="${model_slug//\//-}" + model_slug="${model_slug//./-}" + model_slug="${model_slug//_/-}" + gpu_slug="${GPUS//,/-}" + CONTAINER_NAME="vllm-${model_slug}-g${gpu_slug}-p${PORT}" + if [[ "$ENFORCE_EAGER" -eq 1 ]]; then + CONTAINER_NAME="${CONTAINER_NAME}-eager" + fi +fi + +docker exec sglang_bbuf bash -lc "mkdir -p '$PROFILE_DIR'" +docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true + +profiler_config=$(python3 - </dev/null +cleanup() { + docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +ready=0 +for _ in $(seq 1 180); do + if curl -sf "http://127.0.0.1:${PORT}/v1/models" >/dev/null; then + ready=1 + break + fi + sleep 2 +done +if [[ "$ready" -ne 1 ]]; then + echo "Server did not become ready on port ${PORT}. Recent logs:" >&2 + docker logs "$CONTAINER_NAME" 2>&1 | tail -n 120 >&2 || true + exit 1 +fi + +python3 "$SCRIPT_DIR/analyze_llm_torch_profile.py" \ + --framework vllm \ + --url "http://127.0.0.1:${PORT}" \ + --output-dir "$PROFILE_DIR" \ + --num-steps "$PROFILER_ACTIVE_ITERATIONS" \ + --warmup-steps "$WARMUP_STEPS" \ + --probe-requests 1 \ + --no-profile-by-stage \ + --profile-workload "$PROFILE_WORKLOAD" \ + --probe-prompt "$PROMPT" \ + --probe-max-new-tokens "$REQUEST_MAX_TOKENS" \ + --prefill-input-len "$PREFILL_INPUT_LEN" \ + --prefill-output-len "$PREFILL_OUTPUT_LEN" \ + --decode-input-len "$DECODE_INPUT_LEN" \ + --decode-output-len "$DECODE_OUTPUT_LEN" \ + > "$ANALYSIS_PATH" + +profile_found=0 +for _ in $(seq 1 240); do + if find "$PROFILE_DIR" -type f \( -name '*.pt.trace.json' -o -name '*.pt.trace.json.gz' -o -name '*.trace.json' -o -name '*.trace.json.gz' \) | grep -q .; then + profile_found=1 + break + fi + sleep 2 +done +if [[ "$profile_found" -ne 1 ]]; then + echo "No vLLM profiler traces appeared under $PROFILE_DIR" >&2 + docker logs "$CONTAINER_NAME" 2>&1 | tail -n 120 >&2 || true + exit 1 +fi + +python3 "$SCRIPT_DIR/probe_llm_server.py" \ + --framework vllm \ + --url "http://127.0.0.1:${PORT}" \ + --model "$MODEL" \ + | docker exec -i sglang_bbuf bash -lc "cat > '$BENCHMARK_PATH'" >/dev/null + +docker logs "$CONTAINER_NAME" 2>&1 | docker exec -i sglang_bbuf bash -lc "cat > '$LOG_PATH'" || true +sed -n '1,240p' "$ANALYSIS_PATH" +echo "PROFILE_DIR=$PROFILE_DIR" +echo "LOG_PATH=$LOG_PATH" +echo "ANALYSIS_PATH=$ANALYSIS_PATH" +echo "BENCHMARK_PATH=$BENCHMARK_PATH" diff --git a/.claude/skills/sglang-auto-benchmark/SKILL.md b/.claude/skills/sglang-auto-benchmark/SKILL.md deleted file mode 100644 index 9fef9a571..000000000 --- a/.claude/skills/sglang-auto-benchmark/SKILL.md +++ /dev/null @@ -1,496 +0,0 @@ ---- -name: sglang-auto-benchmark -description: Run SGLang auto benchmark searches with tiered server-flag sweeps, canonical dataset preparation, ShareGPT auto-download, custom-data conversion/validation, SLA or fixed-QPS benchmarking, CSV export, and optional second-stage speculative/EAGLE tuning. Use when the user wants an AI-operated benchmark workflow rather than a one-off bench_serving command. ---- - -# SGLang Auto Benchmark - -This skill is for repeatable, AI-driven SGLang performance tuning. - -The preferred workflow is: -- start from a mostly pure-TP baseline command, -- move the rest of the performance knobs into `search_space`, -- let auto benchmark search and compare candidates under the target SLA. - -The implementation lives in: -- `python -m sglang.auto_benchmark` -- canonical dataset loader in `python -m sglang.bench_serving --dataset-name autobench` -- cookbook-derived LLM reference configs in `.claude/skills/sglang-auto-benchmark/references/cookbook-llm/` - -## Preconditions - -- SGLang can already launch and serve the target model in this environment. -- The model path exists, or the model is otherwise launchable. -- The goal is clear: - - benchmark a fixed QPS list, or - - search the maximum QPS that satisfies `max_ttft_ms` / `max_tpot_ms`. - -If those are not true yet, fix them before running a large search. - -Environment consistency check: -- if the benchmark will run from a remote repo copy or ad-hoc synced workspace, - verify that the remote `python/sglang/bench_serving.py` matches the local - feature level needed by auto benchmark before launching a long run -- at minimum, run a preflight such as `PYTHONPATH=/python python3 -m - sglang.bench_serving --help` and confirm that the dataset choices include - `autobench` -- if `autobench` is missing remotely, do not start the benchmark; sync - `python/sglang/bench_serving.py` and any required dataset modules first - -## Remote Run Logging - -If the benchmark is executed on a remote machine, the progress bar output must be -mirrored back to a local file for humans to watch. - -Scope note: -- use the remote-log mirroring workflow only when the benchmark is running in a - different machine or a different remote container than the one the agent is - actively operating in -- if the agent itself is already running inside the target container where auto - benchmark is executing, do not add a separate log-return loop just for parity; - inspect the live log files and result files directly in the current container -- in other words, "remote container" needs mirrored local logs, while "current - container" should use direct local inspection - -Required behavior: -- start the remote run with a persistent terminal/session log, for example with - `script -q -f -c ""`; on Linux containers that use util-linux - `script`, prefer the explicit `-c` form instead of BSD-style positional - command arguments -- continuously sync a cleaned version of that remote session log back to a local - `progress.log`; this local `progress.log` should already have terminal control - sequences removed, because `script` + `tqdm` progress bars will otherwise leave - ANSI cursor-control bytes and carriage-return redraws that look like garbled text -- if the benchmark itself is executed inside a remote container, the cleaned local - `progress.log` must be refreshed automatically at least once every 30 - seconds while the run is active; do not rely on one-off manual polling -- implement the sync loop as a dedicated local script file checked into neither - git nor the benchmark config; avoid fragile one-line `nohup zsh -lc '...'` - command strings with heavy nested quoting -- prefer running the sync loop inside a long-lived local session such as a - dedicated `tmux` pane, `screen`, or the agent's own persistent PTY session; - detached child processes started from short-lived command runners can be - reaped unexpectedly, so plain `nohup ... &` is not the most stable default -- immediately after starting the sync loop, verify that `progress.log` is - actually updating by checking its timestamp or size twice across a short wait; - if it is not changing, treat that as a broken sync setup and fix it before - telling the user that live log mirroring is working -- tell the user the local log path up front -- keep final result files synced back locally after the run ends -- when scenario-level or top-level markdown summaries are produced, sync those - `summary.md` / `SUMMARY.md` files back locally as first-class result artifacts - rather than leaving them only on the remote machine - -This is important because long searches can run for hours, and people need a -stable local file they can tail without logging into the remote box. The final -local run folder should also be self-contained enough for someone to review the -benchmark outcome without re-entering the remote environment. - -Recommended cleanup pipeline for the local mirrored log: - -```bash -perl -pe 's/\e\[[0-9;?]*[ -\/]*[@-~]//g; s/\r/\n/g; s/\x08//g;' raw_progress.log \ - > progress.log -``` - -Recommended remote-container sync pattern: - -```bash -cat > sync_progress.sh <<'EOF' -#!/bin/zsh -set -euo pipefail -while true; do - ssh "tail -n 200 " > raw_progress.log - perl -pe 's/\e\[[0-9;?]*[ -\/]*[@-~]//g; s/\r/\n/g; s/\x08//g;' raw_progress.log \ - > progress.log - sleep 15 -done -EOF -chmod +x sync_progress.sh -``` - -Run that script from a long-lived local session, for example: - -```bash -tmux new-session -d -s autobench-sync './sync_progress.sh' -``` - -Use a persistent local background job, `tmux` pane, `screen`, or equivalent -long-lived sync process so that humans can watch the cleaned local log in real -time. Use `sleep 15` by default for long runs unless there is a specific need -for tighter polling, and keep the cleaned local `progress.log` within the -required 30-second refresh window while the run is active. - -At the end of the run, make sure the local artifact set includes any generated: -- `results.jsonl` -- `results.csv` -- `summary.md` -- `SUMMARY.md` -- `scenario_summary.jsonl` -- `scenario_summary.csv` - -Required health check after starting the sync script: - -```bash -stat -f '%m %z' progress.log -sleep 5 -stat -f '%m %z' progress.log -``` - -If the timestamp and size both stay unchanged while the remote benchmark is known -to be producing new output, the sync loop is broken. Fix the script before -continuing. - -Do not make the cleaned log optional. The default local progress artifact should -be the cleaned `progress.log` that humans actually read. - -## Most Important Rule - -If the user wants the best command for a **real production or real workload scenario**, the benchmark must use **their real request distribution**. - -That means: -- real prompt lengths, -- real output lengths, -- real multi-turn patterns, -- real tool / reasoning / sampling settings, -- real prefix-sharing behavior if it exists. - -`sharegpt`, `random`, and `generated-shared-prefix` are useful for sanity checks and broad tuning, but they are not a substitute for the user’s real traffic. - -The cookbook reference configs now default to `random` because it is portable and immediately runnable, but that should still be treated as a fallback benchmark shape rather than the final answer for a real deployment. - -## Supported Dataset Kinds - -The current implementation intentionally keeps the dataset surface small: - -- `sharegpt` - - Supports auto-download when no file path is provided. - - Will be prepared into canonical autobench JSONL on disk before benchmarking. -- `custom` - - Supports two cases: - - old `bench_serving` custom conversation JSONL, - - already-converted canonical autobench JSONL. -- `random` - - Uses SGLang’s existing synthetic/random benchmark path. - - This is the default dataset mode in the cookbook reference configs. - - `input_len` and `output_len` can be lists of equal length. - - Each aligned pair becomes one full benchmark scenario, not a cartesian product. - - Example: - -```yaml -dataset: - kind: random - scenario_names: [chat, summarization] - input_len: [1000, 8000] - output_len: [1000, 1000] -``` - - - The workflow will run one full search for `1000 -> 1000` and one full search for `8000 -> 1000`. -- `generated-shared-prefix` - - Uses SGLang’s existing shared-prefix synthetic generator. - -Everything is normalized into one canonical autobench JSONL file before the benchmark loop starts. - -## Canonical Dataset Format - -Canonical format is JSONL, one request per line. - -Minimal rows: - -```json -{"prompt": "Write a summary of this document.", "output_len": 256} -{"prompt": [{"role": "user", "content": "Summarize this document."}], "output_len": 256} -{"prompt": ["first turn", "follow-up turn"], "output_len": 128} -``` - -Optional fields: - -```json -{ - "prompt": [{"role": "user", "content": "Use the weather tool."}], - "output_len": 256, - "extra_request_body": {"temperature": 0.0, "top_p": 0.95}, - "image_data": ["file:///tmp/example.png"], - "timestamp": 1710000000, - "routing_key": "group-a", - "metadata": {"source": "custom-upload"} -} -``` - -Compatibility: -- legacy `messages` -- legacy `prompt_origin` -- legacy `param_send` -- legacy `system + content` - -## ShareGPT Auto-Prepare - -`sharegpt` does not need a full path. - -Example: - -```bash -python3 -m sglang.auto_benchmark convert \ - --kind sharegpt \ - --tokenizer /path/to/tokenizer \ - --num-prompts 200 \ - --output /tmp/sharegpt.autobench.jsonl -``` - -This will: -- auto-download ShareGPT through the existing SGLang cache path when needed, -- convert it into canonical autobench JSONL, -- save it to the requested output path. - -## Custom User Data Workflow - -When the user uploads custom data: - -1. Inspect a few raw rows first. -2. Decide whether the file is: - - already canonical autobench JSONL, - - old `bench_serving` custom format, - - or an unsupported custom schema that must be transformed manually. -3. If manual transformation is needed: - - map it into canonical JSONL, - - never hallucinate missing turns or answers, - - never keep the final assistant answer as part of the benchmark prompt if that answer is the target completion, - - preserve per-request generation settings in `extra_request_body`. -4. Run: - -```bash -python3 -m sglang.auto_benchmark validate \ - --dataset-path /path/to/converted.autobench.jsonl \ - --tokenizer /path/to/tokenizer -``` - -5. Manually inspect at least 3 converted rows and confirm: - - prompt shape is correct, - - final assistant answer was not accidentally left in the prompt, - - `output_len` is sensible, - - request extras were preserved. - -## Search Tiers - -`search.tier` controls search breadth. - -- Tier 1 - - Fastest and smallest sweep. - - Best for quick checks, config validation, and confirming whether a model can run at all. - - Uses a very small subset of the search space and mainly does one-at-a-time changes on top of the baseline. - - Lowest search cost, but also the easiest to miss a better configuration. -- Tier 2 - - Recommended default. - - Good balance between coverage and runtime. - - Runs a small cartesian search on the first few high-priority keys, then expands the rest one at a time. - - Usually the right choice for everyday tuning when you want meaningful search without waiting too long. -- Tier 3 - - Largest search space. - - Runs the full cartesian product of the provided search space. - - Search time is the longest by far. - - Only use it when the search space is already tightly bounded and you intentionally want the most exhaustive sweep. - - This is the best chance of finding the strongest config, but it is also the easiest way to turn a benchmark into a multi-hour or multi-day run. - -`search.max_candidates` still applies at all tiers, including tier 3. -When it is set together with tier 3, the workflow still enumerates the full cartesian order conceptually, but only keeps the first `max_candidates` unique candidates after deduplication. -That makes it useful as a safety valve, but it also means tier 3 is no longer truly exhaustive unless you remove the cap or raise it high enough. - -If `search.max_candidates` is omitted, the workflow now defaults to `8`. -Set it to `null` only when you intentionally want an unbounded sweep. - -The reference configs now default to tier 2 with `search.max_candidates: 8`. - -## Interrupt And Resume - -Long searches may need to be stopped and resumed later. - -Use: - -```yaml -search: - tier: 2 - resume: true -``` - -Behavior: -- every completed trial is appended to `live_results.jsonl` -- if the process receives `SIGINT` or `SIGTERM`, it will first save partial - `results.jsonl`, `results.csv`, and `summary.md` -- on the next run with the same config and `search.resume: true`, completed - trials are reused and only unfinished trials are executed -- resume works per scenario directory, so it is safest to keep the same - `benchmark.output_dir` - -Notes: -- resume assumes the candidate order and dataset are unchanged -- for maximum safety, reuse the same prepared dataset or keep the same dataset - seed/config -- `SIGKILL` cannot be handled gracefully, so only the already-written - `live_results.jsonl` can be reused after a hard kill - -YAML key order matters. Put the most important search keys first. - -## What Is Tunable - -This workflow is not limited to attention backend tuning. - -`server.base_flags` and `server.search_space` are passed directly to `sglang.launch_server`, so in practice any valid server CLI flag can be set or searched. - -There is also a small convenience layer for parallel search: - -- `server.parallel.tp` -- `server.parallel.pp_size` - -When `server.parallel` is used and `dp_size` is not set explicitly, the workflow auto-derives: - -`dp_size = visible_gpus / (tp_size * pp_size)` - -Visible GPU count is inferred from `server.env.CUDA_VISIBLE_DEVICES` by default, or from `server.parallel.gpu_count` if you set it explicitly. - -The most important performance-related groups are: - -- Kernel / backend - - `attention_backend` - - `prefill_attention_backend` - - `decode_attention_backend` - - `sampling_backend` - - `grammar_backend` -- Batching / scheduling - - `max_running_requests` - - `max_queued_requests` - - `chunked_prefill_size` - - `prefill_max_requests` - - `max_prefill_tokens` - - `schedule_conservativeness` - - `num_continuous_decode_steps` - - `stream_interval` -- Memory / cache - - `max_total_tokens` - - `page_size` - - `disable_radix_cache` -- Parallel / distributed execution - - `tp_size` - - `pp_size` - - `dp_size` - - `ep_size` - - `load_balance_method` - - `enable_dp_attention` - - `enable_mixed_chunk` - - `disable_overlap_schedule` -- Runtime / CUDA graph - - keep CUDA graph enabled by default for performance benchmarking - - `cuda_graph_max_bs` - - `disable_cuda_graph_padding` - - `enable_cudagraph_gc` -- Optional speculative / EAGLE stage - - `speculative_num_steps` - - `speculative_eagle_topk` - - `speculative_num_draft_tokens` - - `speculative_attention_mode` - - `speculative_draft_attention_backend` - - `speculative_accept_threshold_single` - - `speculative_accept_threshold_acc` - -For cookbook-derived reference configs, keep `mem_fraction_static` and -`schedule_policy` pinned to the cookbook baseline unless the user explicitly -asks to search them. They are useful knobs, but they add a lot of search width -for relatively low validation value in the default workflow. - -Do not put these into the default search space: -- `mem_fraction_static` -- `schedule_policy` -- `enable_hierarchical_cache` -- `hicache_ratio` -- `hicache_size` -- `enable_lmcache` - -Those features are not treated as standard auto-benchmark sweep knobs in this workflow. - -Budget guardrails for the default workflow: -- use `dataset.num_prompts: 80` unless the user asks for a heavier study -- prefer a coarse QPS search tolerance -- keep `benchmark.qps.max_rounds <= 5` -- keep `search.max_duration_hours <= 12` - -## Base Tuning Before EAGLE - -Never start by tuning EAGLE first. - -Use this order: - -1. Tune the non-speculative base server first. -2. Find the best normal config for the target dataset and SLA. -3. Only if the user explicitly asks for speculative/EAGLE tuning, and provides the required draft model or equivalent assets, run the second-stage speculative search. - -Do not put `disable_cuda_graph` into the default search space. For normal performance tuning, CUDA graph should stay enabled unless the user is debugging compatibility issues. - -When a candidate OOMs, keep it in the final result table as a failed row and add a hint such as: -- increase GPU count, or -- use GPUs with larger memory. - -## Running The Workflow - -Prepare a dataset explicitly: - -```bash -python3 -m sglang.auto_benchmark convert \ - --kind custom \ - --path /path/to/data.jsonl \ - --tokenizer /path/to/tokenizer \ - --output /tmp/data.autobench.jsonl -``` - -Run from config: - -```bash -python3 -m sglang.auto_benchmark run --config /path/to/config.yaml -``` - -Outputs: -- prepared canonical dataset JSONL -- per-run `results.jsonl` -- summary `results.csv` -- per-candidate server logs - -## Config Template - -Standalone example (uses ShareGPT as dataset, a good starting point for non-cookbook models): -- `references/qwen3-32b.yaml` - -Cookbook-derived configs live in `references/cookbook-llm/`. -They default to synthetic `random` traffic and are runnable out of the box. -See `references/cookbook-llm/README.md` for the full list. - -Representative picks from that folder: -- `references/cookbook-llm/llama-3.1-70b-instruct.yaml` -- `references/cookbook-llm/llama-3.3-70b-instruct.yaml` -- `references/cookbook-llm/llama-4-scout-17b-16e-instruct.yaml` -- `references/cookbook-llm/llama-4-maverick-17b-128e-instruct-fp8.yaml` -- `references/cookbook-llm/minimax-m2.5.yaml` -- `references/cookbook-llm/minimax-m2.1.yaml` -- `references/cookbook-llm/deepseek-v3.yaml` -- `references/cookbook-llm/deepseek-v3.1.yaml` -- `references/cookbook-llm/deepseek-v3.2.yaml` -- `references/cookbook-llm/deepseek-r1-0528.yaml` -- `references/cookbook-llm/qwen3-235b-a22b.yaml` -- `references/cookbook-llm/qwen35-397b-a17b-fp8.yaml` -- `references/cookbook-llm/mistral-small-4-119b-2603.yaml` -- `references/cookbook-llm/kimi-k2-instruct.yaml` - -All reference configs use Hugging Face repo IDs by default. -Replace `model_path` and `tokenizer` with local paths when the weights are already on disk. - -## What To Report Back - -After a run, summarize: -- which tier was used, -- which dataset kind was used, -- whether the dataset was synthetic or real user traffic, -- best base config, -- best QPS that satisfied SLA, -- whether speculative tuning was skipped or run, -- paths to: - - prepared dataset JSONL - - `results.jsonl` - - `results.csv` - - key server logs diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/README.md b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/README.md deleted file mode 100644 index 94e25102d..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/README.md +++ /dev/null @@ -1,76 +0,0 @@ -# Cookbook LLM References - -These configs are derived from `sgl-cookbook` autoregressive text-model pages and normalized into the auto-benchmark config format. - -Rules used here: -- Keep the baseline as close as possible to a pure-TP launch command. -- Keep `mem_fraction_static` and `schedule_policy` at the cookbook baseline by - default; search higher-ROI knobs first. -- Move the remaining common performance knobs into `search_space`. -- Add `ep_size` search for relevant MoE pages. -- Keep CUDA graph enabled by default. -- Prefer cookbook H200 defaults first, then H100 defaults when H200 is not available; if neither exists, fall back to the cookbook's published baseline for that model and say so in the config comments. -- Default to synthetic `random` data so every config is runnable out of the box. -- Default to `dataset.num_prompts: 80` so the reference sweep stays cheap enough - for interactive validation. -- Default to a coarse QPS search with `benchmark.qps.max_rounds <= 5`. -- Default to `search.tier: 2` so the shipped configs stay reasonably practical to run. -- Default to `search.max_candidates: 8` so the candidate sweep stays bounded by - default. -- Default to `search.max_duration_hours: 12` because longer searches do not fit - the intended workflow budget. -- Treat `dataset.input_len` and `dataset.output_len` as aligned scenario lists, not a cartesian product. -- If a candidate OOMs, the result table should recommend increasing GPU count or using GPUs with larger memory. - -Default random scenarios in these configs: -- `1000 -> 1000` for a chat-like shape -- `8000 -> 1000` for a summarization-like shape - -Each scenario should run a full search independently, and each scenario should have its own best launch command and summary table. - -Excluded from this folder because they are OCR/VL-oriented rather than text-serving benchmark configs: -- DeepSeekOCR / DeepSeekOCR2 -- GLMOCR -- GLM45V / GLM46V -- Qwen2.5-VL / Qwen3-VL -- Step3-VL-10B - -Configs in this folder: -- `deepseek-v3.2.yaml` -- `deepseek-math-v2.yaml` -- `deepseek-r1-0528.yaml` -- `deepseek-v3.1.yaml` -- `deepseek-v3.yaml` -- `devstral-small-2-24b-instruct-2512.yaml` -- `ernie-4.5-21b-a3b-pt.yaml` -- `glm-4.5.yaml` -- `glm-4.6.yaml` -- `glm-4.7.yaml` -- `glm-4.7-flash.yaml` -- `glm-5-fp8.yaml` -- `gpt-oss-120b.yaml` -- `glyph.yaml` -- `intern-s1.yaml` -- `kimi-k2.5.yaml` -- `kimi-k2-instruct.yaml` -- `kimi-linear-48b-a3b-instruct.yaml` -- `llada2-1-mini.yaml` -- `ling-2.5-1t.yaml` -- `llama-3.1-70b-instruct.yaml` -- `llama-3.3-70b-instruct.yaml` -- `llama-4-scout-17b-16e-instruct.yaml` -- `llama-4-maverick-17b-128e-instruct-fp8.yaml` -- `mimo-v2-flash.yaml` -- `minimax-m2.5.yaml` -- `minimax-m2.1.yaml` -- `ministral-3-8b-instruct-2512.yaml` -- `mistral-small-4-119b-2603.yaml` -- `nemotron-3-nano-30b-a3b-bf16.yaml` -- `nemotron-3-super-120b-a12b-bf16.yaml` -- `qwen35-397b-a17b-fp8.yaml` -- `qwen3-coder-480b-a35b-instruct.yaml` -- `qwen3-coder-next.yaml` -- `qwen3-235b-a22b.yaml` -- `qwen3-next-80b-a3b-instruct.yaml` -- `ring-2.5-1t.yaml` -- `step-3.5-flash.yaml` diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-math-v2.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-math-v2.yaml deleted file mode 100644 index f3e2e4271..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-math-v2.yaml +++ /dev/null @@ -1,81 +0,0 @@ -# Cookbook auto benchmark config for deepseek math v2. -# Cookbook does not expose an H100/H200 default for this model; this config follows the cookbook B200 baseline instead. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: data/models/generated/v0.5.8/deepseek-math-v2.yaml -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 - base_flags: - tp_size: 8 - model_path: deepseek-ai/DeepSeek-Math-V2 - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - prefill_attention_backend: - - flashinfer - decode_attention_backend: - - flashinfer - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 32 - - 48 - - 64 - ep_size: - - 1 - - 4 - - 8 -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: deepseek-ai/DeepSeek-Math-V2 - max_concurrency: - - null - - 4 - - 8 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 4.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/deepseek-math-v2 -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-r1-0528.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-r1-0528.yaml deleted file mode 100644 index 75dce3c0e..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-r1-0528.yaml +++ /dev/null @@ -1,84 +0,0 @@ -# Cookbook auto benchmark config for deepseek r1 0528. -# Baseline follows the cookbook H200 default command. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: data/models/generated/v0.5.6/deepseek-r1.yaml -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 - base_flags: - tp_size: 8 - enable_symm_mem: true - model_path: deepseek-ai/DeepSeek-R1-0528 - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - prefill_attention_backend: - - fa3 - - flashinfer - decode_attention_backend: - - fa3 - - flashinfer - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 32 - - 48 - - 64 - ep_size: - - 1 - - 4 - - 8 -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: deepseek-ai/DeepSeek-R1-0528 - max_concurrency: - - null - - 4 - - 8 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 4.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/deepseek-r1-0528 -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-v3.1.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-v3.1.yaml deleted file mode 100644 index 24e1427a7..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-v3.1.yaml +++ /dev/null @@ -1,83 +0,0 @@ -# Cookbook auto benchmark config for deepseek v3.1. -# Baseline follows the cookbook H200 default command. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: src/components/autoregressive/DeepSeekV31ConfigGenerator/index.js -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 - base_flags: - tp_size: 8 - model_path: deepseek-ai/DeepSeek-V3.1 - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - prefill_attention_backend: - - fa3 - - flashinfer - decode_attention_backend: - - fa3 - - flashinfer - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 32 - - 48 - - 64 - ep_size: - - 1 - - 4 - - 8 -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: deepseek-ai/DeepSeek-V3.1 - max_concurrency: - - null - - 4 - - 8 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 4.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/deepseek-v3.1 -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-v3.2.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-v3.2.yaml deleted file mode 100644 index 1b7959a2e..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-v3.2.yaml +++ /dev/null @@ -1,83 +0,0 @@ -# Cookbook auto benchmark config for deepseek v3.2. -# Baseline follows the cookbook H200 default command. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: data/models/generated/v0.5.6/deepseek.yaml -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 - base_flags: - tp_size: 8 - model_path: deepseek-ai/DeepSeek-V3.2 - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - prefill_attention_backend: - - fa3 - - flashinfer - decode_attention_backend: - - fa3 - - flashinfer - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 32 - - 48 - - 64 - ep_size: - - 1 - - 4 - - 8 -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: deepseek-ai/DeepSeek-V3.2 - max_concurrency: - - null - - 4 - - 8 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 4.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/deepseek-v3.2 -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-v3.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-v3.yaml deleted file mode 100644 index 6808d6b2c..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-v3.yaml +++ /dev/null @@ -1,84 +0,0 @@ -# Cookbook auto benchmark config for deepseek v3. -# Baseline follows the cookbook H200 default command. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: src/components/autoregressive/DeepSeekV3ConfigGenerator/index.js -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 - base_flags: - tp_size: 8 - enable_symm_mem: true - model_path: deepseek-ai/DeepSeek-V3 - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - prefill_attention_backend: - - fa3 - - flashinfer - decode_attention_backend: - - fa3 - - flashinfer - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 32 - - 48 - - 64 - ep_size: - - 1 - - 4 - - 8 -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: deepseek-ai/DeepSeek-V3 - max_concurrency: - - null - - 4 - - 8 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 4.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/deepseek-v3 -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/devstral-small-2-24b-instruct-2512.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/devstral-small-2-24b-instruct-2512.yaml deleted file mode 100644 index 2120ab2ee..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/devstral-small-2-24b-instruct-2512.yaml +++ /dev/null @@ -1,78 +0,0 @@ -# Cookbook auto benchmark config for devstral small 2 24b instruct 2512. -# Baseline follows the cookbook H200 default command. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: src/components/autoregressive/Devstral2ConfigGenerator/index.js -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: '0' - base_flags: - model_path: mistralai/Devstral-Small-2-24B-Instruct-2512 - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - prefill_attention_backend: - - fa3 - - flashinfer - decode_attention_backend: - - fa3 - - flashinfer - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 64 - - 96 - - 128 -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: mistralai/Devstral-Small-2-24B-Instruct-2512 - max_concurrency: - - null - - 16 - - 32 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 16.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/devstral-small-2-24b-instruct-2512 -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/ernie-4.5-21b-a3b-pt.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/ernie-4.5-21b-a3b-pt.yaml deleted file mode 100644 index f9e3ab67f..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/ernie-4.5-21b-a3b-pt.yaml +++ /dev/null @@ -1,76 +0,0 @@ -# Cookbook auto benchmark config for ernie 4.5 21b a3b pt. -# Cookbook does not expose an H100/H200 default for this model; this config follows the cookbook MI300X baseline instead. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: src/components/autoregressive/Ernie45ConfigGenerator/index.js -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: '0' - base_flags: - model_path: baidu/ERNIE-4.5-21B-A3B-PT - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 64 - - 96 - - 128 - command_prefix: - - python3 - - -m - - sglang.launch_server -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: baidu/ERNIE-4.5-21B-A3B-PT - max_concurrency: - - null - - 16 - - 32 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 16.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/ernie-4.5-21b-a3b-pt -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-4.5.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-4.5.yaml deleted file mode 100644 index c0eb3a398..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-4.5.yaml +++ /dev/null @@ -1,74 +0,0 @@ -# Cookbook auto benchmark config for glm 4.5. -# Cookbook does not expose an H100/H200 default for this model; this config follows the cookbook MI300X baseline instead. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: src/components/autoregressive/GLM45ConfigGenerator/index.js -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: 0,1,2,3 - base_flags: - tp_size: 4 - context_length: 8192 - model_path: zai-org/GLM-4.5 - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 64 - - 96 - - 128 -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: zai-org/GLM-4.5 - max_concurrency: - - null - - 8 - - 16 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 8.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/glm-4.5 -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-4.6.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-4.6.yaml deleted file mode 100644 index d9c2270e1..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-4.6.yaml +++ /dev/null @@ -1,83 +0,0 @@ -# Cookbook auto benchmark config for glm 4.6. -# Baseline follows the cookbook H200 default command. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: data/models/generated/v0.5.6/glm46.yaml -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 - base_flags: - tp_size: 8 - model_path: zai-org/GLM-4.6 - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - prefill_attention_backend: - - fa3 - - flashinfer - decode_attention_backend: - - fa3 - - flashinfer - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 32 - - 48 - - 64 - ep_size: - - 1 - - 4 - - 8 -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: zai-org/GLM-4.6 - max_concurrency: - - null - - 4 - - 8 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 4.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/glm-4.6 -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-4.7-flash.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-4.7-flash.yaml deleted file mode 100644 index 426203ac9..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-4.7-flash.yaml +++ /dev/null @@ -1,78 +0,0 @@ -# Cookbook auto benchmark config for glm 4.7 flash. -# Baseline follows the cookbook H100 default command. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: src/components/autoregressive/GLM47FlashConfigGenerator/index.js -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: '0' - base_flags: - model_path: zai-org/GLM-4.7-Flash - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - prefill_attention_backend: - - fa3 - - flashinfer - decode_attention_backend: - - fa3 - - flashinfer - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 64 - - 96 - - 128 -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: zai-org/GLM-4.7-Flash - max_concurrency: - - null - - 16 - - 32 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 16.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/glm-4.7-flash -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-4.7.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-4.7.yaml deleted file mode 100644 index 44640b679..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-4.7.yaml +++ /dev/null @@ -1,78 +0,0 @@ -# Cookbook auto benchmark config for glm 4.7. -# Cookbook does not expose an H100/H200 default for this model; this config follows the cookbook MI300X baseline instead. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: src/components/autoregressive/GLM47ConfigGenerator/index.js -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: 0,1,2,3 - base_flags: - tp_size: 4 - context_length: 8192 - model_path: zai-org/GLM-4.7 - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 64 - - 96 - - 128 - ep_size: - - 1 - - 2 - - 4 -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: zai-org/GLM-4.7 - max_concurrency: - - null - - 8 - - 16 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 8.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/glm-4.7 -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-5-fp8.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-5-fp8.yaml deleted file mode 100644 index 79e6d2241..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-5-fp8.yaml +++ /dev/null @@ -1,83 +0,0 @@ -# Cookbook auto benchmark config for glm 5 fp8. -# Baseline follows the cookbook H200 default command. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: data/models/generated/v0.5.8/glm5.yaml -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 - base_flags: - tp_size: 8 - model_path: zai-org/GLM-5-FP8 - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - prefill_attention_backend: - - fa3 - - flashinfer - decode_attention_backend: - - fa3 - - flashinfer - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 32 - - 48 - - 64 - ep_size: - - 1 - - 4 - - 8 -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: zai-org/GLM-5-FP8 - max_concurrency: - - null - - 4 - - 8 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 4.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/glm-5-fp8 -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glyph.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glyph.yaml deleted file mode 100644 index 25102e092..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glyph.yaml +++ /dev/null @@ -1,81 +0,0 @@ -# Cookbook auto benchmark config for glyph. -# Baseline follows the cookbook H200 default command. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: src/components/autoregressive/GlyphConfigGenerator/index.js -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: 0,1,2,3 - base_flags: - tp_size: 4 - reasoning_parser: glm45 - tool_call_parser: glm45 - model_path: zai-org/Glyph - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - prefill_attention_backend: - - fa3 - - flashinfer - decode_attention_backend: - - fa3 - - flashinfer - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 64 - - 96 - - 128 -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: zai-org/Glyph - max_concurrency: - - null - - 8 - - 16 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 8.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/glyph -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/gpt-oss-120b.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/gpt-oss-120b.yaml deleted file mode 100644 index 7a251b4fa..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/gpt-oss-120b.yaml +++ /dev/null @@ -1,83 +0,0 @@ -# Cookbook auto benchmark config for gpt oss 120b. -# Baseline follows the cookbook H200 default command. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: data/models/generated/v0.5.6/gpt-oss.yaml -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 - base_flags: - tp_size: 8 - model_path: openai/gpt-oss-120b - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - prefill_attention_backend: - - fa3 - - flashinfer - decode_attention_backend: - - fa3 - - flashinfer - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 32 - - 48 - - 64 - ep_size: - - 1 - - 4 - - 8 -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: openai/gpt-oss-120b - max_concurrency: - - null - - 4 - - 8 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 4.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/gpt-oss-120b -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/intern-s1.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/intern-s1.yaml deleted file mode 100644 index aa91be7b4..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/intern-s1.yaml +++ /dev/null @@ -1,84 +0,0 @@ -# Cookbook auto benchmark config for intern s1. -# Baseline follows the cookbook H200 default command. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: data/models/generated/v0.5.6/intern-s1.yaml -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 - base_flags: - tp_size: 8 - trust_remote_code: true - model_path: internlm/Intern-S1 - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - prefill_attention_backend: - - fa3 - - flashinfer - decode_attention_backend: - - fa3 - - flashinfer - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 32 - - 48 - - 64 - ep_size: - - 1 - - 4 - - 8 -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: internlm/Intern-S1 - max_concurrency: - - null - - 4 - - 8 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 4.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/intern-s1 -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/kimi-k2-instruct.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/kimi-k2-instruct.yaml deleted file mode 100644 index b5066bb29..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/kimi-k2-instruct.yaml +++ /dev/null @@ -1,83 +0,0 @@ -# Cookbook auto benchmark config for kimi k2 instruct. -# Baseline follows the cookbook H200 default command. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: data/models/generated/v0.5.6/kimi-k2.yaml -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 - base_flags: - tp_size: 8 - trust_remote_code: true - model_path: moonshotai/Kimi-K2-Instruct - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - prefill_attention_backend: - - fa3 - - flashinfer - decode_attention_backend: - - fa3 - - flashinfer - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 32 - - 48 - - 64 - ep_size: - - 1 - - 4 -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: moonshotai/Kimi-K2-Instruct - max_concurrency: - - null - - 4 - - 8 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 4.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/kimi-k2-instruct -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/kimi-k2.5.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/kimi-k2.5.yaml deleted file mode 100644 index fba48bd72..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/kimi-k2.5.yaml +++ /dev/null @@ -1,80 +0,0 @@ -# Cookbook auto benchmark config for kimi k2.5. -# Baseline follows the cookbook H200 default command. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: data/models/generated/v0.5.8/kimi-k25.yaml -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 - base_flags: - tp_size: 8 - trust_remote_code: true - model_path: moonshotai/Kimi-K2.5 - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - prefill_attention_backend: - - fa3 - - flashinfer - decode_attention_backend: - - fa3 - - flashinfer - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 32 - - 48 - - 64 -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: moonshotai/Kimi-K2.5 - max_concurrency: - - null - - 4 - - 8 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 4.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/kimi-k2.5 -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/kimi-linear-48b-a3b-instruct.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/kimi-linear-48b-a3b-instruct.yaml deleted file mode 100644 index 144fa3ffe..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/kimi-linear-48b-a3b-instruct.yaml +++ /dev/null @@ -1,79 +0,0 @@ -# Cookbook auto benchmark config for kimi linear 48b a3b instruct. -# Cookbook does not expose an H100/H200 default for this model; this config follows the cookbook MI300X baseline instead. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: src/components/autoregressive/KimiK2linearConfigGenerator/index.js -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - SGLANG_ROCM_FUSED_DECODE_MLA: '0' - CUDA_VISIBLE_DEVICES: 0,1,2,3 - base_flags: - tp_size: 4 - trust_remote_code: true - model_path: moonshotai/Kimi-Linear-48B-A3B-Instruct - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 64 - - 96 - - 128 - command_prefix: - - python3 - - -m - - sglang.launch_server -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: moonshotai/Kimi-Linear-48B-A3B-Instruct - max_concurrency: - - null - - 8 - - 16 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 8.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/kimi-linear-48b-a3b-instruct -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/ling-2.5-1t.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/ling-2.5-1t.yaml deleted file mode 100644 index 8bbc2edac..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/ling-2.5-1t.yaml +++ /dev/null @@ -1,91 +0,0 @@ -# Cookbook auto benchmark config for ling 2.5 1t. -# Baseline follows the cookbook H200 default command. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# This cookbook baseline is multi-node or external-service oriented. Keep launch: false and benchmark an already running deployment. -# Source: src/components/autoregressive/Ling25ConfigGenerator/index.js -server: - launch: false - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 - base_flags: - tp_size: 8 - pp_size: 2 - nnodes: 2 - trust_remote_code: true - tool_call_parser: qwen - model_path: inclusionAI/Ling-2.5-1T - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - prefill_attention_backend: - - fa3 - - flashinfer - decode_attention_backend: - - fa3 - - flashinfer - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 32 - - 48 - - 64 - pp_size: - - 1 - - 2 - command_prefix: - - python3 - - -m - - sglang.launch_server -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: inclusionAI/Ling-2.5-1T - max_concurrency: - - null - - 4 - - 8 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 2.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/ling-2.5-1t -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/llada2-1-mini.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/llada2-1-mini.yaml deleted file mode 100644 index 57d6745dc..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/llada2-1-mini.yaml +++ /dev/null @@ -1,83 +0,0 @@ -# Cookbook auto benchmark config for llada2 1 mini. -# Baseline follows the cookbook H200 default command. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: data/models/generated/v0.5.6/llada21.yaml -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: '0' - base_flags: - tp_size: 1 - dllm_algorithm: JointThreshold - trust_remote_code: true - max_running_requests: 1 - attention_backend: flashinfer - model_path: inclusionAI/LLaDA2.1-mini - mem_fraction_static: 0.77 - schedule_policy: lpm - search_space: - prefill_attention_backend: - - fa3 - - flashinfer - decode_attention_backend: - - fa3 - - flashinfer - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 1 - - 2 - - 4 -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: inclusionAI/LLaDA2.1-mini - max_concurrency: - - 1 - - 2 - - 4 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 4.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/llada2-1-mini -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/llama-3.1-70b-instruct.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/llama-3.1-70b-instruct.yaml deleted file mode 100644 index 02109e7d5..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/llama-3.1-70b-instruct.yaml +++ /dev/null @@ -1,79 +0,0 @@ -# Cookbook auto benchmark config for llama 3.1 70b instruct. -# Baseline follows the cookbook H200 default command. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: data/models/generated/v0.5.6/llama31.yaml -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: 0,1,2,3 - base_flags: - tp_size: 4 - model_path: meta-llama/Llama-3.1-70B-Instruct - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - prefill_attention_backend: - - fa3 - - flashinfer - decode_attention_backend: - - fa3 - - flashinfer - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 64 - - 96 - - 128 -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: meta-llama/Llama-3.1-70B-Instruct - max_concurrency: - - null - - 8 - - 16 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 12.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/llama-3.1-70b-instruct -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/llama-3.3-70b-instruct.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/llama-3.3-70b-instruct.yaml deleted file mode 100644 index 52aece47d..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/llama-3.3-70b-instruct.yaml +++ /dev/null @@ -1,73 +0,0 @@ -# Cookbook auto benchmark config for llama 3.3 70b instruct. -# Cookbook does not expose an H100/H200 default for this model; this config follows the cookbook MI300X baseline instead. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: src/components/autoregressive/Llama33ConfigGenerator/index.js -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: '0' - base_flags: - tool_call_parser: llama3 - model_path: meta-llama/Llama-3.3-70B-Instruct - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 64 - - 96 - - 128 -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: meta-llama/Llama-3.3-70B-Instruct - max_concurrency: - - null - - 16 - - 32 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 16.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/llama-3.3-70b-instruct -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/llama-4-maverick-17b-128e-instruct-fp8.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/llama-4-maverick-17b-128e-instruct-fp8.yaml deleted file mode 100644 index b5a1164c4..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/llama-4-maverick-17b-128e-instruct-fp8.yaml +++ /dev/null @@ -1,79 +0,0 @@ -# Cookbook auto benchmark config for llama 4 maverick 17b 128e instruct fp8. -# Cookbook does not expose an H100/H200 default for this model; this config follows the cookbook GENERIC-8GPU baseline instead. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: docs/autoregressive/Llama/Llama4.md -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 - base_flags: - tp_size: 8 - context_length: 1000000 - trust_remote_code: true - enable_multimodal: true - model_path: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 4 - - 8 - - 12 - command_prefix: - - sglang - - serve -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 - max_concurrency: - - null - - 2 - - 4 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 2.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/llama-4-maverick-17b-128e-instruct-fp8 -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE3 - draft_model_path: lmsys/sglang-EAGLE3-Llama-4-Maverick-17B-128E-Instruct-v1 - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/llama-4-scout-17b-16e-instruct.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/llama-4-scout-17b-16e-instruct.yaml deleted file mode 100644 index bf921e8f4..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/llama-4-scout-17b-16e-instruct.yaml +++ /dev/null @@ -1,83 +0,0 @@ -# Cookbook auto benchmark config for llama 4 scout 17b 16e instruct. -# Baseline follows the cookbook H200 default command. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: data/models/generated/v0.5.6/llama4scout.yaml -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 - base_flags: - tp_size: 8 - enable_multimodal: true - context_length: 65536 - dtype: bfloat16 - trust_remote_code: true - model_path: meta-llama/Llama-4-Scout-17B-16E-Instruct - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - prefill_attention_backend: - - fa3 - - flashinfer - decode_attention_backend: - - fa3 - - flashinfer - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 8 - - 16 - - 24 -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: meta-llama/Llama-4-Scout-17B-16E-Instruct - max_concurrency: - - null - - 4 - - 8 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 4.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/llama-4-scout-17b-16e-instruct -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE3 - draft_model_path: lmsys/sglang-EAGLE3-Llama-4-Scout-17B-16E-Instruct-v1 - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/mimo-v2-flash.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/mimo-v2-flash.yaml deleted file mode 100644 index 2a40706c9..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/mimo-v2-flash.yaml +++ /dev/null @@ -1,91 +0,0 @@ -# Cookbook auto benchmark config for mimo v2 flash. -# Baseline follows the cookbook H200 default command. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: src/components/autoregressive/MiMoConfigGenerator/index.js -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 - base_flags: - tp_size: 8 - trust_remote_code: true - max_running_requests: 128 - chunked_prefill_size: 16384 - model_loader_extra_config: '{"enable_multithread_load": "true","num_threads": - 64}' - attention_backend: fa3 - reasoning_parser: qwen3 - tool_call_parser: mimo - model_path: XiaomiMiMo/MiMo-V2-Flash - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - prefill_attention_backend: - - fa3 - - flashinfer - decode_attention_backend: - - fa3 - - flashinfer - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 32 - - 48 - - 64 - command_prefix: - - python3 - - -m - - sglang.launch_server -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: XiaomiMiMo/MiMo-V2-Flash - max_concurrency: - - null - - 4 - - 8 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 4.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/mimo-v2-flash -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/minimax-m2.1.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/minimax-m2.1.yaml deleted file mode 100644 index ff7f7b2d9..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/minimax-m2.1.yaml +++ /dev/null @@ -1,77 +0,0 @@ -# Cookbook auto benchmark config for minimax m2.1. -# Cookbook does not expose an H100/H200 default for this model; this config follows the cookbook MI300X baseline instead. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: src/components/autoregressive/MiniMaxM2ConfigGenerator/index.js -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: 0,1,2,3 - base_flags: - tp_size: 4 - trust_remote_code: true - model_path: MiniMaxAI/MiniMax-M2.1 - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 64 - - 96 - - 128 - command_prefix: - - sglang - - serve -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: MiniMaxAI/MiniMax-M2.1 - max_concurrency: - - null - - 8 - - 16 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 8.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/minimax-m2.1 -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/minimax-m2.5.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/minimax-m2.5.yaml deleted file mode 100644 index d03022c19..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/minimax-m2.5.yaml +++ /dev/null @@ -1,83 +0,0 @@ -# Cookbook auto benchmark config for minimax m2.5. -# Baseline follows the cookbook H200 default command. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: src/components/autoregressive/MiniMaxM25ConfigGenerator/index.js -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: 0,1,2,3 - base_flags: - tp_size: 4 - trust_remote_code: true - model_path: MiniMaxAI/MiniMax-M2.5 - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - prefill_attention_backend: - - fa3 - - flashinfer - decode_attention_backend: - - fa3 - - flashinfer - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 64 - - 96 - - 128 - ep_size: - - 1 - - 4 -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: MiniMaxAI/MiniMax-M2.5 - max_concurrency: - - null - - 8 - - 16 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 4.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/minimax-m2.5 -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/ministral-3-8b-instruct-2512.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/ministral-3-8b-instruct-2512.yaml deleted file mode 100644 index 147bbd180..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/ministral-3-8b-instruct-2512.yaml +++ /dev/null @@ -1,77 +0,0 @@ -# Cookbook auto benchmark config for ministral 3 8b instruct 2512. -# Cookbook does not expose an H100/H200 default for this model; this config follows the cookbook MI300X baseline instead. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: src/components/autoregressive/Ministral3ConfigGenerator/index.js -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: '0' - base_flags: - trust_remote_code: true - tool_call_parser: mistral - model_path: mistralai/Ministral-3-8B-Instruct-2512 - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 64 - - 96 - - 128 - command_prefix: - - sglang - - serve -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: mistralai/Ministral-3-8B-Instruct-2512 - max_concurrency: - - null - - 16 - - 32 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 16.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/ministral-3-8b-instruct-2512 -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/mistral-small-4-119b-2603.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/mistral-small-4-119b-2603.yaml deleted file mode 100644 index 595b61739..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/mistral-small-4-119b-2603.yaml +++ /dev/null @@ -1,79 +0,0 @@ -# Cookbook auto benchmark config for mistral small 4 119b 2603. -# Baseline follows the cookbook H200 default command. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: data/models/generated/v0.5.8/mistral-small-4.yaml -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: 0,1 - base_flags: - tp_size: 2 - model_path: mistralai/Mistral-Small-4-119B-2603 - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - prefill_attention_backend: - - fa3 - - flashinfer - decode_attention_backend: - - fa3 - - flashinfer - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 64 - - 96 - - 128 -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: mistralai/Mistral-Small-4-119B-2603 - max_concurrency: - - null - - 8 - - 16 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 6.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/mistral-small-4-119b-2603 -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/nemotron-3-nano-30b-a3b-bf16.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/nemotron-3-nano-30b-a3b-bf16.yaml deleted file mode 100644 index bba06685c..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/nemotron-3-nano-30b-a3b-bf16.yaml +++ /dev/null @@ -1,81 +0,0 @@ -# Cookbook auto benchmark config for nemotron 3 nano 30b a3b bf16. -# Baseline follows the cookbook H200 default command. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: data/models/generated/v0.5.6/nemotron.yaml -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: '0' - base_flags: - tp_size: 1 - trust_remote_code: true - kv_cache_dtype: fp8_e4m3 - model_path: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - prefill_attention_backend: - - fa3 - - flashinfer - decode_attention_backend: - - fa3 - - flashinfer - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 64 - - 96 - - 128 -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 - max_concurrency: - - null - - 16 - - 32 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 16.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/nemotron-3-nano-30b-a3b-bf16 -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/nemotron-3-super-120b-a12b-bf16.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/nemotron-3-super-120b-a12b-bf16.yaml deleted file mode 100644 index 333b96a63..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/nemotron-3-super-120b-a12b-bf16.yaml +++ /dev/null @@ -1,81 +0,0 @@ -# Cookbook auto benchmark config for nemotron 3 super 120b a12b bf16. -# Baseline follows the cookbook H200 default command. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: data/models/generated/v0.5.8/nemotron-super.yaml -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: 0,1,2,3 - base_flags: - tp_size: 4 - trust_remote_code: true - kv_cache_dtype: fp8_e4m3 - model_path: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16 - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - prefill_attention_backend: - - fa3 - - flashinfer - decode_attention_backend: - - fa3 - - flashinfer - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 64 - - 96 - - 128 -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16 - max_concurrency: - - null - - 8 - - 16 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 6.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/nemotron-3-super-120b-a12b-bf16 -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen3-235b-a22b.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen3-235b-a22b.yaml deleted file mode 100644 index 7b5d25b6a..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen3-235b-a22b.yaml +++ /dev/null @@ -1,83 +0,0 @@ -# Cookbook auto benchmark config for qwen3 235b a22b. -# Baseline follows the cookbook H200 default command. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: data/models/generated/v0.5.6/qwen.yaml -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 - base_flags: - tp_size: 8 - model_path: Qwen/Qwen3-235B-A22B - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - prefill_attention_backend: - - fa3 - - flashinfer - decode_attention_backend: - - fa3 - - flashinfer - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 32 - - 48 - - 64 - ep_size: - - 1 - - 4 - - 8 -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: Qwen/Qwen3-235B-A22B - max_concurrency: - - null - - 4 - - 8 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 4.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/qwen3-235b-a22b -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen3-coder-480b-a35b-instruct.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen3-coder-480b-a35b-instruct.yaml deleted file mode 100644 index 9474058d3..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen3-coder-480b-a35b-instruct.yaml +++ /dev/null @@ -1,82 +0,0 @@ -# Cookbook auto benchmark config for qwen3 coder 480b a35b instruct. -# Cookbook does not expose an H100/H200 default for this model; this config follows the cookbook B200 baseline instead. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: src/components/autoregressive/Qwen3CoderConfigGenerator/index.js -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 - base_flags: - tp_size: 8 - ep_size: 2 - moe_runner_backend: triton - model_path: Qwen/Qwen3-Coder-480B-A35B-Instruct - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - prefill_attention_backend: - - flashinfer - decode_attention_backend: - - flashinfer - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 32 - - 48 - - 64 - ep_size: - - 1 - - 2 -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: Qwen/Qwen3-Coder-480B-A35B-Instruct - max_concurrency: - - null - - 4 - - 8 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 4.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/qwen3-coder-480b-a35b-instruct -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen3-coder-next.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen3-coder-next.yaml deleted file mode 100644 index fead555f0..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen3-coder-next.yaml +++ /dev/null @@ -1,79 +0,0 @@ -# Cookbook auto benchmark config for qwen3 coder next. -# Baseline follows the cookbook H200 default command. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: data/models/generated/v0.5.8/qwen3codernext.yaml -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: 0,1 - base_flags: - tp_size: 2 - model_path: Qwen/Qwen3-Coder-Next - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - prefill_attention_backend: - - fa3 - - flashinfer - decode_attention_backend: - - fa3 - - flashinfer - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 64 - - 96 - - 128 -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: Qwen/Qwen3-Coder-Next - max_concurrency: - - null - - 8 - - 16 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 12.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/qwen3-coder-next -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen3-next-80b-a3b-instruct.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen3-next-80b-a3b-instruct.yaml deleted file mode 100644 index a5490e5d2..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen3-next-80b-a3b-instruct.yaml +++ /dev/null @@ -1,82 +0,0 @@ -# Cookbook auto benchmark config for qwen3 next 80b a3b instruct. -# Baseline follows the cookbook H200 default command. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: data/models/generated/v0.5.6/qwen3next.yaml -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: 0,1 - base_flags: - tp_size: 2 - model_path: Qwen/Qwen3-Next-80B-A3B-Instruct - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - prefill_attention_backend: - - fa3 - - flashinfer - decode_attention_backend: - - fa3 - - flashinfer - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 64 - - 96 - - 128 - ep_size: - - 1 - - 2 -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: Qwen/Qwen3-Next-80B-A3B-Instruct - max_concurrency: - - null - - 8 - - 16 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 12.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/qwen3-next-80b-a3b-instruct -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen35-397b-a17b-fp8.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen35-397b-a17b-fp8.yaml deleted file mode 100644 index 6190ff1d8..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen35-397b-a17b-fp8.yaml +++ /dev/null @@ -1,83 +0,0 @@ -# Cookbook auto benchmark config for qwen35 397b a17b fp8. -# Baseline follows the cookbook H200 default command. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: data/models/generated/v0.5.8/qwen35.yaml -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: 0,1,2,3 - base_flags: - tp_size: 4 - model_path: Qwen/Qwen3.5-397B-A17B-FP8 - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - prefill_attention_backend: - - fa3 - - flashinfer - decode_attention_backend: - - fa3 - - flashinfer - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 32 - - 48 - - 64 - ep_size: - - 1 - - 4 - - 8 -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: Qwen/Qwen3.5-397B-A17B-FP8 - max_concurrency: - - null - - 4 - - 8 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 4.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/qwen35-397b-a17b-fp8 -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/ring-2.5-1t.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/ring-2.5-1t.yaml deleted file mode 100644 index c38bffc81..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/ring-2.5-1t.yaml +++ /dev/null @@ -1,79 +0,0 @@ -# Cookbook auto benchmark config for ring 2.5 1t. -# Baseline follows the cookbook H200 default command. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: data/models/generated/v0.5.8/ring25.yaml -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 - base_flags: - tp_size: 8 - model_path: inclusionAI/Ring-2.5-1T - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - prefill_attention_backend: - - fa3 - - flashinfer - decode_attention_backend: - - fa3 - - flashinfer - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 32 - - 48 - - 64 -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: inclusionAI/Ring-2.5-1T - max_concurrency: - - null - - 4 - - 8 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 2.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/ring-2.5-1t -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/step-3.5-flash.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/step-3.5-flash.yaml deleted file mode 100644 index 00ee40ad7..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/step-3.5-flash.yaml +++ /dev/null @@ -1,83 +0,0 @@ -# Cookbook auto benchmark config for step 3.5 flash. -# Baseline follows the cookbook H200 default command. -# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. -# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. -# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. -# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. -# Source: data/models/generated/v0.5.8/step35.yaml -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: 0,1,2,3 - base_flags: - tp_size: 4 - trust_remote_code: true - model_path: stepfun-ai/Step-3.5-Flash - mem_fraction_static: 0.82 - schedule_policy: lpm - search_space: - prefill_attention_backend: - - fa3 - - flashinfer - decode_attention_backend: - - fa3 - - flashinfer - chunked_prefill_size: - - 4096 - - 8192 - max_running_requests: - - 64 - - 96 - - 128 - ep_size: - - 1 - - 4 -dataset: - kind: random - num_prompts: 80 - scenario_names: - - chat - - summarization - input_len: - - 1000 - - 8000 - output_len: - - 1000 - - 1000 -benchmark: - backend: auto - tokenizer: stepfun-ai/Step-3.5-Flash - max_concurrency: - - null - - 8 - - 16 - extra_request_body: - temperature: 0.0 - qps: - lower: 0.25 - upper: 8.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/cookbook-llm/step-3.5-flash -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true -speculative: - enabled: false - algorithm: EAGLE - draft_model_path: '' - search_space: - speculative_num_steps: - - 3 - - 5 - speculative_eagle_topk: - - 1 - - 2 - speculative_num_draft_tokens: - - 4 - - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/qwen3-32b.yaml b/.claude/skills/sglang-auto-benchmark/references/qwen3-32b.yaml deleted file mode 100644 index 8884c36c9..000000000 --- a/.claude/skills/sglang-auto-benchmark/references/qwen3-32b.yaml +++ /dev/null @@ -1,62 +0,0 @@ -server: - launch: true - host: 127.0.0.1 - port: 30000 - env: - CUDA_VISIBLE_DEVICES: "0" - # Optional parallel search sugar: - # parallel: - # # dp_size is auto-derived as visible_gpus / (tp_size * pp_size). - # tp: [4, 2] - # # pp_size: [1, 2] - base_flags: - # Replace with a local model path if you have one. - model_path: Qwen/Qwen3-32B - tp_size: 1 - trust_remote_code: true - # Keep the baseline close to a pure-TP launch command. - mem_fraction_static: 0.82 - schedule_policy: lpm - # Keep CUDA graph enabled for normal performance tuning. - search_space: - prefill_attention_backend: [fa3, flashinfer] - decode_attention_backend: [fa3, flashinfer] - chunked_prefill_size: [4096, 8192] - max_running_requests: [64, 96, 128] -dataset: - kind: random - num_prompts: 80 - scenario_names: [chat, summarization] - input_len: [1000, 8000] - output_len: [1000, 1000] - -benchmark: - backend: auto - # Replace with a local tokenizer/model path if you have one. - tokenizer: Qwen/Qwen3-32B - max_concurrency: [null, 16, 32] - extra_request_body: - temperature: 0.0 - qps: - lower: 1.0 - upper: 12.0 - tolerance: 0.1 - sla: - max_ttft_ms: 1500 - max_tpot_ms: 30 - output_dir: ./auto_benchmark_results/qwen3-32b - -search: - tier: 2 - max_candidates: 8 # Cap search breadth so full searches stay tractable. - resume: true - -speculative: - enabled: false - algorithm: EAGLE - # Fill this only if you explicitly want a second-stage speculative search. - draft_model_path: "" - search_space: - speculative_num_steps: [3, 5] - speculative_eagle_topk: [1, 4] - speculative_num_draft_tokens: [4, 8] diff --git a/.claude/skills/sglang-prod-incident-triage/SKILL.md b/.claude/skills/sglang-prod-incident-triage/SKILL.md new file mode 100644 index 000000000..708e399f1 --- /dev/null +++ b/.claude/skills/sglang-prod-incident-triage/SKILL.md @@ -0,0 +1,291 @@ +--- +name: sglang-prod-incident-triage +description: Replay-first debug flow for SGLang serving problems. Use when a live or recent server shows health-check failures, latency or throughput regressions, queue growth, timeouts, distributed stalls, crash dumps, wrong outputs after deploys, or PD/EP/HiCache issues, and the job is to turn the problem into a replay plus the right next debug tool. +--- + +# SGLang Serving Debug + +## Overview + +Use this skill to turn a live serving problem into a debug path you can replay. + +Use one loop: + +- collect a baseline bundle +- save the failing request or crash dump +- replay on a clean target +- only then switch tools + +Do not start with profiling. + +This skill should work with more focused skills instead of re-implementing them: + +- `debug-cuda-crash` when replay plus coredump points to a CUDA crash path +- `debug-distributed-hang` when the problem is clearly a TP/PP/DP/EP hang +- `llm-torch-profiler-analysis` when the issue is already narrowed to a + compute-side path + +Three examples are included: + +- TTFT spike with low queue time +- replay-first CUDA crash flow +- request-shaped distributed hang flow + +## Output Contract + +Return: + +- problem class +- what was checked +- strongest signal so far +- current best guess +- what was ruled out +- next step +- production risk + +## When To Use It + +- `/health` or `/health_generate` is unhealthy +- latency or throughput regressed under serving load +- queue size grows while health still looks green +- one request class times out or hangs +- the server crashes only after some requests +- outputs changed after a deploy, topology change, or weight switch +- one older commit is known-good and a newer commit is known-bad + +## Workflow + +### 1. Collect a baseline bundle + +If a live server is reachable, collect a read-only bundle before anything more +intrusive: + +```bash +python3 scripts/incident_artifact_tool.py collect-bundle \ + --base-url http://127.0.0.1:30000 \ + --outdir /tmp/incident_bundle + +python3 scripts/incident_artifact_tool.py summarize-bundle \ + /tmp/incident_bundle +``` + +If the server is protected: + +```bash +python3 scripts/incident_artifact_tool.py collect-bundle \ + --base-url http://127.0.0.1:30000 \ + --token "$SGLANG_BEARER_TOKEN" \ + --outdir /tmp/incident_bundle +``` + +The bundle script collects: + +- `/health` +- `/health_generate` +- `/model_info` +- `/server_info` +- `/v1/loads?include=all` +- `/v1/loads?include=core,queues,disagg,spec` +- `/metrics` +- `/hicache/storage-backend` on a best-effort basis + +Use the summary for a quick read on: + +- health vs. active health state +- topology and runtime flags +- point-in-time queue and token usage +- TTFT / E2E / queue-time heuristics from Prometheus metrics + +If the summary says the bundle was captured while the server was idle, recollect +it during traffic or move quickly to dump plus replay. + +If no live server is reachable, start from the best dump or log already available: + +- crash dump +- request dump +- logs +- CUDA coredump +- OTel trace +- torch profile + +### 2. Save the failing request + +Read [references/decision-tree.md](references/decision-tree.md) only if the +problem class is still unclear: + +- server down or unhealthy +- latency or throughput regression +- wrong output or behavior regression +- intermittent timeout or hang + +Then preserve the request payload that actually triggers the problem: + +- crash path: use `--crash-dump-folder` +- non-crash path: enable request dump or save the exact trigger request + +Do not jump straight from a live symptom to low-level debugging without first +saving something you can replay. + +### 3. Replay on a clean target + +Read [references/endpoints-and-signals.md](references/endpoints-and-signals.md) +when you need help reading the baseline bundle or the replay target. + +Read [references/replay-trace-profile.md](references/replay-trace-profile.md) +when you need the replay, trace, profile, or bisect paths. + +Standard order: + +1. collect baseline bundle +2. capture request dump or crash dump +3. restart a clean debug target if needed +4. replay the same issue +5. collect replay-time logs and dumps + +### 4. Only go deeper after replay + +#### Replay + +Use replay when: + +- a crash dump exists +- a request dump exists +- the problem depends on request shape or workload mix + +If a crash dump exists, summarize it first: + +```bash +python3 scripts/incident_artifact_tool.py summarize-dump \ + --input-file /path/to/crash_dump.pkl +``` + +Then replay: + +```bash +python3 /path/to/sglang/scripts/playground/replay_request_dump.py \ + --input-file /path/to/crash_dump.pkl \ + --host 127.0.0.1 \ + --port 30000 \ + --parallel 128 +``` + +If `safe_pickle_load` blocks a locally captured trusted dump, use: + +```bash +python3 scripts/replay_trusted_request_dump.py \ + --input-file /path/to/request_dump.pkl \ + --host 127.0.0.1 \ + --port 30000 \ + --parallel 1 +``` + +If replay indicates a CUDA crash path, restart the same build with coredumps +enabled before reproducing again: + +```bash +SGLANG_CUDA_COREDUMP=1 \ +SGLANG_CUDA_COREDUMP_DIR=/tmp/sglang_cuda_coredumps \ +python -m sglang.launch_server \ + --model-path ... \ + --crash-dump-folder /tmp/sglang_crash_dump \ + ... +``` + +Then inspect the generated coredump: + +```bash +cuda-gdb "$(which python3)" \ + -ex "target cudacore /tmp/sglang_cuda_coredumps/cuda_coredump_.." +``` + +For a replay-first crash example, read +[references/case-studies.md](references/case-studies.md). + +#### OTel trace + +Use tracing when: + +- request-stage timing is unclear +- router vs. worker attribution is unclear +- PD prefill/decode transfer may be implicated + +If tracing was enabled at startup, you can change the level without restart: + +```bash +curl "http://127.0.0.1:30000/set_trace_level?level=1" +curl "http://127.0.0.1:30000/set_trace_level?level=2" +``` + +#### Torch profile + +Use profiling when: + +- the issue is already narrowed to compute-side ownership +- replay already reproduces the problem +- metrics and loads do not explain the regression + +At that point, switch to `llm-torch-profiler-analysis`. Do not duplicate +its profiling workflow here. + +For a low-noise latency example, read +[references/case-studies.md](references/case-studies.md). + +#### Distributed hang + +If this looks like a collective stall, save the failing request, replay it on a +clean target, collect the replay-time bundle and stacks, then switch to +`debug-distributed-hang`. + +For an example of that flow, read +[references/case-studies.md](references/case-studies.md). + +#### Regression between two commits + +If one commit is known-good and another is known-bad, build a deterministic +harness before doing deeper manual debugging: + +1. choose a stable reproducer: request replay, benchmark command, or correctness check +2. make the harness return `0` on good behavior and non-zero on bad behavior +3. run `git bisect start ` +4. run `git bisect run ` +5. return here only after a candidate commit is isolated + +Prefer replay-backed bisect when the regression depends on request shape or +long-running serving state. + +### 6. Switch tools when the boundary is clear + +Switch tools once the fault class is clear: + +- `llm-torch-profiler-analysis` for kernel and overlap attribution +- `debug-distributed-hang` for collective or rank-divergence hangs +- `debug-cuda-crash` for CUDA crash reproduction and kernel API logging + +Do not switch tools before collecting the first bundle unless the user already has +decisive logs or dumps. + +## References + +Load only what the current step needs: + +- [references/decision-tree.md](references/decision-tree.md) + - problem classes, tool switch points, return shape +- [references/endpoints-and-signals.md](references/endpoints-and-signals.md) + - endpoint behavior, auth notes, field reading +- [references/replay-trace-profile.md](references/replay-trace-profile.md) + - request dump, crash dump, replay, trace, profiler step, bisect +- [references/case-studies.md](references/case-studies.md) + - compact examples for replay-first CUDA crash, latency, and distributed-hang triage + +## Scripts + +- [scripts/incident_artifact_tool.py](scripts/incident_artifact_tool.py) + - collect a read-only live bundle + - summarize a collected bundle into a compact debug note + - summarize a trusted request dump or crash dump before replay +- [scripts/replay_trusted_request_dump.py](scripts/replay_trusted_request_dump.py) + - replay a trusted request dump when `safe_pickle_load` blocks stock replay + +If a live bundle was collected, include its path. + +If replay, trace, or profiling was chosen, say why bundle plus dump were not enough. diff --git a/.claude/skills/sglang-prod-incident-triage/references/case-studies.md b/.claude/skills/sglang-prod-incident-triage/references/case-studies.md new file mode 100644 index 000000000..70de41d51 --- /dev/null +++ b/.claude/skills/sglang-prod-incident-triage/references/case-studies.md @@ -0,0 +1,81 @@ +# Case Studies + +Use these examples only after the live bundle and request dump point toward the +same class of failure. They are patterns for how to reason from replayable +evidence, not recipes to copy blindly. + +## CUDA Crash: Upstream Top-K Corruption, Downstream MoE OOB + +Use when a replayed CUDA crash lands in a MoE align or shared-memory kernel but +the suspicious data was produced by an earlier routing kernel. + +Shape that made the original case useful: + +- model family: Qwen3 MoE +- visible crash: `moe_align_block_size_kernel` +- likely producer: `topkGatingSoftmax` / MoE top-k routing +- evidence path: crash dump -> replay -> CUDA coredump -> walk one kernel + upstream from the visible fault + +Triage loop: + +```text +summarize crash dump + -> replay the exact request + -> enable CUDA coredump on the replay target + -> identify the failing kernel + -> inspect the immediately preceding producer kernel and tensors +``` + +Key lesson: a consumer kernel can be the first one to fault even when the bad +index was produced earlier. Preserve the request shape before changing prompts. + +## Latency: TTFT Spike With Low Queue Time + +Use when `/health` and `/health_generate` are green, queue depth is low, but TTFT +is still high. + +Signals from the original case: + +- `waiting=0` +- average queue time was tiny +- TTFT was high +- scheduler stage timing pointed to prefill forward time + +Triage loop: + +```text +collect live bundle + -> save the slow request + -> replay the same request on a clean target + -> profile only after replay reproduces compute-side ownership +``` + +Key lesson: rule out queue pressure with `/v1/loads`, `/metrics`, and stage +timing before opening a profiler trace. + +## Distributed Hang: Request-Shaped TP Collective Mismatch + +Use when one request hangs, ranks stop making progress differently, and the +failure looks like a generic serving stall until replay isolates it. + +Shape that made the original case useful: + +- a prompt tokenized to a specific extend length +- one TP rank skipped a logits `all_gather` +- the peer rank still entered the real collective +- the request never returned + +Triage loop: + +```text +collect healthy bundle + -> save the trigger request + -> replay on a clean target + -> collect rank stacks and replay-time bundle + -> switch to debug-distributed-hang +``` + +Key lesson: once the symptom looks like rank divergence or a collective mismatch, +do not keep profiling kernels. Preserve the replay and move to distributed-hang +debugging. diff --git a/.claude/skills/sglang-prod-incident-triage/references/decision-tree.md b/.claude/skills/sglang-prod-incident-triage/references/decision-tree.md new file mode 100644 index 000000000..9aa03b567 --- /dev/null +++ b/.claude/skills/sglang-prod-incident-triage/references/decision-tree.md @@ -0,0 +1,197 @@ +# SGLang First Checks + +Use this reference when the problem class is still unclear and you need a fast +starting point. + +## Default Order + +1. classify the symptom +2. collect the fastest useful signal +3. save the failing request or dump +4. replay before you profile + +Do not start with `torch.profiler` unless the issue is already clearly +compute-side. + +If one commit is known-good and another is known-bad, turn the problem into a +stable `git bisect run ` first. + +## Problem Classes + +### Server down or unhealthy + +Check: + +- `/health` +- `/health_generate` +- `/server_info` +- recent stderr/stdout +- crash dump status if `--crash-dump-folder` is enabled + +Likely directions: + +- startup or weight-load failure +- deadlock or blocked scheduler +- CUDA crash or OOM +- auth or routing mismatch + +### High latency or low throughput + +Check: + +- `/v1/loads?include=all` +- `/metrics` +- `/server_info` +- the exact request shape or benchmark command + +Likely directions: + +- queueing or capacity pressure +- cache hit rate collapse +- PD or EP topology mismatch +- speculative decoding disabled or ineffective +- kernel or backend regression + +### Wrong output or behavior regression + +Check: + +- exact request and expected output +- `/model_info` +- `/server_info` +- current weights or recent config change + +Likely directions: + +- wrong weights or wrong revision +- chat template, parser, or tool config drift +- multimodal preprocessing drift +- quantization or kernel correctness bug + +### Timeout or hang + +Check: + +- `/health` +- `/health_generate` +- `/v1/loads?include=all` +- request dumps if enabled +- per-rank logs +- OTel trace if already enabled + +Likely directions: + +- distributed divergence or collective hang +- queue starvation or retraction storm +- PD transfer stall +- storage or HiCache backend stall + +## Quick Paths + +### TTFT spike + +Start with: + +- `/v1/loads?include=all` +- `/metrics` +- `/server_info` + +Watch for: + +- `num_waiting_reqs` growth +- `token_usage` saturation +- `cache_hit_rate` drop +- PD queue buildup + +If queue pressure does not explain the slowdown, save the slow request and +replay it. + +### Throughput collapse + +Start with: + +- `/v1/loads?include=all` +- `/metrics` +- benchmark reproduction if available + +Watch for: + +- low `gen_throughput` +- queue growth +- low cache hit rate +- speculative metrics collapse +- PD transfer or decode prealloc queues backing up + +### Crash after some requests + +Start with: + +- crash dump folder +- stderr/stdout +- request dump folder if available + +Then replay the crash dump or recent request dump. + +### Regression between two commits + +Start with: + +- known-good commit +- known-bad commit +- one stable pass/fail harness + +Best move: + +- `git bisect run ` + +### One request class fails + +Start with: + +- exact request payload +- request dump if available +- smallest reproduction request + +Typical categories: + +- multimodal edge case +- parser or structured output bug +- model-specific kernel path +- tool-call formatting issue + +## When To Switch Tools + +### Use replay when + +- a crash dump or request dump already exists +- the issue depends on request shape or workload mix +- you need one stable reproducer before going deeper + +### Use OTel trace when + +- request-stage timing is unclear +- router vs. worker ownership is unclear +- PD boundaries may be involved + +### Use torch profiler when + +- replay already reproduces the issue +- queueing and routing are mostly ruled out +- you need kernel-level attribution + +At that point, switch to `llm-torch-profiler-analysis`. + +### Use lower-level debug paths when + +- replay plus trace still leave ambiguity +- the problem looks like a specific crash, hang, or correctness bug + +## What To Return + +- problem class +- what was checked +- strongest signal so far +- current best guess +- what was ruled out +- next step +- production risk diff --git a/.claude/skills/sglang-prod-incident-triage/references/endpoints-and-signals.md b/.claude/skills/sglang-prod-incident-triage/references/endpoints-and-signals.md new file mode 100644 index 000000000..c99e95d7a --- /dev/null +++ b/.claude/skills/sglang-prod-incident-triage/references/endpoints-and-signals.md @@ -0,0 +1,218 @@ +# SGLang Endpoints and Signals + +Use this reference when checking a live server. + +## Auth + +Most read endpoints are public unless the server is protected by `api_key` or +`admin_api_key`. + +Use: + +```bash +curl -H "Authorization: Bearer " ... +``` + +Rules: + +- normal protected endpoints require `api_key` +- admin endpoints require `admin_api_key` +- some HiCache endpoints fail if `admin_api_key` is not configured at all +- `/health` and metrics-style health checks are usually still exposed + +## Core Endpoints + +### `/health` + +Cheap liveness check. + +- `200`: process is alive enough to answer health +- `503`: starting, shutting down, or unhealthy + +`/health` alone is not enough for latency or hang diagnosis. + +### `/health_generate` + +Active health check. + +- exercises a real generate or embedding path +- catches stuck schedulers or broken worker paths that `/health` can miss + +Use this when requests time out but `/health` is still green. + +### `/model_info` + +Use for model identity: + +- `model_path` +- `tokenizer_path` +- `is_generation` +- `weight_version` +- multimodal flags +- model type or architectures + +This is the first check for wrong-output or wrong-weight problems. + +### `/server_info` + +Use for runtime shape: + +- serialized `server_args` +- scheduler info +- per-DP `internal_states` +- SGLang version + +This is usually the single best live snapshot. + +## Load And Capacity + +### `/v1/loads?include=all` + +Best structured load endpoint for a first pass. + +Useful fields: + +- `num_running_reqs` +- `num_waiting_reqs` +- `num_total_tokens` +- `num_used_tokens` +- `token_usage` +- `gen_throughput` +- `cache_hit_rate` +- `memory` +- `speculative` +- `disaggregation` +- `queues` + +Useful queries: + +```bash +curl -s http://127.0.0.1:30000/v1/loads +curl -s "http://127.0.0.1:30000/v1/loads?include=all" +curl -s "http://127.0.0.1:30000/v1/loads?include=core,queues,disagg" +curl -s "http://127.0.0.1:30000/v1/loads?format=prometheus" +``` + +What to look for: + +- high `num_waiting_reqs` with low compute throughput usually means queueing or capacity pressure +- `token_usage` near `1.0` usually means KV or token-capacity pressure +- low `cache_hit_rate` after a deploy can explain TTFT regressions +- PD queue fields often explain transfer or prealloc bottlenecks hidden by plain queue size + +### `/metrics` + +Prometheus endpoint. Use it when you need trends rather than one live snapshot. + +High-value metrics: + +- `sglang:time_to_first_token_seconds` +- `sglang:time_per_output_token_seconds` +- `sglang:e2e_request_latency_seconds` +- `sglang:num_running_reqs` +- `sglang:num_queue_reqs` +- `sglang:num_used_tokens` +- `sglang:cache_hit_rate` +- `sglang:gen_throughput` +- `sglang:token_usage` + +## Request Capture + +### `/configure_logging` + +Used by `python -m sglang.srt.managers.configure_logging`. + +Main use: + +- enable request logging +- set request logging level +- enable request dump folder +- set request dump threshold + +Typical payload: + +```json +{ + "log_requests": true, + "log_requests_level": 3, + "dump_requests_folder": "/tmp/sglang_request_dump", + "dump_requests_threshold": 100 +} +``` + +Use this when the problem is ongoing and you need the next failing request +without restarting the service. + +## HiCache + +### `GET /hicache/storage-backend` + +Returns tokenizer-side HiCache storage status: + +- `hicache_storage_backend` +- `hicache_storage_backend_extra_config` +- `hicache_storage_prefetch_policy` +- `hicache_write_policy` + +Use this when long-context or PD problems may involve storage-backed KV reuse. + +### `PUT /hicache/storage-backend` +### `DELETE /hicache/storage-backend` + +Runtime attach or detach. These are operational actions, not passive checks. + +## Profiling And Tracing Controls + +### `/start_profile` +### `/stop_profile` + +Use only after the problem is already narrowed down. + +### `/set_trace_level?level=N` + +Changes trace verbosity when tracing was enabled at startup. + +Levels: + +- `0`: disabled +- `1`: important slices +- `2`: all slices except nested ones +- `3`: all slices + +## Quick Reads By Problem Type + +### TTFT spike + +Read: + +- `/server_info` +- `/v1/loads?include=all` +- `/metrics` + +Compare: + +- queue size +- token usage +- cache hit rate +- PD disaggregation queues + +### Hang or timeout + +Read: + +- `/health` +- `/health_generate` +- `/server_info` +- `/v1/loads?include=all` + +If tracing is already enabled, look at trace data before heavier profiling. + +### Wrong model behavior + +Read: + +- `/model_info` +- `/server_info` +- exact request payload and parser or template config + +Do not jump to kernel profiling until config drift is ruled out. diff --git a/.claude/skills/sglang-prod-incident-triage/references/replay-trace-profile.md b/.claude/skills/sglang-prod-incident-triage/references/replay-trace-profile.md new file mode 100644 index 000000000..6dde7a254 --- /dev/null +++ b/.claude/skills/sglang-prod-incident-triage/references/replay-trace-profile.md @@ -0,0 +1,236 @@ +# Replay, Trace, Profile, and Bisect + +Use this reference after the first live checks. The goal is to turn the problem +into something repeatable. + +## Save Requests + +### Request dump + +```bash +python3 -m sglang.srt.managers.configure_logging \ + --url http://127.0.0.1:30000 \ + --dump-requests-folder /tmp/sglang_request_dump \ + --dump-requests-threshold 100 +``` + +Use this when: + +- the problem is intermittent +- you need the real request shape +- you do not want to restart the server + +### Crash dump + +If the server already runs with: + +```bash +--crash-dump-folder /tmp/crash_dump +``` + +SGLang saves recent requests before a crash. Treat that dump as the best +starting point. + +Summarize it first: + +```bash +python3 scripts/incident_artifact_tool.py summarize-dump \ + --input-file /path/to/crash_dump.pkl +``` + +Current crash-dump tests show at least: + +- `server_args` +- `requests` +- `launch_command` + +## Replay + +Use the stock replay tool: + +```bash +python3 scripts/playground/replay_request_dump.py \ + --input-file /path/to/crash_dump.pkl \ + --host 127.0.0.1 \ + --port 30000 \ + --parallel 128 +``` + +Or replay a folder: + +```bash +python3 scripts/playground/replay_request_dump.py \ + --input-folder /path/to/request_dump_dir \ + --file-number 10 \ + --parallel 128 +``` + +If `safe_pickle_load` blocks a locally captured trusted dump, use: + +```bash +python3 scripts/replay_trusted_request_dump.py \ + --input-file /path/to/request_dump.pkl \ + --host 127.0.0.1 \ + --port 30000 \ + --parallel 1 +``` + +If that happens, the allowlist is the problem, not the dump. + +Use replay before profiling when: + +- the issue depends on workload mix +- it only appears after some number of requests +- you need to compare two builds on the same traffic + +## CUDA Restart-And-Replay + +If replay points to a CUDA crash path, restart the same build with coredumps: + +```bash +SGLANG_CUDA_COREDUMP=1 \ +SGLANG_CUDA_COREDUMP_DIR=/tmp/sglang_cuda_coredumps \ +python -m sglang.launch_server \ + --model-path ... \ + --crash-dump-folder /tmp/sglang_crash_dump \ + ... +``` + +Then inspect the coredump: + +```bash +cuda-gdb "$(which python3)" \ + -ex "target cudacore /tmp/sglang_cuda_coredumps/cuda_coredump_.." +``` + +Good first commands: + +- `where` +- `info cuda kernels` +- `x/10i ` + +Use the coredump to find the failing kernel, not automatically the root-cause +kernel. + +See: + +- [case-studies.md](case-studies.md) + +## Trace + +Tracing must be enabled at startup: + +```bash +python -m sglang.launch_server \ + --enable-trace \ + --otlp-traces-endpoint localhost:4317 \ + ... +``` + +Optional router command: + +```bash +python -m sglang_router.launch_router \ + --enable-trace \ + --otlp-traces-endpoint localhost:4317 \ + ... +``` + +Useful environment variables: + +```bash +export SGLANG_OTLP_EXPORTER_SCHEDULE_DELAY_MILLIS=500 +export SGLANG_OTLP_EXPORTER_MAX_EXPORT_BATCH_SIZE=64 +``` + +If tracing is already enabled, change the level without restart: + +```bash +curl "http://127.0.0.1:30000/set_trace_level?level=1" +curl "http://127.0.0.1:30000/set_trace_level?level=2" +curl "http://127.0.0.1:30000/set_trace_level?level=3" +``` + +Use tracing for: + +- router vs. worker delay +- tokenizer / scheduler / detokenizer timing +- PD transfer timing +- request timing across processes + +If you already have OTEL JSON or JSONL, convert it for timeline inspection: + +```bash +python3 scripts/convert_otel_2_perfetto.py \ + --input /tmp/otel_trace.json \ + --output /tmp/sglang_trace_perfetto.json +``` + +## Torch Profiler + +Switch to `llm-torch-profiler-analysis` when: + +- replay already reproduces the issue +- metrics and loads do not explain it +- the problem now looks compute-side + +This skill should decide when to profile, not duplicate the profiler workflow. + +## Bisect + +If one commit is known-good and a newer commit is known-bad: + +1. build a deterministic harness from the problem +2. prefer replay-based harnesses when the failure depends on request mix +3. use `git bisect run ` +4. only then go back to trace or profile if needed + +Example: + +```bash +git bisect start +git bisect run bash ./repro_or_check.sh +``` + +## Common Paths + +### Crash + +1. crash dump +2. summarize dump +3. replay +4. CUDA coredump plus `cuda-gdb` +5. `debug-cuda-crash` or narrower instrumentation + +### TTFT regression + +1. baseline metrics and loads +2. request dump +3. replay the slow request +4. trace if stage ownership is unclear +5. `llm-torch-profiler-analysis` if it still looks compute-side + +See: + +- [case-studies.md](case-studies.md) + +### Distributed hang + +1. healthy baseline bundle +2. save the trigger request +3. replay on a clean target +4. collect replay-time bundle and stacks +5. identify the NCCL or collective path +6. switch to `debug-distributed-hang` + +See: + +- [case-studies.md](case-studies.md) + +### Throughput regression after deploy + +1. compare `server_info` +2. compare `/metrics` and `/v1/loads` +3. replay stable workload +4. bisect if one older commit is known-good +5. profile only if compute still looks suspicious diff --git a/.claude/skills/sglang-prod-incident-triage/scripts/incident_artifact_tool.py b/.claude/skills/sglang-prod-incident-triage/scripts/incident_artifact_tool.py new file mode 100755 index 000000000..2a9dacc69 --- /dev/null +++ b/.claude/skills/sglang-prod-incident-triage/scripts/incident_artifact_tool.py @@ -0,0 +1,735 @@ +#!/usr/bin/env python3 +"""Collect or inspect serving bundles and dumps for SGLang debug.""" + +from __future__ import annotations + +import argparse +import glob +import json +import math +import os +import pickle +import re +import time +from collections import defaultdict +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, Optional, Sequence +from urllib import error, parse, request + +METRIC_RE = re.compile( + r"^(?P[^{\s]+)(?:\{(?P[^}]*)\})?\s+(?P[-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)$" +) +LABEL_RE = re.compile(r'([a-zA-Z_:][a-zA-Z0-9_:]*)="((?:[^"\\]|\\.)*)"') +ENDPOINT_SPECS = ( + ("text", "health.txt", "/health"), + ("text", "health_generate.txt", "/health_generate"), + ("text", "metrics.txt", "/metrics"), + ("json", "model_info.json", "/model_info"), + ("json", "server_info.json", "/server_info"), + ("json", "loads_all.json", "/v1/loads?include=all"), + ( + "json", + "loads_core_queues_disagg.json", + "/v1/loads?include=core,queues,disagg,spec", + ), + ("json", "hicache_storage_backend.json", "/hicache/storage-backend"), +) +BUNDLE_NOTES = [ + "This bundle is read-only. It does not start profiling or change trace level.", + "HiCache status may fail if admin_api_key is not configured or the wrong bearer token was used.", + "loads_all.json is the best point-in-time load snapshot in this bundle.", + "metrics.txt is raw Prometheus text intended for follow-up parsing.", +] + + +def request_text( + base_url: str, + path: str, + token: Optional[str], + timeout: float = 10.0, +) -> tuple[bool, int, str]: + url = parse.urljoin(base_url.rstrip("/") + "/", path.lstrip("/")) + req = request.Request(url) + if token: + req.add_header("Authorization", f"Bearer {token}") + try: + with request.urlopen(req, timeout=timeout) as resp: + body = resp.read().decode("utf-8", errors="replace") + return True, resp.status, body + except error.HTTPError as e: + body = e.read().decode("utf-8", errors="replace") + return False, e.code, body + except Exception as e: # noqa: BLE001 + return False, -1, f"{type(e).__name__}: {e}" + + +def request_endpoint( + base_url: str, + path: str, + token: Optional[str], + parse_json: bool, + timeout: float = 10.0, +) -> Dict[str, Any]: + ok, status, body = request_text(base_url, path, token, timeout=timeout) + result: Dict[str, Any] = {"ok": ok, "status": status, "path": path} + if not ok: + result["error"] = body + return result + if not parse_json: + result["text"] = body + return result + try: + result["json"] = json.loads(body) + except json.JSONDecodeError: + result["text"] = body + result["decode_error"] = "response was not valid JSON" + return result + + +def write_json(path: Path, obj: Dict[str, Any]) -> None: + path.write_text( + json.dumps(obj, indent=2, ensure_ascii=False) + "\n", encoding="utf-8" + ) + + +def write_text(path: Path, text: str) -> None: + path.write_text(text, encoding="utf-8") + + +def format_summary_line(filename: str, result: Dict[str, Any]) -> str: + if result.get("ok"): + return f"{filename}: ok" + return ( + f"{filename}: failed status={result.get('status')} " + f"error={result.get('error')}" + ) + + +def collect_bundle( + base_url: str, + token: Optional[str], + outdir: Optional[str], + timeout: float, +) -> Path: + timestamp = time.strftime("%Y%m%d_%H%M%S") + bundle_dir = Path(outdir or f"./incident_bundle_{timestamp}").resolve() + bundle_dir.mkdir(parents=True, exist_ok=True) + + metadata = { + "artifact_type": "incident_bundle", + "base_url": base_url, + "collected_at": timestamp, + "token_provided": bool(token), + "timeout_seconds": timeout, + } + write_json(bundle_dir / "metadata.json", metadata) + + summary_lines = [] + for kind, filename, path in ENDPOINT_SPECS: + result = request_endpoint( + base_url, path, token, parse_json=(kind == "json"), timeout=timeout + ) + output_path = bundle_dir / filename + if kind == "text" and result.get("ok"): + write_text(output_path, str(result.get("text", ""))) + else: + write_json( + ( + output_path + if kind == "json" + else bundle_dir / f"{filename}.error.json" + ), + result, + ) + summary_lines.append(format_summary_line(filename, result)) + + write_text( + bundle_dir / "SUMMARY.txt", + "\n".join(summary_lines + [""] + BUNDLE_NOTES) + "\n", + ) + return bundle_dir + + +def load_json(path: Path) -> Optional[Dict[str, Any]]: + if not path.exists(): + return None + return json.loads(path.read_text(encoding="utf-8")) + + +def unwrap_result(path: Path) -> Optional[Dict[str, Any]]: + obj = load_json(path) + if obj is None: + return None + if isinstance(obj, dict) and "json" in obj: + return obj.get("json") + return obj + + +def read_text(path: Path) -> Optional[str]: + if not path.exists(): + return None + return path.read_text(encoding="utf-8") + + +def endpoint_ok(bundle_dir: Path, stem: str) -> bool: + return (bundle_dir / f"{stem}.txt").exists() and not ( + bundle_dir / f"{stem}.txt.error.json" + ).exists() + + +def parse_labels(raw: Optional[str]) -> Dict[str, str]: + if not raw: + return {} + labels = {} + for key, value in LABEL_RE.findall(raw): + labels[key] = bytes(value, "utf-8").decode("unicode_escape") + return labels + + +def parse_metrics(metrics_text: str) -> Dict[str, list[dict[str, Any]]]: + series: Dict[str, list[dict[str, Any]]] = defaultdict(list) + for line in metrics_text.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + match = METRIC_RE.match(line) + if not match: + continue + series[match.group("name")].append( + { + "labels": parse_labels(match.group("labels")), + "value": float(match.group("value")), + } + ) + return series + + +def metric_sum(metrics: Dict[str, list[dict[str, Any]]], name: str) -> float: + return sum(item["value"] for item in metrics.get(name, [])) + + +def safe_div( + numerator: Optional[float], denominator: Optional[float] +) -> Optional[float]: + if numerator is None or denominator in (None, 0): + return None + return numerator / denominator + + +def coalesce(*values: Any) -> Any: + for value in values: + if value is not None: + return value + return None + + +def fmt_float(value: Optional[float], digits: int = 3) -> str: + if value is None or ( + isinstance(value, float) and (math.isnan(value) or math.isinf(value)) + ): + return "n/a" + return f"{value:.{digits}f}" + + +def is_positive_number(value: Any, threshold: float = 0.0) -> bool: + return ( + isinstance(value, (int, float)) + and not math.isnan(value) + and not math.isinf(value) + and value > threshold + ) + + +def compute_stage_averages( + metrics: Dict[str, list[dict[str, Any]]], sum_name: str, count_name: str +) -> Dict[str, float]: + grouped_sum: Dict[str, float] = defaultdict(float) + grouped_count: Dict[str, float] = defaultdict(float) + for item in metrics.get(sum_name, []): + stage = item["labels"].get("stage", "") + rank = item["labels"].get("tp_rank", "") + grouped_sum[f"{stage}|{rank}"] += item["value"] + for item in metrics.get(count_name, []): + stage = item["labels"].get("stage", "") + rank = item["labels"].get("tp_rank", "") + grouped_count[f"{stage}|{rank}"] += item["value"] + + result: Dict[str, float] = {} + for key, total_sum in grouped_sum.items(): + stage, _rank = key.split("|", 1) + avg = safe_div(total_sum, grouped_count.get(key)) + if avg is None: + continue + result[stage] = max(result.get(stage, 0.0), avg) + return result + + +def add_signal(signals: list[str], text: str) -> None: + if text not in signals: + signals.append(text) + + +def build_bundle_summary(bundle_dir: Path) -> Dict[str, Any]: + metadata = load_json(bundle_dir / "metadata.json") or {} + model_info = unwrap_result(bundle_dir / "model_info.json") or {} + server_info = unwrap_result(bundle_dir / "server_info.json") or {} + loads_info = unwrap_result(bundle_dir / "loads_all.json") or {} + metrics_text = read_text(bundle_dir / "metrics.txt") or "" + metrics = parse_metrics(metrics_text) + + aggregate = loads_info.get("aggregate") or {} + loads = loads_info.get("loads") or [] + load0 = loads[0] if loads else {} + internal_states = server_info.get("internal_states") or [] + runtime_state = internal_states[0] if internal_states else {} + memory_usage = runtime_state.get("memory_usage") or load0.get("memory") or {} + + ttft_avg = safe_div( + metric_sum(metrics, "sglang:time_to_first_token_seconds_sum"), + metric_sum(metrics, "sglang:time_to_first_token_seconds_count"), + ) + e2e_avg = safe_div( + metric_sum(metrics, "sglang:e2e_request_latency_seconds_sum"), + metric_sum(metrics, "sglang:e2e_request_latency_seconds_count"), + ) + queue_avg = safe_div( + metric_sum(metrics, "sglang:queue_time_seconds_sum"), + metric_sum(metrics, "sglang:queue_time_seconds_count"), + ) + per_stage_avg = compute_stage_averages( + metrics, + "sglang:per_stage_req_latency_seconds_sum", + "sglang:per_stage_req_latency_seconds_count", + ) + + summary: Dict[str, Any] = { + "artifact_type": "incident_bundle", + "bundle_dir": str(bundle_dir), + "base_url": metadata.get("base_url"), + "collected_at": metadata.get("collected_at"), + "health": { + "health_ok": endpoint_ok(bundle_dir, "health"), + "health_generate_ok": endpoint_ok(bundle_dir, "health_generate"), + }, + "model": { + "model_path": model_info.get("model_path") or server_info.get("model_path"), + "served_model_name": server_info.get("served_model_name"), + "weight_version": model_info.get("weight_version") + or server_info.get("weight_version"), + "model_type": model_info.get("model_type"), + "is_generation": model_info.get("is_generation"), + }, + "topology": { + "tp_size": server_info.get("tp_size"), + "dp_size": server_info.get("dp_size"), + "pp_size": server_info.get("pp_size"), + "ep_size": server_info.get("ep_size"), + "disaggregation_mode": server_info.get("disaggregation_mode"), + "attention_backend": server_info.get("attention_backend"), + "sampling_backend": server_info.get("sampling_backend"), + "schedule_policy": server_info.get("schedule_policy"), + "enable_trace": server_info.get("enable_trace"), + "enable_metrics": server_info.get("enable_metrics"), + }, + "capacity": { + "max_total_num_tokens": server_info.get("max_total_num_tokens"), + "max_req_input_len": server_info.get("max_req_input_len"), + "effective_max_running_requests_per_dp": coalesce( + runtime_state.get("effective_max_running_requests_per_dp"), + load0.get("max_running_requests"), + ), + "weight_gb": coalesce( + memory_usage.get("weight"), memory_usage.get("weight_gb") + ), + "kv_cache_gb": coalesce( + memory_usage.get("kvcache"), memory_usage.get("kv_cache_gb") + ), + "graph_gb": coalesce( + memory_usage.get("graph"), memory_usage.get("graph_gb") + ), + "token_capacity": memory_usage.get("token_capacity"), + }, + "point_in_time_load": { + "running_reqs": coalesce( + aggregate.get("total_running_reqs"), load0.get("num_running_reqs") + ), + "waiting_reqs": coalesce( + aggregate.get("total_waiting_reqs"), load0.get("num_waiting_reqs") + ), + "total_reqs": coalesce( + aggregate.get("total_reqs"), load0.get("num_total_reqs") + ), + "token_usage": coalesce( + aggregate.get("avg_token_usage"), load0.get("token_usage") + ), + "avg_throughput": coalesce( + aggregate.get("avg_throughput"), load0.get("gen_throughput") + ), + "avg_utilization": coalesce( + aggregate.get("avg_utilization"), load0.get("utilization") + ), + "cache_hit_rate": load0.get("cache_hit_rate"), + "queues": load0.get("queues"), + "disaggregation": load0.get("disaggregation"), + }, + "metrics": { + "request_count": metric_sum(metrics, "sglang:num_requests_total"), + "prompt_tokens_total": metric_sum(metrics, "sglang:prompt_tokens_total"), + "generation_tokens_total": metric_sum( + metrics, "sglang:generation_tokens_total" + ), + "avg_ttft_seconds": ttft_avg, + "avg_e2e_seconds": e2e_avg, + "avg_queue_time_seconds": queue_avg, + "stage_avg_seconds_max_tp_rank": per_stage_avg, + }, + "signals": [], + } + + signals = summary["signals"] + health = summary["health"] + point_in_time_load = summary["point_in_time_load"] + running_reqs = point_in_time_load.get("running_reqs") + waiting_reqs = point_in_time_load.get("waiting_reqs") + + if health["health_ok"] and not health["health_generate_ok"]: + add_signal( + signals, + "/health is green but /health_generate failed. Suspect runtime or scheduler path, not just HTTP liveness.", + ) + if not health["health_ok"]: + add_signal( + signals, + "/health failed. Start with startup, crash, or global unhealthy paths.", + ) + if is_positive_number(waiting_reqs): + add_signal( + signals, + f"Point-in-time load shows queue buildup: waiting_reqs={waiting_reqs}.", + ) + if ( + point_in_time_load.get("token_usage") is not None + and point_in_time_load["token_usage"] >= 0.9 + ): + add_signal( + signals, + "Token usage is near saturation. KV or token-capacity pressure may explain latency.", + ) + if ( + ttft_avg is not None + and queue_avg is not None + and ttft_avg > 2.0 + and queue_avg < 0.2 + ): + add_signal( + signals, + f"Average TTFT is high ({fmt_float(ttft_avg)}s) while average queue time is low ({fmt_float(queue_avg)}s). This looks more like prefill or request-path work than queue pressure.", + ) + prefill_forward = per_stage_avg.get("prefill_forward") + request_process = per_stage_avg.get("request_process") + if ( + prefill_forward is not None + and request_process is not None + and prefill_forward > max(0.5, request_process * 10) + ): + add_signal( + signals, + f"Prefill forward dominates quick stage timing: prefill_forward~{fmt_float(prefill_forward)}s vs request_process~{fmt_float(request_process)}s.", + ) + if running_reqs == 0 and waiting_reqs == 0: + add_signal( + signals, + "Bundle snapshot was captured while the server was effectively idle. Reproduce under live traffic or replayed workload if the problem is intermittent.", + ) + + return summary + + +def render_bundle_text(summary: Dict[str, Any]) -> str: + health = summary["health"] + model = summary["model"] + topology = summary["topology"] + capacity = summary["capacity"] + load = summary["point_in_time_load"] + metrics = summary["metrics"] + stage_avgs = metrics["stage_avg_seconds_max_tp_rank"] + + lines = [ + f"Bundle: {summary['bundle_dir']}", + f"Base URL: {summary.get('base_url') or 'n/a'}", + f"Collected At: {summary.get('collected_at') or 'n/a'}", + "", + f"Health: /health={'ok' if health['health_ok'] else 'failed'} /health_generate={'ok' if health['health_generate_ok'] else 'failed'}", + f"Model: {model.get('model_path') or 'n/a'} weight_version={model.get('weight_version') or 'n/a'} type={model.get('model_type') or 'n/a'}", + "Topology: " + f"tp={topology.get('tp_size')} dp={topology.get('dp_size')} pp={topology.get('pp_size')} ep={topology.get('ep_size')} " + f"disagg={topology.get('disaggregation_mode')} trace={topology.get('enable_trace')} metrics={topology.get('enable_metrics')}", + "Capacity: " + f"max_total_tokens={capacity.get('max_total_num_tokens')} " + f"max_running_reqs={capacity.get('effective_max_running_requests_per_dp')} " + f"weight_gb={fmt_float(capacity.get('weight_gb'))} " + f"kv_cache_gb={fmt_float(capacity.get('kv_cache_gb'))} " + f"graph_gb={fmt_float(capacity.get('graph_gb'))}", + "Point-in-time load: " + f"running={load.get('running_reqs')} waiting={load.get('waiting_reqs')} total={load.get('total_reqs')} " + f"token_usage={fmt_float(load.get('token_usage'))} throughput={fmt_float(load.get('avg_throughput'))} " + f"cache_hit_rate={fmt_float(load.get('cache_hit_rate'))}", + "Metrics: " + f"requests={fmt_float(metrics.get('request_count'), 0)} " + f"prompt_tokens={fmt_float(metrics.get('prompt_tokens_total'), 0)} " + f"generation_tokens={fmt_float(metrics.get('generation_tokens_total'), 0)} " + f"avg_ttft_s={fmt_float(metrics.get('avg_ttft_seconds'))} " + f"avg_e2e_s={fmt_float(metrics.get('avg_e2e_seconds'))} " + f"avg_queue_s={fmt_float(metrics.get('avg_queue_time_seconds'))}", + ] + + if stage_avgs: + stage_parts = [ + f"{name}={fmt_float(value)}s" for name, value in sorted(stage_avgs.items()) + ] + lines.append("Stage Averages (max across TP ranks): " + ", ".join(stage_parts)) + + queues = load.get("queues") or {} + if queues: + lines.append( + "Queues: " + + ", ".join(f"{key}={value}" for key, value in sorted(queues.items())) + ) + + disagg = load.get("disaggregation") or {} + if disagg: + lines.append( + "Disaggregation: " + + ", ".join(f"{key}={value}" for key, value in sorted(disagg.items())) + ) + + lines.append("") + lines.append("What stands out:") + if summary["signals"]: + lines.extend(f"- {signal}" for signal in summary["signals"]) + else: + lines.append("- No strong signal from this bundle.") + + return "\n".join(lines) + "\n" + + +def get_field(obj: Any, name: str, default: Any = None) -> Any: + if obj is None: + return default + if isinstance(obj, dict): + return obj.get(name, default) + return getattr(obj, name, default) + + +def iter_dump_files( + input_file: Optional[str], input_folder: Optional[str] +) -> Sequence[Path]: + if input_file: + return [Path(input_file)] + if input_folder: + return [Path(p) for p in sorted(glob.glob(f"{input_folder}/*.pkl"))] + raise SystemExit("Either --input-file or --input-folder must be provided.") + + +def load_dump_payload(path: Path) -> dict[str, Any]: + with path.open("rb") as fh: + payload = pickle.load(fh) + if isinstance(payload, dict): + return payload + return {"requests": payload} + + +def pick_text_preview(req: Any) -> str: + candidates = [ + get_field(req, "origin_input_text"), + get_field(req, "text"), + get_field(req, "prompt"), + ] + for value in candidates: + if isinstance(value, str) and value: + return value + if isinstance(value, list) and value: + first = value[0] + if isinstance(first, str) and first: + return first + return "" + + +def format_timestamp(ts: Any) -> str: + if not isinstance(ts, (int, float)): + return "n/a" + return datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S") + + +def summarize_request( + record: tuple[Any, dict[str, Any], Any, Any], idx: int, preview_chars: int +) -> list[str]: + req, output, start_time, end_time = record + preview = pick_text_preview(req).replace("\n", " ").strip() + if len(preview) > preview_chars: + preview = preview[: preview_chars - 3] + "..." + + output_dict = output if isinstance(output, dict) else {} + meta_info = get_field(output_dict, "meta_info", {}) or {} + rid = get_field(req, "rid") or get_field(meta_info, "id") + stream = bool(get_field(req, "stream", False)) + prompt_tokens = get_field(meta_info, "prompt_tokens") + completion_tokens = get_field(meta_info, "completion_tokens") + duration = ( + end_time - start_time + if isinstance(start_time, (int, float)) and isinstance(end_time, (int, float)) + else None + ) + + elapsed_str = f"{duration:.3f}" if duration is not None else "n/a" + lines = [ + f"[{idx}] rid={rid or 'n/a'} stream={stream} " + f"prompt_tokens={prompt_tokens if prompt_tokens is not None else 'n/a'} " + f"completion_tokens={completion_tokens if completion_tokens is not None else 'n/a'} " + f"start={format_timestamp(start_time)} elapsed_s={elapsed_str}" + ] + if preview: + lines.append(f" text={preview}") + return lines + + +def summarize_dump_file(path: Path, max_requests: int, preview_chars: int) -> str: + payload = load_dump_payload(path) + requests = payload.get("requests") or [] + server_args = payload.get("server_args") + launch_command = payload.get("launch_command") + + model_path = get_field(server_args, "model_path") + tp_size = get_field(server_args, "tp_size") + dp_size = get_field(server_args, "dp_size") + pp_size = get_field(server_args, "pp_size") + host = get_field(server_args, "host") + port = get_field(server_args, "port") + + timestamps = [ + record[2] + for record in requests + if isinstance(record, tuple) + and len(record) >= 4 + and isinstance(record[2], (int, float)) + ] + time_span = ( + max(timestamps) - min(timestamps) + if len(timestamps) >= 2 + else 0.0 if len(timestamps) == 1 else None + ) + + lines = [ + f"File: {path}", + "Dump Type: request_or_crash_dump", + f"Requests: {len(requests)}", + f"Model: {model_path or 'n/a'}", + f"Topology: tp={tp_size if tp_size is not None else 'n/a'} " + f"dp={dp_size if dp_size is not None else 'n/a'} " + f"pp={pp_size if pp_size is not None else 'n/a'}", + f"Endpoint: {host or 'n/a'}:{port if port is not None else 'n/a'}", + ( + f"Time span seconds: {time_span:.3f}" + if time_span is not None + else "Time span seconds: n/a" + ), + ] + if launch_command: + lines.append(f"Launch command: {launch_command}") + + for idx, record in enumerate(requests[:max_requests]): + if not isinstance(record, tuple) or len(record) < 4: + lines.append(f"[{idx}] Unsupported record shape: {type(record)!r}") + continue + lines.extend(summarize_request(record, idx, preview_chars)) + + if len(requests) > max_requests: + lines.append(f"... truncated {len(requests) - max_requests} more requests") + return "\n".join(lines) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Collect or inspect serving bundles and dumps for SGLang debug." + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + collect_parser = subparsers.add_parser( + "collect-bundle", help="Collect a read-only live bundle from a running server" + ) + collect_parser.add_argument("--base-url", required=True) + collect_parser.add_argument( + "--token", + default=os.environ.get("SGLANG_BEARER_TOKEN"), + help="Bearer token for protected endpoints. Defaults to $SGLANG_BEARER_TOKEN.", + ) + collect_parser.add_argument("--outdir", default=None) + collect_parser.add_argument("--timeout", type=float, default=10.0) + + bundle_parser = subparsers.add_parser( + "summarize-bundle", help="Summarize a bundle directory" + ) + bundle_parser.add_argument("bundle_dir") + bundle_parser.add_argument("--out", default=None) + bundle_parser.add_argument("--json-out", default=None) + bundle_parser.add_argument("--stdout-json", action="store_true") + + dump_parser = subparsers.add_parser( + "summarize-dump", help="Summarize a trusted request dump or crash dump" + ) + dump_parser.add_argument("--input-file", default=None) + dump_parser.add_argument("--input-folder", default=None) + dump_parser.add_argument("--max-requests", type=int, default=20) + dump_parser.add_argument("--preview-chars", type=int, default=160) + + args = parser.parse_args() + + if args.command == "collect-bundle": + bundle_dir = collect_bundle( + args.base_url, args.token, args.outdir, args.timeout + ) + print(bundle_dir) + return 0 + + if args.command == "summarize-bundle": + bundle_dir = Path(args.bundle_dir).resolve() + if not bundle_dir.is_dir(): + raise SystemExit( + f"bundle_dir does not exist or is not a directory: {bundle_dir}" + ) + summary = build_bundle_summary(bundle_dir) + out_text = render_bundle_text(summary) + text_path = Path(args.out) if args.out else bundle_dir / "SUMMARY_REPORT.txt" + json_path = ( + Path(args.json_out) if args.json_out else bundle_dir / "SUMMARY_REPORT.json" + ) + text_path.write_text(out_text, encoding="utf-8") + json_path.write_text( + json.dumps(summary, indent=2, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + if args.stdout_json: + print(json.dumps(summary, indent=2, ensure_ascii=False)) + else: + print(out_text, end="") + return 0 + + files = iter_dump_files(args.input_file, args.input_folder) + if not files: + raise SystemExit("No .pkl files matched the provided input.") + for idx, path in enumerate(files): + if idx: + print() + print( + summarize_dump_file( + path=path, + max_requests=args.max_requests, + preview_chars=args.preview_chars, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.claude/skills/sglang-prod-incident-triage/scripts/replay_trusted_request_dump.py b/.claude/skills/sglang-prod-incident-triage/scripts/replay_trusted_request_dump.py new file mode 100755 index 000000000..768b3a33a --- /dev/null +++ b/.claude/skills/sglang-prod-incident-triage/scripts/replay_trusted_request_dump.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +"""Replay a trusted SGLang request dump directly over HTTP. + +Use this only for locally captured or otherwise trusted dump files. +It uses plain pickle loading to bypass SafeUnpickler restrictions that may block +the stock replay helper on newer SGLang builds. +""" + +from __future__ import annotations + +import argparse +import glob +import json +import pickle +import time +from concurrent.futures import ThreadPoolExecutor +from dataclasses import asdict, is_dataclass +from datetime import datetime +from pathlib import Path +from typing import Any, Sequence + +import requests + +Record = tuple[object, dict[str, Any], float, float] + + +def normalize_mm_data_item(item: Any) -> Any: + if isinstance(item, dict) and "url" in item: + return item["url"] + return item + + +def normalize_mm_data(data: Any) -> Any: + if data is None: + return None + if isinstance(data, list): + return [ + ( + [normalize_mm_data_item(item) for item in sublist] + if isinstance(sublist, list) + else normalize_mm_data_item(sublist) + ) + for sublist in data + ] + return normalize_mm_data_item(data) + + +def normalize_request_data(json_data: dict[str, Any]) -> dict[str, Any]: + for field in ["image_data", "video_data", "audio_data"]: + if field in json_data and json_data[field] is not None: + json_data[field] = normalize_mm_data(json_data[field]) + return json_data + + +def to_plain_dict(obj: Any) -> dict[str, Any]: + if obj is None: + return {} + if isinstance(obj, dict): + return dict(obj) + if is_dataclass(obj): + return asdict(obj) + + model_dump = getattr(obj, "model_dump", None) + if callable(model_dump): + dumped = model_dump() + if isinstance(dumped, dict): + return dumped + + dict_method = getattr(obj, "dict", None) + if callable(dict_method): + dumped = dict_method() + if isinstance(dumped, dict): + return dumped + + obj_dict = getattr(obj, "__dict__", None) + if isinstance(obj_dict, dict): + return { + key: value for key, value in obj_dict.items() if not key.startswith("_") + } + + raise TypeError(f"Unsupported request object type: {type(obj)!r}") + + +def request_to_json_data(req: Any) -> dict[str, Any]: + json_data = normalize_request_data(to_plain_dict(req)) + sampling_params = json_data.get("sampling_params") + if sampling_params is not None and not isinstance(sampling_params, dict): + json_data["sampling_params"] = to_plain_dict(sampling_params) + return json_data + + +def load_records(path: Path) -> list[Record]: + with path.open("rb") as fh: + payload = pickle.load(fh) + if isinstance(payload, dict) and "requests" in payload: + return payload["requests"] + return payload + + +def iter_files(args: argparse.Namespace) -> Sequence[Path]: + if args.input_file: + return [Path(args.input_file)] + if args.input_folder: + return [ + Path(p) + for p in sorted(glob.glob(f"{args.input_folder}/*.pkl"))[: args.file_number] + ] + raise SystemExit("Either --input-file or --input-folder must be provided.") + + +def run_one_request( + record: Record, + args: argparse.Namespace, + replay_init_time: float, + base_time: float, + idx: int, +) -> None: + req, output, start_time, end_time = record + relative_start = start_time - base_time + delay = max(0.0, (relative_start - (time.time() - replay_init_time)) / args.speed) + if delay: + time.sleep(delay) + + json_data = request_to_json_data(req) + if args.ignore_eos: + json_data.setdefault("sampling_params", {})["ignore_eos"] = True + completion_tokens = output.get("meta_info", {}).get("completion_tokens") + if completion_tokens: + json_data["sampling_params"]["max_new_tokens"] = completion_tokens + + t0 = time.time() + response = requests.post( + f"http://{args.host}:{args.port}/generate", + json=json_data, + timeout=args.timeout, + stream=bool(json_data.get("stream")), + ) + elapsed = time.time() - t0 + + if json_data.get("stream"): + last = None + for chunk in response.iter_lines(decode_unicode=False): + decoded = chunk.decode("utf-8") + if decoded and decoded.startswith("data:"): + if decoded == "data: [DONE]": + break + last = json.loads(decoded[5:].strip()) + result = last or {} + else: + result = response.json() + + meta = result.get("meta_info", {}) + print( + json.dumps( + { + "idx": idx, + "status_code": response.status_code, + "elapsed_seconds": round(elapsed, 3), + "prompt_tokens": meta.get("prompt_tokens"), + "completion_tokens": meta.get("completion_tokens"), + "rid": meta.get("id"), + }, + ensure_ascii=False, + ) + ) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Replay a trusted SGLang request dump or crash dump directly over HTTP." + ) + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=30000) + parser.add_argument("--input-folder", default=None) + parser.add_argument("--input-file", default=None) + parser.add_argument("--file-number", type=int, default=1) + parser.add_argument("--req-number", type=int, default=1_000_000) + parser.add_argument("--req-start", type=int, default=0) + parser.add_argument("--parallel", type=int, default=1) + parser.add_argument("--ignore-eos", action="store_true") + parser.add_argument("--speed", type=float, default=1.0) + parser.add_argument("--timeout", type=float, default=120.0) + args = parser.parse_args() + + files = iter_files(args) + print(f"Replay files: {[str(p) for p in files]}") + + records: list[Record] = [] + for path in files: + records.extend(load_records(path)) + + if not records: + print("No requests found.") + return 0 + + records.sort(key=lambda x: x[-2]) + records = records[args.req_start : args.req_start + args.req_number] + print(f"Replay requests: {len(records)}") + base_time = records[0][-2] + print( + "Base time: " + datetime.fromtimestamp(base_time).strftime("%Y-%m-%d %H:%M:%S") + ) + + replay_init_time = time.time() + with ThreadPoolExecutor(max_workers=args.parallel) as executor: + futures = [] + for idx, record in enumerate(records): + futures.append( + executor.submit( + run_one_request, record, args, replay_init_time, base_time, idx + ) + ) + for future in futures: + future.result() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.claude/skills/sglang-sota-performance/SKILL.md b/.claude/skills/sglang-sota-performance/SKILL.md new file mode 100644 index 000000000..1432ed3f3 --- /dev/null +++ b/.claude/skills/sglang-sota-performance/SKILL.md @@ -0,0 +1,254 @@ +--- +name: sglang-sota-performance +description: End-to-end SGLang SOTA performance workflow. Use when a user names an LLM model and wants SGLang to match or beat the best observed vLLM and TensorRT-LLM serving performance by searching each framework's best deployment command, benchmarking them fairly, profiling SGLang if it is slower, identifying kernel/overlap/fusion bottlenecks, patching SGLang code, and revalidating with real model runs. +--- + +# SGLang SOTA Performance + +## Overview + +Use this skill as the top-level optimization loop for one model at a time. +It composes two lower-level skills: + +- `llm-serving-auto-benchmark`: search and compare best deployment commands across SGLang, vLLM, and TensorRT-LLM. +- `llm-torch-profiler-analysis`: capture or analyze torch-profiler traces and produce kernel, overlap-opportunity, and fuse-pattern tables. + +This skill's goal is not "run one benchmark." Its goal is a reproducible +SGLang improvement loop: tune every framework fairly, prove whether SGLang is +behind, explain the gap with profiler evidence, patch SGLang, and re-run the +same model workload until the result is SOTA for the target environment. + +Treat "SOTA" as "best observed, reproducible performance under the recorded +model, workload, hardware, framework commits, precision, and SLA." Do not claim +global SOTA without enough external evidence. + +## Required Companion Reads + +Before a real run, read only the needed sections from: + +- `../llm-serving-auto-benchmark/SKILL.md` +- `../llm-torch-profiler-analysis/SKILL.md` + +If the run uses a remote GPU host, also read the matching host skill such as +`h100`, `b200`, `rtx5090`, or another operator-side skill that gives SSH, +container, workspace, and artifact-path conventions. + +## Required Inputs + +Collect or infer these before starting a long search: + +- model id or local checkpoint path, tokenizer path, precision, quantization, + trust-remote-code policy, and max context length +- target GPU type/count, single-node or multi-node allowance, and VRAM budget +- workload distribution: dataset, input/output lengths, request rate or + concurrency mode, sampling settings, endpoint style, and SLA target +- frameworks to compare: default to SGLang, vLLM, and TensorRT-LLM when all are + available in the target environment +- artifact root for commands, logs, benchmark JSONL, profiles, analysis reports, + patches, and final comparison tables + +If the user only provides a model, choose a reasonable first workload and state +it explicitly. Prefer the closest cookbook config from +`llm-serving-auto-benchmark/configs/cookbook-llm/` when available. + +## Artifact Layout + +Use one run directory per model and date, for example: + +```text +runs/YYYYMMDD__sota_loop/ + manifest.txt + help/ + benchmark/ + profiles/ + analysis/ + patches/ + final_report.md +``` + +Record exact framework versions, git commits, container names/images, CUDA/NCCL +versions, GPU ids, launch commands, benchmark commands, and environment knobs. +Never write Hugging Face tokens or other secrets into artifacts. + +## Workflow + +### 1. Preflight The Model And Environment + +Verify the model can be loaded by each framework before launching a sweep. +Capture each framework's current `--help` output and version. Remove candidate +flags that are not accepted by that exact environment. + +For TensorRT-LLM, keep the server backend within the scope of +`llm-serving-auto-benchmark`: `trtllm-serve serve --backend pytorch`. +If that backend is unavailable, mark TensorRT-LLM unsupported for the run +instead of silently switching to a different serving stack. + +### 2. Search Each Framework's Best Command + +Use `llm-serving-auto-benchmark` as the source of truth for benchmark fairness, +candidate generation, result schema, and comparison tables. + +Run a bounded search for every available framework. Do not compare SGLang's +tuned command against competitor defaults. Each framework must get a real chance +to find its best deployment command under the same: + +- model weights and tokenizer +- precision and quantization policy +- GPU type/count and memory budget +- dataset and request distribution +- endpoint path and sampling settings +- SLA target and measurement window + +Keep failed candidates and their failure reasons. The fastest SLA-failing +candidate is not the winner. + +### 3. Compare The Best Commands + +Normalize the benchmark output with +`llm-serving-auto-benchmark/scripts/compare_benchmark_results.py`. + +The comparison must include: + +- best server command per framework +- benchmark command and workload settings +- SLA pass/fail status +- throughput and goodput +- TTFT, ITL, end-to-end latency, and p95/p99 where available +- peak memory or allocator evidence when available +- failed candidate summary + +If SGLang is within benchmark noise of the best framework, rerun enough samples +to decide whether the difference is real. Use a default regression threshold of +3-5% unless the user specifies a tighter target. + +### 4. Profile SGLang When It Is Behind + +If SGLang is meaningfully slower, fails SLA while another framework passes, or +uses much more memory for the same workload, run profiler triage before patching. + +Use `llm-torch-profiler-analysis` against the SGLang best command first: + +- capture live SGLang profiles with `--profile-workload both`; the profiler + skill labels `prefill/` and `decode/` by workload directory for this mode +- keep separate `extend/prefill` and `decode` traces; do not use one mixed + request as the default profiler workload +- set profiler lengths from the slow benchmark scenario instead of the profiler + defaults: prefill uses the slow input length with output `1`, and decode uses + input `1` with the slow output length +- for mixed benchmark datasets, choose the slowest representative bucket + already reported by the benchmark, usually p50 or p95 input/output lengths, + and record that bucket beside the profiler artifact path +- run mapping+formal triage if single-trace output cannot map kernels to useful + Python source locations +- save the kernel, overlap-opportunity, and fuse-pattern tables in artifacts + +Profile the winning competitor too when the SGLang table alone cannot explain +why the other framework is faster. Compare stage by stage, not just total QPS. + +### 5. Turn Tables Into A Root Cause + +Use the profiler tables to identify the narrowest plausible bottleneck. + +Typical signals: + +- kernel table: attention, MoE routing, quantization, sampling, GEMM shape, + cache update, communication, or framework overhead dominates GPU time +- overlap-opportunity table: CPU scheduling, host-to-device work, collectives, + or decode bookkeeping leaves GPU idle time +- fuse-pattern table: a known fusion or overlap path should have applied but did + not, or competitor traces show a fused path SGLang lacks +- source map: hot kernels map to a concrete SGLang Python/CUDA/Triton path that + can be patched + +Do not patch from vibes. State the table row, stage, source location, and +benchmark symptom that justify the code change. + +### 6. Patch SGLang Conservatively + +Patch SGLang only after the benchmark gap and profiler evidence agree. + +Good patch candidates: + +- enable or select a better existing kernel for the model/hardware shape +- fix a missed fast path, fusion, overlap, or batching condition +- reduce unnecessary synchronization, CPU scheduling overhead, or tensor copies +- improve model-specific routing, quantization, attention, or cache handling +- add a guarded heuristic that is backed by benchmark and profiler evidence + +Avoid changes that merely make the benchmark easier: + +- weakening correctness, output quality, safety checks, or tokenizer handling +- changing only the workload or SLA after seeing results +- disabling features for SGLang but not competitors +- claiming SOTA from synthetic data when the user asked for production traffic + +Keep patches minimal and local. Add focused tests when behavior changes, and add +microbenchmarks or profiler evidence when performance is the only intended +change. + +### 7. Revalidate The Patch + +After patching, rerun: + +- the relevant unit or integration tests +- the SGLang candidate that exposed the gap +- the same cross-framework benchmark comparison +- the profiler triage if the original gap was diagnosed from profiler tables + +If the patch changes SGLang's available knobs, re-search SGLang's best command. +If competitor versions or commands changed during the work, rerun their best +commands too. Preserve before/after artifacts. + +## H100 Validation Snapshot + +On 2026-05-01, this workflow was smoke-validated on `h100_sglang` with two +real model runs and two competitor checks per run. Artifacts were saved +under +`/data/bbuf/validate/sglang_sota_performance_skill/runs/20260501_two_model_validation`. + +| Model | GPUs | Workload | SGLang result | vLLM check | TensorRT-LLM check | +| --- | --- | --- | --- | --- | --- | +| `Qwen/Qwen2.5-7B-Instruct` | 2x H100, TP=2 | random, input 512/output 64, 24 prompts, 10 warmup requests | 52.09 req/s, mean TTFT 144.85 ms, mean ITL 4.91 ms | 51.06 req/s, mean TTFT 159.19 ms, mean ITL 4.85 ms | 49.71 req/s, mean TTFT 177.54 ms, mean ITL 4.77 ms | +| `Qwen/Qwen2.5-32B-Instruct` | 4x H100, TP=4 | random, input 512/output 64, 16 prompts, 10 warmup requests | 18.47 req/s, mean TTFT 247.06 ms, mean ITL 9.66 ms | 18.78 req/s, mean TTFT 218.68 ms, mean ITL 9.98 ms | 15.48 req/s, mean TTFT 445.62 ms, mean ITL 9.27 ms | + +Use this only as a workflow health check, not as a universal performance +claim. The TensorRT-LLM checks used `trtllm-serve serve --backend pytorch` and +the same OpenAI-compatible random workload. + +Additional 2-card validation on 2026-05-01 exercised the full handoff from +bounded cross-framework search into SGLang stage-separated profiling. The +benchmark workload was random input `512`, output `64`, 8 prompts, and the +profiler used the same slow-workload lengths: prefill `512->1` and decode +`1->64`, with warmup 10 and capture 5. + +| Model | GPUs | Best SGLang | Best vLLM | Profiler result | Artifact root | +| --- | --- | --- | --- | --- | --- | +| `Qwen/Qwen3-8B` | 2x H100, TP=2 | `sglang_mem086`, 21.64 req/s | `vllm_mem080`, 22.88 req/s | kernel, overlap, and fuse tables rendered with separate `extend/prefill` and `decode` sections | `/data/bbuf/validate/core_skill_validation_20260501/qwen3_8b/sota` | +| `mistralai/Mistral-7B-Instruct-v0.3` | 2x H100, TP=2 | `sglang_mem080`, 24.09 req/s | `vllm_mem090`, 24.76 req/s | kernel, overlap, and fuse tables rendered with separate `extend/prefill` and `decode` sections | `/data/bbuf/validate/core_skill_validation_20260501/mistral_7b_instruct_v03/sota` | + +## Stop Conditions + +Stop with a clear report when any of these is true: + +- SGLang is the best SLA-passing framework for the target workload +- SGLang is within noise of the best framework and the remaining gap is not + statistically stable +- SGLang remains behind but the root cause is external to SGLang, such as missing + model weights, unavailable backend dependencies, or an unsupported hardware + feature +- a patch improves SGLang but still does not reach SOTA; report the next table + row or source path to investigate + +## Final Report Contract + +Return a compact report with: + +- model, hardware, framework versions, workload, and artifact root +- best deployment command per framework +- benchmark comparison table before patch and after patch +- SGLang gap analysis, including exact profiler table rows and source paths +- patch summary with changed files and correctness tests +- real-model validation result and whether SGLang reached target-environment SOTA + +If no code patch was needed, say why and include the benchmark evidence. +If a patch was attempted but not enough, be explicit about the remaining gap.