diff --git a/.claude/skills/sglang-auto-benchmark/SKILL.md b/.claude/skills/sglang-auto-benchmark/SKILL.md new file mode 100644 index 000000000..21dd80320 --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/SKILL.md @@ -0,0 +1,496 @@ +--- +name: sglang-auto-benchmark +description: Run SGLang auto benchmark searches with tiered server-flag sweeps, canonical dataset preparation, ShareGPT auto-download, custom-data conversion/validation, SLA or fixed-QPS benchmarking, CSV export, and optional second-stage speculative/EAGLE tuning. Use when the user wants an AI-operated benchmark workflow rather than a one-off bench_serving command. +--- + +# SGLang Auto Benchmark + +This skill is for repeatable, AI-driven SGLang performance tuning. + +The preferred workflow is: +- start from a mostly pure-TP baseline command, +- move the rest of the performance knobs into `search_space`, +- let auto benchmark search and compare candidates under the target SLA. + +The implementation lives in: +- `python -m sglang.auto_benchmark` +- canonical dataset loader in `python -m sglang.bench_serving --dataset-name autobench` +- cookbook-derived LLM reference configs in `.claude/skills/sglang-auto-benchmark/references/cookbook-llm/` + +## Preconditions + +- SGLang can already launch and serve the target model in this environment. +- The model path exists, or the model is otherwise launchable. +- The goal is clear: + - benchmark a fixed QPS list, or + - search the maximum QPS that satisfies `max_ttft_ms` / `max_tpot_ms`. + +If those are not true yet, fix them before running a large search. + +Environment consistency check: +- if the benchmark will run from a remote repo copy or ad-hoc synced workspace, + verify that the remote `python/sglang/bench_serving.py` matches the local + feature level needed by auto benchmark before launching a long run +- at minimum, run a preflight such as `PYTHONPATH=/python python3 -m + sglang.bench_serving --help` and confirm that the dataset choices include + `autobench` +- if `autobench` is missing remotely, do not start the benchmark; sync + `python/sglang/bench_serving.py` and any required dataset modules first + +## Remote Run Logging + +If the benchmark is executed on a remote machine, the progress bar output must be +mirrored back to a local file for humans to watch. + +Scope note: +- use the remote-log mirroring workflow only when the benchmark is running in a + different machine or a different remote container than the one the agent is + actively operating in +- if the agent itself is already running inside the target container where auto + benchmark is executing, do not add a separate log-return loop just for parity; + inspect the live log files and result files directly in the current container +- in other words, "remote container" needs mirrored local logs, while "current + container" should use direct local inspection + +Required behavior: +- start the remote run with a persistent terminal/session log, for example with + `script -q -f -c ""`; on Linux containers that use util-linux + `script`, prefer the explicit `-c` form instead of BSD-style positional + command arguments +- continuously sync a cleaned version of that remote session log back to a local + `progress.log`; this local `progress.log` should already have terminal control + sequences removed, because `script` + `tqdm` progress bars will otherwise leave + ANSI cursor-control bytes and carriage-return redraws that look like garbled text +- if the benchmark itself is executed inside a remote container, the cleaned local + `progress.log` must be refreshed automatically at least once every 30 + seconds while the run is active; do not rely on one-off manual polling +- implement the sync loop as a dedicated local script file checked into neither + git nor the benchmark config; avoid fragile one-line `nohup zsh -lc '...'` + command strings with heavy nested quoting +- prefer running the sync loop inside a long-lived local session such as a + dedicated `tmux` pane, `screen`, or the agent's own persistent PTY session; + detached child processes started from short-lived command runners can be + reaped unexpectedly, so plain `nohup ... &` is not the most stable default +- immediately after starting the sync loop, verify that `progress.log` is + actually updating by checking its timestamp or size twice across a short wait; + if it is not changing, treat that as a broken sync setup and fix it before + telling the user that live log mirroring is working +- tell the user the local log path up front +- keep final result files synced back locally after the run ends +- when scenario-level or top-level markdown summaries are produced, sync those + `summary.md` / `SUMMARY.md` files back locally as first-class result artifacts + rather than leaving them only on the remote machine + +This is important because long searches can run for hours, and people need a +stable local file they can tail without logging into the remote box. The final +local run folder should also be self-contained enough for someone to review the +benchmark outcome without re-entering the remote environment. + +Recommended cleanup pipeline for the local mirrored log: + +```bash +perl -pe 's/\e\[[0-9;?]*[ -\/]*[@-~]//g; s/\r/\n/g; s/\x08//g;' raw_progress.log \ + > progress.log +``` + +Recommended remote-container sync pattern: + +```bash +cat > sync_progress.sh <<'EOF' +#!/bin/zsh +set -euo pipefail +while true; do + ssh "tail -n 200 " > raw_progress.log + perl -pe 's/\e\[[0-9;?]*[ -\/]*[@-~]//g; s/\r/\n/g; s/\x08//g;' raw_progress.log \ + > progress.log + sleep 15 +done +EOF +chmod +x sync_progress.sh +``` + +Run that script from a long-lived local session, for example: + +```bash +tmux new-session -d -s autobench-sync './sync_progress.sh' +``` + +Use a persistent local background job, `tmux` pane, `screen`, or equivalent +long-lived sync process so that humans can watch the cleaned local log in real +time. Use `sleep 15` by default for long runs unless there is a specific need +for tighter polling, and keep the cleaned local `progress.log` within the +required 30-second refresh window while the run is active. + +At the end of the run, make sure the local artifact set includes any generated: +- `results.jsonl` +- `results.csv` +- `summary.md` +- `SUMMARY.md` +- `scenario_summary.jsonl` +- `scenario_summary.csv` + +Required health check after starting the sync script: + +```bash +stat -f '%m %z' progress.log +sleep 5 +stat -f '%m %z' progress.log +``` + +If the timestamp and size both stay unchanged while the remote benchmark is known +to be producing new output, the sync loop is broken. Fix the script before +continuing. + +Do not make the cleaned log optional. The default local progress artifact should +be the cleaned `progress.log` that humans actually read. + +## Most Important Rule + +If the user wants the best command for a **real production or real workload scenario**, the benchmark must use **their real request distribution**. + +That means: +- real prompt lengths, +- real output lengths, +- real multi-turn patterns, +- real tool / reasoning / sampling settings, +- real prefix-sharing behavior if it exists. + +`sharegpt`, `random`, and `generated-shared-prefix` are useful for sanity checks and broad tuning, but they are not a substitute for the user’s real traffic. + +The cookbook reference configs now default to `random` because it is portable and immediately runnable, but that should still be treated as a fallback benchmark shape rather than the final answer for a real deployment. + +## Supported Dataset Kinds + +The current implementation intentionally keeps the dataset surface small: + +- `sharegpt` + - Supports auto-download when no file path is provided. + - Will be prepared into canonical autobench JSONL on disk before benchmarking. +- `custom` + - Supports two cases: + - old `bench_serving` custom conversation JSONL, + - already-converted canonical autobench JSONL. +- `random` + - Uses SGLang’s existing synthetic/random benchmark path. + - This is the default dataset mode in the cookbook reference configs. + - `input_len` and `output_len` can be lists of equal length. + - Each aligned pair becomes one full benchmark scenario, not a cartesian product. + - Example: + +```yaml +dataset: + kind: random + scenario_names: [chat, summarization] + input_len: [1000, 8000] + output_len: [1000, 1000] +``` + + - The workflow will run one full search for `1000 -> 1000` and one full search for `8000 -> 1000`. +- `generated-shared-prefix` + - Uses SGLang’s existing shared-prefix synthetic generator. + +Everything is normalized into one canonical autobench JSONL file before the benchmark loop starts. + +## Canonical Dataset Format + +Canonical format is JSONL, one request per line. + +Minimal rows: + +```json +{"prompt": "Write a summary of this document.", "output_len": 256} +{"prompt": [{"role": "user", "content": "Summarize this document."}], "output_len": 256} +{"prompt": ["first turn", "follow-up turn"], "output_len": 128} +``` + +Optional fields: + +```json +{ + "prompt": [{"role": "user", "content": "Use the weather tool."}], + "output_len": 256, + "extra_request_body": {"temperature": 0.0, "top_p": 0.95}, + "image_data": ["file:///tmp/example.png"], + "timestamp": 1710000000, + "routing_key": "group-a", + "metadata": {"source": "custom-upload"} +} +``` + +Compatibility: +- legacy `messages` +- legacy `prompt_origin` +- legacy `param_send` +- legacy `system + content` + +## ShareGPT Auto-Prepare + +`sharegpt` does not need a full path. + +Example: + +```bash +python3 -m sglang.auto_benchmark convert \ + --kind sharegpt \ + --tokenizer /path/to/tokenizer \ + --num-prompts 200 \ + --output /tmp/sharegpt.autobench.jsonl +``` + +This will: +- auto-download ShareGPT through the existing SGLang cache path when needed, +- convert it into canonical autobench JSONL, +- save it to the requested output path. + +## Custom User Data Workflow + +When the user uploads custom data: + +1. Inspect a few raw rows first. +2. Decide whether the file is: + - already canonical autobench JSONL, + - old `bench_serving` custom format, + - or an unsupported custom schema that must be transformed manually. +3. If manual transformation is needed: + - map it into canonical JSONL, + - never hallucinate missing turns or answers, + - never keep the final assistant answer as part of the benchmark prompt if that answer is the target completion, + - preserve per-request generation settings in `extra_request_body`. +4. Run: + +```bash +python3 -m sglang.auto_benchmark validate \ + --dataset-path /path/to/converted.autobench.jsonl \ + --tokenizer /path/to/tokenizer +``` + +5. Manually inspect at least 3 converted rows and confirm: + - prompt shape is correct, + - final assistant answer was not accidentally left in the prompt, + - `output_len` is sensible, + - request extras were preserved. + +## Search Tiers + +`search.tier` controls search breadth. + +- Tier 1 + - Fastest and smallest sweep. + - Best for smoke tests, config validation, and quickly checking whether a model can run at all. + - Uses a very small subset of the search space and mainly does one-at-a-time changes on top of the baseline. + - Lowest search cost, but also the easiest to miss a better configuration. +- Tier 2 + - Recommended default. + - Good balance between coverage and runtime. + - Runs a small cartesian search on the first few high-priority keys, then expands the rest one at a time. + - Usually the right choice for everyday tuning when you want meaningful search without waiting too long. +- Tier 3 + - Largest search space. + - Runs the full cartesian product of the provided search space. + - Search time is the longest by far. + - Only use it when the search space is already tightly bounded and you intentionally want the most exhaustive sweep. + - This is the best chance of finding the strongest config, but it is also the easiest way to turn a benchmark into a multi-hour or multi-day run. + +`search.max_candidates` still applies at all tiers, including tier 3. +When it is set together with tier 3, the workflow still enumerates the full cartesian order conceptually, but only keeps the first `max_candidates` unique candidates after deduplication. +That makes it useful as a safety valve, but it also means tier 3 is no longer truly exhaustive unless you remove the cap or raise it high enough. + +If `search.max_candidates` is omitted, the workflow now defaults to `8`. +Set it to `null` only when you intentionally want an unbounded sweep. + +The reference configs now default to tier 2 with `search.max_candidates: 8`. + +## Interrupt And Resume + +Long searches may need to be stopped and resumed later. + +Use: + +```yaml +search: + tier: 2 + resume: true +``` + +Behavior: +- every completed trial is appended to `live_results.jsonl` +- if the process receives `SIGINT` or `SIGTERM`, it will first save partial + `results.jsonl`, `results.csv`, and `summary.md` +- on the next run with the same config and `search.resume: true`, completed + trials are reused and only unfinished trials are executed +- resume works per scenario directory, so it is safest to keep the same + `benchmark.output_dir` + +Notes: +- resume assumes the candidate order and dataset are unchanged +- for maximum safety, reuse the same prepared dataset or keep the same dataset + seed/config +- `SIGKILL` cannot be handled gracefully, so only the already-written + `live_results.jsonl` can be reused after a hard kill + +YAML key order matters. Put the most important search keys first. + +## What Is Tunable + +This workflow is not limited to attention backend tuning. + +`server.base_flags` and `server.search_space` are passed directly to `sglang.launch_server`, so in practice any valid server CLI flag can be set or searched. + +There is also a small convenience layer for parallel search: + +- `server.parallel.tp` +- `server.parallel.pp_size` + +When `server.parallel` is used and `dp_size` is not set explicitly, the workflow auto-derives: + +`dp_size = visible_gpus / (tp_size * pp_size)` + +Visible GPU count is inferred from `server.env.CUDA_VISIBLE_DEVICES` by default, or from `server.parallel.gpu_count` if you set it explicitly. + +The most important performance-related groups are: + +- Kernel / backend + - `attention_backend` + - `prefill_attention_backend` + - `decode_attention_backend` + - `sampling_backend` + - `grammar_backend` +- Batching / scheduling + - `max_running_requests` + - `max_queued_requests` + - `chunked_prefill_size` + - `prefill_max_requests` + - `max_prefill_tokens` + - `schedule_conservativeness` + - `num_continuous_decode_steps` + - `stream_interval` +- Memory / cache + - `max_total_tokens` + - `page_size` + - `disable_radix_cache` +- Parallel / distributed execution + - `tp_size` + - `pp_size` + - `dp_size` + - `ep_size` + - `load_balance_method` + - `enable_dp_attention` + - `enable_mixed_chunk` + - `disable_overlap_schedule` +- Runtime / CUDA graph + - keep CUDA graph enabled by default for performance benchmarking + - `cuda_graph_max_bs` + - `disable_cuda_graph_padding` + - `enable_cudagraph_gc` +- Optional speculative / EAGLE stage + - `speculative_num_steps` + - `speculative_eagle_topk` + - `speculative_num_draft_tokens` + - `speculative_attention_mode` + - `speculative_draft_attention_backend` + - `speculative_accept_threshold_single` + - `speculative_accept_threshold_acc` + +For cookbook-derived reference configs, keep `mem_fraction_static` and +`schedule_policy` pinned to the cookbook baseline unless the user explicitly +asks to search them. They are useful knobs, but they add a lot of search width +for relatively low validation value in the default workflow. + +Do not put these into the default search space: +- `mem_fraction_static` +- `schedule_policy` +- `enable_hierarchical_cache` +- `hicache_ratio` +- `hicache_size` +- `enable_lmcache` + +Those features are not treated as standard auto-benchmark sweep knobs in this workflow. + +Budget guardrails for the default workflow: +- use `dataset.num_prompts: 80` unless the user asks for a heavier study +- prefer a coarse QPS search tolerance +- keep `benchmark.qps.max_rounds <= 5` +- keep `search.max_duration_hours <= 12` + +## Base Tuning Before EAGLE + +Never start by tuning EAGLE first. + +Use this order: + +1. Tune the non-speculative base server first. +2. Find the best normal config for the target dataset and SLA. +3. Only if the user explicitly asks for speculative/EAGLE tuning, and provides the required draft model or equivalent assets, run the second-stage speculative search. + +Do not put `disable_cuda_graph` into the default search space. For normal performance tuning, CUDA graph should stay enabled unless the user is debugging compatibility issues. + +When a candidate OOMs, keep it in the final result table as a failed row and add a hint such as: +- increase GPU count, or +- use GPUs with larger memory. + +## Running The Workflow + +Prepare a dataset explicitly: + +```bash +python3 -m sglang.auto_benchmark convert \ + --kind custom \ + --path /path/to/data.jsonl \ + --tokenizer /path/to/tokenizer \ + --output /tmp/data.autobench.jsonl +``` + +Run from config: + +```bash +python3 -m sglang.auto_benchmark run --config /path/to/config.yaml +``` + +Outputs: +- prepared canonical dataset JSONL +- per-run `results.jsonl` +- summary `results.csv` +- per-candidate server logs + +## Config Template + +Standalone example (uses ShareGPT as dataset, a good starting point for non-cookbook models): +- `references/qwen3-32b.yaml` + +Cookbook-derived configs live in `references/cookbook-llm/`. +They default to synthetic `random` traffic and are runnable out of the box. +See `references/cookbook-llm/README.md` for the full list. + +Representative picks from that folder: +- `references/cookbook-llm/llama-3.1-70b-instruct.yaml` +- `references/cookbook-llm/llama-3.3-70b-instruct.yaml` +- `references/cookbook-llm/llama-4-scout-17b-16e-instruct.yaml` +- `references/cookbook-llm/llama-4-maverick-17b-128e-instruct-fp8.yaml` +- `references/cookbook-llm/minimax-m2.5.yaml` +- `references/cookbook-llm/minimax-m2.1.yaml` +- `references/cookbook-llm/deepseek-v3.yaml` +- `references/cookbook-llm/deepseek-v3.1.yaml` +- `references/cookbook-llm/deepseek-v3.2.yaml` +- `references/cookbook-llm/deepseek-r1-0528.yaml` +- `references/cookbook-llm/qwen3-235b-a22b.yaml` +- `references/cookbook-llm/qwen35-397b-a17b-fp8.yaml` +- `references/cookbook-llm/mistral-small-4-119b-2603.yaml` +- `references/cookbook-llm/kimi-k2-instruct.yaml` + +All reference configs use Hugging Face repo IDs by default. +Replace `model_path` and `tokenizer` with local paths when the weights are already on disk. + +## What To Report Back + +After a run, summarize: +- which tier was used, +- which dataset kind was used, +- whether the dataset was synthetic or real user traffic, +- best base config, +- best QPS that satisfied SLA, +- whether speculative tuning was skipped or run, +- paths to: + - prepared dataset JSONL + - `results.jsonl` + - `results.csv` + - key server logs diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/README.md b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/README.md new file mode 100644 index 000000000..94e25102d --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/README.md @@ -0,0 +1,76 @@ +# Cookbook LLM References + +These configs are derived from `sgl-cookbook` autoregressive text-model pages and normalized into the auto-benchmark config format. + +Rules used here: +- Keep the baseline as close as possible to a pure-TP launch command. +- Keep `mem_fraction_static` and `schedule_policy` at the cookbook baseline by + default; search higher-ROI knobs first. +- Move the remaining common performance knobs into `search_space`. +- Add `ep_size` search for relevant MoE pages. +- Keep CUDA graph enabled by default. +- Prefer cookbook H200 defaults first, then H100 defaults when H200 is not available; if neither exists, fall back to the cookbook's published baseline for that model and say so in the config comments. +- Default to synthetic `random` data so every config is runnable out of the box. +- Default to `dataset.num_prompts: 80` so the reference sweep stays cheap enough + for interactive validation. +- Default to a coarse QPS search with `benchmark.qps.max_rounds <= 5`. +- Default to `search.tier: 2` so the shipped configs stay reasonably practical to run. +- Default to `search.max_candidates: 8` so the candidate sweep stays bounded by + default. +- Default to `search.max_duration_hours: 12` because longer searches do not fit + the intended workflow budget. +- Treat `dataset.input_len` and `dataset.output_len` as aligned scenario lists, not a cartesian product. +- If a candidate OOMs, the result table should recommend increasing GPU count or using GPUs with larger memory. + +Default random scenarios in these configs: +- `1000 -> 1000` for a chat-like shape +- `8000 -> 1000` for a summarization-like shape + +Each scenario should run a full search independently, and each scenario should have its own best launch command and summary table. + +Excluded from this folder because they are OCR/VL-oriented rather than text-serving benchmark configs: +- DeepSeekOCR / DeepSeekOCR2 +- GLMOCR +- GLM45V / GLM46V +- Qwen2.5-VL / Qwen3-VL +- Step3-VL-10B + +Configs in this folder: +- `deepseek-v3.2.yaml` +- `deepseek-math-v2.yaml` +- `deepseek-r1-0528.yaml` +- `deepseek-v3.1.yaml` +- `deepseek-v3.yaml` +- `devstral-small-2-24b-instruct-2512.yaml` +- `ernie-4.5-21b-a3b-pt.yaml` +- `glm-4.5.yaml` +- `glm-4.6.yaml` +- `glm-4.7.yaml` +- `glm-4.7-flash.yaml` +- `glm-5-fp8.yaml` +- `gpt-oss-120b.yaml` +- `glyph.yaml` +- `intern-s1.yaml` +- `kimi-k2.5.yaml` +- `kimi-k2-instruct.yaml` +- `kimi-linear-48b-a3b-instruct.yaml` +- `llada2-1-mini.yaml` +- `ling-2.5-1t.yaml` +- `llama-3.1-70b-instruct.yaml` +- `llama-3.3-70b-instruct.yaml` +- `llama-4-scout-17b-16e-instruct.yaml` +- `llama-4-maverick-17b-128e-instruct-fp8.yaml` +- `mimo-v2-flash.yaml` +- `minimax-m2.5.yaml` +- `minimax-m2.1.yaml` +- `ministral-3-8b-instruct-2512.yaml` +- `mistral-small-4-119b-2603.yaml` +- `nemotron-3-nano-30b-a3b-bf16.yaml` +- `nemotron-3-super-120b-a12b-bf16.yaml` +- `qwen35-397b-a17b-fp8.yaml` +- `qwen3-coder-480b-a35b-instruct.yaml` +- `qwen3-coder-next.yaml` +- `qwen3-235b-a22b.yaml` +- `qwen3-next-80b-a3b-instruct.yaml` +- `ring-2.5-1t.yaml` +- `step-3.5-flash.yaml` diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-math-v2.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-math-v2.yaml new file mode 100644 index 000000000..f3e2e4271 --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-math-v2.yaml @@ -0,0 +1,81 @@ +# Cookbook auto benchmark config for deepseek math v2. +# Cookbook does not expose an H100/H200 default for this model; this config follows the cookbook B200 baseline instead. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: data/models/generated/v0.5.8/deepseek-math-v2.yaml +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 + base_flags: + tp_size: 8 + model_path: deepseek-ai/DeepSeek-Math-V2 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - flashinfer + decode_attention_backend: + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + ep_size: + - 1 + - 4 + - 8 +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: deepseek-ai/DeepSeek-Math-V2 + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/deepseek-math-v2 +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-r1-0528.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-r1-0528.yaml new file mode 100644 index 000000000..75dce3c0e --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-r1-0528.yaml @@ -0,0 +1,84 @@ +# Cookbook auto benchmark config for deepseek r1 0528. +# Baseline follows the cookbook H200 default command. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: data/models/generated/v0.5.6/deepseek-r1.yaml +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 + base_flags: + tp_size: 8 + enable_symm_mem: true + model_path: deepseek-ai/DeepSeek-R1-0528 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + ep_size: + - 1 + - 4 + - 8 +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: deepseek-ai/DeepSeek-R1-0528 + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/deepseek-r1-0528 +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-v3.1.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-v3.1.yaml new file mode 100644 index 000000000..24e1427a7 --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-v3.1.yaml @@ -0,0 +1,83 @@ +# Cookbook auto benchmark config for deepseek v3.1. +# Baseline follows the cookbook H200 default command. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: src/components/autoregressive/DeepSeekV31ConfigGenerator/index.js +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 + base_flags: + tp_size: 8 + model_path: deepseek-ai/DeepSeek-V3.1 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + ep_size: + - 1 + - 4 + - 8 +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: deepseek-ai/DeepSeek-V3.1 + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/deepseek-v3.1 +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-v3.2.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-v3.2.yaml new file mode 100644 index 000000000..1b7959a2e --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-v3.2.yaml @@ -0,0 +1,83 @@ +# Cookbook auto benchmark config for deepseek v3.2. +# Baseline follows the cookbook H200 default command. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: data/models/generated/v0.5.6/deepseek.yaml +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 + base_flags: + tp_size: 8 + model_path: deepseek-ai/DeepSeek-V3.2 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + ep_size: + - 1 + - 4 + - 8 +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: deepseek-ai/DeepSeek-V3.2 + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/deepseek-v3.2 +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-v3.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-v3.yaml new file mode 100644 index 000000000..6808d6b2c --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/deepseek-v3.yaml @@ -0,0 +1,84 @@ +# Cookbook auto benchmark config for deepseek v3. +# Baseline follows the cookbook H200 default command. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: src/components/autoregressive/DeepSeekV3ConfigGenerator/index.js +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 + base_flags: + tp_size: 8 + enable_symm_mem: true + model_path: deepseek-ai/DeepSeek-V3 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + ep_size: + - 1 + - 4 + - 8 +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: deepseek-ai/DeepSeek-V3 + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/deepseek-v3 +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/devstral-small-2-24b-instruct-2512.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/devstral-small-2-24b-instruct-2512.yaml new file mode 100644 index 000000000..2120ab2ee --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/devstral-small-2-24b-instruct-2512.yaml @@ -0,0 +1,78 @@ +# Cookbook auto benchmark config for devstral small 2 24b instruct 2512. +# Baseline follows the cookbook H200 default command. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: src/components/autoregressive/Devstral2ConfigGenerator/index.js +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: '0' + base_flags: + model_path: mistralai/Devstral-Small-2-24B-Instruct-2512 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: mistralai/Devstral-Small-2-24B-Instruct-2512 + max_concurrency: + - null + - 16 + - 32 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 16.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/devstral-small-2-24b-instruct-2512 +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/ernie-4.5-21b-a3b-pt.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/ernie-4.5-21b-a3b-pt.yaml new file mode 100644 index 000000000..f9e3ab67f --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/ernie-4.5-21b-a3b-pt.yaml @@ -0,0 +1,76 @@ +# Cookbook auto benchmark config for ernie 4.5 21b a3b pt. +# Cookbook does not expose an H100/H200 default for this model; this config follows the cookbook MI300X baseline instead. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: src/components/autoregressive/Ernie45ConfigGenerator/index.js +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: '0' + base_flags: + model_path: baidu/ERNIE-4.5-21B-A3B-PT + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 + command_prefix: + - python3 + - -m + - sglang.launch_server +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: baidu/ERNIE-4.5-21B-A3B-PT + max_concurrency: + - null + - 16 + - 32 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 16.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/ernie-4.5-21b-a3b-pt +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-4.5.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-4.5.yaml new file mode 100644 index 000000000..c0eb3a398 --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-4.5.yaml @@ -0,0 +1,74 @@ +# Cookbook auto benchmark config for glm 4.5. +# Cookbook does not expose an H100/H200 default for this model; this config follows the cookbook MI300X baseline instead. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: src/components/autoregressive/GLM45ConfigGenerator/index.js +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: 0,1,2,3 + base_flags: + tp_size: 4 + context_length: 8192 + model_path: zai-org/GLM-4.5 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: zai-org/GLM-4.5 + max_concurrency: + - null + - 8 + - 16 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 8.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/glm-4.5 +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-4.6.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-4.6.yaml new file mode 100644 index 000000000..d9c2270e1 --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-4.6.yaml @@ -0,0 +1,83 @@ +# Cookbook auto benchmark config for glm 4.6. +# Baseline follows the cookbook H200 default command. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: data/models/generated/v0.5.6/glm46.yaml +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 + base_flags: + tp_size: 8 + model_path: zai-org/GLM-4.6 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + ep_size: + - 1 + - 4 + - 8 +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: zai-org/GLM-4.6 + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/glm-4.6 +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-4.7-flash.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-4.7-flash.yaml new file mode 100644 index 000000000..426203ac9 --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-4.7-flash.yaml @@ -0,0 +1,78 @@ +# Cookbook auto benchmark config for glm 4.7 flash. +# Baseline follows the cookbook H100 default command. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: src/components/autoregressive/GLM47FlashConfigGenerator/index.js +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: '0' + base_flags: + model_path: zai-org/GLM-4.7-Flash + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: zai-org/GLM-4.7-Flash + max_concurrency: + - null + - 16 + - 32 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 16.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/glm-4.7-flash +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-4.7.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-4.7.yaml new file mode 100644 index 000000000..44640b679 --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-4.7.yaml @@ -0,0 +1,78 @@ +# Cookbook auto benchmark config for glm 4.7. +# Cookbook does not expose an H100/H200 default for this model; this config follows the cookbook MI300X baseline instead. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: src/components/autoregressive/GLM47ConfigGenerator/index.js +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: 0,1,2,3 + base_flags: + tp_size: 4 + context_length: 8192 + model_path: zai-org/GLM-4.7 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 + ep_size: + - 1 + - 2 + - 4 +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: zai-org/GLM-4.7 + max_concurrency: + - null + - 8 + - 16 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 8.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/glm-4.7 +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-5-fp8.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-5-fp8.yaml new file mode 100644 index 000000000..79e6d2241 --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glm-5-fp8.yaml @@ -0,0 +1,83 @@ +# Cookbook auto benchmark config for glm 5 fp8. +# Baseline follows the cookbook H200 default command. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: data/models/generated/v0.5.8/glm5.yaml +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 + base_flags: + tp_size: 8 + model_path: zai-org/GLM-5-FP8 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + ep_size: + - 1 + - 4 + - 8 +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: zai-org/GLM-5-FP8 + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/glm-5-fp8 +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glyph.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glyph.yaml new file mode 100644 index 000000000..25102e092 --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/glyph.yaml @@ -0,0 +1,81 @@ +# Cookbook auto benchmark config for glyph. +# Baseline follows the cookbook H200 default command. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: src/components/autoregressive/GlyphConfigGenerator/index.js +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: 0,1,2,3 + base_flags: + tp_size: 4 + reasoning_parser: glm45 + tool_call_parser: glm45 + model_path: zai-org/Glyph + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: zai-org/Glyph + max_concurrency: + - null + - 8 + - 16 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 8.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/glyph +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/gpt-oss-120b.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/gpt-oss-120b.yaml new file mode 100644 index 000000000..7a251b4fa --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/gpt-oss-120b.yaml @@ -0,0 +1,83 @@ +# Cookbook auto benchmark config for gpt oss 120b. +# Baseline follows the cookbook H200 default command. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: data/models/generated/v0.5.6/gpt-oss.yaml +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 + base_flags: + tp_size: 8 + model_path: openai/gpt-oss-120b + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + ep_size: + - 1 + - 4 + - 8 +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: openai/gpt-oss-120b + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/gpt-oss-120b +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/intern-s1.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/intern-s1.yaml new file mode 100644 index 000000000..aa91be7b4 --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/intern-s1.yaml @@ -0,0 +1,84 @@ +# Cookbook auto benchmark config for intern s1. +# Baseline follows the cookbook H200 default command. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: data/models/generated/v0.5.6/intern-s1.yaml +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 + base_flags: + tp_size: 8 + trust_remote_code: true + model_path: internlm/Intern-S1 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + ep_size: + - 1 + - 4 + - 8 +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: internlm/Intern-S1 + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/intern-s1 +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/kimi-k2-instruct.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/kimi-k2-instruct.yaml new file mode 100644 index 000000000..b5066bb29 --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/kimi-k2-instruct.yaml @@ -0,0 +1,83 @@ +# Cookbook auto benchmark config for kimi k2 instruct. +# Baseline follows the cookbook H200 default command. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: data/models/generated/v0.5.6/kimi-k2.yaml +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 + base_flags: + tp_size: 8 + trust_remote_code: true + model_path: moonshotai/Kimi-K2-Instruct + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + ep_size: + - 1 + - 4 +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: moonshotai/Kimi-K2-Instruct + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/kimi-k2-instruct +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/kimi-k2.5.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/kimi-k2.5.yaml new file mode 100644 index 000000000..fba48bd72 --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/kimi-k2.5.yaml @@ -0,0 +1,80 @@ +# Cookbook auto benchmark config for kimi k2.5. +# Baseline follows the cookbook H200 default command. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: data/models/generated/v0.5.8/kimi-k25.yaml +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 + base_flags: + tp_size: 8 + trust_remote_code: true + model_path: moonshotai/Kimi-K2.5 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: moonshotai/Kimi-K2.5 + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/kimi-k2.5 +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/kimi-linear-48b-a3b-instruct.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/kimi-linear-48b-a3b-instruct.yaml new file mode 100644 index 000000000..144fa3ffe --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/kimi-linear-48b-a3b-instruct.yaml @@ -0,0 +1,79 @@ +# Cookbook auto benchmark config for kimi linear 48b a3b instruct. +# Cookbook does not expose an H100/H200 default for this model; this config follows the cookbook MI300X baseline instead. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: src/components/autoregressive/KimiK2linearConfigGenerator/index.js +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + SGLANG_ROCM_FUSED_DECODE_MLA: '0' + CUDA_VISIBLE_DEVICES: 0,1,2,3 + base_flags: + tp_size: 4 + trust_remote_code: true + model_path: moonshotai/Kimi-Linear-48B-A3B-Instruct + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 + command_prefix: + - python3 + - -m + - sglang.launch_server +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: moonshotai/Kimi-Linear-48B-A3B-Instruct + max_concurrency: + - null + - 8 + - 16 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 8.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/kimi-linear-48b-a3b-instruct +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/ling-2.5-1t.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/ling-2.5-1t.yaml new file mode 100644 index 000000000..8bbc2edac --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/ling-2.5-1t.yaml @@ -0,0 +1,91 @@ +# Cookbook auto benchmark config for ling 2.5 1t. +# Baseline follows the cookbook H200 default command. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# This cookbook baseline is multi-node or external-service oriented. Keep launch: false and benchmark an already running deployment. +# Source: src/components/autoregressive/Ling25ConfigGenerator/index.js +server: + launch: false + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 + base_flags: + tp_size: 8 + pp_size: 2 + nnodes: 2 + trust_remote_code: true + tool_call_parser: qwen + model_path: inclusionAI/Ling-2.5-1T + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + pp_size: + - 1 + - 2 + command_prefix: + - python3 + - -m + - sglang.launch_server +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: inclusionAI/Ling-2.5-1T + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 2.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/ling-2.5-1t +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/llada2-1-mini.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/llada2-1-mini.yaml new file mode 100644 index 000000000..57d6745dc --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/llada2-1-mini.yaml @@ -0,0 +1,83 @@ +# Cookbook auto benchmark config for llada2 1 mini. +# Baseline follows the cookbook H200 default command. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: data/models/generated/v0.5.6/llada21.yaml +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: '0' + base_flags: + tp_size: 1 + dllm_algorithm: JointThreshold + trust_remote_code: true + max_running_requests: 1 + attention_backend: flashinfer + model_path: inclusionAI/LLaDA2.1-mini + mem_fraction_static: 0.77 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 1 + - 2 + - 4 +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: inclusionAI/LLaDA2.1-mini + max_concurrency: + - 1 + - 2 + - 4 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/llada2-1-mini +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/llama-3.1-70b-instruct.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/llama-3.1-70b-instruct.yaml new file mode 100644 index 000000000..02109e7d5 --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/llama-3.1-70b-instruct.yaml @@ -0,0 +1,79 @@ +# Cookbook auto benchmark config for llama 3.1 70b instruct. +# Baseline follows the cookbook H200 default command. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: data/models/generated/v0.5.6/llama31.yaml +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: 0,1,2,3 + base_flags: + tp_size: 4 + model_path: meta-llama/Llama-3.1-70B-Instruct + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: meta-llama/Llama-3.1-70B-Instruct + max_concurrency: + - null + - 8 + - 16 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 12.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/llama-3.1-70b-instruct +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/llama-3.3-70b-instruct.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/llama-3.3-70b-instruct.yaml new file mode 100644 index 000000000..52aece47d --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/llama-3.3-70b-instruct.yaml @@ -0,0 +1,73 @@ +# Cookbook auto benchmark config for llama 3.3 70b instruct. +# Cookbook does not expose an H100/H200 default for this model; this config follows the cookbook MI300X baseline instead. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: src/components/autoregressive/Llama33ConfigGenerator/index.js +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: '0' + base_flags: + tool_call_parser: llama3 + model_path: meta-llama/Llama-3.3-70B-Instruct + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: meta-llama/Llama-3.3-70B-Instruct + max_concurrency: + - null + - 16 + - 32 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 16.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/llama-3.3-70b-instruct +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/llama-4-maverick-17b-128e-instruct-fp8.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/llama-4-maverick-17b-128e-instruct-fp8.yaml new file mode 100644 index 000000000..b5a1164c4 --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/llama-4-maverick-17b-128e-instruct-fp8.yaml @@ -0,0 +1,79 @@ +# Cookbook auto benchmark config for llama 4 maverick 17b 128e instruct fp8. +# Cookbook does not expose an H100/H200 default for this model; this config follows the cookbook GENERIC-8GPU baseline instead. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: docs/autoregressive/Llama/Llama4.md +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 + base_flags: + tp_size: 8 + context_length: 1000000 + trust_remote_code: true + enable_multimodal: true + model_path: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 4 + - 8 + - 12 + command_prefix: + - sglang + - serve +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 + max_concurrency: + - null + - 2 + - 4 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 2.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/llama-4-maverick-17b-128e-instruct-fp8 +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE3 + draft_model_path: lmsys/sglang-EAGLE3-Llama-4-Maverick-17B-128E-Instruct-v1 + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/llama-4-scout-17b-16e-instruct.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/llama-4-scout-17b-16e-instruct.yaml new file mode 100644 index 000000000..bf921e8f4 --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/llama-4-scout-17b-16e-instruct.yaml @@ -0,0 +1,83 @@ +# Cookbook auto benchmark config for llama 4 scout 17b 16e instruct. +# Baseline follows the cookbook H200 default command. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: data/models/generated/v0.5.6/llama4scout.yaml +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 + base_flags: + tp_size: 8 + enable_multimodal: true + context_length: 65536 + dtype: bfloat16 + trust_remote_code: true + model_path: meta-llama/Llama-4-Scout-17B-16E-Instruct + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 8 + - 16 + - 24 +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: meta-llama/Llama-4-Scout-17B-16E-Instruct + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/llama-4-scout-17b-16e-instruct +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE3 + draft_model_path: lmsys/sglang-EAGLE3-Llama-4-Scout-17B-16E-Instruct-v1 + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/mimo-v2-flash.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/mimo-v2-flash.yaml new file mode 100644 index 000000000..2a40706c9 --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/mimo-v2-flash.yaml @@ -0,0 +1,91 @@ +# Cookbook auto benchmark config for mimo v2 flash. +# Baseline follows the cookbook H200 default command. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: src/components/autoregressive/MiMoConfigGenerator/index.js +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 + base_flags: + tp_size: 8 + trust_remote_code: true + max_running_requests: 128 + chunked_prefill_size: 16384 + model_loader_extra_config: '{"enable_multithread_load": "true","num_threads": + 64}' + attention_backend: fa3 + reasoning_parser: qwen3 + tool_call_parser: mimo + model_path: XiaomiMiMo/MiMo-V2-Flash + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + command_prefix: + - python3 + - -m + - sglang.launch_server +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: XiaomiMiMo/MiMo-V2-Flash + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/mimo-v2-flash +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/minimax-m2.1.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/minimax-m2.1.yaml new file mode 100644 index 000000000..ff7f7b2d9 --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/minimax-m2.1.yaml @@ -0,0 +1,77 @@ +# Cookbook auto benchmark config for minimax m2.1. +# Cookbook does not expose an H100/H200 default for this model; this config follows the cookbook MI300X baseline instead. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: src/components/autoregressive/MiniMaxM2ConfigGenerator/index.js +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: 0,1,2,3 + base_flags: + tp_size: 4 + trust_remote_code: true + model_path: MiniMaxAI/MiniMax-M2.1 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 + command_prefix: + - sglang + - serve +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: MiniMaxAI/MiniMax-M2.1 + max_concurrency: + - null + - 8 + - 16 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 8.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/minimax-m2.1 +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/minimax-m2.5.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/minimax-m2.5.yaml new file mode 100644 index 000000000..d03022c19 --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/minimax-m2.5.yaml @@ -0,0 +1,83 @@ +# Cookbook auto benchmark config for minimax m2.5. +# Baseline follows the cookbook H200 default command. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: src/components/autoregressive/MiniMaxM25ConfigGenerator/index.js +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: 0,1,2,3 + base_flags: + tp_size: 4 + trust_remote_code: true + model_path: MiniMaxAI/MiniMax-M2.5 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 + ep_size: + - 1 + - 4 +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: MiniMaxAI/MiniMax-M2.5 + max_concurrency: + - null + - 8 + - 16 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/minimax-m2.5 +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/ministral-3-8b-instruct-2512.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/ministral-3-8b-instruct-2512.yaml new file mode 100644 index 000000000..147bbd180 --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/ministral-3-8b-instruct-2512.yaml @@ -0,0 +1,77 @@ +# Cookbook auto benchmark config for ministral 3 8b instruct 2512. +# Cookbook does not expose an H100/H200 default for this model; this config follows the cookbook MI300X baseline instead. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: src/components/autoregressive/Ministral3ConfigGenerator/index.js +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: '0' + base_flags: + trust_remote_code: true + tool_call_parser: mistral + model_path: mistralai/Ministral-3-8B-Instruct-2512 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 + command_prefix: + - sglang + - serve +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: mistralai/Ministral-3-8B-Instruct-2512 + max_concurrency: + - null + - 16 + - 32 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 16.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/ministral-3-8b-instruct-2512 +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/mistral-small-4-119b-2603.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/mistral-small-4-119b-2603.yaml new file mode 100644 index 000000000..595b61739 --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/mistral-small-4-119b-2603.yaml @@ -0,0 +1,79 @@ +# Cookbook auto benchmark config for mistral small 4 119b 2603. +# Baseline follows the cookbook H200 default command. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: data/models/generated/v0.5.8/mistral-small-4.yaml +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: 0,1 + base_flags: + tp_size: 2 + model_path: mistralai/Mistral-Small-4-119B-2603 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: mistralai/Mistral-Small-4-119B-2603 + max_concurrency: + - null + - 8 + - 16 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 6.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/mistral-small-4-119b-2603 +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/nemotron-3-nano-30b-a3b-bf16.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/nemotron-3-nano-30b-a3b-bf16.yaml new file mode 100644 index 000000000..bba06685c --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/nemotron-3-nano-30b-a3b-bf16.yaml @@ -0,0 +1,81 @@ +# Cookbook auto benchmark config for nemotron 3 nano 30b a3b bf16. +# Baseline follows the cookbook H200 default command. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: data/models/generated/v0.5.6/nemotron.yaml +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: '0' + base_flags: + tp_size: 1 + trust_remote_code: true + kv_cache_dtype: fp8_e4m3 + model_path: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 + max_concurrency: + - null + - 16 + - 32 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 16.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/nemotron-3-nano-30b-a3b-bf16 +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/nemotron-3-super-120b-a12b-bf16.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/nemotron-3-super-120b-a12b-bf16.yaml new file mode 100644 index 000000000..333b96a63 --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/nemotron-3-super-120b-a12b-bf16.yaml @@ -0,0 +1,81 @@ +# Cookbook auto benchmark config for nemotron 3 super 120b a12b bf16. +# Baseline follows the cookbook H200 default command. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: data/models/generated/v0.5.8/nemotron-super.yaml +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: 0,1,2,3 + base_flags: + tp_size: 4 + trust_remote_code: true + kv_cache_dtype: fp8_e4m3 + model_path: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16 + max_concurrency: + - null + - 8 + - 16 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 6.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/nemotron-3-super-120b-a12b-bf16 +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen3-235b-a22b.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen3-235b-a22b.yaml new file mode 100644 index 000000000..7b5d25b6a --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen3-235b-a22b.yaml @@ -0,0 +1,83 @@ +# Cookbook auto benchmark config for qwen3 235b a22b. +# Baseline follows the cookbook H200 default command. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: data/models/generated/v0.5.6/qwen.yaml +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 + base_flags: + tp_size: 8 + model_path: Qwen/Qwen3-235B-A22B + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + ep_size: + - 1 + - 4 + - 8 +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: Qwen/Qwen3-235B-A22B + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/qwen3-235b-a22b +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen3-coder-480b-a35b-instruct.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen3-coder-480b-a35b-instruct.yaml new file mode 100644 index 000000000..9474058d3 --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen3-coder-480b-a35b-instruct.yaml @@ -0,0 +1,82 @@ +# Cookbook auto benchmark config for qwen3 coder 480b a35b instruct. +# Cookbook does not expose an H100/H200 default for this model; this config follows the cookbook B200 baseline instead. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: src/components/autoregressive/Qwen3CoderConfigGenerator/index.js +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 + base_flags: + tp_size: 8 + ep_size: 2 + moe_runner_backend: triton + model_path: Qwen/Qwen3-Coder-480B-A35B-Instruct + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - flashinfer + decode_attention_backend: + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + ep_size: + - 1 + - 2 +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: Qwen/Qwen3-Coder-480B-A35B-Instruct + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/qwen3-coder-480b-a35b-instruct +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen3-coder-next.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen3-coder-next.yaml new file mode 100644 index 000000000..fead555f0 --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen3-coder-next.yaml @@ -0,0 +1,79 @@ +# Cookbook auto benchmark config for qwen3 coder next. +# Baseline follows the cookbook H200 default command. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: data/models/generated/v0.5.8/qwen3codernext.yaml +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: 0,1 + base_flags: + tp_size: 2 + model_path: Qwen/Qwen3-Coder-Next + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: Qwen/Qwen3-Coder-Next + max_concurrency: + - null + - 8 + - 16 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 12.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/qwen3-coder-next +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen3-next-80b-a3b-instruct.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen3-next-80b-a3b-instruct.yaml new file mode 100644 index 000000000..a5490e5d2 --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen3-next-80b-a3b-instruct.yaml @@ -0,0 +1,82 @@ +# Cookbook auto benchmark config for qwen3 next 80b a3b instruct. +# Baseline follows the cookbook H200 default command. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: data/models/generated/v0.5.6/qwen3next.yaml +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: 0,1 + base_flags: + tp_size: 2 + model_path: Qwen/Qwen3-Next-80B-A3B-Instruct + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 + ep_size: + - 1 + - 2 +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: Qwen/Qwen3-Next-80B-A3B-Instruct + max_concurrency: + - null + - 8 + - 16 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 12.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/qwen3-next-80b-a3b-instruct +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen35-397b-a17b-fp8.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen35-397b-a17b-fp8.yaml new file mode 100644 index 000000000..6190ff1d8 --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/qwen35-397b-a17b-fp8.yaml @@ -0,0 +1,83 @@ +# Cookbook auto benchmark config for qwen35 397b a17b fp8. +# Baseline follows the cookbook H200 default command. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: data/models/generated/v0.5.8/qwen35.yaml +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: 0,1,2,3 + base_flags: + tp_size: 4 + model_path: Qwen/Qwen3.5-397B-A17B-FP8 + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 + ep_size: + - 1 + - 4 + - 8 +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: Qwen/Qwen3.5-397B-A17B-FP8 + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 4.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/qwen35-397b-a17b-fp8 +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/ring-2.5-1t.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/ring-2.5-1t.yaml new file mode 100644 index 000000000..c38bffc81 --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/ring-2.5-1t.yaml @@ -0,0 +1,79 @@ +# Cookbook auto benchmark config for ring 2.5 1t. +# Baseline follows the cookbook H200 default command. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: data/models/generated/v0.5.8/ring25.yaml +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: 0,1,2,3,4,5,6,7 + base_flags: + tp_size: 8 + model_path: inclusionAI/Ring-2.5-1T + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 32 + - 48 + - 64 +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: inclusionAI/Ring-2.5-1T + max_concurrency: + - null + - 4 + - 8 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 2.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/ring-2.5-1t +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/step-3.5-flash.yaml b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/step-3.5-flash.yaml new file mode 100644 index 000000000..00ee40ad7 --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/cookbook-llm/step-3.5-flash.yaml @@ -0,0 +1,83 @@ +# Cookbook auto benchmark config for step 3.5 flash. +# Baseline follows the cookbook H200 default command. +# If you run on larger-memory or stronger GPUs, you can often reduce tp/ep/pp scale or total GPU count. +# Replace server.base_flags.model_path with a local model path if the weights are already present on disk. +# These configs default to synthetic random traffic. Use your real workload data if you want production-faithful tuning. +# Each input_len/output_len pair defines one benchmark scenario, for example [1000, 1000] and [8000, 1000]. +# Source: data/models/generated/v0.5.8/step35.yaml +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: 0,1,2,3 + base_flags: + tp_size: 4 + trust_remote_code: true + model_path: stepfun-ai/Step-3.5-Flash + mem_fraction_static: 0.82 + schedule_policy: lpm + search_space: + prefill_attention_backend: + - fa3 + - flashinfer + decode_attention_backend: + - fa3 + - flashinfer + chunked_prefill_size: + - 4096 + - 8192 + max_running_requests: + - 64 + - 96 + - 128 + ep_size: + - 1 + - 4 +dataset: + kind: random + num_prompts: 80 + scenario_names: + - chat + - summarization + input_len: + - 1000 + - 8000 + output_len: + - 1000 + - 1000 +benchmark: + backend: auto + tokenizer: stepfun-ai/Step-3.5-Flash + max_concurrency: + - null + - 8 + - 16 + extra_request_body: + temperature: 0.0 + qps: + lower: 0.25 + upper: 8.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/cookbook-llm/step-3.5-flash +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true +speculative: + enabled: false + algorithm: EAGLE + draft_model_path: '' + search_space: + speculative_num_steps: + - 3 + - 5 + speculative_eagle_topk: + - 1 + - 2 + speculative_num_draft_tokens: + - 4 + - 8 diff --git a/.claude/skills/sglang-auto-benchmark/references/qwen3-32b.yaml b/.claude/skills/sglang-auto-benchmark/references/qwen3-32b.yaml new file mode 100644 index 000000000..8884c36c9 --- /dev/null +++ b/.claude/skills/sglang-auto-benchmark/references/qwen3-32b.yaml @@ -0,0 +1,62 @@ +server: + launch: true + host: 127.0.0.1 + port: 30000 + env: + CUDA_VISIBLE_DEVICES: "0" + # Optional parallel search sugar: + # parallel: + # # dp_size is auto-derived as visible_gpus / (tp_size * pp_size). + # tp: [4, 2] + # # pp_size: [1, 2] + base_flags: + # Replace with a local model path if you have one. + model_path: Qwen/Qwen3-32B + tp_size: 1 + trust_remote_code: true + # Keep the baseline close to a pure-TP launch command. + mem_fraction_static: 0.82 + schedule_policy: lpm + # Keep CUDA graph enabled for normal performance tuning. + search_space: + prefill_attention_backend: [fa3, flashinfer] + decode_attention_backend: [fa3, flashinfer] + chunked_prefill_size: [4096, 8192] + max_running_requests: [64, 96, 128] +dataset: + kind: random + num_prompts: 80 + scenario_names: [chat, summarization] + input_len: [1000, 8000] + output_len: [1000, 1000] + +benchmark: + backend: auto + # Replace with a local tokenizer/model path if you have one. + tokenizer: Qwen/Qwen3-32B + max_concurrency: [null, 16, 32] + extra_request_body: + temperature: 0.0 + qps: + lower: 1.0 + upper: 12.0 + tolerance: 0.1 + sla: + max_ttft_ms: 1500 + max_tpot_ms: 30 + output_dir: ./auto_benchmark_results/qwen3-32b + +search: + tier: 2 + max_candidates: 8 # Cap search breadth so full searches stay tractable. + resume: true + +speculative: + enabled: false + algorithm: EAGLE + # Fill this only if you explicitly want a second-stage speculative search. + draft_model_path: "" + search_space: + speculative_num_steps: [3, 5] + speculative_eagle_topk: [1, 4] + speculative_num_draft_tokens: [4, 8] diff --git a/.gitignore b/.gitignore index 9f11f1a36..b5917c299 100644 --- a/.gitignore +++ b/.gitignore @@ -178,6 +178,7 @@ benchmark/llava_bench/images benchmark/llava_bench/mme_pack *.jsonl tmp*.txt +/tmp/ # Torch Compile logs tl_out/ diff --git a/python/sglang/auto_benchmark.py b/python/sglang/auto_benchmark.py new file mode 100644 index 000000000..5ceafa464 --- /dev/null +++ b/python/sglang/auto_benchmark.py @@ -0,0 +1,82 @@ +import argparse + +from sglang.auto_benchmark_lib import ( + SUPPORTED_DATASETS, + convert_dataset, + run_auto_benchmark, + validate_dataset, +) + + +def add_dataset_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--kind", + required=True, + choices=sorted(SUPPORTED_DATASETS), + help="Dataset kind: sharegpt, custom, random, or generated-shared-prefix.", + ) + parser.add_argument( + "--path", + default="", + help="Dataset file path. Leave empty for sharegpt auto-download.", + ) + parser.add_argument("--tokenizer", required=True) + parser.add_argument("--model", default=None) + parser.add_argument("--num-prompts", type=int, default=1000) + parser.add_argument("--output-len", type=int, default=None) + parser.add_argument("--context-len", type=int, default=None) + parser.add_argument("--prompt-suffix", type=str, default="") + parser.add_argument("--apply-chat-template", action="store_true") + parser.add_argument("--random-input-len", type=int, default=1024) + parser.add_argument("--random-output-len", type=int, default=256) + parser.add_argument("--random-range-ratio", type=float, default=0.0) + parser.add_argument("--gsp-num-groups", type=int, default=64) + parser.add_argument("--gsp-prompts-per-group", type=int, default=16) + parser.add_argument("--gsp-system-prompt-len", type=int, default=2048) + parser.add_argument("--gsp-question-len", type=int, default=128) + parser.add_argument("--gsp-output-len", type=int, default=256) + parser.add_argument("--gsp-range-ratio", type=float, default=1.0) + parser.add_argument("--gsp-fast-prepare", action="store_true") + parser.add_argument("--gsp-send-routing-key", action="store_true") + parser.add_argument("--gsp-num-turns", type=int, default=1) + parser.add_argument("--gsp-ordered", action="store_true") + parser.add_argument("--seed", type=int, default=1) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="SGLang auto benchmark utilities.") + subparsers = parser.add_subparsers(dest="command", required=True) + + run_parser = subparsers.add_parser( + "run", help="Run auto benchmark from YAML config." + ) + run_parser.add_argument("--config", required=True) + + convert_parser = subparsers.add_parser( + "convert", + help="Prepare sharegpt/custom/random/generated-shared-prefix data into canonical autobench JSONL.", + ) + add_dataset_args(convert_parser) + convert_parser.add_argument("--output", required=True) + + validate_parser = subparsers.add_parser( + "validate", help="Validate a canonical autobench JSONL dataset." + ) + validate_parser.add_argument("--dataset-path", required=True) + validate_parser.add_argument("--tokenizer", required=True) + + return parser + + +def main() -> None: + args = build_parser().parse_args() + if args.command == "run": + run_auto_benchmark(args.config) + elif args.command == "convert": + convert_dataset(args) + elif args.command == "validate": + validate_dataset(args) + + +if __name__ == "__main__": + main() diff --git a/python/sglang/auto_benchmark_lib.py b/python/sglang/auto_benchmark_lib.py new file mode 100644 index 000000000..1b82c9117 --- /dev/null +++ b/python/sglang/auto_benchmark_lib.py @@ -0,0 +1,1985 @@ +import argparse +import csv +import itertools +import json +import os +import shlex +import signal +import subprocess +import sys +import time +from copy import deepcopy +from types import SimpleNamespace +from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple + +import yaml +from tqdm.auto import tqdm + +from sglang.benchmark.datasets import get_dataset +from sglang.benchmark.datasets.autobench import ( + sample_autobench_requests, + serialize_dataset_row_to_autobench, +) +from sglang.benchmark.utils import get_tokenizer + +SUPPORTED_DATASETS = { + "sharegpt", + "custom", + "random", + "generated-shared-prefix", +} + +FLAG_ALIASES = { + "tp": "tp_size", + "pp": "pp_size", + "dp": "dp_size", + "ep": "ep_size", +} + +OOM_HINT = "Candidate likely OOMed. Increase GPU count or use GPUs with larger memory." +PROGRESS_FLAG_KEYS = ( + "tp_size", + "dp_size", + "ep_size", + "pp_size", + "prefill_attention_backend", + "decode_attention_backend", + "attention_backend", + "sampling_backend", + "grammar_backend", + "mem_fraction_static", + "chunked_prefill_size", + "prefill_max_requests", + "max_prefill_tokens", + "max_running_requests", + "max_queued_requests", + "schedule_policy", + "schedule_conservativeness", + "num_continuous_decode_steps", + "stream_interval", + "page_size", + "cuda_graph_max_bs", + "speculative_num_steps", + "speculative_eagle_topk", + "speculative_num_draft_tokens", +) +PROGRESS_FLAG_ALIASES = { + "tp_size": "tp", + "dp_size": "dp", + "ep_size": "ep", + "pp_size": "pp", + "prefill_attention_backend": "prefill", + "decode_attention_backend": "decode", + "attention_backend": "attn", + "sampling_backend": "sampling", + "grammar_backend": "grammar", + "mem_fraction_static": "mfs", + "chunked_prefill_size": "chunk", + "prefill_max_requests": "prefill_req", + "max_prefill_tokens": "prefill_tok", + "max_running_requests": "mrr", + "max_queued_requests": "mqr", + "schedule_policy": "sched", + "schedule_conservativeness": "sched_cons", + "num_continuous_decode_steps": "decode_steps", + "stream_interval": "stream", + "page_size": "page", + "cuda_graph_max_bs": "cg_bs", + "speculative_num_steps": "spec_steps", + "speculative_eagle_topk": "eagle_topk", + "speculative_num_draft_tokens": "draft_tok", +} +SENSITIVE_ENV_MARKERS = ("TOKEN", "KEY", "SECRET", "PASSWORD") +DEFAULT_MAX_CANDIDATES = 8 +MAX_BINARY_SEARCH_ROUNDS = 5 +DEFAULT_BINARY_SEARCH_ROUNDS = 5 +MAX_SEARCH_DURATION_HOURS = 12.0 +DEFAULT_SEARCH_DURATION_HOURS = 12.0 + + +class SearchDeadlineExceeded(RuntimeError): + """Raised when the auto benchmark exhausts its global search budget.""" + + +def load_yaml(path: str) -> Dict[str, Any]: + with open(path, "r", encoding="utf-8") as f: + return yaml.safe_load(f) + + +def as_list(value: Any) -> List[Any]: + return value if isinstance(value, list) else [value] + + +def slugify(text: str) -> str: + return "".join(ch.lower() if ch.isalnum() else "-" for ch in text).strip("-") + + +def canonical_flag_name(name: str) -> str: + return FLAG_ALIASES.get(name, name) + + +def canonicalize_flags(flags: Dict[str, Any]) -> Dict[str, Any]: + return {canonical_flag_name(key): value for key, value in flags.items()} + + +def flatten(data: Dict[str, Any], prefix: str = "") -> Dict[str, Any]: + flat: Dict[str, Any] = {} + for key, value in data.items(): + name = f"{prefix}.{key}" if prefix else key + if isinstance(value, dict): + flat.update(flatten(value, name)) + else: + flat[name] = value + return flat + + +def log_line(message: str) -> None: + tqdm.write(message) + + +def detect_current_cuda_capability() -> Optional[Tuple[int, int]]: + try: + import torch + except ModuleNotFoundError: + return None + + if not torch.cuda.is_available(): + return None + major, minor = torch.cuda.get_device_capability() + return int(major), int(minor) + + +def is_attention_backend_supported( + backend: Any, capability: Optional[Tuple[int, int]] +) -> bool: + if capability is None or backend in (None, ""): + return True + + major, _minor = capability + if backend == "fa3": + return major in (8, 9) + return True + + +def is_candidate_supported_on_current_device( + candidate: Dict[str, Any], capability: Optional[Tuple[int, int]] +) -> bool: + backend_keys = ( + "attention_backend", + "prefill_attention_backend", + "decode_attention_backend", + ) + return all( + is_attention_backend_supported(candidate.get(key), capability) + for key in backend_keys + ) + + +def append_jsonl(path: str, records: Iterable[Dict[str, Any]]) -> None: + with open(path, "a", encoding="utf-8") as f: + for record in records: + f.write(json.dumps(record, ensure_ascii=False) + "\n") + + +def read_jsonl(path: str) -> List[Dict[str, Any]]: + if not path or not os.path.isfile(path): + return [] + records: List[Dict[str, Any]] = [] + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + records.append(json.loads(line)) + return records + + +def describe_search_tier(tier: int) -> str: + descriptions = { + 1: "tier 1: smallest and fastest sanity sweep", + 2: "tier 2: balanced default sweep", + 3: "tier 3: largest and slowest full search", + } + return descriptions.get(tier, f"tier {tier}") + + +def install_interrupt_handlers() -> Dict[signal.Signals, Any]: + previous = {} + + def handler(signum, _frame): # type: ignore[no-untyped-def] + raise KeyboardInterrupt(f"Interrupted by signal {signum}") + + for sig in (signal.SIGINT, signal.SIGTERM): + try: + previous[sig] = signal.getsignal(sig) + signal.signal(sig, handler) + except Exception: + continue + return previous + + +def restore_interrupt_handlers(previous: Dict[signal.Signals, Any]) -> None: + for sig, handler in previous.items(): + try: + signal.signal(sig, handler) + except Exception: + continue + + +def collect_stale_server_pids(port: int) -> List[int]: + patterns = [ + ["lsof", "-ti", f"tcp:{port}", "-sTCP:LISTEN"], + ["pgrep", "-f", f"sglang.launch_server.*--port {port}"], + ["pgrep", "-f", f"sglang.launch_server.*--port={port}"], + ["pgrep", "-f", f"sglang serve .*--port {port}"], + ["pgrep", "-f", f"sglang serve .*--port={port}"], + ] + pids = set() + for command in patterns: + try: + result = subprocess.run( + command, capture_output=True, text=True, check=False + ) + except FileNotFoundError: + continue + if result.returncode not in (0, 1): + continue + for line in result.stdout.splitlines(): + line = line.strip() + if line.isdigit(): + pids.add(int(line)) + return sorted(pids) + + +def kill_pid_or_group(pid: int) -> None: + try: + pgid = os.getpgid(pid) + except ProcessLookupError: + return + + for sig, delay in ((signal.SIGTERM, 1.0), (signal.SIGKILL, 0.0)): + try: + os.killpg(pgid, sig) + except ProcessLookupError: + return + except PermissionError: + try: + os.kill(pid, sig) + except ProcessLookupError: + return + if delay: + time.sleep(delay) + + +def preclean_stale_server(port: int) -> None: + stale_pids = collect_stale_server_pids(port) + if not stale_pids: + return + log_line(f"preclean_port={port} stale_pids={stale_pids}") + for pid in stale_pids: + kill_pid_or_group(pid) + + +def normalize_binary_search_rounds(value: Any) -> int: + if value is None: + return DEFAULT_BINARY_SEARCH_ROUNDS + return max(1, min(int(value), MAX_BINARY_SEARCH_ROUNDS)) + + +def resolve_max_candidates(search_cfg: Dict[str, Any]) -> Optional[int]: + if "max_candidates" not in search_cfg: + return DEFAULT_MAX_CANDIDATES + configured = search_cfg.get("max_candidates") + if configured is None: + return None + value = int(configured) + if value < 1: + raise ValueError("search.max_candidates must be >= 1 or null.") + return value + + +def estimate_binary_search_trials( + lower: float, upper: float, tolerance: float, max_rounds: int +) -> int: + if upper <= lower or tolerance <= 0: + return 1 + + trials = 0 + lo, hi = float(lower), float(upper) + while hi - lo > tolerance and trials < max_rounds: + qps = pick_qps_midpoint(lo, hi) + if qps <= lo or qps >= hi: + break + hi = qps + trials += 1 + return max(trials, 1) + + +def pick_qps_midpoint(lower: float, upper: float) -> float: + midpoint = round((lower + upper) / 2, 4) + if lower < midpoint < upper: + return midpoint + return (lower + upper) / 2 + + +def estimate_trials_per_candidate(benchmark_cfg: Dict[str, Any]) -> int: + mode, values, tolerance, max_rounds = build_qps_plan(benchmark_cfg) + max_concurrency_values = as_list(benchmark_cfg.get("max_concurrency", [None])) + if mode == "fixed": + per_concurrency = len(values) + else: + per_concurrency = estimate_binary_search_trials( + values[0], values[1], tolerance, max_rounds + ) + return max(1, per_concurrency) * len(max_concurrency_values) + + +def describe_qps_plan(benchmark_cfg: Dict[str, Any]) -> str: + mode, values, tolerance, max_rounds = build_qps_plan(benchmark_cfg) + if mode == "fixed": + return f"fixed qps values={values}" + return ( + f"binary search qps lower={values[0]} upper={values[1]} " + f"tolerance={tolerance} max_rounds={max_rounds} " + "estimated_trials_per_max_concurrency=" + f"{estimate_binary_search_trials(values[0], values[1], tolerance, max_rounds)}" + ) + + +def scenario_plan_text(scenario: Dict[str, Any]) -> str: + cfg = scenario["cfg"] + parts = [f"kind={cfg['kind']}", f"num_prompts={cfg.get('num_prompts', '')}"] + if cfg["kind"] == "random": + parts.append(f"input_len={cfg['random_input_len']}") + parts.append(f"output_len={cfg['random_output_len']}") + elif cfg.get("path"): + parts.append(f"path={cfg['path']}") + return ", ".join(str(part) for part in parts if part != "") + + +def print_run_plan( + config_path: str, + output_dir: str, + tier: int, + max_candidates: Optional[int], + benchmark_cfg: Dict[str, Any], + scenarios: Sequence[Dict[str, Any]], + server_cfg: Dict[str, Any], + base_candidates: Sequence[Dict[str, Any]], + speculative_enabled: bool, + search_budget_hours: float, + search_deadline: float, +) -> None: + estimated_base_trials = ( + len(scenarios) + * len(base_candidates) + * estimate_trials_per_candidate(benchmark_cfg) + ) + log_line("=== Auto Benchmark Plan ===") + log_line(f"config={config_path}") + log_line(f"output_dir={output_dir}") + log_line(f"search.tier={tier} ({describe_search_tier(tier)})") + log_line( + "search.max_candidates=" + f"{max_candidates if max_candidates is not None else 'unbounded'}" + ) + log_line( + f"search.max_duration_hours={search_budget_hours:.1f} " + f"(deadline {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(search_deadline))})" + ) + log_line(f"qps_plan={describe_qps_plan(benchmark_cfg)}") + log_line( + "max_concurrency=" + f"{json.dumps(as_list(benchmark_cfg.get('max_concurrency', [None])), ensure_ascii=False)}" + ) + log_line(f"estimated_base_trials={estimated_base_trials}") + log_line("Planned scenarios:") + for index, scenario in enumerate(scenarios, start=1): + log_line( + f" [{index}/{len(scenarios)}] {scenario['display_name']}: " + f"{scenario_plan_text(scenario)}" + ) + log_line("Planned base candidates:") + for index, candidate in enumerate(base_candidates, start=1): + rendered = merge_host_port(server_cfg, candidate) + log_line( + f" [{index}/{len(base_candidates)}] {json.dumps(rendered, ensure_ascii=False)}" + ) + if speculative_enabled: + log_line( + "Speculative stage is enabled. Its candidate list will be printed after " + "the best base configuration is known." + ) + + +def estimated_finish_time( + start_time: float, completed: int, total: Optional[int] +) -> str: + if not total or completed <= 0: + return "?" + remaining_seconds = max( + 0.0, (time.time() - start_time) * (total - completed) / completed + ) + return time.strftime( + "%Y-%m-%d %H:%M:%S", time.localtime(time.time() + remaining_seconds) + ) + + +def current_time_text() -> str: + return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + + +def resolve_search_budget_hours(search_cfg: Dict[str, Any]) -> float: + configured = search_cfg.get("max_duration_hours", DEFAULT_SEARCH_DURATION_HOURS) + return max(0.0, min(float(configured), MAX_SEARCH_DURATION_HOURS)) + + +def format_timestamp(timestamp: float) -> str: + return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(timestamp)) + + +def remaining_search_seconds(search_deadline: Optional[float]) -> Optional[float]: + if search_deadline is None: + return None + return max(0.0, search_deadline - time.time()) + + +def raise_if_search_deadline_reached( + search_deadline: Optional[float], budget_hours: float +) -> None: + remaining = remaining_search_seconds(search_deadline) + if remaining is None or remaining > 0: + return + raise SearchDeadlineExceeded( + "search budget of " + f"{budget_hours:.1f}h reached before the full search completed " + f"(deadline {format_timestamp(search_deadline)})" + ) + + +def summarize_progress_flags(server_flags: Dict[str, Any], limit: int = 6) -> str: + parts = [] + for key in PROGRESS_FLAG_KEYS: + if key not in server_flags: + continue + value = server_flags[key] + if value in (None, "", False): + continue + alias = PROGRESS_FLAG_ALIASES.get(key, key) + parts.append(f"{alias}={value}") + if len(parts) >= limit: + break + if not parts and server_flags.get("candidate_id") is not None: + return f"candidate={server_flags['candidate_id']}" + return ",".join(parts) + + +def format_best_progress(record: Optional[Dict[str, Any]]) -> str: + if not record or not record.get("metrics"): + return "best pending" + + metrics = record["metrics"] + flags = dict(record.get("server_flags", {})) + flags["candidate_id"] = record.get("candidate_id") + return ( + "best " + f"qps={record.get('requested_qps', 0.0):.4f} " + f"tok/s={metrics.get('output_throughput', 0.0):.1f} " + f"ttft={metrics.get('mean_ttft_ms', 0.0):.1f}ms " + f"tpot={metrics.get('mean_tpot_ms', 0.0):.1f}ms " + f"cfg[{summarize_progress_flags(flags)}]" + ) + + +def refresh_progress_eta( + pbar: tqdm, start_time: float, best_record: Optional[Dict[str, Any]] = None +) -> None: + pbar.set_postfix_str( + f"now {current_time_text()} | " + f"finish {estimated_finish_time(start_time, int(pbar.n), pbar.total)} | " + f"{format_best_progress(best_record)}", + refresh=False, + ) + + +def make_progress_bar( + desc: str, total: int, position: int, leave: bool +) -> Tuple[tqdm, float]: + start_time = time.time() + pbar = tqdm( + total=total, + desc=desc, + dynamic_ncols=True, + mininterval=1.0, + position=position, + leave=leave, + bar_format="{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}] {postfix}", + ) + refresh_progress_eta(pbar, start_time) + return pbar, start_time + + +def advance_progress( + pbar: tqdm, + start_time: float, + count: int = 1, + best_record: Optional[Dict[str, Any]] = None, +) -> None: + if pbar.total is not None and pbar.n + count > pbar.total: + pbar.total = pbar.n + count + pbar.update(count) + refresh_progress_eta(pbar, start_time, best_record) + + +def tail_text(path: str, limit: int = 4000) -> str: + if not path or not os.path.isfile(path): + return "" + with open(path, "r", encoding="utf-8", errors="ignore") as f: + text = f.read() + return text[-limit:] + + +def cli_args(flags: Dict[str, Any]) -> List[str]: + args: List[str] = [] + for key, value in flags.items(): + if value is None or value is False: + continue + flag = f"--{key.replace('_', '-')}" + if value is True: + args.append(flag) + elif isinstance(value, list): + args.append(flag) + args.extend(str(item) for item in value) + else: + args.extend([flag, str(value)]) + return args + + +def classify_failure(message: str) -> Tuple[Optional[str], Optional[str]]: + lower = message.lower() + oom_markers = ( + "out of memory", + "cuda out of memory", + "hip out of memory", + "cudnn_status_alloc_failed", + "std::bad_alloc", + "memoryerror", + "memory allocation", + "no available memory", + ) + if any(marker in lower for marker in oom_markers): + return "oom", OOM_HINT + return None, None + + +def prompt_kind(prompt: Any) -> str: + if isinstance(prompt, str): + return "prompt" + if isinstance(prompt, list) and prompt: + if isinstance(prompt[0], dict): + return "messages" + if isinstance(prompt[0], str): + return "multi_turn" + if isinstance(prompt[0], int): + return "token_ids" + return "unknown" + + +def summarize_rows(rows: Sequence[Any]) -> Dict[str, Any]: + kinds: Dict[str, int] = {} + output_lens = [row.output_len for row in rows] + for row in rows: + kind = prompt_kind(row.prompt) + kinds[kind] = kinds.get(kind, 0) + 1 + return { + "num_requests": len(rows), + "prompt_kinds": kinds, + "output_len_min": min(output_lens) if output_lens else 0, + "output_len_max": max(output_lens) if output_lens else 0, + "output_len_avg": ( + round(sum(output_lens) / len(output_lens), 2) if output_lens else 0.0 + ), + } + + +def infer_backend(backend: str, rows: Sequence[Any]) -> str: + if backend != "auto": + return backend + + kinds = {prompt_kind(row.prompt) for row in rows} + if kinds <= {"messages", "multi_turn"}: + return "sglang-oai-chat" + if kinds <= {"prompt"}: + return "sglang-oai" + if kinds <= {"token_ids"}: + return "sglang" + raise ValueError( + f"Cannot infer backend for mixed prompt kinds: {sorted(kinds)}. " + "Set benchmark.backend explicitly." + ) + + +def looks_like_autobench(path: str) -> bool: + if not path or not os.path.isfile(path): + return False + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + return False + return isinstance(row, dict) and any( + key in row for key in ("prompt", "messages", "prompt_origin", "system") + ) + return False + + +def write_autobench_jsonl( + path: str, rows: Sequence[Any], metadata: Optional[Dict[str, Any]] = None +) -> None: + directory = os.path.dirname(path) + if directory: + os.makedirs(directory, exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + for row in rows: + record = serialize_dataset_row_to_autobench(row, metadata=metadata) + f.write(json.dumps(record, ensure_ascii=False) + "\n") + + +def normalize_dataset_cfg( + dataset_cfg: Optional[Dict[str, Any]], benchmark_cfg: Dict[str, Any] +) -> Dict[str, Any]: + raw = {} if dataset_cfg is None else dataset_cfg + if isinstance(raw, str): + raw = {"kind": raw} + cfg = dict(raw) + + if "kind" not in cfg and cfg.get("path") in SUPPORTED_DATASETS: + cfg["kind"] = cfg["path"] + cfg["path"] = "" + + if "kind" not in cfg and benchmark_cfg.get("dataset_path"): + cfg["kind"] = "custom" + cfg["path"] = benchmark_cfg["dataset_path"] + + if "num_prompts" not in cfg and benchmark_cfg.get("num_prompts") is not None: + cfg["num_prompts"] = benchmark_cfg["num_prompts"] + + cfg["kind"] = cfg.get("kind", "custom") + if cfg["kind"] == "autobench": + cfg["kind"] = "custom" + if cfg["kind"] not in SUPPORTED_DATASETS: + raise ValueError( + f"Unsupported dataset kind: {cfg['kind']}. " + f"Supported: {sorted(SUPPORTED_DATASETS)}" + ) + if cfg["kind"] == "custom" and not cfg.get("path"): + raise ValueError("dataset.path is required when dataset.kind=custom.") + return cfg + + +def expand_dataset_scenarios(dataset_cfg: Dict[str, Any]) -> List[Dict[str, Any]]: + if dataset_cfg["kind"] != "random": + name = dataset_cfg.get("scenario_name", "default") + return [ + { + "name": slugify(str(name)) or "default", + "display_name": str(name), + "cfg": dataset_cfg, + } + ] + + input_lens = as_list( + dataset_cfg.get("input_len", dataset_cfg.get("random_input_len", 1024)) + ) + output_lens = as_list( + dataset_cfg.get("output_len", dataset_cfg.get("random_output_len", 256)) + ) + if len(input_lens) != len(output_lens): + raise ValueError( + "random dataset input_len and output_len must have the same number of elements." + ) + + scenario_names = dataset_cfg.get("scenario_names") + if scenario_names is not None and len(as_list(scenario_names)) != len(input_lens): + raise ValueError( + "dataset.scenario_names must match the length of input_len/output_len." + ) + + names = as_list(scenario_names) if scenario_names is not None else None + scenarios = [] + for index, (input_len, output_len) in enumerate(zip(input_lens, output_lens)): + cfg = dict(dataset_cfg) + cfg["random_input_len"] = int(input_len) + cfg["random_output_len"] = int(output_len) + cfg["input_len"] = int(input_len) + cfg["output_len"] = int(output_len) + display_name = ( + str(names[index]) + if names is not None + else f"input{int(input_len)}-output{int(output_len)}" + ) + scenarios.append( + { + "name": slugify(display_name) or f"scenario-{index + 1}", + "display_name": display_name, + "cfg": cfg, + } + ) + return scenarios + + +def build_dataset_args( + dataset_cfg: Dict[str, Any], tokenizer_path: str, model: Optional[str] +) -> SimpleNamespace: + dataset_path = dataset_cfg.get("path", "") + if dataset_cfg["kind"] == "sharegpt" and dataset_path in ("", None, "sharegpt"): + dataset_path = "" + is_random = dataset_cfg["kind"] == "random" + + return SimpleNamespace( + dataset_name=dataset_cfg["kind"], + dataset_path=dataset_path, + tokenizer=tokenizer_path, + model=model, + num_prompts=int(dataset_cfg.get("num_prompts", 1000)), + sharegpt_output_len=(dataset_cfg.get("output_len") if not is_random else None), + sharegpt_context_len=dataset_cfg.get("context_len"), + random_input_len=int( + dataset_cfg.get("input_len", dataset_cfg.get("random_input_len", 1024)) + ), + random_output_len=int( + dataset_cfg.get("output_len", dataset_cfg.get("random_output_len", 256)) + ), + random_range_ratio=float(dataset_cfg.get("random_range_ratio", 0.0)), + prompt_suffix=dataset_cfg.get("prompt_suffix", ""), + apply_chat_template=bool(dataset_cfg.get("apply_chat_template", False)), + gsp_num_groups=int(dataset_cfg.get("gsp_num_groups", 64)), + gsp_prompts_per_group=int(dataset_cfg.get("gsp_prompts_per_group", 16)), + gsp_system_prompt_len=int(dataset_cfg.get("gsp_system_prompt_len", 2048)), + gsp_question_len=int(dataset_cfg.get("gsp_question_len", 128)), + gsp_output_len=int(dataset_cfg.get("gsp_output_len", 256)), + gsp_range_ratio=float(dataset_cfg.get("gsp_range_ratio", 1.0)), + gsp_fast_prepare=bool(dataset_cfg.get("gsp_fast_prepare", False)), + gsp_send_routing_key=bool(dataset_cfg.get("gsp_send_routing_key", False)), + gsp_num_turns=int(dataset_cfg.get("gsp_num_turns", 1)), + gsp_ordered=bool(dataset_cfg.get("gsp_ordered", False)), + seed=int(dataset_cfg.get("seed", 1)), + ) + + +def load_autobench_rows( + dataset_path: str, + tokenizer_path: str, + num_prompts: int = 0, + output_len: Optional[int] = None, +) -> List[Any]: + return sample_autobench_requests( + dataset_path=dataset_path, + num_requests=num_prompts, + tokenizer=get_tokenizer(tokenizer_path), + fixed_output_len=output_len, + ) + + +def prepare_dataset( + dataset_cfg: Dict[str, Any], + tokenizer_path: str, + model: Optional[str], + output_path: str, +) -> Tuple[str, List[Any], Dict[str, Any]]: + dataset_cfg = normalize_dataset_cfg(dataset_cfg, {}) + if dataset_cfg["kind"] == "custom" and looks_like_autobench( + dataset_cfg.get("path", "") + ): + rows = load_autobench_rows( + dataset_path=dataset_cfg["path"], + tokenizer_path=tokenizer_path, + num_prompts=int(dataset_cfg.get("num_prompts", 0)), + output_len=dataset_cfg.get("output_len"), + ) + else: + tokenizer = get_tokenizer(tokenizer_path) + dataset_args = build_dataset_args(dataset_cfg, tokenizer_path, model) + rows = get_dataset(dataset_args, tokenizer=tokenizer, model_id=model) + + if not rows: + raise ValueError("Prepared dataset is empty.") + + write_autobench_jsonl( + output_path, + rows, + metadata={ + "source_dataset_name": dataset_cfg["kind"], + "source_dataset_path": dataset_cfg.get("path") or dataset_cfg["kind"], + }, + ) + return output_path, rows, summarize_rows(rows) + + +def infer_total_gpus(server_cfg: Dict[str, Any]) -> Optional[int]: + parallel_cfg = server_cfg.get("parallel", {}) + for key in ("gpu_count",): + value = parallel_cfg.get(key, server_cfg.get(key)) + if value is not None: + return int(value) + + env = server_cfg.get("env", {}) + for key in ( + "CUDA_VISIBLE_DEVICES", + "ROCR_VISIBLE_DEVICES", + "HIP_VISIBLE_DEVICES", + "NVIDIA_VISIBLE_DEVICES", + ): + value = env.get(key) + if value is None: + continue + value = str(value).strip() + if not value or value.lower() in {"all", "none", "void"}: + continue + return len([item for item in value.split(",") if item.strip()]) + return None + + +def resolve_parallelism( + server_cfg: Dict[str, Any], flags: Dict[str, Any], parallel_requested: bool +) -> Dict[str, Any]: + flags = canonicalize_flags(flags) + if not parallel_requested: + return flags + + tp_size = int(flags.get("tp_size", 1)) + pp_size = int(flags.get("pp_size", 1)) + if "dp_size" in flags: + return flags + + total_gpus = infer_total_gpus(server_cfg) + if total_gpus is None: + raise ValueError( + "Cannot infer total GPU count for parallel search. " + "Set server.parallel.gpu_count or server.env.CUDA_VISIBLE_DEVICES." + ) + + shard_size = tp_size * pp_size + if shard_size <= 0 or total_gpus % shard_size != 0: + raise ValueError( + f"Cannot derive dp_size: total_gpus={total_gpus}, " + f"tp_size={tp_size}, pp_size={pp_size}." + ) + + flags["dp_size"] = total_gpus // shard_size + return flags + + +def build_server_candidates( + server_cfg: Dict[str, Any], tier: int, max_candidates: Optional[int] +) -> List[Dict[str, Any]]: + base_flags = canonicalize_flags(deepcopy(server_cfg.get("base_flags", {}))) + search_space = canonicalize_flags(deepcopy(server_cfg.get("search_space", {}))) + parallel_cfg = canonicalize_flags(deepcopy(server_cfg.get("parallel", {}))) + parallel_requested = bool(parallel_cfg) + for key, value in parallel_cfg.items(): + if key == "gpu_count": + continue + values = as_list(value) + if values: + base_flags.setdefault(key, values[0]) + search_space.update( + {key: value for key, value in parallel_cfg.items() if key != "gpu_count"} + ) + + candidates = build_candidates( + base_flags=base_flags, + search_space=search_space, + tier=tier, + max_candidates=max_candidates, + ) + return [ + resolve_parallelism(server_cfg, candidate, parallel_requested) + for candidate in candidates + ] + + +def build_candidates( + base_flags: Dict[str, Any], + search_space: Dict[str, Sequence[Any]], + tier: int, + max_candidates: Optional[int], +) -> List[Dict[str, Any]]: + base_flags = canonicalize_flags(base_flags) + search_space = canonicalize_flags(search_space) + capability = detect_current_cuda_capability() + items = [(key, as_list(values)) for key, values in search_space.items()] + if tier == 1: + items = [(k, v[:2]) for k, v in items[:6]] + elif tier == 2: + items = [(k, v[:3]) for k, v in items[:8]] + + candidates = [deepcopy(base_flags)] + if tier == 1: + for key, values in items: + for value in values: + candidates.append(deepcopy(base_flags) | {key: value}) + elif tier == 2 and items: + head, tail = items[:3], items[3:] + for combo in itertools.product(*[values for _, values in head]): + candidate = deepcopy(base_flags) + for (key, _), value in zip(head, combo): + candidate[key] = value + candidates.append(candidate) + for key, values in tail: + for value in values: + candidates.append(deepcopy(base_flags) | {key: value}) + elif tier == 3 and items: + for combo in itertools.product(*[values for _, values in items]): + candidate = deepcopy(base_flags) + for (key, _), value in zip(items, combo): + candidate[key] = value + candidates.append(candidate) + + deduped: List[Dict[str, Any]] = [] + seen = set() + for candidate in candidates: + if not is_candidate_supported_on_current_device(candidate, capability): + continue + key = json.dumps(candidate, sort_keys=True, ensure_ascii=False) + if key in seen: + continue + seen.add(key) + deduped.append(candidate) + if max_candidates is not None and len(deduped) >= max_candidates: + break + return deduped + + +def build_qps_plan( + benchmark_cfg: Dict[str, Any], +) -> Tuple[str, List[float], float, int]: + qps_cfg = benchmark_cfg.get("qps", benchmark_cfg.get("request_rate")) + if isinstance(qps_cfg, (int, float)): + return "fixed", [float(qps_cfg)], 0.0, 0 + if isinstance(qps_cfg, list): + return "fixed", [float(value) for value in qps_cfg], 0.0, 0 + if isinstance(qps_cfg, dict) and "values" in qps_cfg: + return "fixed", [float(value) for value in qps_cfg["values"]], 0.0, 0 + if isinstance(qps_cfg, dict) and {"lower", "upper"} <= set(qps_cfg): + return ( + "search", + [float(qps_cfg["lower"]), float(qps_cfg["upper"])], + float(qps_cfg.get("tolerance", 0.1)), + normalize_binary_search_rounds(qps_cfg.get("max_rounds")), + ) + raise ValueError("benchmark.qps must be a list or a {lower, upper, tolerance} map.") + + +def trial_key( + stage_name: str, + candidate_id: int, + request_rate: float, + max_concurrency: Optional[int], + server_flags: Dict[str, Any], +) -> str: + return json.dumps( + { + "stage": stage_name, + "candidate_id": candidate_id, + "requested_qps": request_rate, + "max_concurrency": max_concurrency, + "server_flags": canonicalize_flags(server_flags), + }, + sort_keys=True, + ensure_ascii=False, + ) + + +def record_trial_key(record: Dict[str, Any]) -> str: + return trial_key( + stage_name=str(record.get("stage", "")), + candidate_id=int(record.get("candidate_id", 0)), + request_rate=float(record.get("requested_qps", 0.0)), + max_concurrency=record.get("max_concurrency"), + server_flags=record.get("server_flags", {}), + ) + + +def meets_sla(result: Dict[str, Any], benchmark_cfg: Dict[str, Any]) -> bool: + sla = benchmark_cfg.get("sla", {}) + max_ttft_ms = sla.get("max_ttft_ms") + max_tpot_ms = sla.get("max_tpot_ms") + if ( + max_ttft_ms is not None + and result.get("mean_ttft_ms", float("inf")) > max_ttft_ms + ): + return False + if ( + max_tpot_ms is not None + and result.get("mean_tpot_ms", float("inf")) > max_tpot_ms + ): + return False + return True + + +def result_sort_key(record: Dict[str, Any]) -> Tuple[Any, ...]: + return ( + 1 if record.get("sla_passed") else 0, + record.get("requested_qps", 0.0), + record.get("metrics", {}).get("output_throughput", 0.0), + -record.get("metrics", {}).get("mean_ttft_ms", float("inf")), + -record.get("metrics", {}).get("mean_tpot_ms", float("inf")), + ) + + +def launch_server( + server_cfg: Dict[str, Any], server_flags: Dict[str, Any], log_path: str +) -> subprocess.Popen: + command_prefix = server_cfg.get("command_prefix") + if command_prefix is None: + command = [sys.executable, "-m", "sglang.launch_server"] + elif isinstance(command_prefix, str): + command = shlex.split(command_prefix) + else: + command = [str(item) for item in command_prefix] + + command.extend(cli_args(server_flags)) + command.extend(str(item) for item in server_cfg.get("extra_args", [])) + + env = os.environ.copy() + env.update({key: str(value) for key, value in server_cfg.get("env", {}).items()}) + log_file = open(log_path, "w", encoding="utf-8") + try: + process = subprocess.Popen( + command, + stdout=log_file, + stderr=subprocess.STDOUT, + env=env, + start_new_session=True, + ) + except Exception: + log_file.close() + raise + process._autobench_log_file = log_file # type: ignore[attr-defined] + return process + + +def stop_server(process: Optional[subprocess.Popen]) -> None: + if process is None: + return + try: + os.killpg(process.pid, signal.SIGTERM) + process.wait(timeout=20) + except Exception: + try: + os.killpg(process.pid, signal.SIGKILL) + except Exception: + pass + finally: + log_file = getattr(process, "_autobench_log_file", None) + if log_file is not None: + log_file.close() + + +def build_bench_command( + benchmark_cfg: Dict[str, Any], + dataset_summary: Dict[str, Any], + backend: str, + base_url: str, + dataset_path: str, + tokenizer_path: str, + request_rate: float, + max_concurrency: Optional[int], + output_file: str, +) -> List[str]: + command = [ + sys.executable, + "-m", + "sglang.bench_serving", + "--backend", + backend, + "--base-url", + base_url, + "--dataset-name", + "autobench", + "--dataset-path", + dataset_path, + "--tokenizer", + tokenizer_path, + "--num-prompts", + str(dataset_summary["num_requests"]), + "--request-rate", + str(request_rate), + "--output-file", + output_file, + "--seed", + str(int(benchmark_cfg.get("seed", 1))), + "--ready-check-timeout-sec", + str(int(benchmark_cfg.get("ready_check_timeout_sec", 600))), + ] + if benchmark_cfg.get("model"): + command.extend(["--model", str(benchmark_cfg["model"])]) + if benchmark_cfg.get("served_model_name"): + command.extend(["--served-model-name", str(benchmark_cfg["served_model_name"])]) + if benchmark_cfg.get("disable_tqdm", True): + command.append("--disable-tqdm") + if benchmark_cfg.get("output_details"): + command.append("--output-details") + if benchmark_cfg.get("disable_stream"): + command.append("--disable-stream") + if benchmark_cfg.get("disable_ignore_eos"): + command.append("--disable-ignore-eos") + if benchmark_cfg.get("pd_separated"): + command.append("--pd-separated") + if benchmark_cfg.get("flush_cache"): + command.append("--flush-cache") + if benchmark_cfg.get("tag"): + command.extend(["--tag", str(benchmark_cfg["tag"])]) + if max_concurrency is not None: + command.extend(["--max-concurrency", str(max_concurrency)]) + if benchmark_cfg.get("warmup_requests") is not None: + command.extend( + ["--warmup-requests", str(int(benchmark_cfg["warmup_requests"]))] + ) + if benchmark_cfg.get("extra_request_body") is not None: + command.extend( + [ + "--extra-request-body", + json.dumps(benchmark_cfg["extra_request_body"]), + ] + ) + return command + + +def run_bench_command( + command: List[str], timeout_sec: Optional[float] = None +) -> Dict[str, Any]: + try: + result = subprocess.run( + command, capture_output=True, text=True, timeout=timeout_sec + ) + except subprocess.TimeoutExpired as exc: + raise SearchDeadlineExceeded( + f"search budget expired while waiting for bench_serving: {exc.cmd}" + ) from exc + if result.returncode != 0: + message = (result.stderr or result.stdout).strip() + if len(message) > 4000: + head = message[:2000].rstrip() + tail = message[-2000:].lstrip() + message = f"{head}\n...\n{tail}" + raise RuntimeError(message) + + output_file = command[command.index("--output-file") + 1] + with open(output_file, "r", encoding="utf-8") as f: + lines = [line.strip() for line in f if line.strip()] + if not lines: + raise RuntimeError("bench_serving produced no JSONL output") + return json.loads(lines[-1]) + + +def run_trial( + stage_name: str, + candidate_id: int, + server_cfg: Dict[str, Any], + benchmark_cfg: Dict[str, Any], + dataset_summary: Dict[str, Any], + backend: str, + dataset_path: str, + tokenizer_path: str, + server_flags: Dict[str, Any], + output_dir: str, + request_rate: float, + max_concurrency: Optional[int], + search_deadline: Optional[float] = None, + search_budget_hours: float = DEFAULT_SEARCH_DURATION_HOURS, +) -> Dict[str, Any]: + process = None + log_path = os.path.join( + output_dir, + f"server_{stage_name}_cand{candidate_id}_mc{max_concurrency}_q{request_rate}.log", + ) + bench_path = os.path.join( + output_dir, + f"bench_{stage_name}_cand{candidate_id}_mc{max_concurrency}_q{request_rate}.jsonl", + ) + host = server_cfg.get("host", "127.0.0.1") + port = int(server_flags.get("port", server_cfg.get("port", 30000))) + base_url = benchmark_cfg.get("base_url", f"http://{host}:{port}") + record = { + "stage": stage_name, + "candidate_id": candidate_id, + "requested_qps": request_rate, + "max_concurrency": max_concurrency, + "server_flags": deepcopy(server_flags), + "sla_passed": False, + } + + try: + raise_if_search_deadline_reached(search_deadline, search_budget_hours) + if server_cfg.get("launch", True): + preclean_stale_server(port) + process = launch_server(server_cfg, server_flags, log_path) + metrics = run_bench_command( + build_bench_command( + benchmark_cfg=benchmark_cfg, + dataset_summary=dataset_summary, + backend=backend, + base_url=base_url, + dataset_path=dataset_path, + tokenizer_path=tokenizer_path, + request_rate=request_rate, + max_concurrency=max_concurrency, + output_file=bench_path, + ), + timeout_sec=remaining_search_seconds(search_deadline), + ) + record["sla_passed"] = meets_sla(metrics, benchmark_cfg) + record["metrics"] = metrics + except SearchDeadlineExceeded: + raise + except Exception as exc: # noqa: BLE001 + record["error"] = repr(exc) + diagnosis, hint = classify_failure( + "\n".join(part for part in [repr(exc), tail_text(log_path)] if part) + ) + if diagnosis: + record["diagnosis"] = diagnosis + if hint: + record["hint"] = hint + finally: + stop_server(process) + return record + + +def merge_host_port( + server_cfg: Dict[str, Any], flags: Dict[str, Any] +) -> Dict[str, Any]: + merged = canonicalize_flags(deepcopy(flags)) + if server_cfg.get("host") is not None and "host" not in merged: + merged["host"] = server_cfg["host"] + if server_cfg.get("port") is not None and "port" not in merged: + merged["port"] = server_cfg["port"] + return merged + + +def run_candidate( + stage_name: str, + candidate_id: int, + server_cfg: Dict[str, Any], + benchmark_cfg: Dict[str, Any], + dataset_summary: Dict[str, Any], + backend: str, + dataset_path: str, + tokenizer_path: str, + server_flags: Dict[str, Any], + output_dir: str, + incumbent_record: Optional[Dict[str, Any]] = None, + progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None, + record_callback: Optional[Callable[[Dict[str, Any]], None]] = None, + existing_records: Optional[Sequence[Dict[str, Any]]] = None, + search_deadline: Optional[float] = None, + search_budget_hours: float = DEFAULT_SEARCH_DURATION_HOURS, +) -> List[Dict[str, Any]]: + mode, values, tolerance, max_rounds = build_qps_plan(benchmark_cfg) + max_concurrency_values = as_list(benchmark_cfg.get("max_concurrency", [None])) + records: List[Dict[str, Any]] = [] + existing_by_key = { + record_trial_key(record): deepcopy(record) + for record in (existing_records or []) + } + + def one_trial( + request_rate: float, max_concurrency: Optional[int] + ) -> Tuple[Dict[str, Any], bool]: + key = trial_key( + stage_name=stage_name, + candidate_id=candidate_id, + request_rate=request_rate, + max_concurrency=max_concurrency, + server_flags=server_flags, + ) + if key in existing_by_key: + return deepcopy(existing_by_key[key]), True + return ( + run_trial( + stage_name=stage_name, + candidate_id=candidate_id, + server_cfg=server_cfg, + benchmark_cfg=benchmark_cfg, + dataset_summary=dataset_summary, + backend=backend, + dataset_path=dataset_path, + tokenizer_path=tokenizer_path, + server_flags=server_flags, + output_dir=output_dir, + request_rate=request_rate, + max_concurrency=max_concurrency, + search_deadline=search_deadline, + search_budget_hours=search_budget_hours, + ), + False, + ) + + for max_concurrency in max_concurrency_values: + raise_if_search_deadline_reached(search_deadline, search_budget_hours) + if mode == "fixed": + incumbent_qps = None + if ( + incumbent_record + and incumbent_record.get("metrics") + and incumbent_record.get("sla_passed") + ): + incumbent_qps = float(incumbent_record.get("requested_qps", 0.0)) + for qps in values: + if incumbent_qps is not None and qps < incumbent_qps: + continue + record, reused = one_trial(qps, max_concurrency) + records.append(record) + if record_callback is not None and not reused: + record_callback(record) + if progress_callback is not None: + progress_callback(record) + continue + + lower, upper = values + best: Optional[Dict[str, Any]] = None + incumbent_qps = None + if ( + incumbent_record + and incumbent_record.get("metrics") + and incumbent_record.get("sla_passed") + ): + incumbent_qps = float(incumbent_record.get("requested_qps", 0.0)) + if incumbent_qps is not None and lower < incumbent_qps <= upper: + probe_record, reused = one_trial(incumbent_qps, max_concurrency) + records.append(probe_record) + if record_callback is not None and not reused: + record_callback(probe_record) + if progress_callback is not None: + progress_callback(probe_record) + if probe_record.get("metrics") and probe_record["sla_passed"]: + lower = max(lower, incumbent_qps) + best = probe_record + else: + probe_record["heuristic_pruned"] = True + probe_record["heuristic_reason"] = ( + "Failed incumbent probe; skipped lower-QPS search because " + "it cannot beat the current best candidate." + ) + log_line( + f"[{stage_name}] heuristic prune candidate={candidate_id} " + f"mc={max_concurrency} incumbent_qps={incumbent_qps:.4f}" + ) + continue + rounds_run = 0 + while upper - lower > tolerance and rounds_run < max_rounds: + qps = pick_qps_midpoint(lower, upper) + if qps <= lower or qps >= upper: + break + record, reused = one_trial(qps, max_concurrency) + records.append(record) + if record_callback is not None and not reused: + record_callback(record) + if progress_callback is not None: + progress_callback(record) + if record.get("metrics") and record["sla_passed"]: + lower = qps + best = record + else: + upper = qps + rounds_run += 1 + if best is not None: + best["best_for_candidate"] = True + + return records + + +def write_jsonl(path: str, records: Iterable[Dict[str, Any]]) -> None: + if os.path.exists(path): + os.remove(path) + append_jsonl(path, records) + + +def write_csv(path: str, records: Sequence[Dict[str, Any]]) -> None: + if not records: + return + rows = [flatten(record) for record in records] + headers = sorted({header for row in rows for header in row}) + with open(path, "w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=headers) + writer.writeheader() + writer.writerows(rows) + + +def best_record(records: Sequence[Dict[str, Any]]) -> Optional[Dict[str, Any]]: + successful = [record for record in records if record.get("metrics")] + return max(successful, key=result_sort_key) if successful else None + + +def rendered_launch_command( + server_cfg: Dict[str, Any], server_flags: Dict[str, Any] +) -> str: + prefix = server_cfg.get("command_prefix") + if prefix is None: + command = ["python", "-m", "sglang.launch_server"] + elif isinstance(prefix, str): + command = shlex.split(prefix) + else: + command = [str(item) for item in prefix] + command.extend(cli_args(server_flags)) + command.extend(str(item) for item in server_cfg.get("extra_args", [])) + + env_parts = [] + for key, value in sorted(server_cfg.get("env", {}).items()): + if any(marker in key.upper() for marker in SENSITIVE_ENV_MARKERS): + continue + env_parts.append(f"{key}={shlex.quote(str(value))}") + parts: List[str] = env_parts + i = 0 + while i < len(command): + token = str(command[i]) + if token.startswith("--") and i + 1 < len(command): + nxt = str(command[i + 1]) + if not nxt.startswith("--"): + parts.append(f"{shlex.quote(token)} {shlex.quote(nxt)}") + i += 2 + continue + parts.append(shlex.quote(token)) + i += 1 + return " \\\n ".join(parts) + + +def write_markdown_summary( + path: str, + scenario: Dict[str, Any], + dataset_cfg: Dict[str, Any], + dataset_summary: Dict[str, Any], + records: Sequence[Dict[str, Any]], + best: Optional[Dict[str, Any]], + server_cfg: Dict[str, Any], + partial_reason: Optional[str] = None, +) -> None: + lines = [f"# Auto Benchmark Summary: {scenario['display_name']}", ""] + lines.append(f"- Dataset kind: `{dataset_cfg['kind']}`") + lines.append(f"- Requests: `{dataset_summary['num_requests']}`") + if partial_reason: + lines.append(f"- Status: `partial` ({partial_reason})") + if dataset_cfg["kind"] == "random": + lines.append( + f"- Random distribution: input `{dataset_cfg['random_input_len']}`, output `{dataset_cfg['random_output_len']}`" + ) + lines.append("") + + if best is not None: + lines.extend(["## Best Launch Command", "", "```bash"]) + lines.append(rendered_launch_command(server_cfg, best["server_flags"])) + lines.extend(["```", ""]) + + lines.extend( + [ + "## Results", + "", + "| Candidate | Stage | QPS | Max Conc | Prefill | Decode | TP | EP | PP | Output tok/s | TTFT ms | TPOT ms | SLA | Note |", + "|---|---:|---:|---:|---|---|---:|---:|---:|---:|---:|---:|---|---|", + ] + ) + for record in sorted(records, key=result_sort_key, reverse=True): + flags = record["server_flags"] + metrics = record.get("metrics", {}) + note = record.get("diagnosis") or record.get("hint") or record.get("error", "") + note = note.splitlines()[0][:120] if note else "" + lines.append( + "| {candidate_id} | {stage} | {qps} | {mc} | {prefill} | {decode} | {tp} | {ep} | {pp} | {throughput} | {ttft} | {tpot} | {sla} | {note} |".format( + candidate_id=record["candidate_id"], + stage=record["stage"], + qps=record["requested_qps"], + mc=record["max_concurrency"], + prefill=flags.get("prefill_attention_backend", ""), + decode=flags.get("decode_attention_backend", ""), + tp=flags.get("tp_size", 1), + ep=flags.get("ep_size", ""), + pp=flags.get("pp_size", 1), + throughput=( + round(metrics.get("output_throughput", 0.0), 2) if metrics else "" + ), + ttft=round(metrics.get("mean_ttft_ms", 0.0), 2) if metrics else "", + tpot=round(metrics.get("mean_tpot_ms", 0.0), 2) if metrics else "", + sla="pass" if record.get("sla_passed") else "fail", + note=note.replace("|", "/"), + ) + ) + + with open(path, "w", encoding="utf-8") as f: + f.write("\n".join(lines) + "\n") + + +def render_scenario_summary_markdown( + summary_rows: Sequence[Dict[str, Any]], + run_partial_reason: Optional[str] = None, +) -> str: + lines = ["# Scenario Summary", ""] + if run_partial_reason: + lines.extend([f"- Status: `partial` ({run_partial_reason})", ""]) + lines.extend( + [ + "| Scenario | Status | QPS | Output tok/s | TTFT ms | TPOT ms | Summary |", + "|---|---|---:|---:|---:|---:|---|", + ] + ) + + for row in summary_rows: + summary_path = os.path.join(row["scenario_dir"], "summary.md") + lines.append( + "| {name} | {status} | {qps} | {throughput} | {ttft} | {tpot} | `{path}` |".format( + name=row["scenario_name"], + status=row["status"], + qps=row.get("requested_qps") or "", + throughput=( + round(row.get("output_throughput", 0.0), 2) + if row.get("output_throughput") is not None + else "" + ), + ttft=( + round(row.get("mean_ttft_ms", 0.0), 2) + if row.get("mean_ttft_ms") is not None + else "" + ), + tpot=( + round(row.get("mean_tpot_ms", 0.0), 2) + if row.get("mean_tpot_ms") is not None + else "" + ), + path=summary_path, + ) + ) + + for row in summary_rows: + if row.get("launch_command"): + lines.extend( + [ + "", + f"## {row['scenario_name']}", + "", + "```bash", + row["launch_command"], + "```", + ] + ) + elif row["status"] == "no_successful_runs": + lines.extend( + [ + "", + f"## {row['scenario_name']}", + "", + "No successful run with metrics was produced for this scenario.", + ] + ) + + return "\n".join(lines) + "\n" + + +def run_stage( + scenario_name: str, + stage_name: str, + candidates: Sequence[Dict[str, Any]], + server_cfg: Dict[str, Any], + benchmark_cfg: Dict[str, Any], + dataset_summary: Dict[str, Any], + backend: str, + dataset_path: str, + tokenizer_path: str, + output_dir: str, + live_results_path: Optional[str] = None, + existing_records: Optional[Sequence[Dict[str, Any]]] = None, + search_deadline: Optional[float] = None, + search_budget_hours: float = DEFAULT_SEARCH_DURATION_HOURS, +) -> Tuple[List[Dict[str, Any]], Optional[Dict[str, Any]]]: + records: List[Dict[str, Any]] = [] + existing_stage_records = [ + deepcopy(record) + for record in (existing_records or []) + if record.get("stage") == stage_name + ] + current_best: Optional[Dict[str, Any]] = best_record(existing_stage_records) + stage_label = f"{scenario_name} {stage_name}" + candidate_pbar, candidate_started_at = make_progress_bar( + desc=f"{stage_label} candidates", + total=len(candidates), + position=1, + leave=True, + ) + trial_pbar, trial_started_at = make_progress_bar( + desc=f"{stage_label} trials", + total=len(candidates) * estimate_trials_per_candidate(benchmark_cfg), + position=2, + leave=False, + ) + try: + for candidate_id, candidate_flags in enumerate(candidates): + raise_if_search_deadline_reached(search_deadline, search_budget_hours) + merged = merge_host_port(server_cfg, candidate_flags) + log_line( + f"[{stage_name}] scenario={scenario_name} " + f"candidate {candidate_id + 1}/{len(candidates)}: " + f"{json.dumps(merged, ensure_ascii=False)}" + ) + + def on_trial(record: Dict[str, Any]) -> None: + nonlocal current_best + if record.get("metrics") and ( + current_best is None + or result_sort_key(record) > result_sort_key(current_best) + ): + current_best = record + advance_progress(trial_pbar, trial_started_at, best_record=current_best) + refresh_progress_eta( + candidate_pbar, candidate_started_at, best_record=current_best + ) + + def on_record(record: Dict[str, Any]) -> None: + if live_results_path is not None: + append_jsonl(live_results_path, [record]) + + candidate_records = run_candidate( + stage_name=stage_name, + candidate_id=candidate_id, + server_cfg=server_cfg, + benchmark_cfg=benchmark_cfg, + dataset_summary=dataset_summary, + backend=backend, + dataset_path=dataset_path, + tokenizer_path=tokenizer_path, + server_flags=merged, + output_dir=output_dir, + incumbent_record=current_best, + progress_callback=on_trial, + record_callback=on_record, + existing_records=existing_stage_records, + search_deadline=search_deadline, + search_budget_hours=search_budget_hours, + ) + records.extend(candidate_records) + + advance_progress( + candidate_pbar, + candidate_started_at, + best_record=current_best, + ) + finally: + if trial_pbar.total is not None and trial_pbar.n < trial_pbar.total: + trial_pbar.total = trial_pbar.n + refresh_progress_eta(trial_pbar, trial_started_at, current_best) + candidate_pbar.close() + trial_pbar.close() + + return records, current_best + + +def persist_scenario_outputs( + scenario_output_dir: str, + scenario: Dict[str, Any], + scenario_cfg: Dict[str, Any], + dataset_summary: Dict[str, Any], + records: Sequence[Dict[str, Any]], + server_cfg: Dict[str, Any], + partial_reason: Optional[str] = None, +) -> Optional[Dict[str, Any]]: + if not records: + return None + results_jsonl = os.path.join(scenario_output_dir, "results.jsonl") + results_csv = os.path.join(scenario_output_dir, "results.csv") + best = best_record(records) + write_jsonl(results_jsonl, records) + write_csv(results_csv, records) + write_markdown_summary( + path=os.path.join(scenario_output_dir, "summary.md"), + scenario=scenario, + dataset_cfg=scenario_cfg, + dataset_summary=dataset_summary, + records=records, + best=best, + server_cfg=server_cfg, + partial_reason=partial_reason, + ) + log_line(f"results_jsonl={results_jsonl}") + log_line(f"results_csv={results_csv}") + return best + + +def run_auto_benchmark(config_path: str) -> str: + config = load_yaml(config_path) + server_cfg = config["server"] + benchmark_cfg = config["benchmark"] + search_cfg = config.get("search", {}) + + timestamp = time.strftime("%Y%m%d-%H%M%S") + output_dir = benchmark_cfg.get("output_dir") or os.path.join( + os.getcwd(), "auto_benchmark_results", timestamp + ) + os.makedirs(output_dir, exist_ok=True) + + tokenizer_path = benchmark_cfg.get("tokenizer") or server_cfg.get( + "base_flags", {} + ).get("model_path") + model = benchmark_cfg.get("model") or server_cfg.get("base_flags", {}).get( + "model_path" + ) + if tokenizer_path is None: + raise ValueError( + "benchmark.tokenizer or server.base_flags.model_path is required." + ) + + dataset_cfg = normalize_dataset_cfg(config.get("dataset"), benchmark_cfg) + scenarios = expand_dataset_scenarios(dataset_cfg) + tier = int(search_cfg.get("tier", 2)) + max_candidates = resolve_max_candidates(search_cfg) + resume_enabled = bool(search_cfg.get("resume", True)) + base_candidates = build_server_candidates(server_cfg, tier, max_candidates) + search_budget_hours = resolve_search_budget_hours(search_cfg) + search_deadline = time.time() + (search_budget_hours * 3600) + scenario_records: List[Dict[str, Any]] = [] + interrupted = False + run_partial_reason: Optional[str] = None + print_run_plan( + config_path=config_path, + output_dir=output_dir, + tier=tier, + max_candidates=max_candidates, + benchmark_cfg=benchmark_cfg, + scenarios=scenarios, + server_cfg=server_cfg, + base_candidates=base_candidates, + speculative_enabled=bool(config.get("speculative", {}).get("enabled")), + search_budget_hours=search_budget_hours, + search_deadline=search_deadline, + ) + + scenario_pbar, scenario_started_at = make_progress_bar( + desc="scenarios", + total=len(scenarios), + position=0, + leave=True, + ) + previous_handlers = install_interrupt_handlers() + try: + for scenario in scenarios: + raise_if_search_deadline_reached(search_deadline, search_budget_hours) + scenario_output_dir = ( + output_dir + if len(scenarios) == 1 + else os.path.join(output_dir, scenario["name"]) + ) + os.makedirs(scenario_output_dir, exist_ok=True) + live_results_path = os.path.join(scenario_output_dir, "live_results.jsonl") + if os.path.exists(live_results_path) and not resume_enabled: + os.remove(live_results_path) + prepared_dataset_path = os.path.join( + scenario_output_dir, "prepared_dataset.jsonl" + ) + existing_records = read_jsonl(live_results_path) + if resume_enabled and os.path.exists(prepared_dataset_path): + rows = load_autobench_rows( + dataset_path=prepared_dataset_path, + tokenizer_path=tokenizer_path, + num_prompts=0, + ) + dataset_summary = summarize_rows(rows) + else: + prepared_dataset_path, rows, dataset_summary = prepare_dataset( + dataset_cfg=scenario["cfg"], + tokenizer_path=tokenizer_path, + model=model, + output_path=prepared_dataset_path, + ) + + backend = infer_backend(benchmark_cfg.get("backend", "auto"), rows) + log_line(f"scenario={scenario['display_name']}") + log_line(f"prepared_dataset={prepared_dataset_path}") + log_line( + f"dataset_summary={json.dumps(dataset_summary, ensure_ascii=False)}" + ) + log_line(f"selected_backend={backend}") + if resume_enabled and existing_records: + log_line( + f"resume=true loaded_records={len(existing_records)} " + f"scenario={scenario['display_name']}" + ) + + all_records: List[Dict[str, Any]] = [] + scenario_partial_reason: Optional[str] = None + try: + all_records, best_base = run_stage( + scenario_name=scenario["display_name"], + stage_name="base", + candidates=base_candidates, + server_cfg=server_cfg, + benchmark_cfg=benchmark_cfg, + dataset_summary=dataset_summary, + backend=backend, + dataset_path=prepared_dataset_path, + tokenizer_path=tokenizer_path, + output_dir=scenario_output_dir, + live_results_path=live_results_path, + existing_records=existing_records, + search_deadline=search_deadline, + search_budget_hours=search_budget_hours, + ) + + speculative_cfg = config.get("speculative", {}) + if speculative_cfg.get("enabled"): + if best_base is None: + raise ValueError( + "Speculative search requires at least one successful base run." + ) + if not speculative_cfg.get("draft_model_path"): + raise ValueError("speculative.draft_model_path is required.") + + spec_base_flags = deepcopy(best_base["server_flags"]) + spec_base_flags.update( + deepcopy(speculative_cfg.get("base_flags", {})) + ) + spec_base_flags["speculative_algorithm"] = speculative_cfg.get( + "algorithm", "EAGLE" + ) + spec_base_flags["speculative_draft_model_path"] = speculative_cfg[ + "draft_model_path" + ] + spec_candidates = build_candidates( + base_flags=canonicalize_flags(spec_base_flags), + search_space=deepcopy(speculative_cfg.get("search_space", {})), + tier=tier, + max_candidates=max_candidates, + ) + log_line( + f"Planned speculative candidates for scenario={scenario['display_name']}:" + ) + for index, candidate in enumerate(spec_candidates, start=1): + log_line( + f" [{index}/{len(spec_candidates)}] " + f"{json.dumps(merge_host_port(server_cfg, candidate), ensure_ascii=False)}" + ) + spec_records, _ = run_stage( + scenario_name=scenario["display_name"], + stage_name="speculative", + candidates=spec_candidates, + server_cfg=server_cfg, + benchmark_cfg=benchmark_cfg, + dataset_summary=dataset_summary, + backend=backend, + dataset_path=prepared_dataset_path, + tokenizer_path=tokenizer_path, + output_dir=scenario_output_dir, + live_results_path=live_results_path, + existing_records=read_jsonl(live_results_path), + search_deadline=search_deadline, + search_budget_hours=search_budget_hours, + ) + all_records.extend(spec_records) + except SearchDeadlineExceeded as exc: + interrupted = True + scenario_partial_reason = str(exc) + run_partial_reason = scenario_partial_reason + log_line( + f"search_deadline_reached=true scenario={scenario['display_name']} " + f"detail={scenario_partial_reason}" + ) + except KeyboardInterrupt: + interrupted = True + scenario_partial_reason = "interrupted before the full search completed" + run_partial_reason = scenario_partial_reason + log_line( + f"interrupt_received=true scenario={scenario['display_name']} " + "saving partial results before exit" + ) + finally: + persisted_records = all_records + live_records = read_jsonl(live_results_path) + if len(live_records) > len(persisted_records): + persisted_records = live_records + best = persist_scenario_outputs( + scenario_output_dir=scenario_output_dir, + scenario=scenario, + scenario_cfg=scenario["cfg"], + dataset_summary=dataset_summary, + records=persisted_records, + server_cfg=server_cfg, + partial_reason=scenario_partial_reason, + ) + if persisted_records: + scenario_records.append( + { + "scenario_name": scenario["display_name"], + "scenario_dir": scenario_output_dir, + "best_record": best, + "has_records": True, + } + ) + if interrupted: + break + advance_progress(scenario_pbar, scenario_started_at) + except SearchDeadlineExceeded as exc: + interrupted = True + run_partial_reason = str(exc) + log_line(f"search_deadline_reached=true detail={run_partial_reason}") + finally: + scenario_pbar.close() + restore_interrupt_handlers(previous_handlers) + + if scenario_records and len(scenarios) > 1: + summary_rows = [] + for item in scenario_records: + record = item["best_record"] + metrics = record.get("metrics", {}) if record else {} + summary_rows.append( + { + "scenario_name": item["scenario_name"], + "scenario_dir": item["scenario_dir"], + "status": ( + "ok" + if record and record.get("metrics") + else "no_successful_runs" + ), + "requested_qps": record.get("requested_qps") if record else None, + "mean_ttft_ms": metrics.get("mean_ttft_ms"), + "mean_tpot_ms": metrics.get("mean_tpot_ms"), + "output_throughput": metrics.get("output_throughput"), + "launch_command": ( + rendered_launch_command(server_cfg, record["server_flags"]) + if record + else "" + ), + } + ) + write_jsonl(os.path.join(output_dir, "scenario_summary.jsonl"), summary_rows) + write_csv(os.path.join(output_dir, "scenario_summary.csv"), summary_rows) + with open(os.path.join(output_dir, "SUMMARY.md"), "w", encoding="utf-8") as f: + f.write(render_scenario_summary_markdown(summary_rows, run_partial_reason)) + if interrupted: + log_line(f"interrupted=true partial_output_dir={output_dir}") + return output_dir + + +def convert_dataset(args: argparse.Namespace) -> None: + dataset_cfg = normalize_dataset_cfg( + { + key: value + for key, value in vars(args).items() + if key not in {"command", "output", "tokenizer", "model"} + }, + {}, + ) + output_path, rows, summary = prepare_dataset( + dataset_cfg=dataset_cfg, + tokenizer_path=args.tokenizer, + model=args.model, + output_path=args.output, + ) + print(f"prepared_dataset={output_path}") + print(f"rows={len(rows)}") + print(json.dumps(summary, ensure_ascii=False, indent=2)) + + +def validate_dataset(args: argparse.Namespace) -> None: + rows = load_autobench_rows(args.dataset_path, args.tokenizer, num_prompts=0) + print(json.dumps(summarize_rows(rows), ensure_ascii=False, indent=2)) diff --git a/python/sglang/bench_serving.py b/python/sglang/bench_serving.py index 161b4c3be..7c46d381f 100644 --- a/python/sglang/bench_serving.py +++ b/python/sglang/bench_serving.py @@ -1931,6 +1931,7 @@ if __name__ == "__main__": type=str, default="sharegpt", choices=[ + "autobench", "sharegpt", "custom", "openai", diff --git a/python/sglang/benchmark/datasets/__init__.py b/python/sglang/benchmark/datasets/__init__.py index 615e3a241..2320bb7df 100644 --- a/python/sglang/benchmark/datasets/__init__.py +++ b/python/sglang/benchmark/datasets/__init__.py @@ -1,5 +1,6 @@ from typing import Dict, Type +from sglang.benchmark.datasets.autobench import AutoBenchmarkDataset from sglang.benchmark.datasets.common import BaseDataset, DatasetRow from sglang.benchmark.datasets.custom import CustomDataset from sglang.benchmark.datasets.generated_shared_prefix import ( @@ -14,6 +15,7 @@ from sglang.benchmark.datasets.random import RandomDataset from sglang.benchmark.datasets.sharegpt import ShareGPTDataset DATASET_MAPPING: Dict[str, Type[BaseDataset]] = { + "autobench": AutoBenchmarkDataset, "sharegpt": ShareGPTDataset, "custom": CustomDataset, "openai": OpenAIDataset, diff --git a/python/sglang/benchmark/datasets/autobench.py b/python/sglang/benchmark/datasets/autobench.py new file mode 100644 index 000000000..eb754abca --- /dev/null +++ b/python/sglang/benchmark/datasets/autobench.py @@ -0,0 +1,285 @@ +import json +from argparse import Namespace +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Tuple + +import numpy as np +from transformers import PreTrainedTokenizerBase + +from sglang.benchmark.datasets.common import BaseDataset, DatasetRow + +AUTOBENCH_RESERVED_FIELDS = { + "prompt", + "messages", + "prompt_origin", + "output_len", + "max_tokens", + "max_completion_tokens", + "completion_tokens", + "prompt_len", + "text_prompt_len", + "vision_prompt_len", + "image_data", + "timestamp", + "routing_key", + "metadata", + "extra_request_body", + "param_send", +} + + +def _load_json_if_needed(value: Any) -> Any: + if not isinstance(value, str): + return value + value = value.strip() + if not value: + return value + if value[0] not in "[{": + return value + try: + return json.loads(value) + except json.JSONDecodeError: + return value + + +def _normalize_messages(messages: Any) -> Optional[List[Dict[str, Any]]]: + messages = _load_json_if_needed(messages) + if not isinstance(messages, list) or not messages: + return None + if not all(isinstance(message, dict) for message in messages): + return None + + normalized = [] + for message in messages: + if "role" not in message: + return None + content = message.get("content") + if content is None: + return None + normalized.append({"role": message["role"], "content": content}) + return normalized + + +def _normalize_legacy_system_content( + system_prompt: Any, content_list: Any +) -> Optional[List[Dict[str, Any]]]: + if not isinstance(content_list, list) or not content_list: + return None + + messages: List[Dict[str, Any]] = [] + if system_prompt: + messages.append({"role": "system", "content": str(system_prompt)}) + + turns = [str(item) for item in content_list] + # In the old auto_benchmark helpers, an even number of items usually means the + # last assistant reply is present and should be removed before benchmarking. + if len(turns) % 2 == 0: + turns = turns[:-1] + if not turns: + return None + + for index, turn in enumerate(turns): + role = "user" if index % 2 == 0 else "assistant" + messages.append({"role": role, "content": turn}) + return messages + + +def _normalize_prompt(row: Dict[str, Any]) -> Tuple[Any, str]: + prompt = row.get("prompt") + messages = row.get("messages") + prompt_origin = row.get("prompt_origin") + + if messages is not None: + normalized = _normalize_messages(messages) + if normalized is not None: + return normalized, "messages" + + if prompt is not None: + prompt = _load_json_if_needed(prompt) + if isinstance(prompt, list) and prompt and isinstance(prompt[0], dict): + normalized = _normalize_messages(prompt) + if normalized is not None: + return normalized, "messages" + if ( + isinstance(prompt, list) + and prompt + and all(isinstance(item, str) for item in prompt) + ): + return prompt, "multi_turn" + if ( + isinstance(prompt, list) + and prompt + and all(isinstance(item, int) for item in prompt) + ): + return prompt, "token_ids" + if isinstance(prompt, str) and prompt: + return prompt, "prompt" + + if prompt_origin is not None: + normalized = _normalize_messages(prompt_origin) + if normalized is not None: + return normalized, "messages" + + if "system" in row and "content" in row: + normalized = _normalize_legacy_system_content( + row.get("system"), row.get("content") + ) + if normalized is not None: + return normalized, "messages" + + raise ValueError("Unsupported auto benchmark row: missing prompt/messages") + + +def _estimate_prompt_lens( + prompt: Any, + prompt_kind: str, + tokenizer: PreTrainedTokenizerBase, + row: Dict[str, Any], +) -> Tuple[int, int, int]: + if row.get("prompt_len") is not None: + prompt_len = int(row["prompt_len"]) + text_prompt_len = int(row.get("text_prompt_len", prompt_len)) + vision_prompt_len = int(row.get("vision_prompt_len", 0)) + return prompt_len, text_prompt_len, vision_prompt_len + + if prompt_kind == "messages": + text_prompt_len = len( + tokenizer.apply_chat_template( + prompt, tokenize=True, add_generation_prompt=True + ) + ) + vision_prompt_len = 0 + return text_prompt_len, text_prompt_len, vision_prompt_len + + if prompt_kind == "prompt": + prompt_len = len(tokenizer.encode(prompt, add_special_tokens=False)) + return prompt_len, prompt_len, 0 + + if prompt_kind == "token_ids": + prompt_len = len(prompt) + return prompt_len, prompt_len, 0 + + # Multi-turn prompt lists are handled specially by bench_serving and do not + # contribute reliable static prompt lengths. + return 0, 0, 0 + + +def _collect_extra_request_body(row: Dict[str, Any]) -> Dict[str, Any]: + extra: Dict[str, Any] = {} + + param_send = row.get("param_send") + if param_send is not None: + parsed = _load_json_if_needed(param_send) + if isinstance(parsed, dict): + extra.update(parsed) + + for key, value in row.items(): + if key not in AUTOBENCH_RESERVED_FIELDS: + extra[key] = value + + explicit_extra = row.get("extra_request_body") + explicit_extra = _load_json_if_needed(explicit_extra) + if isinstance(explicit_extra, dict): + extra.update(explicit_extra) + + return extra + + +def serialize_dataset_row_to_autobench( + row: DatasetRow, metadata: Optional[Dict[str, Any]] = None +) -> Dict[str, Any]: + record: Dict[str, Any] = { + "prompt": row.prompt, + "output_len": row.output_len, + } + if row.prompt_len: + record["prompt_len"] = row.prompt_len + if row.text_prompt_len not in (None, row.prompt_len): + record["text_prompt_len"] = row.text_prompt_len + if row.vision_prompt_len: + record["vision_prompt_len"] = row.vision_prompt_len + if row.image_data: + record["image_data"] = row.image_data + if row.timestamp is not None: + record["timestamp"] = row.timestamp + if row.routing_key is not None: + record["routing_key"] = row.routing_key + if row.extra_request_body: + record["extra_request_body"] = row.extra_request_body + if metadata: + record["metadata"] = metadata + return record + + +@dataclass +class AutoBenchmarkDataset(BaseDataset): + dataset_path: str + num_requests: int + fixed_output_len: Optional[int] + + @classmethod + def from_args(cls, args: Namespace) -> "AutoBenchmarkDataset": + return cls( + dataset_path=args.dataset_path, + num_requests=args.num_prompts, + fixed_output_len=args.sharegpt_output_len, + ) + + def load( + self, tokenizer: PreTrainedTokenizerBase, model_id=None + ) -> List[DatasetRow]: + return sample_autobench_requests( + dataset_path=self.dataset_path, + num_requests=self.num_requests, + tokenizer=tokenizer, + fixed_output_len=self.fixed_output_len, + ) + + +def sample_autobench_requests( + dataset_path: str, + num_requests: int, + tokenizer: PreTrainedTokenizerBase, + fixed_output_len: Optional[int] = None, +) -> List[DatasetRow]: + dataset: List[DatasetRow] = [] + + with open(dataset_path, "r", encoding="utf-8") as f: + for line in f: + if num_requests > 0 and len(dataset) >= num_requests: + break + + line = line.strip() + if not line: + continue + + row = json.loads(line) + prompt, prompt_kind = _normalize_prompt(row) + prompt_len, text_prompt_len, vision_prompt_len = _estimate_prompt_lens( + prompt, prompt_kind, tokenizer, row + ) + + output_len = fixed_output_len or row.get("output_len") + output_len = output_len or row.get("max_tokens") + output_len = output_len or row.get("max_completion_tokens") + output_len = output_len or row.get("completion_tokens") + output_len = int(output_len or 256) + + dataset.append( + DatasetRow( + prompt=prompt, + prompt_len=prompt_len, + output_len=output_len, + text_prompt_len=text_prompt_len, + vision_prompt_len=vision_prompt_len, + image_data=row.get("image_data"), + timestamp=row.get("timestamp"), + routing_key=row.get("routing_key"), + extra_request_body=_collect_extra_request_body(row), + ) + ) + + print(f"Loaded {len(dataset)} auto benchmark requests") + print(f"#Input tokens: {np.sum([x.prompt_len for x in dataset])}") + print(f"#Output tokens: {np.sum([x.output_len for x in dataset])}") + return dataset diff --git a/test/registered/unit/test_auto_benchmark_tools.py b/test/registered/unit/test_auto_benchmark_tools.py new file mode 100644 index 000000000..99507da6d --- /dev/null +++ b/test/registered/unit/test_auto_benchmark_tools.py @@ -0,0 +1,615 @@ +import json +import sys +import tempfile +import time +import types +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +from tokenizers import Tokenizer +from tokenizers.models import WordLevel +from tokenizers.pre_tokenizers import Whitespace +from transformers import PreTrainedTokenizerFast + +sys.modules.setdefault("zmq", types.SimpleNamespace()) + +from sglang.auto_benchmark_lib import ( + SearchDeadlineExceeded, + append_jsonl, + build_candidates, + build_qps_plan, + build_server_candidates, + classify_failure, + collect_stale_server_pids, + describe_search_tier, + estimate_trials_per_candidate, + expand_dataset_scenarios, + format_best_progress, + infer_backend, + prepare_dataset, + render_scenario_summary_markdown, + rendered_launch_command, + resolve_max_candidates, + run_candidate, +) +from sglang.benchmark.datasets.autobench import sample_autobench_requests +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=5, suite="stage-b-test-1-gpu-small") + + +def create_lightweight_tokenizer() -> PreTrainedTokenizerFast: + vocab = {"[UNK]": 0, "[PAD]": 1, "[BOS]": 2, "[EOS]": 3} + vocab.update({f"tok_{i}": i + 4 for i in range(4096)}) + + tokenizer = Tokenizer(WordLevel(vocab=vocab, unk_token="[UNK]")) + tokenizer.pre_tokenizer = Whitespace() + + hf_tokenizer = PreTrainedTokenizerFast( + tokenizer_object=tokenizer, + unk_token="[UNK]", + pad_token="[PAD]", + bos_token="[BOS]", + eos_token="[EOS]", + ) + hf_tokenizer.chat_template = ( + "{% for message in messages %}" + "{{ message['role'] }}: {{ message['content'] }}\n" + "{% endfor %}" + "{% if add_generation_prompt %}assistant:{% endif %}" + ) + return hf_tokenizer + + +class TestAutoBenchmarkTools(CustomTestCase): + def setUp(self): + self.tmpdir = tempfile.TemporaryDirectory() + self.tmpdir_path = Path(self.tmpdir.name) + self.tokenizer = create_lightweight_tokenizer() + self.tokenizer_dir = self.tmpdir_path / "tok" + self.tokenizer.save_pretrained(self.tokenizer_dir) + + def tearDown(self): + self.tmpdir.cleanup() + + def _write_autobench_jsonl(self) -> str: + rows = [ + {"prompt": "tok_1 tok_2 tok_3", "output_len": 32}, + { + "messages": [{"role": "user", "content": "tok_4 tok_5"}], + "output_len": 24, + "extra_request_body": {"temperature": 0.0}, + }, + { + "system": "tok_6", + "content": ["tok_7 tok_8", "tok_9", "tok_10 tok_11"], + "output_len": 16, + }, + ] + path = self.tmpdir_path / "sample.autobench.jsonl" + with open(path, "w", encoding="utf-8") as f: + for row in rows: + f.write(json.dumps(row) + "\n") + return str(path) + + def _write_sharegpt_json(self) -> str: + rows = [ + { + "conversations": [ + {"value": "tok_1 tok_2 tok_3"}, + {"value": "tok_4 tok_5"}, + ] + }, + { + "conversations": [ + {"value": "tok_6 tok_7"}, + {"value": "tok_8 tok_9 tok_10"}, + ] + }, + ] + path = self.tmpdir_path / "sharegpt.json" + with open(path, "w", encoding="utf-8") as f: + json.dump(rows, f) + return str(path) + + def test_prepare_custom_autobench_dataset(self): + dataset_path = self._write_autobench_jsonl() + output_path = self.tmpdir_path / "prepared.autobench.jsonl" + + prepared_path, rows, summary = prepare_dataset( + dataset_cfg={ + "kind": "custom", + "path": dataset_path, + "num_prompts": 2, + }, + tokenizer_path=str(self.tokenizer_dir), + model=None, + output_path=str(output_path), + ) + + self.assertEqual(prepared_path, str(output_path)) + self.assertEqual(summary["num_requests"], 2) + self.assertTrue(Path(prepared_path).exists()) + converted_rows = sample_autobench_requests( + dataset_path=prepared_path, + num_requests=0, + tokenizer=self.tokenizer, + ) + self.assertEqual(len(rows), 2) + self.assertEqual(len(converted_rows), 2) + + def test_invalid_json_like_prompt_falls_back_to_plain_text(self): + path = self.tmpdir_path / "jsonlike.autobench.jsonl" + path.write_text( + json.dumps({"prompt": "[not actually json", "output_len": 8}) + "\n", + encoding="utf-8", + ) + + rows = sample_autobench_requests( + dataset_path=str(path), + num_requests=0, + tokenizer=self.tokenizer, + ) + + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0].prompt, "[not actually json") + + def test_prepare_sharegpt_dataset(self): + sharegpt_path = self._write_sharegpt_json() + output_path = self.tmpdir_path / "sharegpt.autobench.jsonl" + + prepared_path, rows, summary = prepare_dataset( + dataset_cfg={ + "kind": "sharegpt", + "path": sharegpt_path, + "num_prompts": 2, + }, + tokenizer_path=str(self.tokenizer_dir), + model=None, + output_path=str(output_path), + ) + + self.assertEqual(prepared_path, str(output_path)) + self.assertEqual(summary["num_requests"], 2) + self.assertEqual(len(rows), 2) + + def test_prepare_custom_dataset_requires_path(self): + with self.assertRaisesRegex(ValueError, "dataset.path is required"): + prepare_dataset( + dataset_cfg={"kind": "custom"}, + tokenizer_path=str(self.tokenizer_dir), + model=None, + output_path=str(self.tmpdir_path / "missing.autobench.jsonl"), + ) + + def test_infer_backend(self): + prompt_rows = [SimpleNamespace(prompt="tok_1 tok_2")] + chat_rows = [SimpleNamespace(prompt=[{"role": "user", "content": "tok_1"}])] + token_id_rows = [SimpleNamespace(prompt=[1, 2, 3])] + + self.assertEqual(infer_backend("auto", prompt_rows), "sglang-oai") + self.assertEqual(infer_backend("auto", chat_rows), "sglang-oai-chat") + self.assertEqual(infer_backend("auto", token_id_rows), "sglang") + + def test_build_candidates_by_tier(self): + base_flags = {"model_path": "/model", "tp_size": 4} + search_space = { + "prefill_attention_backend": ["fa3", "flashinfer", "triton"], + "decode_attention_backend": ["fa3", "flashinfer"], + "chunked_prefill_size": [4096, 8192], + "max_running_requests": [64, 128], + "schedule_policy": ["lpm", "fcfs"], + } + + tier1 = build_candidates(base_flags, search_space, tier=1, max_candidates=None) + tier2 = build_candidates(base_flags, search_space, tier=2, max_candidates=None) + tier3 = build_candidates(base_flags, search_space, tier=3, max_candidates=32) + + self.assertGreater(len(tier1), 1) + self.assertGreater(len(tier2), len(tier1)) + self.assertGreater(len(tier3), len(tier2)) + self.assertEqual(tier1[0]["model_path"], "/model") + + def test_parallel_search_derives_dp_size(self): + server_cfg = { + "env": {"CUDA_VISIBLE_DEVICES": "0,1,2,3,4,5,6,7"}, + "base_flags": {"model_path": "/model"}, + "parallel": { + "tp": [4, 2], + "pp_size": [1], + }, + "search_space": {}, + } + + candidates = build_server_candidates(server_cfg, tier=2, max_candidates=None) + tp_dp_pairs = { + (candidate["tp_size"], candidate["dp_size"]) for candidate in candidates + } + self.assertIn((4, 2), tp_dp_pairs) + self.assertIn((2, 4), tp_dp_pairs) + + def test_build_server_candidates_filters_unsupported_fa3_on_sm100(self): + server_cfg = { + "base_flags": {"model_path": "/model", "tp_size": 1}, + "search_space": { + "prefill_attention_backend": ["fa3", "flashinfer"], + "decode_attention_backend": ["fa3", "flashinfer"], + "chunked_prefill_size": [4096, 8192], + }, + } + + with mock.patch( + "sglang.auto_benchmark_lib.detect_current_cuda_capability", + return_value=(10, 0), + ): + candidates = build_server_candidates( + server_cfg, tier=2, max_candidates=None + ) + + self.assertGreater(len(candidates), 0) + for candidate in candidates: + self.assertNotEqual(candidate.get("attention_backend"), "fa3") + self.assertNotEqual(candidate.get("prefill_attention_backend"), "fa3") + self.assertNotEqual(candidate.get("decode_attention_backend"), "fa3") + + def test_build_server_candidates_keeps_fa3_on_sm90(self): + server_cfg = { + "base_flags": {"model_path": "/model", "tp_size": 1}, + "search_space": { + "prefill_attention_backend": ["fa3", "flashinfer"], + "decode_attention_backend": ["fa3", "flashinfer"], + }, + } + + with mock.patch( + "sglang.auto_benchmark_lib.detect_current_cuda_capability", + return_value=(9, 0), + ): + candidates = build_server_candidates( + server_cfg, tier=2, max_candidates=None + ) + + self.assertTrue( + any( + candidate.get("prefill_attention_backend") == "fa3" + or candidate.get("decode_attention_backend") == "fa3" + for candidate in candidates + ) + ) + + def test_ep_alias_and_oom_classification(self): + server_cfg = { + "base_flags": {"model_path": "/model", "tp_size": 8}, + "search_space": {"ep": [1, 4]}, + } + + candidates = build_server_candidates(server_cfg, tier=2, max_candidates=None) + ep_sizes = {candidate.get("ep_size", 1) for candidate in candidates} + self.assertEqual(ep_sizes, {1, 4}) + + diagnosis, hint = classify_failure("RuntimeError: CUDA out of memory") + self.assertEqual(diagnosis, "oom") + self.assertIn("Increase GPU count", hint) + + def test_expand_random_dataset_scenarios(self): + scenarios = expand_dataset_scenarios( + { + "kind": "random", + "scenario_names": ["chat", "summarization"], + "input_len": [1000, 8000], + "output_len": [1000, 1000], + } + ) + + self.assertEqual(len(scenarios), 2) + self.assertEqual(scenarios[0]["name"], "chat") + self.assertEqual(scenarios[0]["cfg"]["random_input_len"], 1000) + self.assertEqual(scenarios[1]["cfg"]["random_input_len"], 8000) + self.assertEqual(scenarios[1]["cfg"]["random_output_len"], 1000) + + def test_estimate_trials_and_tier_descriptions(self): + benchmark_cfg = { + "qps": {"lower": 0.25, "upper": 4.0, "tolerance": 0.1}, + "max_concurrency": [None, 8, 16], + } + + self.assertEqual(estimate_trials_per_candidate(benchmark_cfg), 15) + self.assertIn("default", describe_search_tier(2)) + self.assertIn("slowest", describe_search_tier(3)) + + def test_resolve_max_candidates_defaults_to_eight(self): + self.assertEqual(resolve_max_candidates({}), 8) + self.assertIsNone(resolve_max_candidates({"max_candidates": None})) + + def test_resolve_max_candidates_rejects_non_positive_values(self): + with self.assertRaisesRegex(ValueError, "search.max_candidates"): + resolve_max_candidates({"max_candidates": 0}) + + def test_build_qps_plan_accepts_numeric_request_rate(self): + mode, values, tolerance, max_rounds = build_qps_plan({"request_rate": 3.5}) + self.assertEqual(mode, "fixed") + self.assertEqual(values, [3.5]) + self.assertEqual(tolerance, 0.0) + self.assertEqual(max_rounds, 0) + + def test_build_qps_plan_clamps_binary_rounds(self): + mode, values, tolerance, max_rounds = build_qps_plan( + {"qps": {"lower": 1.0, "upper": 16.0, "tolerance": 0.1, "max_rounds": 99}} + ) + + self.assertEqual(mode, "search") + self.assertEqual(values, [1.0, 16.0]) + self.assertEqual(tolerance, 0.1) + self.assertEqual(max_rounds, 5) + + def test_format_best_progress(self): + text = format_best_progress( + { + "candidate_id": 3, + "requested_qps": 3.5, + "server_flags": { + "tp_size": 4, + "ep_size": 4, + "mem_fraction_static": 0.84, + "max_running_requests": 96, + }, + "metrics": { + "output_throughput": 1234.56, + "mean_ttft_ms": 250.12, + "mean_tpot_ms": 14.78, + }, + } + ) + + self.assertIn("qps=3.5000", text) + self.assertIn("tok/s=1234.6", text) + self.assertIn("ttft=250.1ms", text) + self.assertIn("tpot=14.8ms", text) + self.assertIn("tp=4", text) + self.assertIn("ep=4", text) + + def test_append_jsonl(self): + path = self.tmpdir_path / "live_results.jsonl" + append_jsonl( + str(path), + [ + {"candidate_id": 1, "requested_qps": 2.0}, + {"candidate_id": 2, "requested_qps": 3.0}, + ], + ) + + lines = path.read_text(encoding="utf-8").strip().splitlines() + self.assertEqual(len(lines), 2) + self.assertEqual(json.loads(lines[0])["candidate_id"], 1) + self.assertEqual(json.loads(lines[1])["requested_qps"], 3.0) + + def test_collect_stale_server_pids_dedups(self): + def fake_run(command, capture_output, text, check): + stdout = "123\n" if command[0] == "lsof" else "123\n456\n" + return SimpleNamespace(returncode=0, stdout=stdout) + + with mock.patch( + "sglang.auto_benchmark_lib.subprocess.run", side_effect=fake_run + ): + self.assertEqual(collect_stale_server_pids(30000), [123, 456]) + + def test_rendered_launch_command_includes_env(self): + text = rendered_launch_command( + { + "env": { + "CUDA_VISIBLE_DEVICES": "0", + "HF_TOKEN": "secret-value", + }, + "extra_args": [], + }, + {"model_path": "Qwen/Qwen3-32B", "tp_size": 1, "port": 30000}, + ) + + self.assertIn("CUDA_VISIBLE_DEVICES=0", text) + self.assertIn("--model-path Qwen/Qwen3-32B", text) + self.assertNotIn("HF_TOKEN", text) + + def test_render_scenario_summary_markdown_keeps_rows_in_single_table(self): + text = render_scenario_summary_markdown( + [ + { + "scenario_name": "chat", + "scenario_dir": "/tmp/chat", + "status": "ok", + "requested_qps": 11.914, + "output_throughput": 1867.28, + "mean_ttft_ms": 99.58, + "mean_tpot_ms": 21.09, + "launch_command": "python -m sglang.launch_server --port 30000", + }, + { + "scenario_name": "summarization", + "scenario_dir": "/tmp/summarization", + "status": "ok", + "requested_qps": 11.914, + "output_throughput": 537.17, + "mean_ttft_ms": 709.99, + "mean_tpot_ms": 26.89, + "launch_command": "python -m sglang.launch_server --port 30001", + }, + ] + ) + + header = ( + "| Scenario | Status | QPS | Output tok/s | TTFT ms | TPOT ms | Summary |" + ) + self.assertEqual(text.count(header), 1) + self.assertLess(text.index("| chat |"), text.index("## chat")) + self.assertLess(text.index("| summarization |"), text.index("## chat")) + self.assertLess(text.index("| summarization |"), text.index("## summarization")) + + def test_run_candidate_binary_search_avoids_rounding_loop(self): + benchmark_cfg = { + "qps": {"lower": 1.0, "upper": 1.00000001, "tolerance": 1e-12}, + "max_concurrency": [None], + } + calls = [] + + def fake_run_trial(**kwargs): + calls.append(kwargs["request_rate"]) + return { + "stage": "base", + "candidate_id": kwargs["candidate_id"], + "requested_qps": kwargs["request_rate"], + "max_concurrency": kwargs["max_concurrency"], + "server_flags": kwargs["server_flags"], + "sla_passed": True, + "metrics": { + "output_throughput": 1.0, + "mean_ttft_ms": 1.0, + "mean_tpot_ms": 1.0, + }, + } + + with mock.patch( + "sglang.auto_benchmark_lib.run_trial", side_effect=fake_run_trial + ): + records = run_candidate( + stage_name="base", + candidate_id=0, + server_cfg={"host": "127.0.0.1", "port": 30000}, + benchmark_cfg=benchmark_cfg, + dataset_summary={"num_requests": 1}, + backend="sglang-oai", + dataset_path="/tmp/fake.jsonl", + tokenizer_path=str(self.tokenizer_dir), + server_flags={"model_path": "/model"}, + output_dir=str(self.tmpdir_path), + ) + + self.assertLess(len(calls), 40) + self.assertEqual(len(records), len(calls)) + + def test_run_candidate_binary_search_respects_max_rounds(self): + benchmark_cfg = { + "qps": {"lower": 1.0, "upper": 32.0, "tolerance": 1e-12, "max_rounds": 2}, + "max_concurrency": [None], + } + calls = [] + + def fake_run_trial(**kwargs): + calls.append(kwargs["request_rate"]) + return { + "stage": "base", + "candidate_id": kwargs["candidate_id"], + "requested_qps": kwargs["request_rate"], + "max_concurrency": kwargs["max_concurrency"], + "server_flags": kwargs["server_flags"], + "sla_passed": True, + "metrics": { + "output_throughput": 1.0, + "mean_ttft_ms": 1.0, + "mean_tpot_ms": 1.0, + }, + } + + with mock.patch( + "sglang.auto_benchmark_lib.run_trial", side_effect=fake_run_trial + ): + records = run_candidate( + stage_name="base", + candidate_id=0, + server_cfg={"host": "127.0.0.1", "port": 30000}, + benchmark_cfg=benchmark_cfg, + dataset_summary={"num_requests": 1}, + backend="sglang-oai", + dataset_path="/tmp/fake.jsonl", + tokenizer_path=str(self.tokenizer_dir), + server_flags={"model_path": "/model"}, + output_dir=str(self.tmpdir_path), + ) + + self.assertEqual(len(calls), 2) + self.assertEqual(len(records), 2) + + def test_run_candidate_stops_when_search_budget_is_exhausted(self): + benchmark_cfg = { + "qps": {"lower": 1.0, "upper": 2.0, "tolerance": 0.1}, + "max_concurrency": [None], + } + + with self.assertRaises(SearchDeadlineExceeded): + run_candidate( + stage_name="base", + candidate_id=0, + server_cfg={"host": "127.0.0.1", "port": 30000}, + benchmark_cfg=benchmark_cfg, + dataset_summary={"num_requests": 1}, + backend="sglang-oai", + dataset_path="/tmp/fake.jsonl", + tokenizer_path=str(self.tokenizer_dir), + server_flags={"model_path": "/model"}, + output_dir=str(self.tmpdir_path), + search_deadline=time.time() - 1.0, + search_budget_hours=0.1, + ) + + def test_run_candidate_resume_skips_existing_fixed_trials(self): + benchmark_cfg = { + "qps": [1.0, 2.0], + "max_concurrency": [None], + } + existing_records = [ + { + "stage": "base", + "candidate_id": 0, + "requested_qps": 1.0, + "max_concurrency": None, + "server_flags": {"model_path": "/model"}, + "sla_passed": True, + "metrics": { + "output_throughput": 1.0, + "mean_ttft_ms": 1.0, + "mean_tpot_ms": 1.0, + }, + } + ] + calls = [] + + def fake_run_trial(**kwargs): + calls.append(kwargs["request_rate"]) + return { + "stage": "base", + "candidate_id": kwargs["candidate_id"], + "requested_qps": kwargs["request_rate"], + "max_concurrency": kwargs["max_concurrency"], + "server_flags": kwargs["server_flags"], + "sla_passed": True, + "metrics": { + "output_throughput": 2.0, + "mean_ttft_ms": 2.0, + "mean_tpot_ms": 2.0, + }, + } + + with mock.patch( + "sglang.auto_benchmark_lib.run_trial", side_effect=fake_run_trial + ): + records = run_candidate( + stage_name="base", + candidate_id=0, + server_cfg={"host": "127.0.0.1", "port": 30000}, + benchmark_cfg=benchmark_cfg, + dataset_summary={"num_requests": 1}, + backend="sglang-oai", + dataset_path="/tmp/fake.jsonl", + tokenizer_path=str(self.tokenizer_dir), + server_flags={"model_path": "/model"}, + output_dir=str(self.tmpdir_path), + existing_records=existing_records, + ) + + self.assertEqual(calls, [2.0]) + self.assertEqual([record["requested_qps"] for record in records], [1.0, 2.0]) + + +if __name__ == "__main__": + unittest.main()