[Feat] simulator: Add baseline collection and ML latency predictor training tools (#44299)

Co-authored-by: shijunyu.sjy <shijunyu.sjy@alibaba-inc.com>
Co-authored-by: baoyunong.byn <baoyunong.byn@alibaba-inc.com>
Co-authored-by: wangyi.wy <aaronwangyi0201@gmail.com>
This commit is contained in:
Ruiyan Ma
2026-10-20 13:02:33 +08:00
committed by GitHub
co-authored by shijunyu.sjy baoyunong.byn wangyi.wy
parent 43a896933e
commit 553836c164
20 changed files with 1182 additions and 74 deletions
@@ -130,6 +130,7 @@ jobs:
simulator:
- ".github/workflows/pr-test-extra.yml"
- ".github/workflows/_pr-test-simulator-cpu.yml"
- ".github/workflows/_pr-test-simulator-gpu.yml"
- "benchmark/simulator/**/!(*.md)"
- "tools/sglang-simulator/**/!(*.md)"
multimodal_gen:
+2 -1
View File
@@ -70,7 +70,7 @@ jobs:
run: |
uv pip install -e "python" --index-strategy unsafe-best-match --prerelease allow
uv pip install pytest
uv pip install -e "tools/sglang-simulator[aic]"
uv pip install "./tools/sglang-simulator[aic]"
- name: Prebuild HiCache native hash extension
timeout-minutes: 5
@@ -91,6 +91,7 @@ jobs:
PYTEST_ADDOPTS: "-s"
TORCH_EXTENSIONS_DIR: ${{ runner.temp }}/torch-extensions
run: |
OMP_NUM_THREADS=2 python3 -m pytest -q tools/sglang-simulator/test/test_train_latency_model.py
python3 -m pytest -q tools/sglang-simulator/test/test_simulation_sglang_runner.py
python3 -m pytest -q tools/sglang-simulator/test/test_simulation_sglang_serving.py
python3 -m pytest -q tools/sglang-simulator/test/test_simulation_cache_hit_ratio.py
@@ -0,0 +1,68 @@
name: PR Test SGLang Simulator (GPU)
on:
workflow_call:
inputs:
check_changes:
description: 'toJson(needs.check-changes.outputs).'
type: string
required: true
caller_inputs:
description: 'toJson(inputs) from the caller workflow.'
type: string
required: true
rust_ext_artifact:
description: 'Artifact of prebuilt Rust extension modules.'
type: string
default: ''
env:
SGLANG_IS_IN_CI: true
SKIP_PR_TEST_HEALTH_CHECK: ${{ (fromJson(inputs.caller_inputs).skip_pr_test_health_check || fromJson(inputs.caller_inputs).test_parallel_dispatch || fromJson(inputs.caller_inputs).run_all_tests) && 'true' || 'false' }}
PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }}
USE_VENV: "1"
jobs:
run:
name: simulator-test-gpu
if: fromJson(inputs.check_changes).main_package == 'true' || fromJson(inputs.check_changes).simulator == 'true' || fromJson(inputs.check_changes).sgl_kernel == 'true'
runs-on: ${{ fromJson(fromJson(inputs.check_changes).runs_on_map)['1-gpu-small'] }}
timeout-minutes: 40
steps:
- uses: actions/checkout@v4
with:
ref: ${{ fromJson(inputs.caller_inputs).git_ref || github.sha }}
- uses: ./.github/actions/check-pr-test-health
- uses: ./.github/actions/check-maintenance
- name: Download kernel wheels
if: fromJson(inputs.check_changes).sgl_kernel == 'true'
uses: actions/download-artifact@v4
with:
path: python/sglang/kernels/aot/dist/
merge-multiple: true
pattern: wheel-python3.10-cuda*
- uses: ./.github/actions/download-rust-ext
with:
artifact_name: ${{ inputs.rust_ext_artifact }}
- name: Install dependencies
timeout-minutes: 20
env:
CUSTOM_BUILD_SGL_KERNEL: ${{ fromJson(inputs.check_changes).sgl_kernel }}
run: bash scripts/ci/cuda/ci_install_dependency.sh
- name: Test baseline collection
timeout-minutes: 10
env:
PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1"
run: |
uv pip install "./tools/sglang-simulator"
python3 -m pytest -q tools/sglang-simulator/test/test_baseline_collection_sglang.py
- name: Cleanup venv
if: always()
run: bash scripts/ci/cuda/ci_cleanup_venv.sh
+10
View File
@@ -150,6 +150,15 @@ jobs:
rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
# No `secrets: inherit`: this hosted CPU job has no secret consumer.
simulator-test-gpu:
needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build]
if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }}
uses: ./.github/workflows/_pr-test-simulator-gpu.yml
with:
check_changes: ${{ toJson(needs.check-changes.outputs) }}
caller_inputs: ${{ toJson(inputs) }}
rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
# =============================================== extra-a / extra-b ===============================================
# `name:` keeps the job names wait-for-jobs and check-pr-test-health match on.
extra-test:
@@ -191,6 +200,7 @@ jobs:
sgl-kernel-build-wheels,
rust-ext-build,
simulator-test-cpu,
simulator-test-gpu,
extra-test,
]
if: always()
+8
View File
@@ -221,6 +221,14 @@ Supported predictors:
Relative predictor paths are resolved from the simulator configuration location.
Environment variables in paths use `${NAME}` syntax.
## Collect baselines and train an ML predictor
Use [collect_baseline](src/sglang_simulator/tools/collect_baseline/README.md) to capture request traces
and synchronized batch steps from a real SGLang server. Then use
[train_ml_time_predictor](src/sglang_simulator/tools/train_ml_time_predictor/README.md) to train an
18-feature latency model. Installing the package provides `sglang-simulator-collect`
and `sglang-simulator-train`, including a small collected training example.
## Workload formats
The Autobench trace format uses timestamps in milliseconds:
+4
View File
@@ -12,6 +12,10 @@ dependencies = [
"joblib",
]
[project.scripts]
sglang-simulator-collect = "sglang_simulator.tools.collect_baseline.collect:main"
sglang-simulator-train = "sglang_simulator.tools.train_ml_time_predictor.train:main"
[project.optional-dependencies]
aic = [
"aiconfigurator==0.10.0",
+7
View File
@@ -18,4 +18,11 @@ setup(
packages=find_packages(where="src"),
package_dir={"": "src"},
py_modules=["usercustomize"],
package_data={
"sglang_simulator.tools.collect_baseline": ["README.md"],
"sglang_simulator.tools.train_ml_time_predictor": [
"README.md",
"example_steps.jsonl",
],
},
)
@@ -1,7 +1,7 @@
"""ML-trained per-iter latency predictor.
Loads a joblib pickle of a sklearn-compatible regressor and predicts forward latency
from batch composition features. Train one with `train_latency_model.py`.
from batch composition features. Train one with `sglang-simulator-train`.
sim_config.json usage:
"predictor": {
@@ -10,7 +10,6 @@ sim_config.json usage:
}
"""
import math
import os
import joblib
@@ -18,47 +17,20 @@ from sglang_simulator.simulation.types import SchedulerConfig
from sglang_simulator.spec.accelerator import AcceleratorInfo
from sglang_simulator.spec.model import ModelInfo
from sglang_simulator.time_predictor.base import InferTimePredictor, ScheduleBatch
from sglang_simulator.time_predictor.ml_features import (
ML_FEATURE_NAMES,
extract_ml_features,
)
from sglang_simulator.utils import get_logger
logger = get_logger("sgl_simulator")
class MLTimePredictor(InferTimePredictor):
"""Per-iter latency predictor backed by an offline-trained sklearn regressor.
"""Predict forward latency using the shared 18-feature batch representation."""
Features (18 dim) extracted from ScheduleBatch:
batch_size, sum/max/min(extend), sum/max/min(past),
sum(extend*past), sum(extend^2), sum(past^2),
sum_attn_flops (= sum(e*(p+e/2))),
sum(extend × max_past), log1p(sum_past), log1p(sum_attn_flops),
batch_size × sum_extend, max_past - min_past,
is_decode, is_prefill
"""
# This ordered list is the ABI between offline training and simulation.
# The concrete regressor algorithm is intentionally unrestricted as long
# as it exposes sklearn-compatible predict([[18 features]]) -> [seconds].
FEATURE_NAMES = [
"batch_size",
"sum_extend",
"max_extend",
"min_extend",
"sum_past",
"max_past",
"min_past",
"sum_extend_x_past",
"sum_extend_squared",
"sum_past_squared",
"sum_attn_flops",
"sum_extend_x_max_past",
"log1p_sum_past",
"log1p_sum_attn_flops",
"batch_size_x_sum_extend",
"max_past_minus_min_past",
"is_decode",
"is_prefill",
]
# Feature order is shared by training and inference, regardless of regressor.
FEATURE_NAMES = list(ML_FEATURE_NAMES)
def __init__(
self,
@@ -74,7 +46,7 @@ class MLTimePredictor(InferTimePredictor):
if not database_path or not os.path.exists(database_path):
raise FileNotFoundError(
f"MLTimePredictor database_path not found: {database_path}. "
"Train one with `train_latency_model.py` first."
"Train one with `sglang-simulator-train` first."
)
bundle = joblib.load(database_path)
@@ -101,14 +73,12 @@ class MLTimePredictor(InferTimePredictor):
"MLTimePredictor model must expose a callable predict() method"
)
self._features = saved_features
self._call_count = 0
self._latency_scale = float(latency_scale)
logger.info(
"MLTimePredictor loaded from %s (model=%s, n_features=%d, latency_scale=%.4f)",
database_path,
type(self._model).__name__,
len(self._features),
len(saved_features),
self._latency_scale,
)
@@ -119,38 +89,6 @@ class MLTimePredictor(InferTimePredictor):
exts = [req.extend_length for req in batch.reqs]
pasts = [req.past_kv_length for req in batch.reqs]
bs = len(exts)
sum_e = sum(exts)
sum_p = sum(pasts)
sum_ep = sum(e * p for e, p in zip(exts, pasts))
sum_e2 = sum(e * e for e in exts)
sum_p2 = sum(p * p for p in pasts)
sum_attn = sum(e * (p + e / 2) for e, p in zip(exts, pasts))
max_e = max(exts)
max_p = max(pasts)
min_e = min(exts)
min_p = min(pasts)
feats = extract_ml_features(exts, pasts)
feats = [
bs,
sum_e,
max_e,
min_e,
sum_p,
max_p,
min_p,
sum_ep,
sum_e2,
sum_p2,
sum_attn,
sum_e * max_p,
math.log1p(sum_p),
math.log1p(sum_attn),
bs * sum_e,
max_p - min_p,
int(all(e == 1 for e in exts)),
int(any(e > 1 for e in exts)),
]
self._call_count += 1
return float(self._model.predict([feats])[0]) * self._latency_scale
@@ -0,0 +1,75 @@
"""Shared feature contract for ML latency prediction and offline training."""
import math
from collections.abc import Sequence
ML_FEATURE_NAMES = (
"batch_size",
"sum_extend",
"max_extend",
"min_extend",
"sum_past",
"max_past",
"min_past",
"sum_extend_x_past",
"sum_extend_squared",
"sum_past_squared",
"sum_attn_flops",
"sum_extend_x_max_past",
"log1p_sum_past",
"log1p_sum_attn_flops",
"batch_size_x_sum_extend",
"max_past_minus_min_past",
"is_decode",
"is_prefill",
)
def extract_ml_features(
extend_lengths: Sequence[int], past_kv_lengths: Sequence[int]
) -> list[float]:
"""Build the ordered 18-feature vector consumed by ``MLTimePredictor``."""
if not extend_lengths:
raise ValueError("cannot extract ML features from an empty batch")
if len(extend_lengths) != len(past_kv_lengths):
raise ValueError("extend and past-KV length vectors must have equal size")
if any(value < 0 for value in (*extend_lengths, *past_kv_lengths)):
raise ValueError("token lengths must be non-negative")
batch_size = len(extend_lengths)
sum_extend = sum(extend_lengths)
sum_past = sum(past_kv_lengths)
sum_extend_x_past = sum(
extend * past for extend, past in zip(extend_lengths, past_kv_lengths)
)
sum_extend_squared = sum(value * value for value in extend_lengths)
sum_past_squared = sum(value * value for value in past_kv_lengths)
sum_attn_flops = sum(
extend * (past + extend / 2)
for extend, past in zip(extend_lengths, past_kv_lengths)
)
max_extend = max(extend_lengths)
max_past = max(past_kv_lengths)
min_extend = min(extend_lengths)
min_past = min(past_kv_lengths)
return [
batch_size,
sum_extend,
max_extend,
min_extend,
sum_past,
max_past,
min_past,
sum_extend_x_past,
sum_extend_squared,
sum_past_squared,
sum_attn_flops,
sum_extend * max_past,
math.log1p(sum_past),
math.log1p(sum_attn_flops),
batch_size * sum_extend,
max_past - min_past,
int(all(value == 1 for value in extend_lengths)),
int(any(value > 1 for value in extend_lengths)),
]
@@ -0,0 +1 @@
"""Baseline collection and latency-model training tools."""
@@ -0,0 +1,60 @@
# Collect a baseline
Install the simulator package in an environment with the matching SGLang version
and its serving dependencies. From a source checkout, use
`pip install ./tools/sglang-simulator`; a wheel installation provides the same
command, which can run from any directory:
```bash
sglang-simulator-collect \
--collection-output-dir /tmp/baseline-001 \
--model-path /path/to/model \
--disable-overlap-schedule --host 127.0.0.1 --port 30000
```
Other flags are standard SGLang server arguments. The output directory must be
new or empty. After warmup, collect a request with:
```bash
curl -f -X POST http://127.0.0.1:30000/start_profile -H 'Content-Type: application/json' -d '{}'
curl -f http://127.0.0.1:30000/generate -H 'Content-Type: application/json' \
-d '{"text":"Hello","sampling_params":{"max_new_tokens":16,"ignore_eos":true}}'
curl -f -X POST http://127.0.0.1:30000/stop_profile -H 'Content-Type: application/json' -d '{}'
```
Replace the generate call with your benchmark; wait for all requests before stop.
These endpoints control collection instead of the PyTorch profiler. Hooks reuse
the simulator infrastructure without patching SGLang source. Records stay in
memory until `stop_profile`; use a fresh output directory/server for each run.
Each scheduler writes three JSONL files (`TP0` shown):
| File | Contents |
| --- | --- |
| `TP0.request.jsonl` | Request ID, Unix arrival time in milliseconds, input/output lengths, disjoint device/host/disk cache-hit lengths. |
| `TP0.raw_request.jsonl` | Arrival/queue times in seconds, completion flag, token IDs, output length and token emission latencies in seconds. |
| `TP0.schedule_batch.jsonl` | Forward mode, per-request extend/past-KV lengths and synchronized `iter_latency` in seconds. |
DP/PP/EP ranks are appended when their parallel size exceeds one. Cache counts
come from SGLang's first scheduled chunk. The first token latency starts at server
arrival; subsequent entries measure time between output-producing steps.
`client_created_time` in sampling `custom_params`, when supplied in Unix seconds,
is preferred as the normalized arrival timestamp.
Step timing synchronizes the device before and after `Scheduler.run_batch`.
Features are captured before execution; start/end step timestamps use
`perf_counter`. This requires `--disable-overlap-schedule` and changes execution
timing. The initial scope is ordinary text generation, not speculative decoding
or multimodal feature modeling. Health probes are excluded from request traces;
steps containing any health probe are excluded from training data.
Read files after a successful stop response. Export never overwrites existing
files and retains records on failure; remove any logged partial files before
retrying. Raw traces include token IDs: review them before sharing. Continue with
[the training guide](../train_ml_time_predictor/README.md).
For a minimal GPU smoke test with a small dummy model and four requests, run:
```bash
python3 -m pytest -q tools/sglang-simulator/test/test_baseline_collection_sglang.py
```
@@ -0,0 +1 @@
"""Collect baseline traces from a real SGLang server."""
@@ -0,0 +1,74 @@
"""Launch a real SGLang server instrumented for simulator baseline collection."""
import argparse
import os
from pathlib import Path
from sglang_simulator.tools.collect_baseline.hooks import (
COLLECTION_OUTPUT_DIR_ENV,
TokenizerManagerCollectionHook,
install_frontend_collection_hooks,
run_collection_scheduler_process,
validate_collection_hook,
)
# This must happen before importing the SGLang HTTP server.
install_frontend_collection_hooks()
def _prepare_output_dir(parser: argparse.ArgumentParser, value: str) -> Path:
output_dir = Path(value).expanduser().resolve()
if output_dir.exists() and not output_dir.is_dir():
parser.error(
f"baseline collection output path is not a directory: {output_dir}"
)
if output_dir.is_dir() and any(output_dir.iterdir()):
parser.error(
f"baseline collection output directory must be empty: {output_dir}"
)
output_dir.mkdir(parents=True, exist_ok=True)
os.environ[COLLECTION_OUTPUT_DIR_ENV] = str(output_dir)
return output_dir
def main(argv: list[str] | None = None) -> None:
from sglang.srt.entrypoints.http_server import launch_server
from sglang.srt.server_args import ServerArgs
from sglang.srt.utils import kill_process_tree
validate_collection_hook(TokenizerManagerCollectionHook)
parser = argparse.ArgumentParser()
collection = parser.add_argument_group("baseline collection")
collection.add_argument(
"--collection-output-dir",
default=os.getenv(COLLECTION_OUTPUT_DIR_ENV),
required=os.getenv(COLLECTION_OUTPUT_DIR_ENV) is None,
help=(
"Empty directory for TP*.request.jsonl, raw_request.jsonl, "
"and schedule_batch.jsonl"
),
)
ServerArgs.add_cli_args(parser.add_argument_group("sglang"))
raw_args = parser.parse_args(argv)
output_dir = _prepare_output_dir(parser, raw_args.collection_output_dir)
server_args = ServerArgs.from_cli_args(raw_args)
if not server_args.disable_overlap_schedule:
parser.error(
"--disable-overlap-schedule is required so synchronized batch "
"latencies match ScheduleBatch execution boundaries"
)
print(f"SGLang Simulator baseline collection directory: {output_dir}")
try:
launch_server(
server_args,
run_scheduler_process_func=run_collection_scheduler_process,
)
finally:
kill_process_tree(os.getpid(), include_parent=False)
if __name__ == "__main__":
main()
@@ -0,0 +1,409 @@
"""Hooks for collecting requests and synchronized steps from a real SGLang server."""
from __future__ import annotations
import json
import logging
import os
import time
from dataclasses import asdict, dataclass, field
from functools import wraps
from pathlib import Path
from typing import Any, Iterable
from sglang_simulator.hook import BaseHook, install_class_hooks
from sglang_simulator.hook.class_hook_entry import is_class_hook_matched
logger = logging.getLogger(__name__)
COLLECTION_OUTPUT_DIR_ENV = "SGLANG_SIMULATOR_COLLECTION_DIR"
@dataclass
class RequestRecord:
rid: str = ""
created_time: float | None = None
client_created_time: float | None = None
server_created_time: float | None = None
queue_start: float = 0.0
queue_end: float = 0.0
completed: bool = False
output_length: int = 0
input_length: int = 0
before_adder_device_hit_len: int = 0
final_device_hit_len: int = 0
final_host_hit_len: int = 0
final_disk_hit_len: int = 0
last_event_time: float = 0.0
gen_token_latencies: list[float] = field(default_factory=list)
input_ids: list[int] = field(default_factory=list)
output_ids: list[int] = field(default_factory=list)
@dataclass
class CollectionState:
active: bool = False
requests: dict[str, RequestRecord] = field(default_factory=dict)
schedule_batches: list[dict[str, Any]] = field(default_factory=list)
def start(self) -> None:
if self.active:
raise RuntimeError("baseline collection is already active")
self.requests.clear()
self.schedule_batches.clear()
self.active = True
def get_request(self, rid: str) -> RequestRecord:
record = self.requests.get(rid)
if record is None:
record = RequestRecord(rid=rid)
self.requests[rid] = record
return record
def stop_and_export(self) -> list[Path]:
if not self.active:
raise RuntimeError("baseline collection is not active")
output_dir = os.getenv(COLLECTION_OUTPUT_DIR_ENV)
if not output_dir:
raise RuntimeError(
f"{COLLECTION_OUTPUT_DIR_ENV} is required for baseline collection"
)
root = Path(output_dir).expanduser()
root.mkdir(parents=True, exist_ok=True)
prefix = _rank_prefix()
exports = {
"request": (_normalized_request(r) for r in self.requests.values()),
"raw_request": (asdict(r) for r in self.requests.values()),
"schedule_batch": iter(self.schedule_batches),
}
paths = []
try:
for kind, rows in exports.items():
path = root / f"{prefix}.{kind}.jsonl"
with path.open("x", encoding="utf-8") as output:
paths.append(path)
for row in rows:
output.write(json.dumps(row) + "\n")
except (OSError, TypeError, ValueError):
# Keep in-memory records for retry; never remove an existing file.
for path in paths:
try:
path.unlink()
except OSError:
logger.warning(
"Failed to remove partial baseline file %s", path, exc_info=True
)
raise
self.active = False
self.requests.clear()
self.schedule_batches.clear()
return paths
def _rank_prefix() -> str:
from sglang.srt.runtime_context import get_parallel
parallel = get_parallel()
prefix = f"TP{parallel.tp_rank}"
for size_name, rank_name, label in (
("dp_size", "dp_rank", "DP"),
("pp_size", "pp_rank", "PP"),
("moe_ep_size", "moe_ep_rank", "EP"),
):
if getattr(parallel, size_name) > 1:
prefix += f"-{label}{getattr(parallel, rank_name)}"
return prefix
def _normalized_request(record: RequestRecord) -> dict[str, Any]:
timestamp = record.created_time
if timestamp is None:
timestamp = record.queue_start
return {
"rid": record.rid,
"timestamp": timestamp * 1000,
"input_length": record.input_length,
"output_length": record.output_length,
"device_cache_hit_length": record.before_adder_device_hit_len,
"host_cache_hit_length": record.final_host_hit_len - record.final_disk_hit_len,
"disk_cache_hit_length": record.final_disk_hit_len,
}
COLLECTION_STATE = CollectionState()
def _custom_params(sampling_params: Any) -> dict[str, Any]:
if isinstance(sampling_params, dict):
return sampling_params.get("custom_params") or {}
return getattr(sampling_params, "custom_params", None) or {}
def _stamp_created_time(tokenized_obj: Any) -> None:
from sglang.srt.observability.req_time_stats import convert_time_to_realtime
time_stats = getattr(tokenized_obj, "time_stats", None)
created_time = getattr(time_stats, "created_time", None)
if not created_time:
return
sampling_params = tokenized_obj.sampling_params
params = dict(_custom_params(sampling_params))
# Convert SGLang's perf_counter arrival to Unix time before IPC.
params["server_created_time"] = convert_time_to_realtime(created_time)
if isinstance(sampling_params, dict):
sampling_params["custom_params"] = params
else:
sampling_params.custom_params = params
def _iter_received_requests(recv_reqs: Iterable[Any]) -> Iterable[Any]:
for recv_req in recv_reqs:
if recv_req.__class__.__name__.startswith("BatchTokenized"):
yield from recv_req
else:
yield recv_req
def _record_received_requests(recv_reqs: Iterable[Any]) -> None:
if not COLLECTION_STATE.active:
return
from sglang.srt.managers.utils import is_health_check_generate_req
recv_time = time.time()
for req in _iter_received_requests(recv_reqs):
# Control messages (abort, profile, RPC, etc.) must not become requests.
if not hasattr(req, "sampling_params") or not getattr(req, "rid", None):
continue
if is_health_check_generate_req(req):
continue
record = COLLECTION_STATE.get_request(str(req.rid))
record.queue_start = recv_time
params = _custom_params(req.sampling_params)
record.server_created_time = params.get("server_created_time")
record.client_created_time = params.get("client_created_time")
record.created_time = next(
value
for value in (
record.client_created_time,
record.server_created_time,
recv_time,
)
if value is not None
)
record.last_event_time = record.server_created_time or recv_time
record.input_ids = list(getattr(req, "input_ids", None) or [])
record.input_length = len(record.input_ids)
def _batch_request_lengths(batch: Any) -> list[tuple[int, int]]:
"""Read prepared batch vectors; Req lengths may lag in decode/mixed batches."""
if not batch.reqs:
return []
if batch.forward_mode.is_decode():
lengths = [(1, max(length - 1, 0)) for length in batch.seq_lens_cpu.tolist()]
else:
lengths = list(zip(batch.extend_lens, batch.prefix_lens, strict=True))
if len(lengths) != len(batch.reqs):
raise ValueError("ScheduleBatch length vectors and reqs must have equal size")
return lengths
def _synchronize_device() -> None:
# Defer SGLang imports until the class hooks are installed.
from sglang.srt.platforms import current_platform
current_platform.synchronize()
def _profile_output(message: str, success: bool = True) -> Any:
from sglang.srt.managers.io_struct import ProfileReqOutput
return ProfileReqOutput(success=success, message=message)
def _handle_profile_request(recv_req: Any) -> Any:
try:
if recv_req.req_type.name == "START_PROFILE":
COLLECTION_STATE.start()
return _profile_output("Baseline collection started")
paths = COLLECTION_STATE.stop_and_export()
return _profile_output(
"Baseline collection exported: " + ", ".join(path.name for path in paths)
)
except (RuntimeError, OSError, TypeError, ValueError) as exc:
# Report API errors without terminating the scheduler.
return _profile_output(str(exc), success=False)
class TokenizerManagerCollectionHook(BaseHook):
HOOK_CLASS_NAME = "TokenizerManager"
HOOK_MODULE_NAME = "sglang.srt.managers.tokenizer_manager"
@classmethod
def hook(cls, target: Any) -> None:
original_one = target._send_one_request
original_batch = target._send_batch_request
@wraps(original_one)
async def send_one(
self: Any, tokenized_obj: Any, *args: Any, **kwargs: Any
) -> Any:
_stamp_created_time(tokenized_obj)
return await original_one(self, tokenized_obj, *args, **kwargs)
@wraps(original_batch)
async def send_batch(
self: Any, tokenized_objs: list[Any], *args: Any, **kwargs: Any
) -> Any:
for obj in tokenized_objs:
_stamp_created_time(obj)
return await original_batch(self, tokenized_objs, *args, **kwargs)
target._send_one_request = send_one
target._send_batch_request = send_batch
class SchedulerCollectionHook(BaseHook):
HOOK_CLASS_NAME = "Scheduler"
HOOK_MODULE_NAME = "sglang.srt.managers.scheduler"
@classmethod
def hook(cls, target: Any) -> None:
from sglang.srt.managers.utils import is_health_check_generate_req
original_input = target.process_input_requests
original_run = target.run_batch
original_result = target.process_batch_result
original_profiler = target.init_profiler
@wraps(original_input)
def process_input(
self: Any, recv_reqs: list[Any], *args: Any, **kwargs: Any
) -> Any:
_record_received_requests(recv_reqs)
return original_input(self, recv_reqs, *args, **kwargs)
@wraps(original_run)
def run_batch(self: Any, batch: Any, *args: Any, **kwargs: Any) -> Any:
if not COLLECTION_STATE.active:
return original_run(self, batch, *args, **kwargs)
# Workers may mutate the batch, so snapshot before forward.
requests = tuple(batch.reqs or ())
scheduled_time = time.time()
for req in requests:
record = COLLECTION_STATE.requests.get(str(req.rid))
if record is not None and record.queue_end == 0:
record.queue_end = scheduled_time
record.input_ids = list(req.origin_input_ids)
record.input_length = len(record.input_ids)
# Preserve SGLang's first-chunk counts across later chunks/retractions.
device = int(req.cached_tokens_device)
host = int(req.cached_tokens_host)
storage = int(req.cached_tokens_storage)
record.before_adder_device_hit_len = device
record.final_device_hit_len = device + host + storage
record.final_host_hit_len = host + storage
record.final_disk_hit_len = storage
# A mixed step's latency includes the probe: exclude the entire step.
if any(is_health_check_generate_req(req) for req in requests):
return original_run(self, batch, *args, **kwargs)
lengths = _batch_request_lengths(batch)
row = {
"forward_mode": int(batch.forward_mode),
"request_infos": [
{
"rid": str(req.rid),
"extend_input_len": extend,
"prefix_indices_len": past,
"output_ids_len": len(req.output_ids),
}
for req, (extend, past) in zip(requests, lengths, strict=True)
],
}
_synchronize_device()
start = time.perf_counter()
result = original_run(self, batch, *args, **kwargs)
_synchronize_device()
end = time.perf_counter()
row.update(
start_timestamp=start, end_timestamp=end, iter_latency=end - start
)
COLLECTION_STATE.schedule_batches.append(row)
return result
@wraps(original_result)
def process_result(self: Any, batch: Any, *args: Any, **kwargs: Any) -> Any:
if not COLLECTION_STATE.active:
return original_result(self, batch, *args, **kwargs)
requests = [(req, len(req.output_ids)) for req in (batch.reqs or ())]
result = original_result(self, batch, *args, **kwargs)
event_time = time.time()
for req, previous_length in requests:
record = COLLECTION_STATE.requests.get(str(req.rid))
if record is None:
continue
record.output_length = len(req.output_ids)
if record.output_length > previous_length:
record.gen_token_latencies.append(
max(event_time - record.last_event_time, 0.0)
)
record.last_event_time = event_time
if req.finished():
record.output_ids = list(req.output_ids)
record.completed = True
return result
@wraps(original_profiler)
def init_profiler(self: Any, *args: Any, **kwargs: Any) -> Any:
result = original_profiler(self, *args, **kwargs)
self.profiler_manager._profile = _handle_profile_request
return result
target.process_input_requests = process_input
target.run_batch = run_batch
target.process_batch_result = process_result
target.init_profiler = init_profiler
_FRONTEND_HOOKS_INSTALLED = False
_SCHEDULER_HOOKS_INSTALLED = False
def install_frontend_collection_hooks() -> None:
global _FRONTEND_HOOKS_INSTALLED
if not _FRONTEND_HOOKS_INSTALLED:
install_class_hooks(TokenizerManagerCollectionHook)
_FRONTEND_HOOKS_INSTALLED = True
def install_scheduler_collection_hooks() -> None:
global _SCHEDULER_HOOKS_INSTALLED
if not _SCHEDULER_HOOKS_INSTALLED:
install_class_hooks(SchedulerCollectionHook)
_SCHEDULER_HOOKS_INSTALLED = True
def validate_collection_hook(hook: type[BaseHook]) -> None:
if is_class_hook_matched(hook):
return
raise RuntimeError(
"Required SGLang Simulator hook did not match an imported SGLang class: "
f"{hook.__name__} ({hook.HOOK_MODULE_NAME}.{hook.HOOK_CLASS_NAME}). "
"The collector must be adapted to this SGLang revision."
)
def run_collection_scheduler_process(*args: Any, **kwargs: Any) -> Any:
"""Install collection hooks before importing and running Scheduler."""
install_scheduler_collection_hooks()
from sglang.srt.managers.scheduler import run_scheduler_process
# TokenizerManager belongs to the API process; validate only Scheduler here.
validate_collection_hook(SchedulerCollectionHook)
return run_scheduler_process(*args, **kwargs)
@@ -0,0 +1,61 @@
# Train an ML time predictor
Install the simulator package (`pip install ./tools/sglang-simulator` from a
source checkout, or install its wheel). The command and example data are included
in the package, so this works from any directory:
```bash
example=$(python -c "from importlib.resources import files; print(files('sglang_simulator.tools.train_ml_time_predictor') / 'example_steps.jsonl')")
sglang-simulator-train --data-root "$example" --out-dir /tmp/example-predictor
```
`example_steps.jsonl` contains collected EXTEND steps from a tiny, two-layer Qwen3
model with dummy weights on one A100, using repeated synthetic token sequences.
Only numeric feature inputs and measured latencies are retained. It demonstrates
the file format and training flow; it cannot produce a useful production predictor
or demonstrate prediction accuracy.
For real data, replace `--data-root` with a [collection](../collect_baseline/README.md)
directory. The trainer recursively reads `TP0*.schedule_batch.jsonl` to avoid
counting duplicate tensor-parallel ranks. Repeat `--data-root` for more cases, or
pass explicit files for a different rank selection. Use data collected for the
same target model, hardware and serving configuration.
The default filter keeps EXTEND (`forward_mode=1`) steps with nonempty batches and
finite latencies in `(0, 30]` seconds. Use `--forward-modes 1 2` to include DECODE
and `--max-latency` to change the upper bound. Malformed records fail with their
file and line; skipped row counts are included in the summary.
Training shares the simulator's ordered 18-feature definition and uses monotonic
HistGradientBoosting regressors. The new or empty output directory receives:
- `latency_model_hgbmono_l2.pkl`: squared-error model.
- `latency_model_hgbmono_p50.pkl`: median-quantile model.
- `summary.json`: seed, split sizes, skipped counts and held-out MAE (ms)/MAPE (%). <!-- codespell:ignore mape -->
Both models predict seconds. `l2` names the squared-error loss, not an L2 cache;
`p50` estimates the median. Compare them on independent data for your workload.
By default, each collection directory is split 50/50 with seed 42 (single-row
cases stay in training). Metrics evaluate this split; saved models are then
refitted on all input rows. This measures interpolation within collected cases.
Use repeatable `--test-data-root` arguments for independent evaluation cases;
those rows never enter fitting or the saved models. Same-directory overlaps are
rejected. Copied data is not detected: keep independent collections separate.
Choose one loss with `--loss l2` or `--loss p50` if needed.
```bash
sglang-simulator-train \
--data-root /path/to/train-collection \
--test-data-root /path/to/test-collection --out-dir /tmp/predictor
```
Point the simulator configuration at the selected bundle:
```json
{"predictor": {"name": "ml", "database_path": "/tmp/example-predictor/latency_model_hgbmono_l2.pkl"}}
```
Use the same scikit-learn version for training and inference, and load only
trusted joblib files. Before production use, validate the selected model on
independent step data and complete simulator workloads.
@@ -0,0 +1 @@
"""Train latency predictors from collected baseline steps."""
@@ -0,0 +1,14 @@
{"forward_mode": 1, "request_infos": [{"extend_input_len": 1, "prefix_indices_len": 255}], "iter_latency": 0.002675119787454605}
{"forward_mode": 1, "request_infos": [{"extend_input_len": 8, "prefix_indices_len": 256}], "iter_latency": 0.0030074845999479294}
{"forward_mode": 1, "request_infos": [{"extend_input_len": 128, "prefix_indices_len": 0}], "iter_latency": 0.0027154013514518738}
{"forward_mode": 1, "request_infos": [{"extend_input_len": 128, "prefix_indices_len": 128}], "iter_latency": 0.0027412977069616318}
{"forward_mode": 1, "request_infos": [{"extend_input_len": 16, "prefix_indices_len": 256}], "iter_latency": 0.002909639850258827}
{"forward_mode": 1, "request_infos": [{"extend_input_len": 8, "prefix_indices_len": 272}], "iter_latency": 0.002743590623140335}
{"forward_mode": 1, "request_infos": [{"extend_input_len": 128, "prefix_indices_len": 0}], "iter_latency": 0.002713024616241455}
{"forward_mode": 1, "request_infos": [{"extend_input_len": 128, "prefix_indices_len": 128}], "iter_latency": 0.0027074161916971207}
{"forward_mode": 1, "request_infos": [{"extend_input_len": 32, "prefix_indices_len": 256}], "iter_latency": 0.0030285529792308807}
{"forward_mode": 1, "request_infos": [{"extend_input_len": 8, "prefix_indices_len": 288}], "iter_latency": 0.0027687866240739822}
{"forward_mode": 1, "request_infos": [{"extend_input_len": 128, "prefix_indices_len": 0}], "iter_latency": 0.002656087279319763}
{"forward_mode": 1, "request_infos": [{"extend_input_len": 64, "prefix_indices_len": 128}, {"extend_input_len": 64, "prefix_indices_len": 0}], "iter_latency": 1.545724280178547}
{"forward_mode": 1, "request_infos": [{"extend_input_len": 128, "prefix_indices_len": 64}], "iter_latency": 0.0031625479459762573}
{"forward_mode": 1, "request_infos": [{"extend_input_len": 8, "prefix_indices_len": 192}], "iter_latency": 0.0028553567826747894}
@@ -0,0 +1,216 @@
"""Train MLTimePredictor bundles from synchronized SGLang schedule-batch JSONL."""
from __future__ import annotations
import argparse
import json
import math
from collections import Counter, defaultdict
from pathlib import Path
import joblib
import numpy as np
import sklearn
from sglang_simulator.time_predictor.ml_features import (
ML_FEATURE_NAMES,
extract_ml_features,
)
from sklearn.ensemble import HistGradientBoostingRegressor
MONOTONIC_FEATURES = {
"batch_size",
"sum_extend",
"sum_past",
"sum_extend_x_past",
"sum_extend_squared",
"sum_past_squared",
"sum_attn_flops",
"sum_extend_x_max_past",
"log1p_sum_past",
"log1p_sum_attn_flops",
"batch_size_x_sum_extend",
}
def load_rows(roots, forward_modes, max_latency):
files = set()
for value in roots:
root = Path(value).expanduser().resolve()
if not root.exists():
raise FileNotFoundError(root)
files.update(
[root] if root.is_file() else root.rglob("TP0*.schedule_batch.jsonl")
)
if not files:
raise ValueError(f"no schedule files found under {roots}")
rows, skipped = [], Counter()
for path in sorted(files):
with path.open() as source:
for number, line in enumerate(source, 1):
try:
item = json.loads(line)
if not isinstance(item, dict):
raise ValueError("expected a JSON object")
if item.get("forward_mode") not in forward_modes:
skipped["mode"] += 1
continue
requests = item["request_infos"]
if not requests:
skipped["empty"] += 1
continue
latency = float(item["iter_latency"])
if not math.isfinite(latency) or not 0 < latency <= max_latency:
skipped["latency"] += 1
continue
extend = [r["extend_input_len"] for r in requests]
past = [r["prefix_indices_len"] for r in requests]
if any(type(v) is not int or v < 0 for v in extend + past):
raise ValueError("token lengths must be non-negative integers")
features = extract_ml_features(extend, past)
except (KeyError, TypeError, ValueError, OverflowError) as exc:
raise ValueError(f"{path}:{number}: {exc}") from exc
rows.append((path.parent, features, latency))
return rows, dict(skipped)
def split_rows(rows, seed):
"""Deterministic 50/50 split within each case (collection directory)."""
by_case = defaultdict(list)
for row in rows:
by_case[row[0]].append(row)
random = np.random.RandomState(seed)
train, test = [], []
for case in by_case.values():
order = random.permutation(len(case))
test_count = len(case) // 2
test.extend(case[i] for i in order[:test_count])
train.extend(case[i] for i in order[test_count:])
return train, test
def as_arrays(rows):
return np.asarray([r[1] for r in rows]), np.asarray([r[2] for r in rows])
def train_model(rows, loss, seed):
model = HistGradientBoostingRegressor(
loss="quantile" if loss == "p50" else "squared_error",
quantile=0.5 if loss == "p50" else None,
max_iter=400,
learning_rate=0.04,
max_leaf_nodes=63,
l2_regularization=0.1,
monotonic_cst=[int(name in MONOTONIC_FEATURES) for name in ML_FEATURE_NAMES],
early_stopping=len(rows) > 1,
validation_fraction=0.15,
n_iter_no_change=20,
random_state=seed,
)
return model.fit(*as_arrays(rows))
def parse_args(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--data-root",
action="append",
required=True,
help="Collection directory or JSONL file; repeatable.",
)
parser.add_argument(
"--test-data-root",
action="append",
default=[],
help="Independent evaluation data, excluded from fitting; repeatable.",
)
parser.add_argument(
"--out-dir", required=True, help="New or empty output directory."
)
parser.add_argument(
"--forward-modes",
type=int,
nargs="+",
default=[1],
help="SGLang ForwardMode values; default: EXTEND (1).",
)
parser.add_argument(
"--max-latency",
type=float,
default=30.0,
help="Largest retained latency, in seconds.",
)
parser.add_argument("--seed", type=int, default=42)
parser.add_argument("--loss", choices=("l2", "p50", "both"), default="both")
args = parser.parse_args(argv)
if not math.isfinite(args.max_latency) or args.max_latency <= 0:
parser.error("--max-latency must be finite and positive")
return args
def main(argv=None):
args = parse_args(argv)
output = Path(args.out_dir).expanduser()
if output.exists() and (not output.is_dir() or any(output.iterdir())):
raise FileExistsError(f"training output directory must be empty: {output}")
rows, skipped = load_rows(args.data_root, args.forward_modes, args.max_latency)
if not rows:
raise ValueError(
f"no usable training rows under {args.data_root}; skipped={skipped}"
)
if args.test_data_root:
test, test_skipped = load_rows(
args.test_data_root, args.forward_modes, args.max_latency
)
if not test:
raise ValueError(
f"no usable test rows under {args.test_data_root}; skipped={test_skipped}"
)
if {r[0] for r in rows} & {r[0] for r in test}:
raise ValueError(
"training and test roots contain overlapping collection cases"
)
train = rows
else:
train, test = split_rows(rows, args.seed)
test_skipped = {}
if not train or not test:
raise ValueError(
"need nonempty training and test data; provide a case with at least two usable rows or --test-data-root"
)
output.mkdir(parents=True, exist_ok=True)
summary = {
"seed": args.seed,
"split": "independent" if args.test_data_root else "within_case_50_50",
"train_rows": len(train),
"test_rows": len(test),
"final_fit_rows": len(rows),
"skipped": skipped,
"test_skipped": test_skipped,
"metrics": {},
}
features, latency = as_arrays(test)
for loss in ("l2", "p50") if args.loss == "both" else (args.loss,):
model = train_model(train, loss, args.seed)
errors = np.abs(model.predict(features) - latency)
summary["metrics"][loss] = {
"MAE_ms": float(errors.mean() * 1000),
"MAPE_pct": float((errors / latency).mean() * 100),
} # codespell:ignore mape
if not args.test_data_root:
model = train_model(rows, loss, args.seed)
# Runtime validates the ordered feature ABI before making predictions.
bundle = {
"model": model,
"features": list(ML_FEATURE_NAMES),
"sklearn_version": sklearn.__version__,
}
with (output / f"latency_model_hgbmono_{loss}.pkl").open("xb") as destination:
joblib.dump(bundle, destination)
with (output / "summary.json").open("x") as destination:
json.dump(summary, destination, indent=2)
destination.write("\n")
print(json.dumps(summary, indent=2))
if __name__ == "__main__":
main()
@@ -0,0 +1,127 @@
"""GPU example: start collection, generate responses, export the traces."""
import json
import os
import signal
import socket
import subprocess
import time
from pathlib import Path
import pytest
import requests
@pytest.fixture
def collection_server(tmp_path):
simulator_root = Path(__file__).resolve().parents[1]
model = tmp_path / "model"
model.mkdir()
config = json.loads(
(simulator_root / "test/assets/qwen3-8b/config.json").read_text()
)
# Small dummy model: this tests lifecycle and data contracts, not accuracy.
config.update(
hidden_size=256,
intermediate_size=512,
num_hidden_layers=2,
num_attention_heads=2,
num_key_value_heads=2,
max_window_layers=2,
vocab_size=1024,
bos_token_id=1,
eos_token_id=2,
)
(model / "config.json").write_text(json.dumps(config))
with socket.socket() as sock:
sock.bind(("127.0.0.1", 0))
port = sock.getsockname()[1]
url = f"http://127.0.0.1:{port}"
output = tmp_path / "baseline"
command = [
"sglang-simulator-collect",
f"--collection-output-dir={output}",
f"--model-path={model}",
"--load-format=dummy",
"--skip-tokenizer-init",
"--disable-overlap-schedule",
"--device=cuda",
"--cuda-graph-backend-decode=disabled",
"--cuda-graph-backend-prefill=disabled",
"--chunked-prefill-size=128",
"--max-total-tokens=4096",
"--mem-fraction-static=0.1",
"--host=127.0.0.1",
f"--port={port}",
]
log_path = tmp_path / "server.log"
with log_path.open("w") as log:
server = subprocess.Popen(
command,
stdout=log,
stderr=subprocess.STDOUT,
env={**os.environ, "OMP_NUM_THREADS": "2"},
start_new_session=True,
)
try:
deadline = time.monotonic() + 180
while time.monotonic() < deadline:
assert server.poll() is None, log_path.read_text()
try:
if requests.get(url + "/health", timeout=1).ok:
break
except requests.RequestException:
pass
time.sleep(0.2)
else:
raise AssertionError(log_path.read_text())
yield url, output, log_path
finally:
if server.poll() is None:
os.killpg(server.pid, signal.SIGTERM)
try:
server.wait(timeout=15)
except subprocess.TimeoutExpired:
os.killpg(server.pid, signal.SIGKILL)
server.wait()
def test_baseline_collection(collection_server):
url, output, log_path = collection_server
def post(path, payload=None):
response = requests.post(url + path, json=payload or {}, timeout=60)
assert response.ok, response.text + "\n" + log_path.read_text()
post("/start_profile")
for inputs in ([10] * 256, [20] * 192, [[30] * 128, [40] * 200]):
post(
"/generate",
{
"input_ids": inputs,
"sampling_params": {"max_new_tokens": 4, "ignore_eos": True},
},
)
post("/stop_profile")
def read(kind):
return [
json.loads(line)
for line in (output / f"TP0.{kind}.jsonl").read_text().splitlines()
]
requests_trace, raw, steps = (
read("request"),
read("raw_request"),
read("schedule_batch"),
)
request_ids = {r["rid"] for r in raw}
assert len(requests_trace) == len(raw) == len(request_ids) == 4
assert sorted(r["input_length"] for r in requests_trace) == [128, 192, 200, 256]
assert all(r["output_length"] == 4 for r in requests_trace)
assert all(r["completed"] and len(r["output_ids"]) == 4 for r in raw)
assert {r["rid"] for r in requests_trace} == request_ids
assert {r["rid"] for step in steps for r in step["request_infos"]} == request_ids
assert {step["forward_mode"] for step in steps} == {1, 2} # Prefill and decode.
assert all(step["iter_latency"] > 0 for step in steps)
@@ -0,0 +1,32 @@
"""Minimal example: train on collected steps, then load the models for prediction."""
import json
import math
from pathlib import Path
from sglang_simulator.time_predictor.base import ScheduleBatch, ScheduleRequest
from sglang_simulator.time_predictor.ml import MLTimePredictor
from sglang_simulator.tools.train_ml_time_predictor import train as trainer
def test_train_and_predict(tmp_path):
example = Path(trainer.__file__).with_name("example_steps.jsonl")
output = tmp_path / "models"
trainer.main(["--data-root", str(example), "--out-dir", str(output)])
summary = json.loads((output / "summary.json").read_text())
assert summary["train_rows"] > 0 and summary["test_rows"] > 0
assert summary["train_rows"] + summary["test_rows"] == summary["final_fit_rows"]
assert set(summary["metrics"]) == {"l2", "p50"}
batch = ScheduleBatch(
[
ScheduleRequest(extend_length=4, past_kv_length=12),
ScheduleRequest(extend_length=8, past_kv_length=0),
]
)
for loss in ("l2", "p50"):
model = output / f"latency_model_hgbmono_{loss}.pkl"
predictor = MLTimePredictor(None, None, None, str(model))
latency = predictor.predict_infer_time(batch)
assert math.isfinite(latency) and latency > 0