[Feat] simulator: Add baseline collection and ML latency predictor training tools (#44299)
Co-authored-by: shijunyu.sjy <shijunyu.sjy@alibaba-inc.com> Co-authored-by: baoyunong.byn <baoyunong.byn@alibaba-inc.com> Co-authored-by: wangyi.wy <aaronwangyi0201@gmail.com>
This commit is contained in:
co-authored by
shijunyu.sjy
baoyunong.byn
wangyi.wy
parent
43a896933e
commit
553836c164
@@ -130,6 +130,7 @@ jobs:
|
||||
simulator:
|
||||
- ".github/workflows/pr-test-extra.yml"
|
||||
- ".github/workflows/_pr-test-simulator-cpu.yml"
|
||||
- ".github/workflows/_pr-test-simulator-gpu.yml"
|
||||
- "benchmark/simulator/**/!(*.md)"
|
||||
- "tools/sglang-simulator/**/!(*.md)"
|
||||
multimodal_gen:
|
||||
|
||||
@@ -70,7 +70,7 @@ jobs:
|
||||
run: |
|
||||
uv pip install -e "python" --index-strategy unsafe-best-match --prerelease allow
|
||||
uv pip install pytest
|
||||
uv pip install -e "tools/sglang-simulator[aic]"
|
||||
uv pip install "./tools/sglang-simulator[aic]"
|
||||
|
||||
- name: Prebuild HiCache native hash extension
|
||||
timeout-minutes: 5
|
||||
@@ -91,6 +91,7 @@ jobs:
|
||||
PYTEST_ADDOPTS: "-s"
|
||||
TORCH_EXTENSIONS_DIR: ${{ runner.temp }}/torch-extensions
|
||||
run: |
|
||||
OMP_NUM_THREADS=2 python3 -m pytest -q tools/sglang-simulator/test/test_train_latency_model.py
|
||||
python3 -m pytest -q tools/sglang-simulator/test/test_simulation_sglang_runner.py
|
||||
python3 -m pytest -q tools/sglang-simulator/test/test_simulation_sglang_serving.py
|
||||
python3 -m pytest -q tools/sglang-simulator/test/test_simulation_cache_hit_ratio.py
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
name: PR Test SGLang Simulator (GPU)
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
check_changes:
|
||||
description: 'toJson(needs.check-changes.outputs).'
|
||||
type: string
|
||||
required: true
|
||||
caller_inputs:
|
||||
description: 'toJson(inputs) from the caller workflow.'
|
||||
type: string
|
||||
required: true
|
||||
rust_ext_artifact:
|
||||
description: 'Artifact of prebuilt Rust extension modules.'
|
||||
type: string
|
||||
default: ''
|
||||
|
||||
env:
|
||||
SGLANG_IS_IN_CI: true
|
||||
SKIP_PR_TEST_HEALTH_CHECK: ${{ (fromJson(inputs.caller_inputs).skip_pr_test_health_check || fromJson(inputs.caller_inputs).test_parallel_dispatch || fromJson(inputs.caller_inputs).run_all_tests) && 'true' || 'false' }}
|
||||
PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }}
|
||||
USE_VENV: "1"
|
||||
|
||||
jobs:
|
||||
run:
|
||||
name: simulator-test-gpu
|
||||
if: fromJson(inputs.check_changes).main_package == 'true' || fromJson(inputs.check_changes).simulator == 'true' || fromJson(inputs.check_changes).sgl_kernel == 'true'
|
||||
runs-on: ${{ fromJson(fromJson(inputs.check_changes).runs_on_map)['1-gpu-small'] }}
|
||||
timeout-minutes: 40
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ fromJson(inputs.caller_inputs).git_ref || github.sha }}
|
||||
|
||||
- uses: ./.github/actions/check-pr-test-health
|
||||
|
||||
- uses: ./.github/actions/check-maintenance
|
||||
|
||||
- name: Download kernel wheels
|
||||
if: fromJson(inputs.check_changes).sgl_kernel == 'true'
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: python/sglang/kernels/aot/dist/
|
||||
merge-multiple: true
|
||||
pattern: wheel-python3.10-cuda*
|
||||
|
||||
- uses: ./.github/actions/download-rust-ext
|
||||
with:
|
||||
artifact_name: ${{ inputs.rust_ext_artifact }}
|
||||
|
||||
- name: Install dependencies
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
CUSTOM_BUILD_SGL_KERNEL: ${{ fromJson(inputs.check_changes).sgl_kernel }}
|
||||
run: bash scripts/ci/cuda/ci_install_dependency.sh
|
||||
|
||||
- name: Test baseline collection
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1"
|
||||
run: |
|
||||
uv pip install "./tools/sglang-simulator"
|
||||
python3 -m pytest -q tools/sglang-simulator/test/test_baseline_collection_sglang.py
|
||||
|
||||
- name: Cleanup venv
|
||||
if: always()
|
||||
run: bash scripts/ci/cuda/ci_cleanup_venv.sh
|
||||
@@ -150,6 +150,15 @@ jobs:
|
||||
rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
|
||||
# No `secrets: inherit`: this hosted CPU job has no secret consumer.
|
||||
|
||||
simulator-test-gpu:
|
||||
needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build]
|
||||
if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }}
|
||||
uses: ./.github/workflows/_pr-test-simulator-gpu.yml
|
||||
with:
|
||||
check_changes: ${{ toJson(needs.check-changes.outputs) }}
|
||||
caller_inputs: ${{ toJson(inputs) }}
|
||||
rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
|
||||
|
||||
# =============================================== extra-a / extra-b ===============================================
|
||||
# `name:` keeps the job names wait-for-jobs and check-pr-test-health match on.
|
||||
extra-test:
|
||||
@@ -191,6 +200,7 @@ jobs:
|
||||
sgl-kernel-build-wheels,
|
||||
rust-ext-build,
|
||||
simulator-test-cpu,
|
||||
simulator-test-gpu,
|
||||
extra-test,
|
||||
]
|
||||
if: always()
|
||||
|
||||
@@ -221,6 +221,14 @@ Supported predictors:
|
||||
Relative predictor paths are resolved from the simulator configuration location.
|
||||
Environment variables in paths use `${NAME}` syntax.
|
||||
|
||||
## Collect baselines and train an ML predictor
|
||||
|
||||
Use [collect_baseline](src/sglang_simulator/tools/collect_baseline/README.md) to capture request traces
|
||||
and synchronized batch steps from a real SGLang server. Then use
|
||||
[train_ml_time_predictor](src/sglang_simulator/tools/train_ml_time_predictor/README.md) to train an
|
||||
18-feature latency model. Installing the package provides `sglang-simulator-collect`
|
||||
and `sglang-simulator-train`, including a small collected training example.
|
||||
|
||||
## Workload formats
|
||||
|
||||
The Autobench trace format uses timestamps in milliseconds:
|
||||
|
||||
@@ -12,6 +12,10 @@ dependencies = [
|
||||
"joblib",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
sglang-simulator-collect = "sglang_simulator.tools.collect_baseline.collect:main"
|
||||
sglang-simulator-train = "sglang_simulator.tools.train_ml_time_predictor.train:main"
|
||||
|
||||
[project.optional-dependencies]
|
||||
aic = [
|
||||
"aiconfigurator==0.10.0",
|
||||
|
||||
@@ -18,4 +18,11 @@ setup(
|
||||
packages=find_packages(where="src"),
|
||||
package_dir={"": "src"},
|
||||
py_modules=["usercustomize"],
|
||||
package_data={
|
||||
"sglang_simulator.tools.collect_baseline": ["README.md"],
|
||||
"sglang_simulator.tools.train_ml_time_predictor": [
|
||||
"README.md",
|
||||
"example_steps.jsonl",
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""ML-trained per-iter latency predictor.
|
||||
|
||||
Loads a joblib pickle of a sklearn-compatible regressor and predicts forward latency
|
||||
from batch composition features. Train one with `train_latency_model.py`.
|
||||
from batch composition features. Train one with `sglang-simulator-train`.
|
||||
|
||||
sim_config.json usage:
|
||||
"predictor": {
|
||||
@@ -10,7 +10,6 @@ sim_config.json usage:
|
||||
}
|
||||
"""
|
||||
|
||||
import math
|
||||
import os
|
||||
|
||||
import joblib
|
||||
@@ -18,47 +17,20 @@ from sglang_simulator.simulation.types import SchedulerConfig
|
||||
from sglang_simulator.spec.accelerator import AcceleratorInfo
|
||||
from sglang_simulator.spec.model import ModelInfo
|
||||
from sglang_simulator.time_predictor.base import InferTimePredictor, ScheduleBatch
|
||||
from sglang_simulator.time_predictor.ml_features import (
|
||||
ML_FEATURE_NAMES,
|
||||
extract_ml_features,
|
||||
)
|
||||
from sglang_simulator.utils import get_logger
|
||||
|
||||
logger = get_logger("sgl_simulator")
|
||||
|
||||
|
||||
class MLTimePredictor(InferTimePredictor):
|
||||
"""Per-iter latency predictor backed by an offline-trained sklearn regressor.
|
||||
"""Predict forward latency using the shared 18-feature batch representation."""
|
||||
|
||||
Features (18 dim) extracted from ScheduleBatch:
|
||||
batch_size, sum/max/min(extend), sum/max/min(past),
|
||||
sum(extend*past), sum(extend^2), sum(past^2),
|
||||
sum_attn_flops (= sum(e*(p+e/2))),
|
||||
sum(extend × max_past), log1p(sum_past), log1p(sum_attn_flops),
|
||||
batch_size × sum_extend, max_past - min_past,
|
||||
is_decode, is_prefill
|
||||
"""
|
||||
|
||||
# This ordered list is the ABI between offline training and simulation.
|
||||
# The concrete regressor algorithm is intentionally unrestricted as long
|
||||
# as it exposes sklearn-compatible predict([[18 features]]) -> [seconds].
|
||||
|
||||
FEATURE_NAMES = [
|
||||
"batch_size",
|
||||
"sum_extend",
|
||||
"max_extend",
|
||||
"min_extend",
|
||||
"sum_past",
|
||||
"max_past",
|
||||
"min_past",
|
||||
"sum_extend_x_past",
|
||||
"sum_extend_squared",
|
||||
"sum_past_squared",
|
||||
"sum_attn_flops",
|
||||
"sum_extend_x_max_past",
|
||||
"log1p_sum_past",
|
||||
"log1p_sum_attn_flops",
|
||||
"batch_size_x_sum_extend",
|
||||
"max_past_minus_min_past",
|
||||
"is_decode",
|
||||
"is_prefill",
|
||||
]
|
||||
# Feature order is shared by training and inference, regardless of regressor.
|
||||
FEATURE_NAMES = list(ML_FEATURE_NAMES)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -74,7 +46,7 @@ class MLTimePredictor(InferTimePredictor):
|
||||
if not database_path or not os.path.exists(database_path):
|
||||
raise FileNotFoundError(
|
||||
f"MLTimePredictor database_path not found: {database_path}. "
|
||||
"Train one with `train_latency_model.py` first."
|
||||
"Train one with `sglang-simulator-train` first."
|
||||
)
|
||||
|
||||
bundle = joblib.load(database_path)
|
||||
@@ -101,14 +73,12 @@ class MLTimePredictor(InferTimePredictor):
|
||||
"MLTimePredictor model must expose a callable predict() method"
|
||||
)
|
||||
|
||||
self._features = saved_features
|
||||
self._call_count = 0
|
||||
self._latency_scale = float(latency_scale)
|
||||
logger.info(
|
||||
"MLTimePredictor loaded from %s (model=%s, n_features=%d, latency_scale=%.4f)",
|
||||
database_path,
|
||||
type(self._model).__name__,
|
||||
len(self._features),
|
||||
len(saved_features),
|
||||
self._latency_scale,
|
||||
)
|
||||
|
||||
@@ -119,38 +89,6 @@ class MLTimePredictor(InferTimePredictor):
|
||||
exts = [req.extend_length for req in batch.reqs]
|
||||
pasts = [req.past_kv_length for req in batch.reqs]
|
||||
|
||||
bs = len(exts)
|
||||
sum_e = sum(exts)
|
||||
sum_p = sum(pasts)
|
||||
sum_ep = sum(e * p for e, p in zip(exts, pasts))
|
||||
sum_e2 = sum(e * e for e in exts)
|
||||
sum_p2 = sum(p * p for p in pasts)
|
||||
sum_attn = sum(e * (p + e / 2) for e, p in zip(exts, pasts))
|
||||
max_e = max(exts)
|
||||
max_p = max(pasts)
|
||||
min_e = min(exts)
|
||||
min_p = min(pasts)
|
||||
feats = extract_ml_features(exts, pasts)
|
||||
|
||||
feats = [
|
||||
bs,
|
||||
sum_e,
|
||||
max_e,
|
||||
min_e,
|
||||
sum_p,
|
||||
max_p,
|
||||
min_p,
|
||||
sum_ep,
|
||||
sum_e2,
|
||||
sum_p2,
|
||||
sum_attn,
|
||||
sum_e * max_p,
|
||||
math.log1p(sum_p),
|
||||
math.log1p(sum_attn),
|
||||
bs * sum_e,
|
||||
max_p - min_p,
|
||||
int(all(e == 1 for e in exts)),
|
||||
int(any(e > 1 for e in exts)),
|
||||
]
|
||||
|
||||
self._call_count += 1
|
||||
return float(self._model.predict([feats])[0]) * self._latency_scale
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Shared feature contract for ML latency prediction and offline training."""
|
||||
|
||||
import math
|
||||
from collections.abc import Sequence
|
||||
|
||||
ML_FEATURE_NAMES = (
|
||||
"batch_size",
|
||||
"sum_extend",
|
||||
"max_extend",
|
||||
"min_extend",
|
||||
"sum_past",
|
||||
"max_past",
|
||||
"min_past",
|
||||
"sum_extend_x_past",
|
||||
"sum_extend_squared",
|
||||
"sum_past_squared",
|
||||
"sum_attn_flops",
|
||||
"sum_extend_x_max_past",
|
||||
"log1p_sum_past",
|
||||
"log1p_sum_attn_flops",
|
||||
"batch_size_x_sum_extend",
|
||||
"max_past_minus_min_past",
|
||||
"is_decode",
|
||||
"is_prefill",
|
||||
)
|
||||
|
||||
|
||||
def extract_ml_features(
|
||||
extend_lengths: Sequence[int], past_kv_lengths: Sequence[int]
|
||||
) -> list[float]:
|
||||
"""Build the ordered 18-feature vector consumed by ``MLTimePredictor``."""
|
||||
if not extend_lengths:
|
||||
raise ValueError("cannot extract ML features from an empty batch")
|
||||
if len(extend_lengths) != len(past_kv_lengths):
|
||||
raise ValueError("extend and past-KV length vectors must have equal size")
|
||||
if any(value < 0 for value in (*extend_lengths, *past_kv_lengths)):
|
||||
raise ValueError("token lengths must be non-negative")
|
||||
|
||||
batch_size = len(extend_lengths)
|
||||
sum_extend = sum(extend_lengths)
|
||||
sum_past = sum(past_kv_lengths)
|
||||
sum_extend_x_past = sum(
|
||||
extend * past for extend, past in zip(extend_lengths, past_kv_lengths)
|
||||
)
|
||||
sum_extend_squared = sum(value * value for value in extend_lengths)
|
||||
sum_past_squared = sum(value * value for value in past_kv_lengths)
|
||||
sum_attn_flops = sum(
|
||||
extend * (past + extend / 2)
|
||||
for extend, past in zip(extend_lengths, past_kv_lengths)
|
||||
)
|
||||
max_extend = max(extend_lengths)
|
||||
max_past = max(past_kv_lengths)
|
||||
min_extend = min(extend_lengths)
|
||||
min_past = min(past_kv_lengths)
|
||||
|
||||
return [
|
||||
batch_size,
|
||||
sum_extend,
|
||||
max_extend,
|
||||
min_extend,
|
||||
sum_past,
|
||||
max_past,
|
||||
min_past,
|
||||
sum_extend_x_past,
|
||||
sum_extend_squared,
|
||||
sum_past_squared,
|
||||
sum_attn_flops,
|
||||
sum_extend * max_past,
|
||||
math.log1p(sum_past),
|
||||
math.log1p(sum_attn_flops),
|
||||
batch_size * sum_extend,
|
||||
max_past - min_past,
|
||||
int(all(value == 1 for value in extend_lengths)),
|
||||
int(any(value > 1 for value in extend_lengths)),
|
||||
]
|
||||
@@ -0,0 +1 @@
|
||||
"""Baseline collection and latency-model training tools."""
|
||||
@@ -0,0 +1,60 @@
|
||||
# Collect a baseline
|
||||
|
||||
Install the simulator package in an environment with the matching SGLang version
|
||||
and its serving dependencies. From a source checkout, use
|
||||
`pip install ./tools/sglang-simulator`; a wheel installation provides the same
|
||||
command, which can run from any directory:
|
||||
|
||||
```bash
|
||||
sglang-simulator-collect \
|
||||
--collection-output-dir /tmp/baseline-001 \
|
||||
--model-path /path/to/model \
|
||||
--disable-overlap-schedule --host 127.0.0.1 --port 30000
|
||||
```
|
||||
|
||||
Other flags are standard SGLang server arguments. The output directory must be
|
||||
new or empty. After warmup, collect a request with:
|
||||
|
||||
```bash
|
||||
curl -f -X POST http://127.0.0.1:30000/start_profile -H 'Content-Type: application/json' -d '{}'
|
||||
curl -f http://127.0.0.1:30000/generate -H 'Content-Type: application/json' \
|
||||
-d '{"text":"Hello","sampling_params":{"max_new_tokens":16,"ignore_eos":true}}'
|
||||
curl -f -X POST http://127.0.0.1:30000/stop_profile -H 'Content-Type: application/json' -d '{}'
|
||||
```
|
||||
|
||||
Replace the generate call with your benchmark; wait for all requests before stop.
|
||||
These endpoints control collection instead of the PyTorch profiler. Hooks reuse
|
||||
the simulator infrastructure without patching SGLang source. Records stay in
|
||||
memory until `stop_profile`; use a fresh output directory/server for each run.
|
||||
|
||||
Each scheduler writes three JSONL files (`TP0` shown):
|
||||
|
||||
| File | Contents |
|
||||
| --- | --- |
|
||||
| `TP0.request.jsonl` | Request ID, Unix arrival time in milliseconds, input/output lengths, disjoint device/host/disk cache-hit lengths. |
|
||||
| `TP0.raw_request.jsonl` | Arrival/queue times in seconds, completion flag, token IDs, output length and token emission latencies in seconds. |
|
||||
| `TP0.schedule_batch.jsonl` | Forward mode, per-request extend/past-KV lengths and synchronized `iter_latency` in seconds. |
|
||||
|
||||
DP/PP/EP ranks are appended when their parallel size exceeds one. Cache counts
|
||||
come from SGLang's first scheduled chunk. The first token latency starts at server
|
||||
arrival; subsequent entries measure time between output-producing steps.
|
||||
`client_created_time` in sampling `custom_params`, when supplied in Unix seconds,
|
||||
is preferred as the normalized arrival timestamp.
|
||||
|
||||
Step timing synchronizes the device before and after `Scheduler.run_batch`.
|
||||
Features are captured before execution; start/end step timestamps use
|
||||
`perf_counter`. This requires `--disable-overlap-schedule` and changes execution
|
||||
timing. The initial scope is ordinary text generation, not speculative decoding
|
||||
or multimodal feature modeling. Health probes are excluded from request traces;
|
||||
steps containing any health probe are excluded from training data.
|
||||
|
||||
Read files after a successful stop response. Export never overwrites existing
|
||||
files and retains records on failure; remove any logged partial files before
|
||||
retrying. Raw traces include token IDs: review them before sharing. Continue with
|
||||
[the training guide](../train_ml_time_predictor/README.md).
|
||||
|
||||
For a minimal GPU smoke test with a small dummy model and four requests, run:
|
||||
|
||||
```bash
|
||||
python3 -m pytest -q tools/sglang-simulator/test/test_baseline_collection_sglang.py
|
||||
```
|
||||
@@ -0,0 +1 @@
|
||||
"""Collect baseline traces from a real SGLang server."""
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Launch a real SGLang server instrumented for simulator baseline collection."""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from sglang_simulator.tools.collect_baseline.hooks import (
|
||||
COLLECTION_OUTPUT_DIR_ENV,
|
||||
TokenizerManagerCollectionHook,
|
||||
install_frontend_collection_hooks,
|
||||
run_collection_scheduler_process,
|
||||
validate_collection_hook,
|
||||
)
|
||||
|
||||
# This must happen before importing the SGLang HTTP server.
|
||||
install_frontend_collection_hooks()
|
||||
|
||||
|
||||
def _prepare_output_dir(parser: argparse.ArgumentParser, value: str) -> Path:
|
||||
output_dir = Path(value).expanduser().resolve()
|
||||
if output_dir.exists() and not output_dir.is_dir():
|
||||
parser.error(
|
||||
f"baseline collection output path is not a directory: {output_dir}"
|
||||
)
|
||||
if output_dir.is_dir() and any(output_dir.iterdir()):
|
||||
parser.error(
|
||||
f"baseline collection output directory must be empty: {output_dir}"
|
||||
)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
os.environ[COLLECTION_OUTPUT_DIR_ENV] = str(output_dir)
|
||||
return output_dir
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> None:
|
||||
from sglang.srt.entrypoints.http_server import launch_server
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
|
||||
validate_collection_hook(TokenizerManagerCollectionHook)
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
collection = parser.add_argument_group("baseline collection")
|
||||
collection.add_argument(
|
||||
"--collection-output-dir",
|
||||
default=os.getenv(COLLECTION_OUTPUT_DIR_ENV),
|
||||
required=os.getenv(COLLECTION_OUTPUT_DIR_ENV) is None,
|
||||
help=(
|
||||
"Empty directory for TP*.request.jsonl, raw_request.jsonl, "
|
||||
"and schedule_batch.jsonl"
|
||||
),
|
||||
)
|
||||
ServerArgs.add_cli_args(parser.add_argument_group("sglang"))
|
||||
|
||||
raw_args = parser.parse_args(argv)
|
||||
output_dir = _prepare_output_dir(parser, raw_args.collection_output_dir)
|
||||
server_args = ServerArgs.from_cli_args(raw_args)
|
||||
if not server_args.disable_overlap_schedule:
|
||||
parser.error(
|
||||
"--disable-overlap-schedule is required so synchronized batch "
|
||||
"latencies match ScheduleBatch execution boundaries"
|
||||
)
|
||||
|
||||
print(f"SGLang Simulator baseline collection directory: {output_dir}")
|
||||
try:
|
||||
launch_server(
|
||||
server_args,
|
||||
run_scheduler_process_func=run_collection_scheduler_process,
|
||||
)
|
||||
finally:
|
||||
kill_process_tree(os.getpid(), include_parent=False)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,409 @@
|
||||
"""Hooks for collecting requests and synchronized steps from a real SGLang server."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from functools import wraps
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
from sglang_simulator.hook import BaseHook, install_class_hooks
|
||||
from sglang_simulator.hook.class_hook_entry import is_class_hook_matched
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
COLLECTION_OUTPUT_DIR_ENV = "SGLANG_SIMULATOR_COLLECTION_DIR"
|
||||
|
||||
|
||||
@dataclass
|
||||
class RequestRecord:
|
||||
rid: str = ""
|
||||
created_time: float | None = None
|
||||
client_created_time: float | None = None
|
||||
server_created_time: float | None = None
|
||||
queue_start: float = 0.0
|
||||
queue_end: float = 0.0
|
||||
completed: bool = False
|
||||
output_length: int = 0
|
||||
input_length: int = 0
|
||||
before_adder_device_hit_len: int = 0
|
||||
final_device_hit_len: int = 0
|
||||
final_host_hit_len: int = 0
|
||||
final_disk_hit_len: int = 0
|
||||
last_event_time: float = 0.0
|
||||
gen_token_latencies: list[float] = field(default_factory=list)
|
||||
input_ids: list[int] = field(default_factory=list)
|
||||
output_ids: list[int] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CollectionState:
|
||||
active: bool = False
|
||||
requests: dict[str, RequestRecord] = field(default_factory=dict)
|
||||
schedule_batches: list[dict[str, Any]] = field(default_factory=list)
|
||||
|
||||
def start(self) -> None:
|
||||
if self.active:
|
||||
raise RuntimeError("baseline collection is already active")
|
||||
self.requests.clear()
|
||||
self.schedule_batches.clear()
|
||||
self.active = True
|
||||
|
||||
def get_request(self, rid: str) -> RequestRecord:
|
||||
record = self.requests.get(rid)
|
||||
if record is None:
|
||||
record = RequestRecord(rid=rid)
|
||||
self.requests[rid] = record
|
||||
return record
|
||||
|
||||
def stop_and_export(self) -> list[Path]:
|
||||
if not self.active:
|
||||
raise RuntimeError("baseline collection is not active")
|
||||
output_dir = os.getenv(COLLECTION_OUTPUT_DIR_ENV)
|
||||
if not output_dir:
|
||||
raise RuntimeError(
|
||||
f"{COLLECTION_OUTPUT_DIR_ENV} is required for baseline collection"
|
||||
)
|
||||
|
||||
root = Path(output_dir).expanduser()
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
prefix = _rank_prefix()
|
||||
exports = {
|
||||
"request": (_normalized_request(r) for r in self.requests.values()),
|
||||
"raw_request": (asdict(r) for r in self.requests.values()),
|
||||
"schedule_batch": iter(self.schedule_batches),
|
||||
}
|
||||
paths = []
|
||||
try:
|
||||
for kind, rows in exports.items():
|
||||
path = root / f"{prefix}.{kind}.jsonl"
|
||||
with path.open("x", encoding="utf-8") as output:
|
||||
paths.append(path)
|
||||
for row in rows:
|
||||
output.write(json.dumps(row) + "\n")
|
||||
except (OSError, TypeError, ValueError):
|
||||
# Keep in-memory records for retry; never remove an existing file.
|
||||
for path in paths:
|
||||
try:
|
||||
path.unlink()
|
||||
except OSError:
|
||||
logger.warning(
|
||||
"Failed to remove partial baseline file %s", path, exc_info=True
|
||||
)
|
||||
raise
|
||||
|
||||
self.active = False
|
||||
self.requests.clear()
|
||||
self.schedule_batches.clear()
|
||||
return paths
|
||||
|
||||
|
||||
def _rank_prefix() -> str:
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
parallel = get_parallel()
|
||||
prefix = f"TP{parallel.tp_rank}"
|
||||
for size_name, rank_name, label in (
|
||||
("dp_size", "dp_rank", "DP"),
|
||||
("pp_size", "pp_rank", "PP"),
|
||||
("moe_ep_size", "moe_ep_rank", "EP"),
|
||||
):
|
||||
if getattr(parallel, size_name) > 1:
|
||||
prefix += f"-{label}{getattr(parallel, rank_name)}"
|
||||
return prefix
|
||||
|
||||
|
||||
def _normalized_request(record: RequestRecord) -> dict[str, Any]:
|
||||
timestamp = record.created_time
|
||||
if timestamp is None:
|
||||
timestamp = record.queue_start
|
||||
return {
|
||||
"rid": record.rid,
|
||||
"timestamp": timestamp * 1000,
|
||||
"input_length": record.input_length,
|
||||
"output_length": record.output_length,
|
||||
"device_cache_hit_length": record.before_adder_device_hit_len,
|
||||
"host_cache_hit_length": record.final_host_hit_len - record.final_disk_hit_len,
|
||||
"disk_cache_hit_length": record.final_disk_hit_len,
|
||||
}
|
||||
|
||||
|
||||
COLLECTION_STATE = CollectionState()
|
||||
|
||||
|
||||
def _custom_params(sampling_params: Any) -> dict[str, Any]:
|
||||
if isinstance(sampling_params, dict):
|
||||
return sampling_params.get("custom_params") or {}
|
||||
return getattr(sampling_params, "custom_params", None) or {}
|
||||
|
||||
|
||||
def _stamp_created_time(tokenized_obj: Any) -> None:
|
||||
from sglang.srt.observability.req_time_stats import convert_time_to_realtime
|
||||
|
||||
time_stats = getattr(tokenized_obj, "time_stats", None)
|
||||
created_time = getattr(time_stats, "created_time", None)
|
||||
if not created_time:
|
||||
return
|
||||
sampling_params = tokenized_obj.sampling_params
|
||||
params = dict(_custom_params(sampling_params))
|
||||
# Convert SGLang's perf_counter arrival to Unix time before IPC.
|
||||
params["server_created_time"] = convert_time_to_realtime(created_time)
|
||||
if isinstance(sampling_params, dict):
|
||||
sampling_params["custom_params"] = params
|
||||
else:
|
||||
sampling_params.custom_params = params
|
||||
|
||||
|
||||
def _iter_received_requests(recv_reqs: Iterable[Any]) -> Iterable[Any]:
|
||||
for recv_req in recv_reqs:
|
||||
if recv_req.__class__.__name__.startswith("BatchTokenized"):
|
||||
yield from recv_req
|
||||
else:
|
||||
yield recv_req
|
||||
|
||||
|
||||
def _record_received_requests(recv_reqs: Iterable[Any]) -> None:
|
||||
if not COLLECTION_STATE.active:
|
||||
return
|
||||
from sglang.srt.managers.utils import is_health_check_generate_req
|
||||
|
||||
recv_time = time.time()
|
||||
for req in _iter_received_requests(recv_reqs):
|
||||
# Control messages (abort, profile, RPC, etc.) must not become requests.
|
||||
if not hasattr(req, "sampling_params") or not getattr(req, "rid", None):
|
||||
continue
|
||||
if is_health_check_generate_req(req):
|
||||
continue
|
||||
record = COLLECTION_STATE.get_request(str(req.rid))
|
||||
record.queue_start = recv_time
|
||||
params = _custom_params(req.sampling_params)
|
||||
record.server_created_time = params.get("server_created_time")
|
||||
record.client_created_time = params.get("client_created_time")
|
||||
record.created_time = next(
|
||||
value
|
||||
for value in (
|
||||
record.client_created_time,
|
||||
record.server_created_time,
|
||||
recv_time,
|
||||
)
|
||||
if value is not None
|
||||
)
|
||||
record.last_event_time = record.server_created_time or recv_time
|
||||
record.input_ids = list(getattr(req, "input_ids", None) or [])
|
||||
record.input_length = len(record.input_ids)
|
||||
|
||||
|
||||
def _batch_request_lengths(batch: Any) -> list[tuple[int, int]]:
|
||||
"""Read prepared batch vectors; Req lengths may lag in decode/mixed batches."""
|
||||
if not batch.reqs:
|
||||
return []
|
||||
if batch.forward_mode.is_decode():
|
||||
lengths = [(1, max(length - 1, 0)) for length in batch.seq_lens_cpu.tolist()]
|
||||
else:
|
||||
lengths = list(zip(batch.extend_lens, batch.prefix_lens, strict=True))
|
||||
if len(lengths) != len(batch.reqs):
|
||||
raise ValueError("ScheduleBatch length vectors and reqs must have equal size")
|
||||
return lengths
|
||||
|
||||
|
||||
def _synchronize_device() -> None:
|
||||
# Defer SGLang imports until the class hooks are installed.
|
||||
from sglang.srt.platforms import current_platform
|
||||
|
||||
current_platform.synchronize()
|
||||
|
||||
|
||||
def _profile_output(message: str, success: bool = True) -> Any:
|
||||
from sglang.srt.managers.io_struct import ProfileReqOutput
|
||||
|
||||
return ProfileReqOutput(success=success, message=message)
|
||||
|
||||
|
||||
def _handle_profile_request(recv_req: Any) -> Any:
|
||||
try:
|
||||
if recv_req.req_type.name == "START_PROFILE":
|
||||
COLLECTION_STATE.start()
|
||||
return _profile_output("Baseline collection started")
|
||||
paths = COLLECTION_STATE.stop_and_export()
|
||||
return _profile_output(
|
||||
"Baseline collection exported: " + ", ".join(path.name for path in paths)
|
||||
)
|
||||
except (RuntimeError, OSError, TypeError, ValueError) as exc:
|
||||
# Report API errors without terminating the scheduler.
|
||||
return _profile_output(str(exc), success=False)
|
||||
|
||||
|
||||
class TokenizerManagerCollectionHook(BaseHook):
|
||||
HOOK_CLASS_NAME = "TokenizerManager"
|
||||
HOOK_MODULE_NAME = "sglang.srt.managers.tokenizer_manager"
|
||||
|
||||
@classmethod
|
||||
def hook(cls, target: Any) -> None:
|
||||
original_one = target._send_one_request
|
||||
original_batch = target._send_batch_request
|
||||
|
||||
@wraps(original_one)
|
||||
async def send_one(
|
||||
self: Any, tokenized_obj: Any, *args: Any, **kwargs: Any
|
||||
) -> Any:
|
||||
_stamp_created_time(tokenized_obj)
|
||||
return await original_one(self, tokenized_obj, *args, **kwargs)
|
||||
|
||||
@wraps(original_batch)
|
||||
async def send_batch(
|
||||
self: Any, tokenized_objs: list[Any], *args: Any, **kwargs: Any
|
||||
) -> Any:
|
||||
for obj in tokenized_objs:
|
||||
_stamp_created_time(obj)
|
||||
return await original_batch(self, tokenized_objs, *args, **kwargs)
|
||||
|
||||
target._send_one_request = send_one
|
||||
target._send_batch_request = send_batch
|
||||
|
||||
|
||||
class SchedulerCollectionHook(BaseHook):
|
||||
HOOK_CLASS_NAME = "Scheduler"
|
||||
HOOK_MODULE_NAME = "sglang.srt.managers.scheduler"
|
||||
|
||||
@classmethod
|
||||
def hook(cls, target: Any) -> None:
|
||||
from sglang.srt.managers.utils import is_health_check_generate_req
|
||||
|
||||
original_input = target.process_input_requests
|
||||
original_run = target.run_batch
|
||||
original_result = target.process_batch_result
|
||||
original_profiler = target.init_profiler
|
||||
|
||||
@wraps(original_input)
|
||||
def process_input(
|
||||
self: Any, recv_reqs: list[Any], *args: Any, **kwargs: Any
|
||||
) -> Any:
|
||||
_record_received_requests(recv_reqs)
|
||||
return original_input(self, recv_reqs, *args, **kwargs)
|
||||
|
||||
@wraps(original_run)
|
||||
def run_batch(self: Any, batch: Any, *args: Any, **kwargs: Any) -> Any:
|
||||
if not COLLECTION_STATE.active:
|
||||
return original_run(self, batch, *args, **kwargs)
|
||||
|
||||
# Workers may mutate the batch, so snapshot before forward.
|
||||
requests = tuple(batch.reqs or ())
|
||||
scheduled_time = time.time()
|
||||
for req in requests:
|
||||
record = COLLECTION_STATE.requests.get(str(req.rid))
|
||||
if record is not None and record.queue_end == 0:
|
||||
record.queue_end = scheduled_time
|
||||
record.input_ids = list(req.origin_input_ids)
|
||||
record.input_length = len(record.input_ids)
|
||||
# Preserve SGLang's first-chunk counts across later chunks/retractions.
|
||||
device = int(req.cached_tokens_device)
|
||||
host = int(req.cached_tokens_host)
|
||||
storage = int(req.cached_tokens_storage)
|
||||
record.before_adder_device_hit_len = device
|
||||
record.final_device_hit_len = device + host + storage
|
||||
record.final_host_hit_len = host + storage
|
||||
record.final_disk_hit_len = storage
|
||||
|
||||
# A mixed step's latency includes the probe: exclude the entire step.
|
||||
if any(is_health_check_generate_req(req) for req in requests):
|
||||
return original_run(self, batch, *args, **kwargs)
|
||||
|
||||
lengths = _batch_request_lengths(batch)
|
||||
row = {
|
||||
"forward_mode": int(batch.forward_mode),
|
||||
"request_infos": [
|
||||
{
|
||||
"rid": str(req.rid),
|
||||
"extend_input_len": extend,
|
||||
"prefix_indices_len": past,
|
||||
"output_ids_len": len(req.output_ids),
|
||||
}
|
||||
for req, (extend, past) in zip(requests, lengths, strict=True)
|
||||
],
|
||||
}
|
||||
|
||||
_synchronize_device()
|
||||
start = time.perf_counter()
|
||||
result = original_run(self, batch, *args, **kwargs)
|
||||
_synchronize_device()
|
||||
end = time.perf_counter()
|
||||
row.update(
|
||||
start_timestamp=start, end_timestamp=end, iter_latency=end - start
|
||||
)
|
||||
COLLECTION_STATE.schedule_batches.append(row)
|
||||
return result
|
||||
|
||||
@wraps(original_result)
|
||||
def process_result(self: Any, batch: Any, *args: Any, **kwargs: Any) -> Any:
|
||||
if not COLLECTION_STATE.active:
|
||||
return original_result(self, batch, *args, **kwargs)
|
||||
requests = [(req, len(req.output_ids)) for req in (batch.reqs or ())]
|
||||
result = original_result(self, batch, *args, **kwargs)
|
||||
event_time = time.time()
|
||||
for req, previous_length in requests:
|
||||
record = COLLECTION_STATE.requests.get(str(req.rid))
|
||||
if record is None:
|
||||
continue
|
||||
record.output_length = len(req.output_ids)
|
||||
if record.output_length > previous_length:
|
||||
record.gen_token_latencies.append(
|
||||
max(event_time - record.last_event_time, 0.0)
|
||||
)
|
||||
record.last_event_time = event_time
|
||||
if req.finished():
|
||||
record.output_ids = list(req.output_ids)
|
||||
record.completed = True
|
||||
return result
|
||||
|
||||
@wraps(original_profiler)
|
||||
def init_profiler(self: Any, *args: Any, **kwargs: Any) -> Any:
|
||||
result = original_profiler(self, *args, **kwargs)
|
||||
self.profiler_manager._profile = _handle_profile_request
|
||||
return result
|
||||
|
||||
target.process_input_requests = process_input
|
||||
target.run_batch = run_batch
|
||||
target.process_batch_result = process_result
|
||||
target.init_profiler = init_profiler
|
||||
|
||||
|
||||
_FRONTEND_HOOKS_INSTALLED = False
|
||||
_SCHEDULER_HOOKS_INSTALLED = False
|
||||
|
||||
|
||||
def install_frontend_collection_hooks() -> None:
|
||||
global _FRONTEND_HOOKS_INSTALLED
|
||||
if not _FRONTEND_HOOKS_INSTALLED:
|
||||
install_class_hooks(TokenizerManagerCollectionHook)
|
||||
_FRONTEND_HOOKS_INSTALLED = True
|
||||
|
||||
|
||||
def install_scheduler_collection_hooks() -> None:
|
||||
global _SCHEDULER_HOOKS_INSTALLED
|
||||
if not _SCHEDULER_HOOKS_INSTALLED:
|
||||
install_class_hooks(SchedulerCollectionHook)
|
||||
_SCHEDULER_HOOKS_INSTALLED = True
|
||||
|
||||
|
||||
def validate_collection_hook(hook: type[BaseHook]) -> None:
|
||||
if is_class_hook_matched(hook):
|
||||
return
|
||||
raise RuntimeError(
|
||||
"Required SGLang Simulator hook did not match an imported SGLang class: "
|
||||
f"{hook.__name__} ({hook.HOOK_MODULE_NAME}.{hook.HOOK_CLASS_NAME}). "
|
||||
"The collector must be adapted to this SGLang revision."
|
||||
)
|
||||
|
||||
|
||||
def run_collection_scheduler_process(*args: Any, **kwargs: Any) -> Any:
|
||||
"""Install collection hooks before importing and running Scheduler."""
|
||||
install_scheduler_collection_hooks()
|
||||
from sglang.srt.managers.scheduler import run_scheduler_process
|
||||
|
||||
# TokenizerManager belongs to the API process; validate only Scheduler here.
|
||||
validate_collection_hook(SchedulerCollectionHook)
|
||||
return run_scheduler_process(*args, **kwargs)
|
||||
@@ -0,0 +1,61 @@
|
||||
# Train an ML time predictor
|
||||
|
||||
Install the simulator package (`pip install ./tools/sglang-simulator` from a
|
||||
source checkout, or install its wheel). The command and example data are included
|
||||
in the package, so this works from any directory:
|
||||
|
||||
```bash
|
||||
example=$(python -c "from importlib.resources import files; print(files('sglang_simulator.tools.train_ml_time_predictor') / 'example_steps.jsonl')")
|
||||
sglang-simulator-train --data-root "$example" --out-dir /tmp/example-predictor
|
||||
```
|
||||
|
||||
`example_steps.jsonl` contains collected EXTEND steps from a tiny, two-layer Qwen3
|
||||
model with dummy weights on one A100, using repeated synthetic token sequences.
|
||||
Only numeric feature inputs and measured latencies are retained. It demonstrates
|
||||
the file format and training flow; it cannot produce a useful production predictor
|
||||
or demonstrate prediction accuracy.
|
||||
|
||||
For real data, replace `--data-root` with a [collection](../collect_baseline/README.md)
|
||||
directory. The trainer recursively reads `TP0*.schedule_batch.jsonl` to avoid
|
||||
counting duplicate tensor-parallel ranks. Repeat `--data-root` for more cases, or
|
||||
pass explicit files for a different rank selection. Use data collected for the
|
||||
same target model, hardware and serving configuration.
|
||||
|
||||
The default filter keeps EXTEND (`forward_mode=1`) steps with nonempty batches and
|
||||
finite latencies in `(0, 30]` seconds. Use `--forward-modes 1 2` to include DECODE
|
||||
and `--max-latency` to change the upper bound. Malformed records fail with their
|
||||
file and line; skipped row counts are included in the summary.
|
||||
|
||||
Training shares the simulator's ordered 18-feature definition and uses monotonic
|
||||
HistGradientBoosting regressors. The new or empty output directory receives:
|
||||
|
||||
- `latency_model_hgbmono_l2.pkl`: squared-error model.
|
||||
- `latency_model_hgbmono_p50.pkl`: median-quantile model.
|
||||
- `summary.json`: seed, split sizes, skipped counts and held-out MAE (ms)/MAPE (%). <!-- codespell:ignore mape -->
|
||||
|
||||
Both models predict seconds. `l2` names the squared-error loss, not an L2 cache;
|
||||
`p50` estimates the median. Compare them on independent data for your workload.
|
||||
|
||||
By default, each collection directory is split 50/50 with seed 42 (single-row
|
||||
cases stay in training). Metrics evaluate this split; saved models are then
|
||||
refitted on all input rows. This measures interpolation within collected cases.
|
||||
Use repeatable `--test-data-root` arguments for independent evaluation cases;
|
||||
those rows never enter fitting or the saved models. Same-directory overlaps are
|
||||
rejected. Copied data is not detected: keep independent collections separate.
|
||||
Choose one loss with `--loss l2` or `--loss p50` if needed.
|
||||
|
||||
```bash
|
||||
sglang-simulator-train \
|
||||
--data-root /path/to/train-collection \
|
||||
--test-data-root /path/to/test-collection --out-dir /tmp/predictor
|
||||
```
|
||||
|
||||
Point the simulator configuration at the selected bundle:
|
||||
|
||||
```json
|
||||
{"predictor": {"name": "ml", "database_path": "/tmp/example-predictor/latency_model_hgbmono_l2.pkl"}}
|
||||
```
|
||||
|
||||
Use the same scikit-learn version for training and inference, and load only
|
||||
trusted joblib files. Before production use, validate the selected model on
|
||||
independent step data and complete simulator workloads.
|
||||
@@ -0,0 +1 @@
|
||||
"""Train latency predictors from collected baseline steps."""
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
{"forward_mode": 1, "request_infos": [{"extend_input_len": 1, "prefix_indices_len": 255}], "iter_latency": 0.002675119787454605}
|
||||
{"forward_mode": 1, "request_infos": [{"extend_input_len": 8, "prefix_indices_len": 256}], "iter_latency": 0.0030074845999479294}
|
||||
{"forward_mode": 1, "request_infos": [{"extend_input_len": 128, "prefix_indices_len": 0}], "iter_latency": 0.0027154013514518738}
|
||||
{"forward_mode": 1, "request_infos": [{"extend_input_len": 128, "prefix_indices_len": 128}], "iter_latency": 0.0027412977069616318}
|
||||
{"forward_mode": 1, "request_infos": [{"extend_input_len": 16, "prefix_indices_len": 256}], "iter_latency": 0.002909639850258827}
|
||||
{"forward_mode": 1, "request_infos": [{"extend_input_len": 8, "prefix_indices_len": 272}], "iter_latency": 0.002743590623140335}
|
||||
{"forward_mode": 1, "request_infos": [{"extend_input_len": 128, "prefix_indices_len": 0}], "iter_latency": 0.002713024616241455}
|
||||
{"forward_mode": 1, "request_infos": [{"extend_input_len": 128, "prefix_indices_len": 128}], "iter_latency": 0.0027074161916971207}
|
||||
{"forward_mode": 1, "request_infos": [{"extend_input_len": 32, "prefix_indices_len": 256}], "iter_latency": 0.0030285529792308807}
|
||||
{"forward_mode": 1, "request_infos": [{"extend_input_len": 8, "prefix_indices_len": 288}], "iter_latency": 0.0027687866240739822}
|
||||
{"forward_mode": 1, "request_infos": [{"extend_input_len": 128, "prefix_indices_len": 0}], "iter_latency": 0.002656087279319763}
|
||||
{"forward_mode": 1, "request_infos": [{"extend_input_len": 64, "prefix_indices_len": 128}, {"extend_input_len": 64, "prefix_indices_len": 0}], "iter_latency": 1.545724280178547}
|
||||
{"forward_mode": 1, "request_infos": [{"extend_input_len": 128, "prefix_indices_len": 64}], "iter_latency": 0.0031625479459762573}
|
||||
{"forward_mode": 1, "request_infos": [{"extend_input_len": 8, "prefix_indices_len": 192}], "iter_latency": 0.0028553567826747894}
|
||||
@@ -0,0 +1,216 @@
|
||||
"""Train MLTimePredictor bundles from synchronized SGLang schedule-batch JSONL."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
import joblib
|
||||
import numpy as np
|
||||
import sklearn
|
||||
from sglang_simulator.time_predictor.ml_features import (
|
||||
ML_FEATURE_NAMES,
|
||||
extract_ml_features,
|
||||
)
|
||||
from sklearn.ensemble import HistGradientBoostingRegressor
|
||||
|
||||
MONOTONIC_FEATURES = {
|
||||
"batch_size",
|
||||
"sum_extend",
|
||||
"sum_past",
|
||||
"sum_extend_x_past",
|
||||
"sum_extend_squared",
|
||||
"sum_past_squared",
|
||||
"sum_attn_flops",
|
||||
"sum_extend_x_max_past",
|
||||
"log1p_sum_past",
|
||||
"log1p_sum_attn_flops",
|
||||
"batch_size_x_sum_extend",
|
||||
}
|
||||
|
||||
|
||||
def load_rows(roots, forward_modes, max_latency):
|
||||
files = set()
|
||||
for value in roots:
|
||||
root = Path(value).expanduser().resolve()
|
||||
if not root.exists():
|
||||
raise FileNotFoundError(root)
|
||||
files.update(
|
||||
[root] if root.is_file() else root.rglob("TP0*.schedule_batch.jsonl")
|
||||
)
|
||||
if not files:
|
||||
raise ValueError(f"no schedule files found under {roots}")
|
||||
rows, skipped = [], Counter()
|
||||
for path in sorted(files):
|
||||
with path.open() as source:
|
||||
for number, line in enumerate(source, 1):
|
||||
try:
|
||||
item = json.loads(line)
|
||||
if not isinstance(item, dict):
|
||||
raise ValueError("expected a JSON object")
|
||||
if item.get("forward_mode") not in forward_modes:
|
||||
skipped["mode"] += 1
|
||||
continue
|
||||
requests = item["request_infos"]
|
||||
if not requests:
|
||||
skipped["empty"] += 1
|
||||
continue
|
||||
latency = float(item["iter_latency"])
|
||||
if not math.isfinite(latency) or not 0 < latency <= max_latency:
|
||||
skipped["latency"] += 1
|
||||
continue
|
||||
extend = [r["extend_input_len"] for r in requests]
|
||||
past = [r["prefix_indices_len"] for r in requests]
|
||||
if any(type(v) is not int or v < 0 for v in extend + past):
|
||||
raise ValueError("token lengths must be non-negative integers")
|
||||
features = extract_ml_features(extend, past)
|
||||
except (KeyError, TypeError, ValueError, OverflowError) as exc:
|
||||
raise ValueError(f"{path}:{number}: {exc}") from exc
|
||||
rows.append((path.parent, features, latency))
|
||||
return rows, dict(skipped)
|
||||
|
||||
|
||||
def split_rows(rows, seed):
|
||||
"""Deterministic 50/50 split within each case (collection directory)."""
|
||||
by_case = defaultdict(list)
|
||||
for row in rows:
|
||||
by_case[row[0]].append(row)
|
||||
random = np.random.RandomState(seed)
|
||||
train, test = [], []
|
||||
for case in by_case.values():
|
||||
order = random.permutation(len(case))
|
||||
test_count = len(case) // 2
|
||||
test.extend(case[i] for i in order[:test_count])
|
||||
train.extend(case[i] for i in order[test_count:])
|
||||
return train, test
|
||||
|
||||
|
||||
def as_arrays(rows):
|
||||
return np.asarray([r[1] for r in rows]), np.asarray([r[2] for r in rows])
|
||||
|
||||
|
||||
def train_model(rows, loss, seed):
|
||||
model = HistGradientBoostingRegressor(
|
||||
loss="quantile" if loss == "p50" else "squared_error",
|
||||
quantile=0.5 if loss == "p50" else None,
|
||||
max_iter=400,
|
||||
learning_rate=0.04,
|
||||
max_leaf_nodes=63,
|
||||
l2_regularization=0.1,
|
||||
monotonic_cst=[int(name in MONOTONIC_FEATURES) for name in ML_FEATURE_NAMES],
|
||||
early_stopping=len(rows) > 1,
|
||||
validation_fraction=0.15,
|
||||
n_iter_no_change=20,
|
||||
random_state=seed,
|
||||
)
|
||||
return model.fit(*as_arrays(rows))
|
||||
|
||||
|
||||
def parse_args(argv=None):
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--data-root",
|
||||
action="append",
|
||||
required=True,
|
||||
help="Collection directory or JSONL file; repeatable.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--test-data-root",
|
||||
action="append",
|
||||
default=[],
|
||||
help="Independent evaluation data, excluded from fitting; repeatable.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--out-dir", required=True, help="New or empty output directory."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--forward-modes",
|
||||
type=int,
|
||||
nargs="+",
|
||||
default=[1],
|
||||
help="SGLang ForwardMode values; default: EXTEND (1).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-latency",
|
||||
type=float,
|
||||
default=30.0,
|
||||
help="Largest retained latency, in seconds.",
|
||||
)
|
||||
parser.add_argument("--seed", type=int, default=42)
|
||||
parser.add_argument("--loss", choices=("l2", "p50", "both"), default="both")
|
||||
args = parser.parse_args(argv)
|
||||
if not math.isfinite(args.max_latency) or args.max_latency <= 0:
|
||||
parser.error("--max-latency must be finite and positive")
|
||||
return args
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
args = parse_args(argv)
|
||||
output = Path(args.out_dir).expanduser()
|
||||
if output.exists() and (not output.is_dir() or any(output.iterdir())):
|
||||
raise FileExistsError(f"training output directory must be empty: {output}")
|
||||
rows, skipped = load_rows(args.data_root, args.forward_modes, args.max_latency)
|
||||
if not rows:
|
||||
raise ValueError(
|
||||
f"no usable training rows under {args.data_root}; skipped={skipped}"
|
||||
)
|
||||
if args.test_data_root:
|
||||
test, test_skipped = load_rows(
|
||||
args.test_data_root, args.forward_modes, args.max_latency
|
||||
)
|
||||
if not test:
|
||||
raise ValueError(
|
||||
f"no usable test rows under {args.test_data_root}; skipped={test_skipped}"
|
||||
)
|
||||
if {r[0] for r in rows} & {r[0] for r in test}:
|
||||
raise ValueError(
|
||||
"training and test roots contain overlapping collection cases"
|
||||
)
|
||||
train = rows
|
||||
else:
|
||||
train, test = split_rows(rows, args.seed)
|
||||
test_skipped = {}
|
||||
if not train or not test:
|
||||
raise ValueError(
|
||||
"need nonempty training and test data; provide a case with at least two usable rows or --test-data-root"
|
||||
)
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
summary = {
|
||||
"seed": args.seed,
|
||||
"split": "independent" if args.test_data_root else "within_case_50_50",
|
||||
"train_rows": len(train),
|
||||
"test_rows": len(test),
|
||||
"final_fit_rows": len(rows),
|
||||
"skipped": skipped,
|
||||
"test_skipped": test_skipped,
|
||||
"metrics": {},
|
||||
}
|
||||
features, latency = as_arrays(test)
|
||||
for loss in ("l2", "p50") if args.loss == "both" else (args.loss,):
|
||||
model = train_model(train, loss, args.seed)
|
||||
errors = np.abs(model.predict(features) - latency)
|
||||
summary["metrics"][loss] = {
|
||||
"MAE_ms": float(errors.mean() * 1000),
|
||||
"MAPE_pct": float((errors / latency).mean() * 100),
|
||||
} # codespell:ignore mape
|
||||
if not args.test_data_root:
|
||||
model = train_model(rows, loss, args.seed)
|
||||
# Runtime validates the ordered feature ABI before making predictions.
|
||||
bundle = {
|
||||
"model": model,
|
||||
"features": list(ML_FEATURE_NAMES),
|
||||
"sklearn_version": sklearn.__version__,
|
||||
}
|
||||
with (output / f"latency_model_hgbmono_{loss}.pkl").open("xb") as destination:
|
||||
joblib.dump(bundle, destination)
|
||||
with (output / "summary.json").open("x") as destination:
|
||||
json.dump(summary, destination, indent=2)
|
||||
destination.write("\n")
|
||||
print(json.dumps(summary, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,127 @@
|
||||
"""GPU example: start collection, generate responses, export the traces."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def collection_server(tmp_path):
|
||||
simulator_root = Path(__file__).resolve().parents[1]
|
||||
model = tmp_path / "model"
|
||||
model.mkdir()
|
||||
config = json.loads(
|
||||
(simulator_root / "test/assets/qwen3-8b/config.json").read_text()
|
||||
)
|
||||
# Small dummy model: this tests lifecycle and data contracts, not accuracy.
|
||||
config.update(
|
||||
hidden_size=256,
|
||||
intermediate_size=512,
|
||||
num_hidden_layers=2,
|
||||
num_attention_heads=2,
|
||||
num_key_value_heads=2,
|
||||
max_window_layers=2,
|
||||
vocab_size=1024,
|
||||
bos_token_id=1,
|
||||
eos_token_id=2,
|
||||
)
|
||||
(model / "config.json").write_text(json.dumps(config))
|
||||
with socket.socket() as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
port = sock.getsockname()[1]
|
||||
url = f"http://127.0.0.1:{port}"
|
||||
output = tmp_path / "baseline"
|
||||
command = [
|
||||
"sglang-simulator-collect",
|
||||
f"--collection-output-dir={output}",
|
||||
f"--model-path={model}",
|
||||
"--load-format=dummy",
|
||||
"--skip-tokenizer-init",
|
||||
"--disable-overlap-schedule",
|
||||
"--device=cuda",
|
||||
"--cuda-graph-backend-decode=disabled",
|
||||
"--cuda-graph-backend-prefill=disabled",
|
||||
"--chunked-prefill-size=128",
|
||||
"--max-total-tokens=4096",
|
||||
"--mem-fraction-static=0.1",
|
||||
"--host=127.0.0.1",
|
||||
f"--port={port}",
|
||||
]
|
||||
log_path = tmp_path / "server.log"
|
||||
with log_path.open("w") as log:
|
||||
server = subprocess.Popen(
|
||||
command,
|
||||
stdout=log,
|
||||
stderr=subprocess.STDOUT,
|
||||
env={**os.environ, "OMP_NUM_THREADS": "2"},
|
||||
start_new_session=True,
|
||||
)
|
||||
try:
|
||||
deadline = time.monotonic() + 180
|
||||
while time.monotonic() < deadline:
|
||||
assert server.poll() is None, log_path.read_text()
|
||||
try:
|
||||
if requests.get(url + "/health", timeout=1).ok:
|
||||
break
|
||||
except requests.RequestException:
|
||||
pass
|
||||
time.sleep(0.2)
|
||||
else:
|
||||
raise AssertionError(log_path.read_text())
|
||||
|
||||
yield url, output, log_path
|
||||
finally:
|
||||
if server.poll() is None:
|
||||
os.killpg(server.pid, signal.SIGTERM)
|
||||
try:
|
||||
server.wait(timeout=15)
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(server.pid, signal.SIGKILL)
|
||||
server.wait()
|
||||
|
||||
|
||||
def test_baseline_collection(collection_server):
|
||||
url, output, log_path = collection_server
|
||||
|
||||
def post(path, payload=None):
|
||||
response = requests.post(url + path, json=payload or {}, timeout=60)
|
||||
assert response.ok, response.text + "\n" + log_path.read_text()
|
||||
|
||||
post("/start_profile")
|
||||
for inputs in ([10] * 256, [20] * 192, [[30] * 128, [40] * 200]):
|
||||
post(
|
||||
"/generate",
|
||||
{
|
||||
"input_ids": inputs,
|
||||
"sampling_params": {"max_new_tokens": 4, "ignore_eos": True},
|
||||
},
|
||||
)
|
||||
post("/stop_profile")
|
||||
|
||||
def read(kind):
|
||||
return [
|
||||
json.loads(line)
|
||||
for line in (output / f"TP0.{kind}.jsonl").read_text().splitlines()
|
||||
]
|
||||
|
||||
requests_trace, raw, steps = (
|
||||
read("request"),
|
||||
read("raw_request"),
|
||||
read("schedule_batch"),
|
||||
)
|
||||
request_ids = {r["rid"] for r in raw}
|
||||
assert len(requests_trace) == len(raw) == len(request_ids) == 4
|
||||
assert sorted(r["input_length"] for r in requests_trace) == [128, 192, 200, 256]
|
||||
assert all(r["output_length"] == 4 for r in requests_trace)
|
||||
assert all(r["completed"] and len(r["output_ids"]) == 4 for r in raw)
|
||||
assert {r["rid"] for r in requests_trace} == request_ids
|
||||
assert {r["rid"] for step in steps for r in step["request_infos"]} == request_ids
|
||||
assert {step["forward_mode"] for step in steps} == {1, 2} # Prefill and decode.
|
||||
assert all(step["iter_latency"] > 0 for step in steps)
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Minimal example: train on collected steps, then load the models for prediction."""
|
||||
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
|
||||
from sglang_simulator.time_predictor.base import ScheduleBatch, ScheduleRequest
|
||||
from sglang_simulator.time_predictor.ml import MLTimePredictor
|
||||
from sglang_simulator.tools.train_ml_time_predictor import train as trainer
|
||||
|
||||
|
||||
def test_train_and_predict(tmp_path):
|
||||
example = Path(trainer.__file__).with_name("example_steps.jsonl")
|
||||
output = tmp_path / "models"
|
||||
trainer.main(["--data-root", str(example), "--out-dir", str(output)])
|
||||
|
||||
summary = json.loads((output / "summary.json").read_text())
|
||||
assert summary["train_rows"] > 0 and summary["test_rows"] > 0
|
||||
assert summary["train_rows"] + summary["test_rows"] == summary["final_fit_rows"]
|
||||
assert set(summary["metrics"]) == {"l2", "p50"}
|
||||
|
||||
batch = ScheduleBatch(
|
||||
[
|
||||
ScheduleRequest(extend_length=4, past_kv_length=12),
|
||||
ScheduleRequest(extend_length=8, past_kv_length=0),
|
||||
]
|
||||
)
|
||||
for loss in ("l2", "p50"):
|
||||
model = output / f"latency_model_hgbmono_{loss}.pkl"
|
||||
predictor = MLTimePredictor(None, None, None, str(model))
|
||||
latency = predictor.predict_infer_time(batch)
|
||||
assert math.isfinite(latency) and latency > 0
|
||||
Reference in New Issue
Block a user