diff --git a/.github/workflows/_pr-test-check-changes.yml b/.github/workflows/_pr-test-check-changes.yml index 5670f4cddb..8f117225e4 100644 --- a/.github/workflows/_pr-test-check-changes.yml +++ b/.github/workflows/_pr-test-check-changes.yml @@ -130,6 +130,7 @@ jobs: simulator: - ".github/workflows/pr-test-extra.yml" - ".github/workflows/_pr-test-simulator-cpu.yml" + - ".github/workflows/_pr-test-simulator-gpu.yml" - "benchmark/simulator/**/!(*.md)" - "tools/sglang-simulator/**/!(*.md)" multimodal_gen: diff --git a/.github/workflows/_pr-test-simulator-cpu.yml b/.github/workflows/_pr-test-simulator-cpu.yml index 40886e6b77..b7098417c6 100644 --- a/.github/workflows/_pr-test-simulator-cpu.yml +++ b/.github/workflows/_pr-test-simulator-cpu.yml @@ -70,7 +70,7 @@ jobs: run: | uv pip install -e "python" --index-strategy unsafe-best-match --prerelease allow uv pip install pytest - uv pip install -e "tools/sglang-simulator[aic]" + uv pip install "./tools/sglang-simulator[aic]" - name: Prebuild HiCache native hash extension timeout-minutes: 5 @@ -91,6 +91,7 @@ jobs: PYTEST_ADDOPTS: "-s" TORCH_EXTENSIONS_DIR: ${{ runner.temp }}/torch-extensions run: | + OMP_NUM_THREADS=2 python3 -m pytest -q tools/sglang-simulator/test/test_train_latency_model.py python3 -m pytest -q tools/sglang-simulator/test/test_simulation_sglang_runner.py python3 -m pytest -q tools/sglang-simulator/test/test_simulation_sglang_serving.py python3 -m pytest -q tools/sglang-simulator/test/test_simulation_cache_hit_ratio.py diff --git a/.github/workflows/_pr-test-simulator-gpu.yml b/.github/workflows/_pr-test-simulator-gpu.yml new file mode 100644 index 0000000000..6d247dfe66 --- /dev/null +++ b/.github/workflows/_pr-test-simulator-gpu.yml @@ -0,0 +1,68 @@ +name: PR Test SGLang Simulator (GPU) + +on: + workflow_call: + inputs: + check_changes: + description: 'toJson(needs.check-changes.outputs).' + type: string + required: true + caller_inputs: + description: 'toJson(inputs) from the caller workflow.' + type: string + required: true + rust_ext_artifact: + description: 'Artifact of prebuilt Rust extension modules.' + type: string + default: '' + +env: + SGLANG_IS_IN_CI: true + SKIP_PR_TEST_HEALTH_CHECK: ${{ (fromJson(inputs.caller_inputs).skip_pr_test_health_check || fromJson(inputs.caller_inputs).test_parallel_dispatch || fromJson(inputs.caller_inputs).run_all_tests) && 'true' || 'false' }} + PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }} + USE_VENV: "1" + +jobs: + run: + name: simulator-test-gpu + if: fromJson(inputs.check_changes).main_package == 'true' || fromJson(inputs.check_changes).simulator == 'true' || fromJson(inputs.check_changes).sgl_kernel == 'true' + runs-on: ${{ fromJson(fromJson(inputs.check_changes).runs_on_map)['1-gpu-small'] }} + timeout-minutes: 40 + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ fromJson(inputs.caller_inputs).git_ref || github.sha }} + + - uses: ./.github/actions/check-pr-test-health + + - uses: ./.github/actions/check-maintenance + + - name: Download kernel wheels + if: fromJson(inputs.check_changes).sgl_kernel == 'true' + uses: actions/download-artifact@v4 + with: + path: python/sglang/kernels/aot/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda* + + - uses: ./.github/actions/download-rust-ext + with: + artifact_name: ${{ inputs.rust_ext_artifact }} + + - name: Install dependencies + timeout-minutes: 20 + env: + CUSTOM_BUILD_SGL_KERNEL: ${{ fromJson(inputs.check_changes).sgl_kernel }} + run: bash scripts/ci/cuda/ci_install_dependency.sh + + - name: Test baseline collection + timeout-minutes: 10 + env: + PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1" + run: | + uv pip install "./tools/sglang-simulator" + python3 -m pytest -q tools/sglang-simulator/test/test_baseline_collection_sglang.py + + - name: Cleanup venv + if: always() + run: bash scripts/ci/cuda/ci_cleanup_venv.sh diff --git a/.github/workflows/pr-test-extra.yml b/.github/workflows/pr-test-extra.yml index 278ea3bf29..58668d3089 100644 --- a/.github/workflows/pr-test-extra.yml +++ b/.github/workflows/pr-test-extra.yml @@ -150,6 +150,15 @@ jobs: rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} # No `secrets: inherit`: this hosted CPU job has no secret consumer. + simulator-test-gpu: + needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build] + if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }} + uses: ./.github/workflows/_pr-test-simulator-gpu.yml + with: + check_changes: ${{ toJson(needs.check-changes.outputs) }} + caller_inputs: ${{ toJson(inputs) }} + rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} + # =============================================== extra-a / extra-b =============================================== # `name:` keeps the job names wait-for-jobs and check-pr-test-health match on. extra-test: @@ -191,6 +200,7 @@ jobs: sgl-kernel-build-wheels, rust-ext-build, simulator-test-cpu, + simulator-test-gpu, extra-test, ] if: always() diff --git a/tools/sglang-simulator/README.md b/tools/sglang-simulator/README.md index 2b453e7b4b..7b91f89ddd 100644 --- a/tools/sglang-simulator/README.md +++ b/tools/sglang-simulator/README.md @@ -221,6 +221,14 @@ Supported predictors: Relative predictor paths are resolved from the simulator configuration location. Environment variables in paths use `${NAME}` syntax. +## Collect baselines and train an ML predictor + +Use [collect_baseline](src/sglang_simulator/tools/collect_baseline/README.md) to capture request traces +and synchronized batch steps from a real SGLang server. Then use +[train_ml_time_predictor](src/sglang_simulator/tools/train_ml_time_predictor/README.md) to train an +18-feature latency model. Installing the package provides `sglang-simulator-collect` +and `sglang-simulator-train`, including a small collected training example. + ## Workload formats The Autobench trace format uses timestamps in milliseconds: diff --git a/tools/sglang-simulator/pyproject.toml b/tools/sglang-simulator/pyproject.toml index dcbd400044..8b31c1649e 100644 --- a/tools/sglang-simulator/pyproject.toml +++ b/tools/sglang-simulator/pyproject.toml @@ -12,6 +12,10 @@ dependencies = [ "joblib", ] +[project.scripts] +sglang-simulator-collect = "sglang_simulator.tools.collect_baseline.collect:main" +sglang-simulator-train = "sglang_simulator.tools.train_ml_time_predictor.train:main" + [project.optional-dependencies] aic = [ "aiconfigurator==0.10.0", diff --git a/tools/sglang-simulator/setup.py b/tools/sglang-simulator/setup.py index bc7ecf81a8..89a18b19f9 100644 --- a/tools/sglang-simulator/setup.py +++ b/tools/sglang-simulator/setup.py @@ -18,4 +18,11 @@ setup( packages=find_packages(where="src"), package_dir={"": "src"}, py_modules=["usercustomize"], + package_data={ + "sglang_simulator.tools.collect_baseline": ["README.md"], + "sglang_simulator.tools.train_ml_time_predictor": [ + "README.md", + "example_steps.jsonl", + ], + }, ) diff --git a/tools/sglang-simulator/src/sglang_simulator/time_predictor/ml.py b/tools/sglang-simulator/src/sglang_simulator/time_predictor/ml.py index d495695567..5df3a9e5a3 100644 --- a/tools/sglang-simulator/src/sglang_simulator/time_predictor/ml.py +++ b/tools/sglang-simulator/src/sglang_simulator/time_predictor/ml.py @@ -1,7 +1,7 @@ """ML-trained per-iter latency predictor. Loads a joblib pickle of a sklearn-compatible regressor and predicts forward latency -from batch composition features. Train one with `train_latency_model.py`. +from batch composition features. Train one with `sglang-simulator-train`. sim_config.json usage: "predictor": { @@ -10,7 +10,6 @@ sim_config.json usage: } """ -import math import os import joblib @@ -18,47 +17,20 @@ from sglang_simulator.simulation.types import SchedulerConfig from sglang_simulator.spec.accelerator import AcceleratorInfo from sglang_simulator.spec.model import ModelInfo from sglang_simulator.time_predictor.base import InferTimePredictor, ScheduleBatch +from sglang_simulator.time_predictor.ml_features import ( + ML_FEATURE_NAMES, + extract_ml_features, +) from sglang_simulator.utils import get_logger logger = get_logger("sgl_simulator") class MLTimePredictor(InferTimePredictor): - """Per-iter latency predictor backed by an offline-trained sklearn regressor. + """Predict forward latency using the shared 18-feature batch representation.""" - Features (18 dim) extracted from ScheduleBatch: - batch_size, sum/max/min(extend), sum/max/min(past), - sum(extend*past), sum(extend^2), sum(past^2), - sum_attn_flops (= sum(e*(p+e/2))), - sum(extend × max_past), log1p(sum_past), log1p(sum_attn_flops), - batch_size × sum_extend, max_past - min_past, - is_decode, is_prefill - """ - - # This ordered list is the ABI between offline training and simulation. - # The concrete regressor algorithm is intentionally unrestricted as long - # as it exposes sklearn-compatible predict([[18 features]]) -> [seconds]. - - FEATURE_NAMES = [ - "batch_size", - "sum_extend", - "max_extend", - "min_extend", - "sum_past", - "max_past", - "min_past", - "sum_extend_x_past", - "sum_extend_squared", - "sum_past_squared", - "sum_attn_flops", - "sum_extend_x_max_past", - "log1p_sum_past", - "log1p_sum_attn_flops", - "batch_size_x_sum_extend", - "max_past_minus_min_past", - "is_decode", - "is_prefill", - ] + # Feature order is shared by training and inference, regardless of regressor. + FEATURE_NAMES = list(ML_FEATURE_NAMES) def __init__( self, @@ -74,7 +46,7 @@ class MLTimePredictor(InferTimePredictor): if not database_path or not os.path.exists(database_path): raise FileNotFoundError( f"MLTimePredictor database_path not found: {database_path}. " - "Train one with `train_latency_model.py` first." + "Train one with `sglang-simulator-train` first." ) bundle = joblib.load(database_path) @@ -101,14 +73,12 @@ class MLTimePredictor(InferTimePredictor): "MLTimePredictor model must expose a callable predict() method" ) - self._features = saved_features - self._call_count = 0 self._latency_scale = float(latency_scale) logger.info( "MLTimePredictor loaded from %s (model=%s, n_features=%d, latency_scale=%.4f)", database_path, type(self._model).__name__, - len(self._features), + len(saved_features), self._latency_scale, ) @@ -119,38 +89,6 @@ class MLTimePredictor(InferTimePredictor): exts = [req.extend_length for req in batch.reqs] pasts = [req.past_kv_length for req in batch.reqs] - bs = len(exts) - sum_e = sum(exts) - sum_p = sum(pasts) - sum_ep = sum(e * p for e, p in zip(exts, pasts)) - sum_e2 = sum(e * e for e in exts) - sum_p2 = sum(p * p for p in pasts) - sum_attn = sum(e * (p + e / 2) for e, p in zip(exts, pasts)) - max_e = max(exts) - max_p = max(pasts) - min_e = min(exts) - min_p = min(pasts) + feats = extract_ml_features(exts, pasts) - feats = [ - bs, - sum_e, - max_e, - min_e, - sum_p, - max_p, - min_p, - sum_ep, - sum_e2, - sum_p2, - sum_attn, - sum_e * max_p, - math.log1p(sum_p), - math.log1p(sum_attn), - bs * sum_e, - max_p - min_p, - int(all(e == 1 for e in exts)), - int(any(e > 1 for e in exts)), - ] - - self._call_count += 1 return float(self._model.predict([feats])[0]) * self._latency_scale diff --git a/tools/sglang-simulator/src/sglang_simulator/time_predictor/ml_features.py b/tools/sglang-simulator/src/sglang_simulator/time_predictor/ml_features.py new file mode 100644 index 0000000000..fc2ff565fa --- /dev/null +++ b/tools/sglang-simulator/src/sglang_simulator/time_predictor/ml_features.py @@ -0,0 +1,75 @@ +"""Shared feature contract for ML latency prediction and offline training.""" + +import math +from collections.abc import Sequence + +ML_FEATURE_NAMES = ( + "batch_size", + "sum_extend", + "max_extend", + "min_extend", + "sum_past", + "max_past", + "min_past", + "sum_extend_x_past", + "sum_extend_squared", + "sum_past_squared", + "sum_attn_flops", + "sum_extend_x_max_past", + "log1p_sum_past", + "log1p_sum_attn_flops", + "batch_size_x_sum_extend", + "max_past_minus_min_past", + "is_decode", + "is_prefill", +) + + +def extract_ml_features( + extend_lengths: Sequence[int], past_kv_lengths: Sequence[int] +) -> list[float]: + """Build the ordered 18-feature vector consumed by ``MLTimePredictor``.""" + if not extend_lengths: + raise ValueError("cannot extract ML features from an empty batch") + if len(extend_lengths) != len(past_kv_lengths): + raise ValueError("extend and past-KV length vectors must have equal size") + if any(value < 0 for value in (*extend_lengths, *past_kv_lengths)): + raise ValueError("token lengths must be non-negative") + + batch_size = len(extend_lengths) + sum_extend = sum(extend_lengths) + sum_past = sum(past_kv_lengths) + sum_extend_x_past = sum( + extend * past for extend, past in zip(extend_lengths, past_kv_lengths) + ) + sum_extend_squared = sum(value * value for value in extend_lengths) + sum_past_squared = sum(value * value for value in past_kv_lengths) + sum_attn_flops = sum( + extend * (past + extend / 2) + for extend, past in zip(extend_lengths, past_kv_lengths) + ) + max_extend = max(extend_lengths) + max_past = max(past_kv_lengths) + min_extend = min(extend_lengths) + min_past = min(past_kv_lengths) + + return [ + batch_size, + sum_extend, + max_extend, + min_extend, + sum_past, + max_past, + min_past, + sum_extend_x_past, + sum_extend_squared, + sum_past_squared, + sum_attn_flops, + sum_extend * max_past, + math.log1p(sum_past), + math.log1p(sum_attn_flops), + batch_size * sum_extend, + max_past - min_past, + int(all(value == 1 for value in extend_lengths)), + int(any(value > 1 for value in extend_lengths)), + ] diff --git a/tools/sglang-simulator/src/sglang_simulator/tools/__init__.py b/tools/sglang-simulator/src/sglang_simulator/tools/__init__.py new file mode 100644 index 0000000000..ce0fead7af --- /dev/null +++ b/tools/sglang-simulator/src/sglang_simulator/tools/__init__.py @@ -0,0 +1 @@ +"""Baseline collection and latency-model training tools.""" diff --git a/tools/sglang-simulator/src/sglang_simulator/tools/collect_baseline/README.md b/tools/sglang-simulator/src/sglang_simulator/tools/collect_baseline/README.md new file mode 100644 index 0000000000..210d628b3c --- /dev/null +++ b/tools/sglang-simulator/src/sglang_simulator/tools/collect_baseline/README.md @@ -0,0 +1,60 @@ +# Collect a baseline + +Install the simulator package in an environment with the matching SGLang version +and its serving dependencies. From a source checkout, use +`pip install ./tools/sglang-simulator`; a wheel installation provides the same +command, which can run from any directory: + +```bash +sglang-simulator-collect \ + --collection-output-dir /tmp/baseline-001 \ + --model-path /path/to/model \ + --disable-overlap-schedule --host 127.0.0.1 --port 30000 +``` + +Other flags are standard SGLang server arguments. The output directory must be +new or empty. After warmup, collect a request with: + +```bash +curl -f -X POST http://127.0.0.1:30000/start_profile -H 'Content-Type: application/json' -d '{}' +curl -f http://127.0.0.1:30000/generate -H 'Content-Type: application/json' \ + -d '{"text":"Hello","sampling_params":{"max_new_tokens":16,"ignore_eos":true}}' +curl -f -X POST http://127.0.0.1:30000/stop_profile -H 'Content-Type: application/json' -d '{}' +``` + +Replace the generate call with your benchmark; wait for all requests before stop. +These endpoints control collection instead of the PyTorch profiler. Hooks reuse +the simulator infrastructure without patching SGLang source. Records stay in +memory until `stop_profile`; use a fresh output directory/server for each run. + +Each scheduler writes three JSONL files (`TP0` shown): + +| File | Contents | +| --- | --- | +| `TP0.request.jsonl` | Request ID, Unix arrival time in milliseconds, input/output lengths, disjoint device/host/disk cache-hit lengths. | +| `TP0.raw_request.jsonl` | Arrival/queue times in seconds, completion flag, token IDs, output length and token emission latencies in seconds. | +| `TP0.schedule_batch.jsonl` | Forward mode, per-request extend/past-KV lengths and synchronized `iter_latency` in seconds. | + +DP/PP/EP ranks are appended when their parallel size exceeds one. Cache counts +come from SGLang's first scheduled chunk. The first token latency starts at server +arrival; subsequent entries measure time between output-producing steps. +`client_created_time` in sampling `custom_params`, when supplied in Unix seconds, +is preferred as the normalized arrival timestamp. + +Step timing synchronizes the device before and after `Scheduler.run_batch`. +Features are captured before execution; start/end step timestamps use +`perf_counter`. This requires `--disable-overlap-schedule` and changes execution +timing. The initial scope is ordinary text generation, not speculative decoding +or multimodal feature modeling. Health probes are excluded from request traces; +steps containing any health probe are excluded from training data. + +Read files after a successful stop response. Export never overwrites existing +files and retains records on failure; remove any logged partial files before +retrying. Raw traces include token IDs: review them before sharing. Continue with +[the training guide](../train_ml_time_predictor/README.md). + +For a minimal GPU smoke test with a small dummy model and four requests, run: + +```bash +python3 -m pytest -q tools/sglang-simulator/test/test_baseline_collection_sglang.py +``` diff --git a/tools/sglang-simulator/src/sglang_simulator/tools/collect_baseline/__init__.py b/tools/sglang-simulator/src/sglang_simulator/tools/collect_baseline/__init__.py new file mode 100644 index 0000000000..0613be056d --- /dev/null +++ b/tools/sglang-simulator/src/sglang_simulator/tools/collect_baseline/__init__.py @@ -0,0 +1 @@ +"""Collect baseline traces from a real SGLang server.""" diff --git a/tools/sglang-simulator/src/sglang_simulator/tools/collect_baseline/collect.py b/tools/sglang-simulator/src/sglang_simulator/tools/collect_baseline/collect.py new file mode 100644 index 0000000000..267ac0f534 --- /dev/null +++ b/tools/sglang-simulator/src/sglang_simulator/tools/collect_baseline/collect.py @@ -0,0 +1,74 @@ +"""Launch a real SGLang server instrumented for simulator baseline collection.""" + +import argparse +import os +from pathlib import Path + +from sglang_simulator.tools.collect_baseline.hooks import ( + COLLECTION_OUTPUT_DIR_ENV, + TokenizerManagerCollectionHook, + install_frontend_collection_hooks, + run_collection_scheduler_process, + validate_collection_hook, +) + +# This must happen before importing the SGLang HTTP server. +install_frontend_collection_hooks() + + +def _prepare_output_dir(parser: argparse.ArgumentParser, value: str) -> Path: + output_dir = Path(value).expanduser().resolve() + if output_dir.exists() and not output_dir.is_dir(): + parser.error( + f"baseline collection output path is not a directory: {output_dir}" + ) + if output_dir.is_dir() and any(output_dir.iterdir()): + parser.error( + f"baseline collection output directory must be empty: {output_dir}" + ) + output_dir.mkdir(parents=True, exist_ok=True) + os.environ[COLLECTION_OUTPUT_DIR_ENV] = str(output_dir) + return output_dir + + +def main(argv: list[str] | None = None) -> None: + from sglang.srt.entrypoints.http_server import launch_server + from sglang.srt.server_args import ServerArgs + from sglang.srt.utils import kill_process_tree + + validate_collection_hook(TokenizerManagerCollectionHook) + + parser = argparse.ArgumentParser() + collection = parser.add_argument_group("baseline collection") + collection.add_argument( + "--collection-output-dir", + default=os.getenv(COLLECTION_OUTPUT_DIR_ENV), + required=os.getenv(COLLECTION_OUTPUT_DIR_ENV) is None, + help=( + "Empty directory for TP*.request.jsonl, raw_request.jsonl, " + "and schedule_batch.jsonl" + ), + ) + ServerArgs.add_cli_args(parser.add_argument_group("sglang")) + + raw_args = parser.parse_args(argv) + output_dir = _prepare_output_dir(parser, raw_args.collection_output_dir) + server_args = ServerArgs.from_cli_args(raw_args) + if not server_args.disable_overlap_schedule: + parser.error( + "--disable-overlap-schedule is required so synchronized batch " + "latencies match ScheduleBatch execution boundaries" + ) + + print(f"SGLang Simulator baseline collection directory: {output_dir}") + try: + launch_server( + server_args, + run_scheduler_process_func=run_collection_scheduler_process, + ) + finally: + kill_process_tree(os.getpid(), include_parent=False) + + +if __name__ == "__main__": + main() diff --git a/tools/sglang-simulator/src/sglang_simulator/tools/collect_baseline/hooks.py b/tools/sglang-simulator/src/sglang_simulator/tools/collect_baseline/hooks.py new file mode 100644 index 0000000000..378f0a170a --- /dev/null +++ b/tools/sglang-simulator/src/sglang_simulator/tools/collect_baseline/hooks.py @@ -0,0 +1,409 @@ +"""Hooks for collecting requests and synchronized steps from a real SGLang server.""" + +from __future__ import annotations + +import json +import logging +import os +import time +from dataclasses import asdict, dataclass, field +from functools import wraps +from pathlib import Path +from typing import Any, Iterable + +from sglang_simulator.hook import BaseHook, install_class_hooks +from sglang_simulator.hook.class_hook_entry import is_class_hook_matched + +logger = logging.getLogger(__name__) + +COLLECTION_OUTPUT_DIR_ENV = "SGLANG_SIMULATOR_COLLECTION_DIR" + + +@dataclass +class RequestRecord: + rid: str = "" + created_time: float | None = None + client_created_time: float | None = None + server_created_time: float | None = None + queue_start: float = 0.0 + queue_end: float = 0.0 + completed: bool = False + output_length: int = 0 + input_length: int = 0 + before_adder_device_hit_len: int = 0 + final_device_hit_len: int = 0 + final_host_hit_len: int = 0 + final_disk_hit_len: int = 0 + last_event_time: float = 0.0 + gen_token_latencies: list[float] = field(default_factory=list) + input_ids: list[int] = field(default_factory=list) + output_ids: list[int] = field(default_factory=list) + + +@dataclass +class CollectionState: + active: bool = False + requests: dict[str, RequestRecord] = field(default_factory=dict) + schedule_batches: list[dict[str, Any]] = field(default_factory=list) + + def start(self) -> None: + if self.active: + raise RuntimeError("baseline collection is already active") + self.requests.clear() + self.schedule_batches.clear() + self.active = True + + def get_request(self, rid: str) -> RequestRecord: + record = self.requests.get(rid) + if record is None: + record = RequestRecord(rid=rid) + self.requests[rid] = record + return record + + def stop_and_export(self) -> list[Path]: + if not self.active: + raise RuntimeError("baseline collection is not active") + output_dir = os.getenv(COLLECTION_OUTPUT_DIR_ENV) + if not output_dir: + raise RuntimeError( + f"{COLLECTION_OUTPUT_DIR_ENV} is required for baseline collection" + ) + + root = Path(output_dir).expanduser() + root.mkdir(parents=True, exist_ok=True) + prefix = _rank_prefix() + exports = { + "request": (_normalized_request(r) for r in self.requests.values()), + "raw_request": (asdict(r) for r in self.requests.values()), + "schedule_batch": iter(self.schedule_batches), + } + paths = [] + try: + for kind, rows in exports.items(): + path = root / f"{prefix}.{kind}.jsonl" + with path.open("x", encoding="utf-8") as output: + paths.append(path) + for row in rows: + output.write(json.dumps(row) + "\n") + except (OSError, TypeError, ValueError): + # Keep in-memory records for retry; never remove an existing file. + for path in paths: + try: + path.unlink() + except OSError: + logger.warning( + "Failed to remove partial baseline file %s", path, exc_info=True + ) + raise + + self.active = False + self.requests.clear() + self.schedule_batches.clear() + return paths + + +def _rank_prefix() -> str: + from sglang.srt.runtime_context import get_parallel + + parallel = get_parallel() + prefix = f"TP{parallel.tp_rank}" + for size_name, rank_name, label in ( + ("dp_size", "dp_rank", "DP"), + ("pp_size", "pp_rank", "PP"), + ("moe_ep_size", "moe_ep_rank", "EP"), + ): + if getattr(parallel, size_name) > 1: + prefix += f"-{label}{getattr(parallel, rank_name)}" + return prefix + + +def _normalized_request(record: RequestRecord) -> dict[str, Any]: + timestamp = record.created_time + if timestamp is None: + timestamp = record.queue_start + return { + "rid": record.rid, + "timestamp": timestamp * 1000, + "input_length": record.input_length, + "output_length": record.output_length, + "device_cache_hit_length": record.before_adder_device_hit_len, + "host_cache_hit_length": record.final_host_hit_len - record.final_disk_hit_len, + "disk_cache_hit_length": record.final_disk_hit_len, + } + + +COLLECTION_STATE = CollectionState() + + +def _custom_params(sampling_params: Any) -> dict[str, Any]: + if isinstance(sampling_params, dict): + return sampling_params.get("custom_params") or {} + return getattr(sampling_params, "custom_params", None) or {} + + +def _stamp_created_time(tokenized_obj: Any) -> None: + from sglang.srt.observability.req_time_stats import convert_time_to_realtime + + time_stats = getattr(tokenized_obj, "time_stats", None) + created_time = getattr(time_stats, "created_time", None) + if not created_time: + return + sampling_params = tokenized_obj.sampling_params + params = dict(_custom_params(sampling_params)) + # Convert SGLang's perf_counter arrival to Unix time before IPC. + params["server_created_time"] = convert_time_to_realtime(created_time) + if isinstance(sampling_params, dict): + sampling_params["custom_params"] = params + else: + sampling_params.custom_params = params + + +def _iter_received_requests(recv_reqs: Iterable[Any]) -> Iterable[Any]: + for recv_req in recv_reqs: + if recv_req.__class__.__name__.startswith("BatchTokenized"): + yield from recv_req + else: + yield recv_req + + +def _record_received_requests(recv_reqs: Iterable[Any]) -> None: + if not COLLECTION_STATE.active: + return + from sglang.srt.managers.utils import is_health_check_generate_req + + recv_time = time.time() + for req in _iter_received_requests(recv_reqs): + # Control messages (abort, profile, RPC, etc.) must not become requests. + if not hasattr(req, "sampling_params") or not getattr(req, "rid", None): + continue + if is_health_check_generate_req(req): + continue + record = COLLECTION_STATE.get_request(str(req.rid)) + record.queue_start = recv_time + params = _custom_params(req.sampling_params) + record.server_created_time = params.get("server_created_time") + record.client_created_time = params.get("client_created_time") + record.created_time = next( + value + for value in ( + record.client_created_time, + record.server_created_time, + recv_time, + ) + if value is not None + ) + record.last_event_time = record.server_created_time or recv_time + record.input_ids = list(getattr(req, "input_ids", None) or []) + record.input_length = len(record.input_ids) + + +def _batch_request_lengths(batch: Any) -> list[tuple[int, int]]: + """Read prepared batch vectors; Req lengths may lag in decode/mixed batches.""" + if not batch.reqs: + return [] + if batch.forward_mode.is_decode(): + lengths = [(1, max(length - 1, 0)) for length in batch.seq_lens_cpu.tolist()] + else: + lengths = list(zip(batch.extend_lens, batch.prefix_lens, strict=True)) + if len(lengths) != len(batch.reqs): + raise ValueError("ScheduleBatch length vectors and reqs must have equal size") + return lengths + + +def _synchronize_device() -> None: + # Defer SGLang imports until the class hooks are installed. + from sglang.srt.platforms import current_platform + + current_platform.synchronize() + + +def _profile_output(message: str, success: bool = True) -> Any: + from sglang.srt.managers.io_struct import ProfileReqOutput + + return ProfileReqOutput(success=success, message=message) + + +def _handle_profile_request(recv_req: Any) -> Any: + try: + if recv_req.req_type.name == "START_PROFILE": + COLLECTION_STATE.start() + return _profile_output("Baseline collection started") + paths = COLLECTION_STATE.stop_and_export() + return _profile_output( + "Baseline collection exported: " + ", ".join(path.name for path in paths) + ) + except (RuntimeError, OSError, TypeError, ValueError) as exc: + # Report API errors without terminating the scheduler. + return _profile_output(str(exc), success=False) + + +class TokenizerManagerCollectionHook(BaseHook): + HOOK_CLASS_NAME = "TokenizerManager" + HOOK_MODULE_NAME = "sglang.srt.managers.tokenizer_manager" + + @classmethod + def hook(cls, target: Any) -> None: + original_one = target._send_one_request + original_batch = target._send_batch_request + + @wraps(original_one) + async def send_one( + self: Any, tokenized_obj: Any, *args: Any, **kwargs: Any + ) -> Any: + _stamp_created_time(tokenized_obj) + return await original_one(self, tokenized_obj, *args, **kwargs) + + @wraps(original_batch) + async def send_batch( + self: Any, tokenized_objs: list[Any], *args: Any, **kwargs: Any + ) -> Any: + for obj in tokenized_objs: + _stamp_created_time(obj) + return await original_batch(self, tokenized_objs, *args, **kwargs) + + target._send_one_request = send_one + target._send_batch_request = send_batch + + +class SchedulerCollectionHook(BaseHook): + HOOK_CLASS_NAME = "Scheduler" + HOOK_MODULE_NAME = "sglang.srt.managers.scheduler" + + @classmethod + def hook(cls, target: Any) -> None: + from sglang.srt.managers.utils import is_health_check_generate_req + + original_input = target.process_input_requests + original_run = target.run_batch + original_result = target.process_batch_result + original_profiler = target.init_profiler + + @wraps(original_input) + def process_input( + self: Any, recv_reqs: list[Any], *args: Any, **kwargs: Any + ) -> Any: + _record_received_requests(recv_reqs) + return original_input(self, recv_reqs, *args, **kwargs) + + @wraps(original_run) + def run_batch(self: Any, batch: Any, *args: Any, **kwargs: Any) -> Any: + if not COLLECTION_STATE.active: + return original_run(self, batch, *args, **kwargs) + + # Workers may mutate the batch, so snapshot before forward. + requests = tuple(batch.reqs or ()) + scheduled_time = time.time() + for req in requests: + record = COLLECTION_STATE.requests.get(str(req.rid)) + if record is not None and record.queue_end == 0: + record.queue_end = scheduled_time + record.input_ids = list(req.origin_input_ids) + record.input_length = len(record.input_ids) + # Preserve SGLang's first-chunk counts across later chunks/retractions. + device = int(req.cached_tokens_device) + host = int(req.cached_tokens_host) + storage = int(req.cached_tokens_storage) + record.before_adder_device_hit_len = device + record.final_device_hit_len = device + host + storage + record.final_host_hit_len = host + storage + record.final_disk_hit_len = storage + + # A mixed step's latency includes the probe: exclude the entire step. + if any(is_health_check_generate_req(req) for req in requests): + return original_run(self, batch, *args, **kwargs) + + lengths = _batch_request_lengths(batch) + row = { + "forward_mode": int(batch.forward_mode), + "request_infos": [ + { + "rid": str(req.rid), + "extend_input_len": extend, + "prefix_indices_len": past, + "output_ids_len": len(req.output_ids), + } + for req, (extend, past) in zip(requests, lengths, strict=True) + ], + } + + _synchronize_device() + start = time.perf_counter() + result = original_run(self, batch, *args, **kwargs) + _synchronize_device() + end = time.perf_counter() + row.update( + start_timestamp=start, end_timestamp=end, iter_latency=end - start + ) + COLLECTION_STATE.schedule_batches.append(row) + return result + + @wraps(original_result) + def process_result(self: Any, batch: Any, *args: Any, **kwargs: Any) -> Any: + if not COLLECTION_STATE.active: + return original_result(self, batch, *args, **kwargs) + requests = [(req, len(req.output_ids)) for req in (batch.reqs or ())] + result = original_result(self, batch, *args, **kwargs) + event_time = time.time() + for req, previous_length in requests: + record = COLLECTION_STATE.requests.get(str(req.rid)) + if record is None: + continue + record.output_length = len(req.output_ids) + if record.output_length > previous_length: + record.gen_token_latencies.append( + max(event_time - record.last_event_time, 0.0) + ) + record.last_event_time = event_time + if req.finished(): + record.output_ids = list(req.output_ids) + record.completed = True + return result + + @wraps(original_profiler) + def init_profiler(self: Any, *args: Any, **kwargs: Any) -> Any: + result = original_profiler(self, *args, **kwargs) + self.profiler_manager._profile = _handle_profile_request + return result + + target.process_input_requests = process_input + target.run_batch = run_batch + target.process_batch_result = process_result + target.init_profiler = init_profiler + + +_FRONTEND_HOOKS_INSTALLED = False +_SCHEDULER_HOOKS_INSTALLED = False + + +def install_frontend_collection_hooks() -> None: + global _FRONTEND_HOOKS_INSTALLED + if not _FRONTEND_HOOKS_INSTALLED: + install_class_hooks(TokenizerManagerCollectionHook) + _FRONTEND_HOOKS_INSTALLED = True + + +def install_scheduler_collection_hooks() -> None: + global _SCHEDULER_HOOKS_INSTALLED + if not _SCHEDULER_HOOKS_INSTALLED: + install_class_hooks(SchedulerCollectionHook) + _SCHEDULER_HOOKS_INSTALLED = True + + +def validate_collection_hook(hook: type[BaseHook]) -> None: + if is_class_hook_matched(hook): + return + raise RuntimeError( + "Required SGLang Simulator hook did not match an imported SGLang class: " + f"{hook.__name__} ({hook.HOOK_MODULE_NAME}.{hook.HOOK_CLASS_NAME}). " + "The collector must be adapted to this SGLang revision." + ) + + +def run_collection_scheduler_process(*args: Any, **kwargs: Any) -> Any: + """Install collection hooks before importing and running Scheduler.""" + install_scheduler_collection_hooks() + from sglang.srt.managers.scheduler import run_scheduler_process + + # TokenizerManager belongs to the API process; validate only Scheduler here. + validate_collection_hook(SchedulerCollectionHook) + return run_scheduler_process(*args, **kwargs) diff --git a/tools/sglang-simulator/src/sglang_simulator/tools/train_ml_time_predictor/README.md b/tools/sglang-simulator/src/sglang_simulator/tools/train_ml_time_predictor/README.md new file mode 100644 index 0000000000..12daede560 --- /dev/null +++ b/tools/sglang-simulator/src/sglang_simulator/tools/train_ml_time_predictor/README.md @@ -0,0 +1,61 @@ +# Train an ML time predictor + +Install the simulator package (`pip install ./tools/sglang-simulator` from a +source checkout, or install its wheel). The command and example data are included +in the package, so this works from any directory: + +```bash +example=$(python -c "from importlib.resources import files; print(files('sglang_simulator.tools.train_ml_time_predictor') / 'example_steps.jsonl')") +sglang-simulator-train --data-root "$example" --out-dir /tmp/example-predictor +``` + +`example_steps.jsonl` contains collected EXTEND steps from a tiny, two-layer Qwen3 +model with dummy weights on one A100, using repeated synthetic token sequences. +Only numeric feature inputs and measured latencies are retained. It demonstrates +the file format and training flow; it cannot produce a useful production predictor +or demonstrate prediction accuracy. + +For real data, replace `--data-root` with a [collection](../collect_baseline/README.md) +directory. The trainer recursively reads `TP0*.schedule_batch.jsonl` to avoid +counting duplicate tensor-parallel ranks. Repeat `--data-root` for more cases, or +pass explicit files for a different rank selection. Use data collected for the +same target model, hardware and serving configuration. + +The default filter keeps EXTEND (`forward_mode=1`) steps with nonempty batches and +finite latencies in `(0, 30]` seconds. Use `--forward-modes 1 2` to include DECODE +and `--max-latency` to change the upper bound. Malformed records fail with their +file and line; skipped row counts are included in the summary. + +Training shares the simulator's ordered 18-feature definition and uses monotonic +HistGradientBoosting regressors. The new or empty output directory receives: + +- `latency_model_hgbmono_l2.pkl`: squared-error model. +- `latency_model_hgbmono_p50.pkl`: median-quantile model. +- `summary.json`: seed, split sizes, skipped counts and held-out MAE (ms)/MAPE (%). + +Both models predict seconds. `l2` names the squared-error loss, not an L2 cache; +`p50` estimates the median. Compare them on independent data for your workload. + +By default, each collection directory is split 50/50 with seed 42 (single-row +cases stay in training). Metrics evaluate this split; saved models are then +refitted on all input rows. This measures interpolation within collected cases. +Use repeatable `--test-data-root` arguments for independent evaluation cases; +those rows never enter fitting or the saved models. Same-directory overlaps are +rejected. Copied data is not detected: keep independent collections separate. +Choose one loss with `--loss l2` or `--loss p50` if needed. + +```bash +sglang-simulator-train \ + --data-root /path/to/train-collection \ + --test-data-root /path/to/test-collection --out-dir /tmp/predictor +``` + +Point the simulator configuration at the selected bundle: + +```json +{"predictor": {"name": "ml", "database_path": "/tmp/example-predictor/latency_model_hgbmono_l2.pkl"}} +``` + +Use the same scikit-learn version for training and inference, and load only +trusted joblib files. Before production use, validate the selected model on +independent step data and complete simulator workloads. diff --git a/tools/sglang-simulator/src/sglang_simulator/tools/train_ml_time_predictor/__init__.py b/tools/sglang-simulator/src/sglang_simulator/tools/train_ml_time_predictor/__init__.py new file mode 100644 index 0000000000..e9c2287520 --- /dev/null +++ b/tools/sglang-simulator/src/sglang_simulator/tools/train_ml_time_predictor/__init__.py @@ -0,0 +1 @@ +"""Train latency predictors from collected baseline steps.""" diff --git a/tools/sglang-simulator/src/sglang_simulator/tools/train_ml_time_predictor/example_steps.jsonl b/tools/sglang-simulator/src/sglang_simulator/tools/train_ml_time_predictor/example_steps.jsonl new file mode 100644 index 0000000000..8767cce398 --- /dev/null +++ b/tools/sglang-simulator/src/sglang_simulator/tools/train_ml_time_predictor/example_steps.jsonl @@ -0,0 +1,14 @@ +{"forward_mode": 1, "request_infos": [{"extend_input_len": 1, "prefix_indices_len": 255}], "iter_latency": 0.002675119787454605} +{"forward_mode": 1, "request_infos": [{"extend_input_len": 8, "prefix_indices_len": 256}], "iter_latency": 0.0030074845999479294} +{"forward_mode": 1, "request_infos": [{"extend_input_len": 128, "prefix_indices_len": 0}], "iter_latency": 0.0027154013514518738} +{"forward_mode": 1, "request_infos": [{"extend_input_len": 128, "prefix_indices_len": 128}], "iter_latency": 0.0027412977069616318} +{"forward_mode": 1, "request_infos": [{"extend_input_len": 16, "prefix_indices_len": 256}], "iter_latency": 0.002909639850258827} +{"forward_mode": 1, "request_infos": [{"extend_input_len": 8, "prefix_indices_len": 272}], "iter_latency": 0.002743590623140335} +{"forward_mode": 1, "request_infos": [{"extend_input_len": 128, "prefix_indices_len": 0}], "iter_latency": 0.002713024616241455} +{"forward_mode": 1, "request_infos": [{"extend_input_len": 128, "prefix_indices_len": 128}], "iter_latency": 0.0027074161916971207} +{"forward_mode": 1, "request_infos": [{"extend_input_len": 32, "prefix_indices_len": 256}], "iter_latency": 0.0030285529792308807} +{"forward_mode": 1, "request_infos": [{"extend_input_len": 8, "prefix_indices_len": 288}], "iter_latency": 0.0027687866240739822} +{"forward_mode": 1, "request_infos": [{"extend_input_len": 128, "prefix_indices_len": 0}], "iter_latency": 0.002656087279319763} +{"forward_mode": 1, "request_infos": [{"extend_input_len": 64, "prefix_indices_len": 128}, {"extend_input_len": 64, "prefix_indices_len": 0}], "iter_latency": 1.545724280178547} +{"forward_mode": 1, "request_infos": [{"extend_input_len": 128, "prefix_indices_len": 64}], "iter_latency": 0.0031625479459762573} +{"forward_mode": 1, "request_infos": [{"extend_input_len": 8, "prefix_indices_len": 192}], "iter_latency": 0.0028553567826747894} diff --git a/tools/sglang-simulator/src/sglang_simulator/tools/train_ml_time_predictor/train.py b/tools/sglang-simulator/src/sglang_simulator/tools/train_ml_time_predictor/train.py new file mode 100644 index 0000000000..978f05dd0b --- /dev/null +++ b/tools/sglang-simulator/src/sglang_simulator/tools/train_ml_time_predictor/train.py @@ -0,0 +1,216 @@ +"""Train MLTimePredictor bundles from synchronized SGLang schedule-batch JSONL.""" + +from __future__ import annotations + +import argparse +import json +import math +from collections import Counter, defaultdict +from pathlib import Path + +import joblib +import numpy as np +import sklearn +from sglang_simulator.time_predictor.ml_features import ( + ML_FEATURE_NAMES, + extract_ml_features, +) +from sklearn.ensemble import HistGradientBoostingRegressor + +MONOTONIC_FEATURES = { + "batch_size", + "sum_extend", + "sum_past", + "sum_extend_x_past", + "sum_extend_squared", + "sum_past_squared", + "sum_attn_flops", + "sum_extend_x_max_past", + "log1p_sum_past", + "log1p_sum_attn_flops", + "batch_size_x_sum_extend", +} + + +def load_rows(roots, forward_modes, max_latency): + files = set() + for value in roots: + root = Path(value).expanduser().resolve() + if not root.exists(): + raise FileNotFoundError(root) + files.update( + [root] if root.is_file() else root.rglob("TP0*.schedule_batch.jsonl") + ) + if not files: + raise ValueError(f"no schedule files found under {roots}") + rows, skipped = [], Counter() + for path in sorted(files): + with path.open() as source: + for number, line in enumerate(source, 1): + try: + item = json.loads(line) + if not isinstance(item, dict): + raise ValueError("expected a JSON object") + if item.get("forward_mode") not in forward_modes: + skipped["mode"] += 1 + continue + requests = item["request_infos"] + if not requests: + skipped["empty"] += 1 + continue + latency = float(item["iter_latency"]) + if not math.isfinite(latency) or not 0 < latency <= max_latency: + skipped["latency"] += 1 + continue + extend = [r["extend_input_len"] for r in requests] + past = [r["prefix_indices_len"] for r in requests] + if any(type(v) is not int or v < 0 for v in extend + past): + raise ValueError("token lengths must be non-negative integers") + features = extract_ml_features(extend, past) + except (KeyError, TypeError, ValueError, OverflowError) as exc: + raise ValueError(f"{path}:{number}: {exc}") from exc + rows.append((path.parent, features, latency)) + return rows, dict(skipped) + + +def split_rows(rows, seed): + """Deterministic 50/50 split within each case (collection directory).""" + by_case = defaultdict(list) + for row in rows: + by_case[row[0]].append(row) + random = np.random.RandomState(seed) + train, test = [], [] + for case in by_case.values(): + order = random.permutation(len(case)) + test_count = len(case) // 2 + test.extend(case[i] for i in order[:test_count]) + train.extend(case[i] for i in order[test_count:]) + return train, test + + +def as_arrays(rows): + return np.asarray([r[1] for r in rows]), np.asarray([r[2] for r in rows]) + + +def train_model(rows, loss, seed): + model = HistGradientBoostingRegressor( + loss="quantile" if loss == "p50" else "squared_error", + quantile=0.5 if loss == "p50" else None, + max_iter=400, + learning_rate=0.04, + max_leaf_nodes=63, + l2_regularization=0.1, + monotonic_cst=[int(name in MONOTONIC_FEATURES) for name in ML_FEATURE_NAMES], + early_stopping=len(rows) > 1, + validation_fraction=0.15, + n_iter_no_change=20, + random_state=seed, + ) + return model.fit(*as_arrays(rows)) + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--data-root", + action="append", + required=True, + help="Collection directory or JSONL file; repeatable.", + ) + parser.add_argument( + "--test-data-root", + action="append", + default=[], + help="Independent evaluation data, excluded from fitting; repeatable.", + ) + parser.add_argument( + "--out-dir", required=True, help="New or empty output directory." + ) + parser.add_argument( + "--forward-modes", + type=int, + nargs="+", + default=[1], + help="SGLang ForwardMode values; default: EXTEND (1).", + ) + parser.add_argument( + "--max-latency", + type=float, + default=30.0, + help="Largest retained latency, in seconds.", + ) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--loss", choices=("l2", "p50", "both"), default="both") + args = parser.parse_args(argv) + if not math.isfinite(args.max_latency) or args.max_latency <= 0: + parser.error("--max-latency must be finite and positive") + return args + + +def main(argv=None): + args = parse_args(argv) + output = Path(args.out_dir).expanduser() + if output.exists() and (not output.is_dir() or any(output.iterdir())): + raise FileExistsError(f"training output directory must be empty: {output}") + rows, skipped = load_rows(args.data_root, args.forward_modes, args.max_latency) + if not rows: + raise ValueError( + f"no usable training rows under {args.data_root}; skipped={skipped}" + ) + if args.test_data_root: + test, test_skipped = load_rows( + args.test_data_root, args.forward_modes, args.max_latency + ) + if not test: + raise ValueError( + f"no usable test rows under {args.test_data_root}; skipped={test_skipped}" + ) + if {r[0] for r in rows} & {r[0] for r in test}: + raise ValueError( + "training and test roots contain overlapping collection cases" + ) + train = rows + else: + train, test = split_rows(rows, args.seed) + test_skipped = {} + if not train or not test: + raise ValueError( + "need nonempty training and test data; provide a case with at least two usable rows or --test-data-root" + ) + output.mkdir(parents=True, exist_ok=True) + summary = { + "seed": args.seed, + "split": "independent" if args.test_data_root else "within_case_50_50", + "train_rows": len(train), + "test_rows": len(test), + "final_fit_rows": len(rows), + "skipped": skipped, + "test_skipped": test_skipped, + "metrics": {}, + } + features, latency = as_arrays(test) + for loss in ("l2", "p50") if args.loss == "both" else (args.loss,): + model = train_model(train, loss, args.seed) + errors = np.abs(model.predict(features) - latency) + summary["metrics"][loss] = { + "MAE_ms": float(errors.mean() * 1000), + "MAPE_pct": float((errors / latency).mean() * 100), + } # codespell:ignore mape + if not args.test_data_root: + model = train_model(rows, loss, args.seed) + # Runtime validates the ordered feature ABI before making predictions. + bundle = { + "model": model, + "features": list(ML_FEATURE_NAMES), + "sklearn_version": sklearn.__version__, + } + with (output / f"latency_model_hgbmono_{loss}.pkl").open("xb") as destination: + joblib.dump(bundle, destination) + with (output / "summary.json").open("x") as destination: + json.dump(summary, destination, indent=2) + destination.write("\n") + print(json.dumps(summary, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tools/sglang-simulator/test/test_baseline_collection_sglang.py b/tools/sglang-simulator/test/test_baseline_collection_sglang.py new file mode 100644 index 0000000000..4fc83722ba --- /dev/null +++ b/tools/sglang-simulator/test/test_baseline_collection_sglang.py @@ -0,0 +1,127 @@ +"""GPU example: start collection, generate responses, export the traces.""" + +import json +import os +import signal +import socket +import subprocess +import time +from pathlib import Path + +import pytest +import requests + + +@pytest.fixture +def collection_server(tmp_path): + simulator_root = Path(__file__).resolve().parents[1] + model = tmp_path / "model" + model.mkdir() + config = json.loads( + (simulator_root / "test/assets/qwen3-8b/config.json").read_text() + ) + # Small dummy model: this tests lifecycle and data contracts, not accuracy. + config.update( + hidden_size=256, + intermediate_size=512, + num_hidden_layers=2, + num_attention_heads=2, + num_key_value_heads=2, + max_window_layers=2, + vocab_size=1024, + bos_token_id=1, + eos_token_id=2, + ) + (model / "config.json").write_text(json.dumps(config)) + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + url = f"http://127.0.0.1:{port}" + output = tmp_path / "baseline" + command = [ + "sglang-simulator-collect", + f"--collection-output-dir={output}", + f"--model-path={model}", + "--load-format=dummy", + "--skip-tokenizer-init", + "--disable-overlap-schedule", + "--device=cuda", + "--cuda-graph-backend-decode=disabled", + "--cuda-graph-backend-prefill=disabled", + "--chunked-prefill-size=128", + "--max-total-tokens=4096", + "--mem-fraction-static=0.1", + "--host=127.0.0.1", + f"--port={port}", + ] + log_path = tmp_path / "server.log" + with log_path.open("w") as log: + server = subprocess.Popen( + command, + stdout=log, + stderr=subprocess.STDOUT, + env={**os.environ, "OMP_NUM_THREADS": "2"}, + start_new_session=True, + ) + try: + deadline = time.monotonic() + 180 + while time.monotonic() < deadline: + assert server.poll() is None, log_path.read_text() + try: + if requests.get(url + "/health", timeout=1).ok: + break + except requests.RequestException: + pass + time.sleep(0.2) + else: + raise AssertionError(log_path.read_text()) + + yield url, output, log_path + finally: + if server.poll() is None: + os.killpg(server.pid, signal.SIGTERM) + try: + server.wait(timeout=15) + except subprocess.TimeoutExpired: + os.killpg(server.pid, signal.SIGKILL) + server.wait() + + +def test_baseline_collection(collection_server): + url, output, log_path = collection_server + + def post(path, payload=None): + response = requests.post(url + path, json=payload or {}, timeout=60) + assert response.ok, response.text + "\n" + log_path.read_text() + + post("/start_profile") + for inputs in ([10] * 256, [20] * 192, [[30] * 128, [40] * 200]): + post( + "/generate", + { + "input_ids": inputs, + "sampling_params": {"max_new_tokens": 4, "ignore_eos": True}, + }, + ) + post("/stop_profile") + + def read(kind): + return [ + json.loads(line) + for line in (output / f"TP0.{kind}.jsonl").read_text().splitlines() + ] + + requests_trace, raw, steps = ( + read("request"), + read("raw_request"), + read("schedule_batch"), + ) + request_ids = {r["rid"] for r in raw} + assert len(requests_trace) == len(raw) == len(request_ids) == 4 + assert sorted(r["input_length"] for r in requests_trace) == [128, 192, 200, 256] + assert all(r["output_length"] == 4 for r in requests_trace) + assert all(r["completed"] and len(r["output_ids"]) == 4 for r in raw) + assert {r["rid"] for r in requests_trace} == request_ids + assert {r["rid"] for step in steps for r in step["request_infos"]} == request_ids + assert {step["forward_mode"] for step in steps} == {1, 2} # Prefill and decode. + assert all(step["iter_latency"] > 0 for step in steps) diff --git a/tools/sglang-simulator/test/test_train_latency_model.py b/tools/sglang-simulator/test/test_train_latency_model.py new file mode 100644 index 0000000000..c1def72ad0 --- /dev/null +++ b/tools/sglang-simulator/test/test_train_latency_model.py @@ -0,0 +1,32 @@ +"""Minimal example: train on collected steps, then load the models for prediction.""" + +import json +import math +from pathlib import Path + +from sglang_simulator.time_predictor.base import ScheduleBatch, ScheduleRequest +from sglang_simulator.time_predictor.ml import MLTimePredictor +from sglang_simulator.tools.train_ml_time_predictor import train as trainer + + +def test_train_and_predict(tmp_path): + example = Path(trainer.__file__).with_name("example_steps.jsonl") + output = tmp_path / "models" + trainer.main(["--data-root", str(example), "--out-dir", str(output)]) + + summary = json.loads((output / "summary.json").read_text()) + assert summary["train_rows"] > 0 and summary["test_rows"] > 0 + assert summary["train_rows"] + summary["test_rows"] == summary["final_fit_rows"] + assert set(summary["metrics"]) == {"l2", "p50"} + + batch = ScheduleBatch( + [ + ScheduleRequest(extend_length=4, past_kv_length=12), + ScheduleRequest(extend_length=8, past_kv_length=0), + ] + ) + for loss in ("l2", "p50"): + model = output / f"latency_model_hgbmono_{loss}.pkl" + predictor = MLTimePredictor(None, None, None, str(model)) + latency = predictor.predict_infer_time(batch) + assert math.isfinite(latency) and latency > 0