[Simulator] Add high-fidelity CPU-based inference simulator (#33824)

Co-authored-by: zhouhaizhu.zhz <zhouhaizhu.zhz@alibaba-inc.com> Co-authored-by: LinSiyuan814 <linsiyuan.lsy@alibaba-inc.com> Co-authored-by: hzh0425 <hzh0425@apache.org>
2026-09-04 11:12:11 +08:00
parent a5f07b1241
commit 59799a3687
81 changed files with 6624 additions and 0 deletions
@@ -0,0 +1,50 @@
+# SGLang Simulator examples
+
+The example assets are organized by purpose:
+
+- `sim_configs/`: standalone AIC SOL, AIC SILICON, ML, and replay simulator configs;
+- `assets/`: the small illustrative ML model, replay table, and test tokenizer;
+- `workloads/`: ShareGPT and timestamped simulator/Autobench workload examples;
+
+The ML model is an illustrative constant-latency sklearn model, not a calibrated
+hardware predictor. Rebuild it and the tokenizer with:
+
+```bash
+python3 examples/build_example_assets.py
+```
+
+Only load pickle/joblib assets from sources you trust.
+
+For maintained direct-run and serving examples, see
+[`test_simulation_sglang_runner.py`](../test/test_simulation_sglang_runner.py) and
+[`test_simulation_sglang_serving.py`](../test/test_simulation_sglang_serving.py).
+
+Start a server with any example config:
+
+```bash
+python3 -m sglang_simulator.simulation.sglang.launch_server \
+  --model-path /path/to/model \
+  --sim-config-path examples/sim_configs/aic_sol.json \
+  --port 30000
+```
+
+Run a ShareGPT workload with at least four output tokens so decode and TPOT are
+measured:
+
+```bash
+cd /path/to/sglang
+python3 benchmark/simulator/bench_serving.py \
+  --simulator-mode=offline \
+  --backend=sglang \
+  --base-url=http://127.0.0.1:30000 \
+  --model=/path/to/model \
+  --tokenizer=/path/to/model \
+  --dataset-name=sharegpt \
+  --dataset-path=examples/workloads/sharegpt-example.json \
+  --sharegpt-output-len=4 \
+  --num-prompts=3 \
+  --profile
+```
+
+The timestamp trace uses the simulator-owned Autobench JSONL contract. Its
+`timestamp` values are request-arrival times in milliseconds.
@@ -0,0 +1,5 @@
+{
+  "[[1, 3]]": 0.001,
+  "[[4, 0]]": 0.005,
+  "[[8, 0]]": 0.008
+}
@@ -0,0 +1,59 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "[UNK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": null,
+  "pre_tokenizer": {
+    "type": "Whitespace"
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {}
+  },
+  "decoder": null,
+  "model": {
+    "type": "WordLevel",
+    "vocab": {
+      "[UNK]": 0,
+      "prefix": 1,
+      "caching": 2,
+      "latency": 3,
+      "decode": 4,
+      "token": 5
+    },
+    "unk_token": "[UNK]"
+  }
+}
@@ -0,0 +1,6 @@
+{
+  "backend": "tokenizers",
+  "model_max_length": 1000000000000000019884624838656,
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "[UNK]"
+}
@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+"""Rebuild the small ML and tokenizer assets used by examples and tests."""
+
+from pathlib import Path
+
+import joblib
+import numpy as np
+from sglang_simulator.time_predictor.ml import MLTimePredictor
+from sklearn.dummy import DummyRegressor
+from tokenizers import Tokenizer
+from tokenizers.models import WordLevel
+from tokenizers.pre_tokenizers import Whitespace
+from transformers import PreTrainedTokenizerFast
+
+ASSETS = Path(__file__).parent / "assets"
+
+
+def build_ml_model() -> None:
+    model = DummyRegressor(strategy="constant", constant=0.001)
+    model.fit(np.zeros((1, len(MLTimePredictor.FEATURE_NAMES))), [0.001])
+    joblib.dump(
+        {"model": model, "features": MLTimePredictor.FEATURE_NAMES},
+        ASSETS / "model.pkl",
+    )
+
+
+def build_tokenizer() -> None:
+    tokenizer = Tokenizer(
+        WordLevel(
+            {
+                "[UNK]": 0,
+                "prefix": 1,
+                "caching": 2,
+                "latency": 3,
+                "decode": 4,
+                "token": 5,
+            },
+            unk_token="[UNK]",
+        )
+    )
+    tokenizer.pre_tokenizer = Whitespace()
+    PreTrainedTokenizerFast(
+        tokenizer_object=tokenizer,
+        unk_token="[UNK]",
+    ).save_pretrained(ASSETS / "tokenizer")
+
+
+def main() -> None:
+    ASSETS.mkdir(parents=True, exist_ok=True)
+    build_ml_model()
+    build_tokenizer()
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,21 @@
+{
+  "platform": {
+    "accelerator": {"name": "a100_sxm", "hbm_capacity_gb": 80},
+    "disk_read_bandwidth_gb": 8,
+    "disk_write_bandwidth_gb": 8,
+    "memory_read_bandwidth_gb": 64,
+    "memory_write_bandwidth_gb": 64,
+    "num_device_per_node": 8
+  },
+  "predictor": {
+    "name": "aiconfigurator",
+    "database_mode": "SILICON"
+  },
+  "scheduler": {
+    "tp_size": 1,
+    "ep_size": 1,
+    "dp_size": 1,
+    "backend_name": "sglang",
+    "backend_version": "0.5.9"
+  }
+}
@@ -0,0 +1,21 @@
+{
+  "platform": {
+    "accelerator": {"name": "a100_sxm", "hbm_capacity_gb": 80},
+    "disk_read_bandwidth_gb": 8,
+    "disk_write_bandwidth_gb": 8,
+    "memory_read_bandwidth_gb": 64,
+    "memory_write_bandwidth_gb": 64,
+    "num_device_per_node": 8
+  },
+  "predictor": {
+    "name": "aiconfigurator",
+    "database_mode": "SOL"
+  },
+  "scheduler": {
+    "tp_size": 1,
+    "ep_size": 1,
+    "dp_size": 1,
+    "backend_name": "sglang",
+    "backend_version": "0.5.9"
+  }
+}
@@ -0,0 +1,22 @@
+{
+  "platform": {
+    "accelerator": {"name": "a100_sxm", "hbm_capacity_gb": 80},
+    "disk_read_bandwidth_gb": 8,
+    "disk_write_bandwidth_gb": 8,
+    "memory_read_bandwidth_gb": 64,
+    "memory_write_bandwidth_gb": 64,
+    "num_device_per_node": 8
+  },
+  "predictor": {
+    "name": "ml",
+    "database_path": "../assets/model.pkl",
+    "latency_scale": 1.0
+  },
+  "scheduler": {
+    "tp_size": 1,
+    "ep_size": 1,
+    "dp_size": 1,
+    "backend_name": "sglang",
+    "backend_version": "0.5.9"
+  }
+}
@@ -0,0 +1,23 @@
+{
+  "platform": {
+    "accelerator": {"name": "a100_sxm", "hbm_capacity_gb": 80},
+    "disk_read_bandwidth_gb": 8,
+    "disk_write_bandwidth_gb": 8,
+    "memory_read_bandwidth_gb": 64,
+    "memory_write_bandwidth_gb": 64,
+    "num_device_per_node": 8
+  },
+  "predictor": {
+    "name": "replay",
+    "database_path": "../assets/replay_table.json",
+    "miss_strategy": "knn",
+    "miss_knn_k": 1
+  },
+  "scheduler": {
+    "tp_size": 1,
+    "ep_size": 1,
+    "dp_size": 1,
+    "backend_name": "sglang",
+    "backend_version": "0.5.9"
+  }
+}
@@ -0,0 +1,38 @@
+[
+  {
+    "conversations": [
+      {
+        "from": "human",
+        "value": "Explain prefix caching in one concise sentence."
+      },
+      {
+        "from": "gpt",
+        "value": "Prefix caching reuses KV states shared by prompt prefixes."
+      }
+    ]
+  },
+  {
+    "conversations": [
+      {
+        "from": "human",
+        "value": "What does time to first token measure?"
+      },
+      {
+        "from": "gpt",
+        "value": "It measures latency from request arrival to the first generated token."
+      }
+    ]
+  },
+  {
+    "conversations": [
+      {
+        "from": "human",
+        "value": "Why does decode latency matter for serving?"
+      },
+      {
+        "from": "gpt",
+        "value": "Decode latency determines the cadence at which later tokens reach the user."
+      }
+    ]
+  }
+]
@@ -0,0 +1,3 @@
+{"prompt":[100,101,102,103],"prompt_len":4,"output_len":4,"timestamp":0}
+{"prompt":[100,101,102,104],"prompt_len":4,"output_len":4,"timestamp":75}
+{"prompt":[200,201,202,203],"prompt_len":4,"output_len":4,"timestamp":250}