[Simulator] Add high-fidelity CPU-based inference simulator (#33824)

Co-authored-by: zhouhaizhu.zhz <zhouhaizhu.zhz@alibaba-inc.com>
Co-authored-by: LinSiyuan814 <linsiyuan.lsy@alibaba-inc.com>
Co-authored-by: hzh0425 <hzh0425@apache.org>

.github/workflows/_pr-test-check-changes.yml

@@ -19,6 +19,8 @@ on:
     outputs:
       main_package:
         value: ${{ jobs.run.outputs.main_package }}
+      simulator:
+        value: ${{ jobs.run.outputs.simulator }}
       sgl_kernel:
         value: ${{ jobs.run.outputs.sgl_kernel }}
       jit_kernel:
@@ -44,6 +46,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       main_package: ${{ steps.filter.outputs.main_package || steps.run-mode.outputs.run_all_tests }}
+      simulator: ${{ steps.filter.outputs.simulator || steps.run-mode.outputs.run_all_tests }}
       sgl_kernel: ${{ steps.filter.outputs.sgl_kernel }}
       jit_kernel: ${{ steps.filter.outputs.jit_kernel || steps.run-mode.outputs.run_all_tests }}
       multimodal_gen: ${{ steps.filter.outputs.multimodal_gen || steps.run-mode.outputs.run_all_tests }}
@@ -94,6 +97,11 @@ jobs:
               - "test/**/!(*.md)"
               - "rust/**"
               - "proto/sglang/runtime/v1/sglang.proto"
+            simulator:
+              - ".github/workflows/pr-test-extra.yml"
+              - ".github/workflows/_pr-test-simulator-cpu.yml"
+              - "benchmark/simulator/**/!(*.md)"
+              - "tools/sglang-simulator/**/!(*.md)"
             multimodal_gen:
               - ".github/workflows/pr-test.yml"
               - ".github/workflows/pr-test-multimodal-gen.yml"

.github/workflows/_pr-test-simulator-cpu.yml

@@ -0,0 +1,96 @@
+name: PR Test SGLang Simulator (CPU)
+
+on:
+  workflow_call:
+    inputs:
+      check_changes:
+        description: 'toJson(needs.check-changes.outputs).'
+        type: string
+        required: true
+      caller_inputs:
+        description: 'toJson(inputs) from the caller workflow.'
+        type: string
+        required: true
+      rust_ext_artifact:
+        description: 'Artifact of prebuilt Rust extension modules.'
+        type: string
+        default: ''
+
+env:
+  SGLANG_IS_IN_CI: true
+  SKIP_PR_TEST_HEALTH_CHECK: ${{ (fromJson(inputs.caller_inputs).skip_pr_test_health_check || fromJson(inputs.caller_inputs).test_parallel_dispatch || fromJson(inputs.caller_inputs).run_all_tests) && 'true' || 'false' }}
+  PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }}
+  USE_VENV: false
+
+jobs:
+  run:
+    name: simulator-test-cpu
+    if: fromJson(inputs.check_changes).main_package == 'true' || fromJson(inputs.check_changes).simulator == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 40
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ fromJson(inputs.caller_inputs).git_ref || github.sha }}
+
+      - uses: ./.github/actions/check-pr-test-health
+
+      - uses: ./.github/actions/check-maintenance
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - uses: ./.github/actions/download-rust-ext
+        id: rust_ext
+        with:
+          artifact_name: ${{ inputs.rust_ext_artifact }}
+
+      - name: Install protoc + Rust toolchain
+        if: ${{ steps.rust_ext.outputs.hit != 'true' }}
+        timeout-minutes: 10
+        run: bash scripts/ci/utils/install_rust_protoc.sh
+
+      - name: Rust cache (rust/ workspace)
+        if: ${{ steps.rust_ext.outputs.hit != 'true' }}
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: rust
+          shared-key: "sglang-grpc-cpu"
+
+      - name: Install dependencies
+        timeout-minutes: 20
+        env:
+          UV_SYSTEM_PYTHON: "1"
+        run: |
+          uv pip install -e "python" --index-strategy unsafe-best-match --prerelease allow
+          uv pip install pytest
+          uv pip install -e "tools/sglang-simulator[aic]"
+
+      - name: Prebuild HiCache native hash extension
+        timeout-minutes: 5
+        env:
+          MALLOC_ARENA_MAX: "2"
+          MAX_JOBS: "1"
+          TORCH_EXTENSIONS_DIR: ${{ runner.temp }}/torch-extensions
+        run: |
+          python3 -c \
+            'from sglang.srt.mem_cache.cpp_utils.native_hash import get_native_hash; get_native_hash([1], None)'
+
+      - name: Run SGLang Simulator compatibility tests
+        timeout-minutes: 10
+        env:
+          MALLOC_ARENA_MAX: "2"
+          MAX_JOBS: "1"
+          PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1"
+          TORCH_EXTENSIONS_DIR: ${{ runner.temp }}/torch-extensions
+        run: |
+          python3 -m pytest -q tools/sglang-simulator/test/test_simulation_sglang_runner.py
+          python3 -m pytest -q tools/sglang-simulator/test/test_simulation_sglang_serving.py
+          python3 -m pytest -q tools/sglang-simulator/test/test_simulation_cache_hit_ratio.py
+          python3 -m pytest -q tools/sglang-simulator/test/test_simulation_offline_blocking.py

.github/workflows/pr-test-extra.yml

@@ -149,6 +149,16 @@ jobs:
       skip_pr_test_health_check: ${{ inputs.skip_pr_test_health_check == true }}
     secrets: inherit
 
+  simulator-test-cpu:
+    needs: [check-changes, call-gate, rust-ext-build]
+    if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }}
+    uses: ./.github/workflows/_pr-test-simulator-cpu.yml
+    with:
+      check_changes: ${{ toJson(needs.check-changes.outputs) }}
+      caller_inputs: ${{ toJson(inputs) }}
+      rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
+    # No `secrets: inherit`: this hosted CPU job has no secret consumer.
+
   # =============================================== extra-a (1-/2-gpu) ===============================================
   extra-a-test-1-gpu-small:
     needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build]
@@ -249,6 +259,7 @@ jobs:
         call-gate,
         sgl-kernel-build-wheels,
         rust-ext-build,
+        simulator-test-cpu,
         extra-a-test-1-gpu-small,
         extra-a-test-1-gpu-large,
         extra-a-test-2-gpu-large,