Files
sglang/.github/workflows/pr-test-sgl-router.yml
T

442 lines
18 KiB
YAML

name: PR Test (sgl-router)
# Trigger contract — modeled on `pr-test.yml`:
#
# * No `paths:` filter at the trigger level. The workflow ALWAYS
# fires on every PR synchronize / push, so the run record always
# appears as a check on the PR (no more "workflow silently didn't
# fire" mystery debugging). Path-based skip decisions are made
# inside the `sgl-router-changes` job below, where we can log the
# reason. Same for the `run-ci` label gate — moved off the
# workflow trigger and into the gate job so its outcome is
# observable, and now fails (rather than skips) when it is missing.
#
# * `push` on `main` keeps firing so post-merge runs still record
# against the default branch.
on:
push:
branches: [main]
pull_request:
branches: [main]
types: [opened, synchronize, reopened, labeled]
workflow_dispatch:
concurrency:
group: sgl-router-${{ github.event_name }}-${{ github.head_ref || github.ref_name || 'default' }}
cancel-in-progress: true
env:
SGLANG_IS_IN_CI: true
jobs:
# Stage 0a — path detection. Always runs (cheap, ubuntu-latest) and
# emits a single `should_run` output consumed by every tier below.
# It only answers "does this PR touch sgl-router?"; the `run-ci`
# label is checked by the separate gate job so that a missing label
# fails that job instead of silently skipping the tiers (see below).
sgl-router-changes:
name: changes
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
should_run: ${{ steps.decide.outputs.should_run }}
steps:
- uses: actions/checkout@v4
# `dorny/paths-filter` computes the diff against the PR base
# (for pull_request) or the push's `before` SHA (for push) and
# exposes `sgl_router=true|false` on whether any tracked path
# changed. Replaces the old workflow-level `paths:` filter so
# the gate is observable.
- name: Detect sgl-router path changes
id: paths
if: github.event_name != 'workflow_dispatch'
uses: dorny/paths-filter@v3
with:
filters: |
sgl_router:
- 'sgl-router/**'
- 'rust/Cargo.toml'
- 'rust/sglang-processor/**'
- 'rust/sglang-api-types/**'
- 'proto/**'
- '.github/workflows/pr-test-sgl-router.yml'
- 'docker/sgl-router.Dockerfile'
- 'scripts/ci/cuda/ci_install_dependency.sh'
- name: Decide
id: decide
env:
EVENT_NAME: ${{ github.event_name }}
PATHS_CHANGED: ${{ steps.paths.outputs.sgl_router }}
run: |
if [[ "$EVENT_NAME" == "workflow_dispatch" ]]; then
echo "should_run=true" >> "$GITHUB_OUTPUT"
echo "Manual dispatch; running sgl-router tiers."
elif [[ "$PATHS_CHANGED" != "true" ]]; then
echo "should_run=false" >> "$GITHUB_OUTPUT"
echo "No sgl-router-related changes; skipping tiers."
else
echo "should_run=true" >> "$GITHUB_OUTPUT"
echo "sgl-router-related changes detected; running tiers."
fi
# Stage 0b — `run-ci` label gate, shared with `pr-test.yml`. Cheap
# workflow runs (e.g. docs-only PRs that happen to touch the
# sgl-router dir) still need the opt-in label before consuming the
# H100 / kind tiers below. `push` on main and `workflow_dispatch`
# pass straight through.
#
# `pr-gate.yml` fetches the PR's labels live and FAILS when `run-ci`
# is missing. Both matter for `/tag-and-rerun-ci`: the slash handler
# adds the label with GITHUB_TOKEN, which fires no `labeled` event,
# then reruns the PR's unsuccessful runs. A rerun replays the original
# event payload, so a gate reading `github.event.pull_request.labels`
# would skip again, and a run that concluded `success` with every
# tier skipped is never rerun at all. Failing here on the live label
# set makes the rerun both happen and pick the label up.
sgl-router-gate:
name: gate
needs: sgl-router-changes
if: needs.sgl-router-changes.outputs.should_run == 'true'
uses: ./.github/workflows/pr-gate.yml
with:
require-run-ci: true
sgl-router-lint:
name: tier-1 — lint
needs: [sgl-router-changes, sgl-router-gate]
if: needs.sgl-router-changes.outputs.should_run == 'true'
runs-on: ubuntu-latest
env:
# Scoped per-job (not workflow-wide) to avoid SMG's documented breakage
# of self-hosted-runner pip installs that compile Rust extensions.
RUSTC_WRAPPER: sccache
SCCACHE_GHA_ENABLED: "true"
steps:
- uses: actions/checkout@v4
- name: Install protoc
run: sudo bash scripts/ci/utils/install_protoc.sh
# sccache install is best-effort: the GitHub release CDN intermittently
# returns 5xx and the action's built-in retry is shallow. If it fails,
# we unset RUSTC_WRAPPER below and proceed without compile caching —
# rust-cache still covers the bulk of warm-cache wins.
- name: Configure sccache
id: sccache
continue-on-error: true
uses: mozilla-actions/sccache-action@v0.0.9
with:
version: "v0.12.0"
disable_annotations: true
- name: Disable sccache if install failed
if: steps.sccache.outcome != 'success'
run: |
echo "::warning::sccache install failed (${{ steps.sccache.outcome }}); proceeding without compile caching"
echo "RUSTC_WRAPPER=" >> "$GITHUB_ENV"
- name: Rust cache
uses: Swatinem/rust-cache@v2
with:
workspaces: sgl-router
shared-key: "sgl-router-cache"
cache-all-crates: true
cache-on-failure: true
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: cargo check
working-directory: sgl-router
run: cargo check --locked --workspace --all-targets
- name: cargo clippy
working-directory: sgl-router
run: cargo clippy --locked --workspace --all-targets -- -D warnings
- name: cargo fmt
working-directory: sgl-router
run: |
rustup toolchain install nightly --profile minimal --component rustfmt
cargo +nightly fmt --all -- --check
# cargo-deny 0.19.6+ required for CVSS 4.0 parsing.
- name: Install cargo-deny
run: cargo install --locked cargo-deny@0.19.6
- name: cargo deny check
working-directory: sgl-router
run: cargo deny check licenses bans sources advisories
- name: Show sccache stats
if: always() && steps.sccache.outcome == 'success'
run: sccache --show-stats
sgl-router-build-and-test:
name: tier-2 — build + test
needs: [sgl-router-changes, sgl-router-gate, sgl-router-lint]
if: needs.sgl-router-changes.outputs.should_run == 'true'
runs-on: ubuntu-latest
env:
RUSTC_WRAPPER: sccache
SCCACHE_GHA_ENABLED: "true"
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
- name: Install protoc
run: sudo bash scripts/ci/utils/install_protoc.sh
- name: Install OS deps (libssl, pkg-config)
run: |
sudo apt-get update
sudo apt-get install -y libssl-dev pkg-config
# sccache install is best-effort: the GitHub release CDN intermittently
# returns 5xx and the action's built-in retry is shallow. If it fails,
# we unset RUSTC_WRAPPER below and proceed without compile caching —
# rust-cache still covers the bulk of warm-cache wins.
- name: Configure sccache
id: sccache
continue-on-error: true
uses: mozilla-actions/sccache-action@v0.0.9
with:
version: "v0.12.0"
disable_annotations: true
- name: Disable sccache if install failed
if: steps.sccache.outcome != 'success'
run: |
echo "::warning::sccache install failed (${{ steps.sccache.outcome }}); proceeding without compile caching"
echo "RUSTC_WRAPPER=" >> "$GITHUB_ENV"
- name: Rust cache
uses: Swatinem/rust-cache@v2
with:
workspaces: sgl-router
shared-key: "sgl-router-cache"
cache-all-crates: true
cache-on-failure: true
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: cargo build (record wall time)
working-directory: sgl-router
run: |
START=$(date +%s)
cargo build --locked --release --all-targets
END=$(date +%s)
BUILD_SECONDS=$((END - START))
echo "tier_2_build_duration_seconds=${BUILD_SECONDS}" >> "$GITHUB_ENV"
echo "::notice title=Tier-2 build duration::${BUILD_SECONDS}s"
- name: cargo test
working-directory: sgl-router
# Skip the tokenizer parity_matrix test here: ubuntu-latest has
# no HuggingFace cache, so the matrix would hard-fail (see the
# test docstring). The e2e job runs it after pytest populates
# the Qwen3-0.6B tokenizer.json. Filter is matched against the
# full test path `tokenizer::parity::parity_matrix`; substring
# `parity_matrix` is unique to that test.
run: cargo test --locked --release --workspace -- --skip parity_matrix
# Regenerate the cross-impl block-hash parity fixture and fail if it
# differs from the committed file. The Python script replicates
# SGLang's `radix_cache.hash_page` algorithm verbatim (no
# `import sglang`); a diff here means EITHER the Rust port's algorithm
# drifted from the script, OR the script drifted from SGLang's actual
# code — in both cases the operator must investigate and re-commit
# the fixture in the same PR as the algorithm change.
#
# `set -euo pipefail` makes a Python-side failure (syntax error,
# missing import) cause this step to fail instead of falling
# through to a no-diff "pass" when no fixture was actually
# regenerated.
- name: kv_events hash parity fixture (drift check)
working-directory: sgl-router
run: |
set -euo pipefail
python3 tests/scripts/generate_kv_events_hash_parity.py
if ! git diff --exit-code -- tests/fixtures/kv_events_hash_parity.json; then
echo "::error::kv_events hash parity fixture drifted." \
"Re-run sgl-router/tests/scripts/generate_kv_events_hash_parity.py" \
"and commit the updated tests/fixtures/kv_events_hash_parity.json."
exit 1
fi
- name: Show sccache stats
if: always() && steps.sccache.outcome == 'success'
run: sccache --show-stats
- name: Summary
if: always()
run: |
echo "## Tier-2 results" >> "$GITHUB_STEP_SUMMARY"
echo "- Build duration: \`${tier_2_build_duration_seconds:-(not recorded)}\` s" >> "$GITHUB_STEP_SUMMARY"
sgl-router-docker-build-test:
name: tier-3 — docker (placeholder)
needs: [sgl-router-changes, sgl-router-gate, sgl-router-build-and-test]
if: needs.sgl-router-changes.outputs.should_run == 'true'
runs-on: ubuntu-24.04
timeout-minutes: 15
steps:
- name: Placeholder
run: |
echo "docker-build-test not implemented yet."
exit 0
# tier-3 k8s integration is decoupled from tier-3 e2e: each runs on a
# different runner type (ubuntu kind cluster vs H100), each manages its
# own test scope (k8s_integration/ vs everything-else-in-e2e), and
# each is allowed to fail without blocking the other. They both still
# gate on tier-2's build+test passing — a Rust compile failure blocks
# the GPU/cluster runners from spinning up.
sgl-router-k8s-integration:
name: tier-3 — k8s integration
needs: [sgl-router-changes, sgl-router-gate, sgl-router-build-and-test]
if: needs.sgl-router-changes.outputs.should_run == 'true'
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
- name: Setup Docker buildx
uses: docker/setup-buildx-action@v3
- name: Install kind
run: |
curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v0.24.0/kind-linux-amd64
chmod +x /tmp/kind && sudo mv /tmp/kind /usr/local/bin/kind
- name: Install kubectl
run: |
curl -Lo /tmp/kubectl https://dl.k8s.io/release/v1.31.0/bin/linux/amd64/kubectl
chmod +x /tmp/kubectl && sudo mv /tmp/kubectl /usr/local/bin/kubectl
- name: Build router image
run: docker build -t sgl-router:e2e -f sgl-router/tests/e2e/k8s_integration/Dockerfile.router .
- name: Build fake_worker image
run: |
docker build -t sgl-router-fake-worker:e2e \
-f sgl-router/tests/e2e/k8s_integration/Dockerfile.fake_worker \
sgl-router/tests/e2e/k8s_integration/
- name: Bootstrap kind + deploy
run: bash sgl-router/tests/e2e/k8s_integration/setup.sh
- name: Set up Python
run: |
python3 -m venv /tmp/e2e-venv
/tmp/e2e-venv/bin/pip install -r sgl-router/tests/e2e/k8s_integration/requirements.txt
- name: Run E2E
run: /tmp/e2e-venv/bin/pytest sgl-router/tests/e2e/k8s_integration/ -v --tb=short
- name: Dump cluster + router diagnostics on failure
if: failure()
run: |
set +e
kubectl -n sgl-router-test get pods -o wide
kubectl -n sgl-router-test describe pod -l app=sgl-router
kubectl -n sgl-router-test describe pod -l app=sglang
kubectl -n sgl-router-test logs deploy/sgl-router --tail=300
kubectl -n sgl-router-test logs deploy/sgl-router --tail=300 --previous
kubectl -n sgl-router-test get events --sort-by=.lastTimestamp
kubectl -n sgl-router-test get endpointslices -o wide
kubectl -n sgl-router-test get svc
true
sgl-router-e2e:
name: tier-3 — e2e
needs: [sgl-router-changes, sgl-router-gate, sgl-router-build-and-test]
if: needs.sgl-router-changes.outputs.should_run == 'true'
runs-on: 2-gpu-h100
timeout-minutes: 45
steps:
- uses: actions/checkout@v4
- name: Install Rust toolchain
run: bash scripts/ci/utils/install_rust_protoc.sh
- name: Rust cache
uses: Swatinem/rust-cache@v2
with:
workspaces: sgl-router
shared-key: "sgl-router-cache"
- name: cargo build (release)
working-directory: sgl-router
run: |
source "$HOME/.cargo/env"
cargo build --locked --release --workspace
# Install SGLang from the local checkout in editable mode (no PyPI
# version pin) — mirrors `pr-test-rust.yml` so the router e2e runs
# against whatever SGLang version is on the branch. Replaces the
# previous `pip install sglang[srt]==X.Y.Z` pin, which forced the
# e2e to test against a stale wheel and would silently miss any
# new ServerArgs field (e.g. `disaggregation_mode` /
# `disaggregation_bootstrap_port`) added in the branch.
- name: Install SGLang dependencies
run: |
bash scripts/ci/cuda/ci_install_dependency.sh
- name: Install e2e test dependencies
run: |
python3 -m pip install -r sgl-router/tests/e2e/requirements.txt
# IMPORTANT: --ignore=tests/e2e/k8s_integration. The k8s integration
# suite is owned by the `sgl-router-k8s-integration` job which has
# kind + kubectl installed and a real cluster running. The e2e GPU
# runner has neither, so pytest's recursive collection of
# tests/e2e/ would ERROR every k8s test at setup. Both tiers must
# be allowed to fail independently — k8s flakes (cluster boot,
# image pull) must not surface as e2e failures on H100, and e2e
# flakes (model load, HF auth) must not surface as k8s failures.
- name: Run e2e
working-directory: sgl-router
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
run: |
python3 -m pytest tests/e2e/ -v -s --tb=short \
--ignore=tests/e2e/k8s_integration
- name: Tokenizer parity matrix (uses HF cache populated by e2e)
working-directory: sgl-router
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
run: |
source "$HOME/.cargo/env"
cargo test --locked --release --test component tokenizer::parity
sgl-router-finish:
name: finish
needs:
- sgl-router-changes
- sgl-router-gate
- sgl-router-lint
- sgl-router-build-and-test
- sgl-router-docker-build-test
- sgl-router-k8s-integration
- sgl-router-e2e
# `always()` lets `finish` run after upstream `skipped` outcomes.
# Two outcomes are a success: every tier passed, or the PR touches
# no sgl-router path and every tier was skipped. Anything else,
# including a gate that failed for a missing `run-ci` label, fails
# the run, so `/tag-and-rerun-ci` sees an unsuccessful run to rerun.
if: always()
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Check required results
env:
CHANGES_RESULT: ${{ needs.sgl-router-changes.result }}
SHOULD_RUN: ${{ needs.sgl-router-changes.outputs.should_run }}
GATE_RESULT: ${{ needs.sgl-router-gate.result }}
LINT_RESULT: ${{ needs.sgl-router-lint.result }}
BUILD_RESULT: ${{ needs.sgl-router-build-and-test.result }}
DOCKER_RESULT: ${{ needs.sgl-router-docker-build-test.result }}
K8S_RESULT: ${{ needs.sgl-router-k8s-integration.result }}
E2E_RESULT: ${{ needs.sgl-router-e2e.result }}
run: |
if [[ "$CHANGES_RESULT" != "success" ]]; then
echo "::error::sgl-router path detection did not succeed ($CHANGES_RESULT)."
exit 1
fi
tiers="$GATE_RESULT $LINT_RESULT $BUILD_RESULT $DOCKER_RESULT $K8S_RESULT $E2E_RESULT"
if [[ "$SHOULD_RUN" == "true" && "$tiers" == "success success success success success success" ]]; then
echo "All sgl-router checks passed."
elif [[ "$SHOULD_RUN" == "false" && "$tiers" == "skipped skipped skipped skipped skipped skipped" ]]; then
echo "No sgl-router-related changes; tiers skipped."
else
echo "::error::Unexpected sgl-router results: should_run=$SHOULD_RUN, gate=$GATE_RESULT, lint=$LINT_RESULT, build=$BUILD_RESULT, docker=$DOCKER_RESULT, k8s=$K8S_RESULT, e2e=$E2E_RESULT."
exit 1
fi