442 lines
18 KiB
YAML
442 lines
18 KiB
YAML
name: PR Test (sgl-router)
|
|
|
|
# Trigger contract — modeled on `pr-test.yml`:
|
|
#
|
|
# * No `paths:` filter at the trigger level. The workflow ALWAYS
|
|
# fires on every PR synchronize / push, so the run record always
|
|
# appears as a check on the PR (no more "workflow silently didn't
|
|
# fire" mystery debugging). Path-based skip decisions are made
|
|
# inside the `sgl-router-changes` job below, where we can log the
|
|
# reason. Same for the `run-ci` label gate — moved off the
|
|
# workflow trigger and into the gate job so its outcome is
|
|
# observable, and now fails (rather than skips) when it is missing.
|
|
#
|
|
# * `push` on `main` keeps firing so post-merge runs still record
|
|
# against the default branch.
|
|
on:
|
|
push:
|
|
branches: [main]
|
|
pull_request:
|
|
branches: [main]
|
|
types: [opened, synchronize, reopened, labeled]
|
|
workflow_dispatch:
|
|
|
|
concurrency:
|
|
group: sgl-router-${{ github.event_name }}-${{ github.head_ref || github.ref_name || 'default' }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
SGLANG_IS_IN_CI: true
|
|
|
|
jobs:
|
|
# Stage 0a — path detection. Always runs (cheap, ubuntu-latest) and
|
|
# emits a single `should_run` output consumed by every tier below.
|
|
# It only answers "does this PR touch sgl-router?"; the `run-ci`
|
|
# label is checked by the separate gate job so that a missing label
|
|
# fails that job instead of silently skipping the tiers (see below).
|
|
sgl-router-changes:
|
|
name: changes
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 5
|
|
outputs:
|
|
should_run: ${{ steps.decide.outputs.should_run }}
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
# `dorny/paths-filter` computes the diff against the PR base
|
|
# (for pull_request) or the push's `before` SHA (for push) and
|
|
# exposes `sgl_router=true|false` on whether any tracked path
|
|
# changed. Replaces the old workflow-level `paths:` filter so
|
|
# the gate is observable.
|
|
- name: Detect sgl-router path changes
|
|
id: paths
|
|
if: github.event_name != 'workflow_dispatch'
|
|
uses: dorny/paths-filter@v3
|
|
with:
|
|
filters: |
|
|
sgl_router:
|
|
- 'sgl-router/**'
|
|
- 'rust/Cargo.toml'
|
|
- 'rust/sglang-processor/**'
|
|
- 'rust/sglang-api-types/**'
|
|
- 'proto/**'
|
|
- '.github/workflows/pr-test-sgl-router.yml'
|
|
- 'docker/sgl-router.Dockerfile'
|
|
- 'scripts/ci/cuda/ci_install_dependency.sh'
|
|
- name: Decide
|
|
id: decide
|
|
env:
|
|
EVENT_NAME: ${{ github.event_name }}
|
|
PATHS_CHANGED: ${{ steps.paths.outputs.sgl_router }}
|
|
run: |
|
|
if [[ "$EVENT_NAME" == "workflow_dispatch" ]]; then
|
|
echo "should_run=true" >> "$GITHUB_OUTPUT"
|
|
echo "Manual dispatch; running sgl-router tiers."
|
|
elif [[ "$PATHS_CHANGED" != "true" ]]; then
|
|
echo "should_run=false" >> "$GITHUB_OUTPUT"
|
|
echo "No sgl-router-related changes; skipping tiers."
|
|
else
|
|
echo "should_run=true" >> "$GITHUB_OUTPUT"
|
|
echo "sgl-router-related changes detected; running tiers."
|
|
fi
|
|
|
|
# Stage 0b — `run-ci` label gate, shared with `pr-test.yml`. Cheap
|
|
# workflow runs (e.g. docs-only PRs that happen to touch the
|
|
# sgl-router dir) still need the opt-in label before consuming the
|
|
# H100 / kind tiers below. `push` on main and `workflow_dispatch`
|
|
# pass straight through.
|
|
#
|
|
# `pr-gate.yml` fetches the PR's labels live and FAILS when `run-ci`
|
|
# is missing. Both matter for `/tag-and-rerun-ci`: the slash handler
|
|
# adds the label with GITHUB_TOKEN, which fires no `labeled` event,
|
|
# then reruns the PR's unsuccessful runs. A rerun replays the original
|
|
# event payload, so a gate reading `github.event.pull_request.labels`
|
|
# would skip again, and a run that concluded `success` with every
|
|
# tier skipped is never rerun at all. Failing here on the live label
|
|
# set makes the rerun both happen and pick the label up.
|
|
sgl-router-gate:
|
|
name: gate
|
|
needs: sgl-router-changes
|
|
if: needs.sgl-router-changes.outputs.should_run == 'true'
|
|
uses: ./.github/workflows/pr-gate.yml
|
|
with:
|
|
require-run-ci: true
|
|
|
|
sgl-router-lint:
|
|
name: tier-1 — lint
|
|
needs: [sgl-router-changes, sgl-router-gate]
|
|
if: needs.sgl-router-changes.outputs.should_run == 'true'
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
# Scoped per-job (not workflow-wide) to avoid SMG's documented breakage
|
|
# of self-hosted-runner pip installs that compile Rust extensions.
|
|
RUSTC_WRAPPER: sccache
|
|
SCCACHE_GHA_ENABLED: "true"
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- name: Install protoc
|
|
run: sudo bash scripts/ci/utils/install_protoc.sh
|
|
|
|
# sccache install is best-effort: the GitHub release CDN intermittently
|
|
# returns 5xx and the action's built-in retry is shallow. If it fails,
|
|
# we unset RUSTC_WRAPPER below and proceed without compile caching —
|
|
# rust-cache still covers the bulk of warm-cache wins.
|
|
- name: Configure sccache
|
|
id: sccache
|
|
continue-on-error: true
|
|
uses: mozilla-actions/sccache-action@v0.0.9
|
|
with:
|
|
version: "v0.12.0"
|
|
disable_annotations: true
|
|
|
|
- name: Disable sccache if install failed
|
|
if: steps.sccache.outcome != 'success'
|
|
run: |
|
|
echo "::warning::sccache install failed (${{ steps.sccache.outcome }}); proceeding without compile caching"
|
|
echo "RUSTC_WRAPPER=" >> "$GITHUB_ENV"
|
|
|
|
- name: Rust cache
|
|
uses: Swatinem/rust-cache@v2
|
|
with:
|
|
workspaces: sgl-router
|
|
shared-key: "sgl-router-cache"
|
|
cache-all-crates: true
|
|
cache-on-failure: true
|
|
save-if: ${{ github.ref == 'refs/heads/main' }}
|
|
|
|
- name: cargo check
|
|
working-directory: sgl-router
|
|
run: cargo check --locked --workspace --all-targets
|
|
|
|
- name: cargo clippy
|
|
working-directory: sgl-router
|
|
run: cargo clippy --locked --workspace --all-targets -- -D warnings
|
|
|
|
- name: cargo fmt
|
|
working-directory: sgl-router
|
|
run: |
|
|
rustup toolchain install nightly --profile minimal --component rustfmt
|
|
cargo +nightly fmt --all -- --check
|
|
|
|
# cargo-deny 0.19.6+ required for CVSS 4.0 parsing.
|
|
- name: Install cargo-deny
|
|
run: cargo install --locked cargo-deny@0.19.6
|
|
|
|
- name: cargo deny check
|
|
working-directory: sgl-router
|
|
run: cargo deny check licenses bans sources advisories
|
|
|
|
- name: Show sccache stats
|
|
if: always() && steps.sccache.outcome == 'success'
|
|
run: sccache --show-stats
|
|
|
|
sgl-router-build-and-test:
|
|
name: tier-2 — build + test
|
|
needs: [sgl-router-changes, sgl-router-gate, sgl-router-lint]
|
|
if: needs.sgl-router-changes.outputs.should_run == 'true'
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
RUSTC_WRAPPER: sccache
|
|
SCCACHE_GHA_ENABLED: "true"
|
|
timeout-minutes: 30
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- name: Install protoc
|
|
run: sudo bash scripts/ci/utils/install_protoc.sh
|
|
|
|
- name: Install OS deps (libssl, pkg-config)
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y libssl-dev pkg-config
|
|
|
|
# sccache install is best-effort: the GitHub release CDN intermittently
|
|
# returns 5xx and the action's built-in retry is shallow. If it fails,
|
|
# we unset RUSTC_WRAPPER below and proceed without compile caching —
|
|
# rust-cache still covers the bulk of warm-cache wins.
|
|
- name: Configure sccache
|
|
id: sccache
|
|
continue-on-error: true
|
|
uses: mozilla-actions/sccache-action@v0.0.9
|
|
with:
|
|
version: "v0.12.0"
|
|
disable_annotations: true
|
|
|
|
- name: Disable sccache if install failed
|
|
if: steps.sccache.outcome != 'success'
|
|
run: |
|
|
echo "::warning::sccache install failed (${{ steps.sccache.outcome }}); proceeding without compile caching"
|
|
echo "RUSTC_WRAPPER=" >> "$GITHUB_ENV"
|
|
|
|
- name: Rust cache
|
|
uses: Swatinem/rust-cache@v2
|
|
with:
|
|
workspaces: sgl-router
|
|
shared-key: "sgl-router-cache"
|
|
cache-all-crates: true
|
|
cache-on-failure: true
|
|
save-if: ${{ github.ref == 'refs/heads/main' }}
|
|
|
|
- name: cargo build (record wall time)
|
|
working-directory: sgl-router
|
|
run: |
|
|
START=$(date +%s)
|
|
cargo build --locked --release --all-targets
|
|
END=$(date +%s)
|
|
BUILD_SECONDS=$((END - START))
|
|
echo "tier_2_build_duration_seconds=${BUILD_SECONDS}" >> "$GITHUB_ENV"
|
|
echo "::notice title=Tier-2 build duration::${BUILD_SECONDS}s"
|
|
|
|
- name: cargo test
|
|
working-directory: sgl-router
|
|
# Skip the tokenizer parity_matrix test here: ubuntu-latest has
|
|
# no HuggingFace cache, so the matrix would hard-fail (see the
|
|
# test docstring). The e2e job runs it after pytest populates
|
|
# the Qwen3-0.6B tokenizer.json. Filter is matched against the
|
|
# full test path `tokenizer::parity::parity_matrix`; substring
|
|
# `parity_matrix` is unique to that test.
|
|
run: cargo test --locked --release --workspace -- --skip parity_matrix
|
|
|
|
# Regenerate the cross-impl block-hash parity fixture and fail if it
|
|
# differs from the committed file. The Python script replicates
|
|
# SGLang's `radix_cache.hash_page` algorithm verbatim (no
|
|
# `import sglang`); a diff here means EITHER the Rust port's algorithm
|
|
# drifted from the script, OR the script drifted from SGLang's actual
|
|
# code — in both cases the operator must investigate and re-commit
|
|
# the fixture in the same PR as the algorithm change.
|
|
#
|
|
# `set -euo pipefail` makes a Python-side failure (syntax error,
|
|
# missing import) cause this step to fail instead of falling
|
|
# through to a no-diff "pass" when no fixture was actually
|
|
# regenerated.
|
|
- name: kv_events hash parity fixture (drift check)
|
|
working-directory: sgl-router
|
|
run: |
|
|
set -euo pipefail
|
|
python3 tests/scripts/generate_kv_events_hash_parity.py
|
|
if ! git diff --exit-code -- tests/fixtures/kv_events_hash_parity.json; then
|
|
echo "::error::kv_events hash parity fixture drifted." \
|
|
"Re-run sgl-router/tests/scripts/generate_kv_events_hash_parity.py" \
|
|
"and commit the updated tests/fixtures/kv_events_hash_parity.json."
|
|
exit 1
|
|
fi
|
|
|
|
- name: Show sccache stats
|
|
if: always() && steps.sccache.outcome == 'success'
|
|
run: sccache --show-stats
|
|
|
|
- name: Summary
|
|
if: always()
|
|
run: |
|
|
echo "## Tier-2 results" >> "$GITHUB_STEP_SUMMARY"
|
|
echo "- Build duration: \`${tier_2_build_duration_seconds:-(not recorded)}\` s" >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
sgl-router-docker-build-test:
|
|
name: tier-3 — docker (placeholder)
|
|
needs: [sgl-router-changes, sgl-router-gate, sgl-router-build-and-test]
|
|
if: needs.sgl-router-changes.outputs.should_run == 'true'
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 15
|
|
steps:
|
|
- name: Placeholder
|
|
run: |
|
|
echo "docker-build-test not implemented yet."
|
|
exit 0
|
|
|
|
# tier-3 k8s integration is decoupled from tier-3 e2e: each runs on a
|
|
# different runner type (ubuntu kind cluster vs H100), each manages its
|
|
# own test scope (k8s_integration/ vs everything-else-in-e2e), and
|
|
# each is allowed to fail without blocking the other. They both still
|
|
# gate on tier-2's build+test passing — a Rust compile failure blocks
|
|
# the GPU/cluster runners from spinning up.
|
|
sgl-router-k8s-integration:
|
|
name: tier-3 — k8s integration
|
|
needs: [sgl-router-changes, sgl-router-gate, sgl-router-build-and-test]
|
|
if: needs.sgl-router-changes.outputs.should_run == 'true'
|
|
runs-on: ubuntu-22.04
|
|
timeout-minutes: 30
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Setup Docker buildx
|
|
uses: docker/setup-buildx-action@v3
|
|
- name: Install kind
|
|
run: |
|
|
curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v0.24.0/kind-linux-amd64
|
|
chmod +x /tmp/kind && sudo mv /tmp/kind /usr/local/bin/kind
|
|
- name: Install kubectl
|
|
run: |
|
|
curl -Lo /tmp/kubectl https://dl.k8s.io/release/v1.31.0/bin/linux/amd64/kubectl
|
|
chmod +x /tmp/kubectl && sudo mv /tmp/kubectl /usr/local/bin/kubectl
|
|
- name: Build router image
|
|
run: docker build -t sgl-router:e2e -f sgl-router/tests/e2e/k8s_integration/Dockerfile.router .
|
|
- name: Build fake_worker image
|
|
run: |
|
|
docker build -t sgl-router-fake-worker:e2e \
|
|
-f sgl-router/tests/e2e/k8s_integration/Dockerfile.fake_worker \
|
|
sgl-router/tests/e2e/k8s_integration/
|
|
- name: Bootstrap kind + deploy
|
|
run: bash sgl-router/tests/e2e/k8s_integration/setup.sh
|
|
- name: Set up Python
|
|
run: |
|
|
python3 -m venv /tmp/e2e-venv
|
|
/tmp/e2e-venv/bin/pip install -r sgl-router/tests/e2e/k8s_integration/requirements.txt
|
|
- name: Run E2E
|
|
run: /tmp/e2e-venv/bin/pytest sgl-router/tests/e2e/k8s_integration/ -v --tb=short
|
|
- name: Dump cluster + router diagnostics on failure
|
|
if: failure()
|
|
run: |
|
|
set +e
|
|
kubectl -n sgl-router-test get pods -o wide
|
|
kubectl -n sgl-router-test describe pod -l app=sgl-router
|
|
kubectl -n sgl-router-test describe pod -l app=sglang
|
|
kubectl -n sgl-router-test logs deploy/sgl-router --tail=300
|
|
kubectl -n sgl-router-test logs deploy/sgl-router --tail=300 --previous
|
|
kubectl -n sgl-router-test get events --sort-by=.lastTimestamp
|
|
kubectl -n sgl-router-test get endpointslices -o wide
|
|
kubectl -n sgl-router-test get svc
|
|
true
|
|
|
|
sgl-router-e2e:
|
|
name: tier-3 — e2e
|
|
needs: [sgl-router-changes, sgl-router-gate, sgl-router-build-and-test]
|
|
if: needs.sgl-router-changes.outputs.should_run == 'true'
|
|
runs-on: 2-gpu-h100
|
|
timeout-minutes: 45
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Install Rust toolchain
|
|
run: bash scripts/ci/utils/install_rust_protoc.sh
|
|
- name: Rust cache
|
|
uses: Swatinem/rust-cache@v2
|
|
with:
|
|
workspaces: sgl-router
|
|
shared-key: "sgl-router-cache"
|
|
- name: cargo build (release)
|
|
working-directory: sgl-router
|
|
run: |
|
|
source "$HOME/.cargo/env"
|
|
cargo build --locked --release --workspace
|
|
|
|
# Install SGLang from the local checkout in editable mode (no PyPI
|
|
# version pin) — mirrors `pr-test-rust.yml` so the router e2e runs
|
|
# against whatever SGLang version is on the branch. Replaces the
|
|
# previous `pip install sglang[srt]==X.Y.Z` pin, which forced the
|
|
# e2e to test against a stale wheel and would silently miss any
|
|
# new ServerArgs field (e.g. `disaggregation_mode` /
|
|
# `disaggregation_bootstrap_port`) added in the branch.
|
|
- name: Install SGLang dependencies
|
|
run: |
|
|
bash scripts/ci/cuda/ci_install_dependency.sh
|
|
|
|
- name: Install e2e test dependencies
|
|
run: |
|
|
python3 -m pip install -r sgl-router/tests/e2e/requirements.txt
|
|
|
|
# IMPORTANT: --ignore=tests/e2e/k8s_integration. The k8s integration
|
|
# suite is owned by the `sgl-router-k8s-integration` job which has
|
|
# kind + kubectl installed and a real cluster running. The e2e GPU
|
|
# runner has neither, so pytest's recursive collection of
|
|
# tests/e2e/ would ERROR every k8s test at setup. Both tiers must
|
|
# be allowed to fail independently — k8s flakes (cluster boot,
|
|
# image pull) must not surface as e2e failures on H100, and e2e
|
|
# flakes (model load, HF auth) must not surface as k8s failures.
|
|
- name: Run e2e
|
|
working-directory: sgl-router
|
|
env:
|
|
HF_TOKEN: ${{ secrets.HF_TOKEN }}
|
|
run: |
|
|
python3 -m pytest tests/e2e/ -v -s --tb=short \
|
|
--ignore=tests/e2e/k8s_integration
|
|
|
|
- name: Tokenizer parity matrix (uses HF cache populated by e2e)
|
|
working-directory: sgl-router
|
|
env:
|
|
HF_TOKEN: ${{ secrets.HF_TOKEN }}
|
|
run: |
|
|
source "$HOME/.cargo/env"
|
|
cargo test --locked --release --test component tokenizer::parity
|
|
|
|
sgl-router-finish:
|
|
name: finish
|
|
needs:
|
|
- sgl-router-changes
|
|
- sgl-router-gate
|
|
- sgl-router-lint
|
|
- sgl-router-build-and-test
|
|
- sgl-router-docker-build-test
|
|
- sgl-router-k8s-integration
|
|
- sgl-router-e2e
|
|
# `always()` lets `finish` run after upstream `skipped` outcomes.
|
|
# Two outcomes are a success: every tier passed, or the PR touches
|
|
# no sgl-router path and every tier was skipped. Anything else,
|
|
# including a gate that failed for a missing `run-ci` label, fails
|
|
# the run, so `/tag-and-rerun-ci` sees an unsuccessful run to rerun.
|
|
if: always()
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 5
|
|
steps:
|
|
- name: Check required results
|
|
env:
|
|
CHANGES_RESULT: ${{ needs.sgl-router-changes.result }}
|
|
SHOULD_RUN: ${{ needs.sgl-router-changes.outputs.should_run }}
|
|
GATE_RESULT: ${{ needs.sgl-router-gate.result }}
|
|
LINT_RESULT: ${{ needs.sgl-router-lint.result }}
|
|
BUILD_RESULT: ${{ needs.sgl-router-build-and-test.result }}
|
|
DOCKER_RESULT: ${{ needs.sgl-router-docker-build-test.result }}
|
|
K8S_RESULT: ${{ needs.sgl-router-k8s-integration.result }}
|
|
E2E_RESULT: ${{ needs.sgl-router-e2e.result }}
|
|
run: |
|
|
if [[ "$CHANGES_RESULT" != "success" ]]; then
|
|
echo "::error::sgl-router path detection did not succeed ($CHANGES_RESULT)."
|
|
exit 1
|
|
fi
|
|
tiers="$GATE_RESULT $LINT_RESULT $BUILD_RESULT $DOCKER_RESULT $K8S_RESULT $E2E_RESULT"
|
|
if [[ "$SHOULD_RUN" == "true" && "$tiers" == "success success success success success success" ]]; then
|
|
echo "All sgl-router checks passed."
|
|
elif [[ "$SHOULD_RUN" == "false" && "$tiers" == "skipped skipped skipped skipped skipped skipped" ]]; then
|
|
echo "No sgl-router-related changes; tiers skipped."
|
|
else
|
|
echo "::error::Unexpected sgl-router results: should_run=$SHOULD_RUN, gate=$GATE_RESULT, lint=$LINT_RESULT, build=$BUILD_RESULT, docker=$DOCKER_RESULT, k8s=$K8S_RESULT, e2e=$E2E_RESULT."
|
|
exit 1
|
|
fi
|