diff --git a/.github/workflows/_pr-test-rust-ext-build.yml b/.github/workflows/_pr-test-rust-ext-build.yml new file mode 100644 index 000000000..5d832d947 --- /dev/null +++ b/.github/workflows/_pr-test-rust-ext-build.yml @@ -0,0 +1,188 @@ +name: PR Test - Build Rust Extensions + +# Builds the rust/ workspace's PyO3 extension modules once so the CUDA test +# stages install with SGLANG_BUILD_RUST_EXTS=none instead of each rebuilding them. +# +# Why: uv holds the editable sdist lock in the shared ~/.cache/uv for the whole +# build, cargo included, so CUDA jobs on one host serialize on it - and once the +# Rust build passed a few minutes, the queue's tail hit uv's lock timeout. +# +# The win is compiling once per run instead of in all ~25 install steps; the two +# layers below only decide who pays for it. +# - The cache, keyed by a source hash, carries a build across runs - no version to +# bump, unlike sgl-kernel, which publishes one because it ships to users. +# - The artifact hands it to the stages, which the cache cannot: it is best-effort, +# evictable, and this repo is at its 10 GB limit. Artifacts are durable per run. +# +# Two jobs, because only compiling needs the build node: exactly one runner carries +# that label and it also serves the sgl-kernel and docker builds, so queueing there +# on a cache hit would put its wait in front of every stage. Republishing bytes +# needs no particular host - the glibc a module requires is recorded in the module, +# not decided by whoever uploads it. + +on: + workflow_call: + inputs: + runs_on: + description: 'Runner label for the compile job. Must be self-hosted: its glibc has to satisfy max_glibc, and the cargo build cache needs a persistent ~/.cache.' + type: string + required: true + restore_runs_on: + description: 'Runner label for the cache-hit path. Wants free capacity and nothing else, since it neither compiles nor imports the modules.' + type: string + default: ubuntu-latest + artifact_name: + description: 'Artifact name. Suffix it per caller: artifacts are immutable per name per run, so two callers sharing a run would collide.' + type: string + default: rust-ext-x86_64 + cache_key_prefix: + description: 'Cache key prefix. Callers share it on purpose to reuse each other''s build; it encodes the arch, since the modules are not portable across architectures.' + type: string + default: rust-ext-x86_64 + max_glibc: + description: 'Highest GLIBC symbol version the built .so files may require. Set by the oldest test runner image, jammy at glibc 2.35 - the pools are not all on one image.' + type: string + default: '2.35' + git_ref: + type: string + default: '' + skip_pr_test_health_check: + description: 'Forwarded from the caller for the check-maintenance action.' + type: boolean + default: false + outputs: + artifact_name: + description: 'Artifact holding the built modules. Empty when neither job published one, whether skipped or failed; consumers that still run compile them during install.' + value: ${{ jobs.restore.outputs.artifact_name || jobs.compile.outputs.artifact_name }} + +# Reusable workflows do not inherit the caller's env; mirror what +# check-maintenance reads. +env: + SGLANG_IS_IN_CI: true + SKIP_PR_TEST_HEALTH_CHECK: ${{ inputs.skip_pr_test_health_check && 'true' || 'false' }} + PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }} + +jobs: + restore: + runs-on: ${{ inputs.restore_runs_on }} + timeout-minutes: 15 + name: Restore Rust Ext + outputs: + hit: ${{ steps.cache.outputs.cache-hit }} + # From the last step, so a failed job publishes no name for always() consumers. + artifact_name: ${{ steps.publish.outputs.name }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.git_ref || github.sha }} + # Just what the cache key hashes, plus the action and script this job + # runs: the workspace is cold here and the rest of the tree is mostly + # docs. Both jobs must hash the same rust/** set, which this preserves. + # Cone mode off is what allows naming a single file. + sparse-checkout: | + rust + python/setup.py + .github + scripts/ci/utils + sparse-checkout-cone-mode: false + + - uses: ./.github/actions/check-maintenance + + # setup.py counts because it selects which crates get built. pyproject.toml + # is left out - it churns on bumps that cannot affect these modules. + - name: Restore built modules + id: cache + uses: actions/cache/restore@v4 + with: + path: python/sglang/srt/*/_core*.so + key: ${{ inputs.cache_key_prefix }}-${{ hashFiles('rust/**', 'python/setup.py') }} + + # No MAX_GLIBC: these are the bytes the compile job already checked before + # saving them under this key. The module count is still worth re-checking, + # so a truncated entry fails here rather than as a test import error. + - name: Stage modules for upload + if: steps.cache.outputs.cache-hit == 'true' + run: bash scripts/ci/utils/stage_rust_ext_modules.sh + + - name: Upload extension modules + if: steps.cache.outputs.cache-hit == 'true' + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.artifact_name }} + # Archive holds /_core*.so, so it unpacks into python/sglang/srt/. + path: rust-ext-staging/ + if-no-files-found: error + retention-days: 1 + + - name: Publish artifact name + id: publish + if: steps.cache.outputs.cache-hit == 'true' + run: echo "name=${{ inputs.artifact_name }}" >> "$GITHUB_OUTPUT" + + compile: + needs: restore + if: needs.restore.outputs.hit != 'true' + runs-on: ${{ inputs.runs_on }} + timeout-minutes: 60 + name: Build Rust Ext + outputs: + artifact_name: ${{ steps.publish.outputs.name }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.git_ref || github.sha }} + + - uses: ./.github/actions/check-maintenance + + # No crate sets abi3, so the ABI tag is minor-version specific and only the + # pools on this version can use the result - h20 ships 3.12 and falls back to + # compiling during install. Pinned rather than left to the image so the tag is + # at least predictable. + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install protoc and Rust toolchain + run: bash scripts/ci/utils/install_rust_protoc.sh + + - name: Build extension modules + run: | + set -euxo pipefail + export PATH="${CARGO_HOME:-$HOME/.cargo}/bin:${PATH}" + # Same path ci_install_dependency.sh uses: one warm cargo cache per host. + export CARGO_TARGET_DIR="${HOME}/.cache/sglang-cargo-target" + python3 -m pip install --upgrade pip + command -v uv >/dev/null 2>&1 || pip install uv + # build_rust needs only the build backend, not sglang's ~294 runtime deps. + uv venv /tmp/rust-ext-build --python python3.10 --seed + # shellcheck disable=SC1091 + source /tmp/rust-ext-build/bin/activate + uv pip install "setuptools>=61.0" "setuptools-rust>=1.10" "setuptools-scm>=8.0" wheel + cd python + SGLANG_BUILD_RUST_EXTS=all python setup.py build_rust --inplace + + - name: Verify modules and stage for upload + env: + MAX_GLIBC: ${{ inputs.max_glibc }} + run: bash scripts/ci/utils/stage_rust_ext_modules.sh + + # After the verify step, so a rejected build cannot poison this key for every + # later run. + - name: Save built modules + uses: actions/cache/save@v4 + with: + path: python/sglang/srt/*/_core*.so + key: ${{ inputs.cache_key_prefix }}-${{ hashFiles('rust/**', 'python/setup.py') }} + + - name: Upload extension modules + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.artifact_name }} + path: rust-ext-staging/ + if-no-files-found: error + retention-days: 1 + + - name: Publish artifact name + id: publish + run: echo "name=${{ inputs.artifact_name }}" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/_pr-test-stage.yml b/.github/workflows/_pr-test-stage.yml index 3dd77c072..f0861b30c 100644 --- a/.github/workflows/_pr-test-stage.yml +++ b/.github/workflows/_pr-test-stage.yml @@ -54,6 +54,10 @@ on: description: 'pytest path to run after the suite (b200 FA4 jit_kernel tests). Empty skips.' type: string default: '' + rust_ext_artifact: + description: 'Artifact of prebuilt Rust extension modules, from rust-ext-build. Empty, or a download that fails, means this stage compiles them during install.' + type: string + default: '' # Mirror pr-test.yml top-level env. Reusable workflows do NOT inherit caller's # workflow-level env across the workflow_call boundary, so anything pr-test.yml @@ -120,10 +124,23 @@ jobs: merge-multiple: true pattern: wheel-python3.10-cuda* + # continue-on-error: the artifact expires in a day, so a later re-run just + # compiles during install. download-artifact creates srt/server/, which + # exists only as build output. + - name: Download prebuilt Rust extensions + id: rust_ext + if: ${{ inputs.rust_ext_artifact != '' }} + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.rust_ext_artifact }} + path: python/sglang/srt/ + - name: Install dependencies timeout-minutes: ${{ fromJson(steps.rc.outputs.install_timeout) }} env: GRACE_BLACKWELL: ${{ steps.rc.outputs.grace_blackwell || '0' }} + SGLANG_BUILD_RUST_EXTS: ${{ steps.rust_ext.outcome == 'success' && 'none' || '' }} run: | CUSTOM_BUILD_SGL_KERNEL=${{ fromJson(inputs.check_changes).sgl_kernel }} bash ${{ steps.rc.outputs.install }} diff --git a/.github/workflows/pr-test-extra.yml b/.github/workflows/pr-test-extra.yml index c004b923c..944d1f236 100644 --- a/.github/workflows/pr-test-extra.yml +++ b/.github/workflows/pr-test-extra.yml @@ -131,9 +131,28 @@ jobs: skip_pr_test_health_check: ${{ inputs.skip_pr_test_health_check == true }} secrets: inherit + # =============================================== Rust extensions ==================================================== + + # Own build job because artifacts are per-run: a standalone `run-ci-extra` run + # cannot reach the base run's. Only artifact_name is suffixed; the cache prefix + # stays at the default so the two workflows share one cache entry. + rust-ext-build: + needs: [check-changes, call-gate] + if: | + !cancelled() && + needs.check-changes.result == 'success' && + (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') + uses: ./.github/workflows/_pr-test-rust-ext-build.yml + with: + runs_on: x64-kernel-build-node + artifact_name: rust-ext-x86_64-extra + git_ref: ${{ inputs.git_ref || '' }} + skip_pr_test_health_check: ${{ inputs.skip_pr_test_health_check == true }} + secrets: inherit + # =============================================== extra-a (1-/2-gpu) =============================================== extra-a-test-1-gpu-small: - needs: [check-changes, call-gate, sgl-kernel-build-wheels] + needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build] if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }} uses: ./.github/workflows/_pr-test-stage.yml with: @@ -143,10 +162,11 @@ jobs: caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} run_timeout_minutes: '60' + rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit extra-a-test-1-gpu-large: - needs: [check-changes, call-gate, sgl-kernel-build-wheels] + needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build] if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }} uses: ./.github/workflows/_pr-test-stage.yml with: @@ -157,10 +177,11 @@ jobs: partitions: ${{ needs.check-changes.outputs.partitions }} run_timeout_minutes: '60' timeout_per_file: '1800' + rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit extra-a-test-2-gpu-large: - needs: [check-changes, call-gate, sgl-kernel-build-wheels] + needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build] if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }} uses: ./.github/workflows/_pr-test-stage.yml with: @@ -170,11 +191,12 @@ jobs: caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} run_timeout_minutes: '60' + rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit # =============================================== extra-b (4-/8-gpu) =============================================== extra-b-test-4-gpu-h100: - needs: [check-changes, call-gate, sgl-kernel-build-wheels] + needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build] if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }} uses: ./.github/workflows/_pr-test-stage.yml with: @@ -184,10 +206,11 @@ jobs: caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} run_timeout_minutes: '60' + rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit extra-b-test-4-gpu-b200: - needs: [check-changes, call-gate, sgl-kernel-build-wheels] + needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build] if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }} uses: ./.github/workflows/_pr-test-stage.yml with: @@ -198,10 +221,11 @@ jobs: partitions: ${{ needs.check-changes.outputs.partitions }} run_timeout_minutes: '60' timeout_per_file: '1800' + rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit extra-b-test-8-gpu-h200: - needs: [check-changes, call-gate, sgl-kernel-build-wheels] + needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build] if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }} uses: ./.github/workflows/_pr-test-stage.yml with: @@ -211,10 +235,11 @@ jobs: caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} run_timeout_minutes: '60' + rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit extra-b-test-deepep-4-gpu-h100: - needs: [check-changes, call-gate, sgl-kernel-build-wheels] + needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build] if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }} uses: ./.github/workflows/_pr-test-stage.yml with: @@ -224,10 +249,11 @@ jobs: caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} run_timeout_minutes: '60' + rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit extra-b-test-deepep-4-gpu-b200: - needs: [check-changes, call-gate, sgl-kernel-build-wheels] + needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build] if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }} uses: ./.github/workflows/_pr-test-stage.yml with: @@ -238,10 +264,11 @@ jobs: partitions: ${{ needs.check-changes.outputs.partitions }} run_timeout_minutes: '60' timeout_per_file: '1800' + rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit extra-b-test-deepep-8-gpu-h200: - needs: [check-changes, call-gate, sgl-kernel-build-wheels] + needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build] if: ${{ !failure() && !cancelled() && needs.check-changes.result == 'success' && (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') }} uses: ./.github/workflows/_pr-test-stage.yml with: @@ -251,6 +278,7 @@ jobs: caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} run_timeout_minutes: '60' + rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit # =============================================== aggregator ==================================================== @@ -263,6 +291,7 @@ jobs: check-changes, call-gate, sgl-kernel-build-wheels, + rust-ext-build, extra-a-test-1-gpu-small, extra-a-test-1-gpu-large, extra-a-test-2-gpu-large, diff --git a/.github/workflows/pr-test-jit-kernel.yml b/.github/workflows/pr-test-jit-kernel.yml index 3c91479b9..5d8c59316 100644 --- a/.github/workflows/pr-test-jit-kernel.yml +++ b/.github/workflows/pr-test-jit-kernel.yml @@ -9,6 +9,10 @@ on: sgl_kernel: required: true type: string + rust_ext_artifact: + description: 'Artifact of prebuilt Rust extension modules, from rust-ext-build. Empty, or a download that fails, means this job compiles them during install.' + type: string + default: '' runner_config: required: true type: string @@ -67,8 +71,19 @@ jobs: merge-multiple: true pattern: wheel-python3.10-cuda13.0 + - name: Download prebuilt Rust extensions + id: rust_ext + if: ${{ inputs.rust_ext_artifact != '' }} + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.rust_ext_artifact }} + path: python/sglang/srt/ + - name: Install dependencies timeout-minutes: 20 + env: + SGLANG_BUILD_RUST_EXTS: ${{ steps.rust_ext.outcome == 'success' && 'none' || '' }} run: | CUSTOM_BUILD_SGL_KERNEL=${{ inputs.sgl_kernel }} bash scripts/ci/cuda/ci_install_dependency.sh diffusion @@ -105,8 +120,19 @@ jobs: merge-multiple: true pattern: wheel-python3.10-cuda13.0 + - name: Download prebuilt Rust extensions + id: rust_ext + if: ${{ inputs.rust_ext_artifact != '' }} + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.rust_ext_artifact }} + path: python/sglang/srt/ + - name: Install dependencies timeout-minutes: 20 + env: + SGLANG_BUILD_RUST_EXTS: ${{ steps.rust_ext.outcome == 'success' && 'none' || '' }} run: | CUSTOM_BUILD_SGL_KERNEL=${{ inputs.sgl_kernel }} bash scripts/ci/cuda/ci_install_dependency.sh diffusion @@ -145,8 +171,19 @@ jobs: merge-multiple: true pattern: wheel-python3.10-cuda13.0 + - name: Download prebuilt Rust extensions + id: rust_ext + if: ${{ inputs.rust_ext_artifact != '' }} + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.rust_ext_artifact }} + path: python/sglang/srt/ + - name: Install dependencies timeout-minutes: 20 + env: + SGLANG_BUILD_RUST_EXTS: ${{ steps.rust_ext.outcome == 'success' && 'none' || '' }} run: | CUSTOM_BUILD_SGL_KERNEL=${{ inputs.sgl_kernel }} bash scripts/ci/cuda/ci_install_dependency.sh diffusion @@ -185,8 +222,19 @@ jobs: merge-multiple: true pattern: wheel-python3.10-cuda13.0 + - name: Download prebuilt Rust extensions + id: rust_ext + if: ${{ inputs.rust_ext_artifact != '' }} + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.rust_ext_artifact }} + path: python/sglang/srt/ + - name: Install dependencies timeout-minutes: 20 + env: + SGLANG_BUILD_RUST_EXTS: ${{ steps.rust_ext.outcome == 'success' && 'none' || '' }} run: | CUSTOM_BUILD_SGL_KERNEL=${{ inputs.sgl_kernel }} bash scripts/ci/cuda/ci_install_dependency.sh diffusion diff --git a/.github/workflows/pr-test-multimodal-gen.yml b/.github/workflows/pr-test-multimodal-gen.yml index c91356a30..eea4b4ba3 100644 --- a/.github/workflows/pr-test-multimodal-gen.yml +++ b/.github/workflows/pr-test-multimodal-gen.yml @@ -9,6 +9,10 @@ on: sgl_kernel: required: true type: string + rust_ext_artifact: + description: 'Artifact of prebuilt Rust extension modules, from rust-ext-build. Empty, or a download that fails, means this job compiles them during install.' + type: string + default: '' runner_config: required: true type: string @@ -102,8 +106,19 @@ jobs: merge-multiple: true pattern: wheel-python3.10-cuda* + - name: Download prebuilt Rust extensions + id: rust_ext + if: ${{ inputs.rust_ext_artifact != '' }} + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.rust_ext_artifact }} + path: python/sglang/srt/ + - name: Install dependencies timeout-minutes: 20 + env: + SGLANG_BUILD_RUST_EXTS: ${{ steps.rust_ext.outcome == 'success' && 'none' || '' }} run: | CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion - name: Run diffusion server tests @@ -168,8 +183,19 @@ jobs: merge-multiple: true pattern: wheel-python3.10-cuda* + - name: Download prebuilt Rust extensions + id: rust_ext + if: ${{ inputs.rust_ext_artifact != '' }} + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.rust_ext_artifact }} + path: python/sglang/srt/ + - name: Install dependencies timeout-minutes: 20 + env: + SGLANG_BUILD_RUST_EXTS: ${{ steps.rust_ext.outcome == 'success' && 'none' || '' }} run: | CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion @@ -231,8 +257,19 @@ jobs: merge-multiple: true pattern: wheel-python3.10-cuda* + - name: Download prebuilt Rust extensions + id: rust_ext + if: ${{ inputs.rust_ext_artifact != '' }} + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.rust_ext_artifact }} + path: python/sglang/srt/ + - name: Install dependencies timeout-minutes: 20 + env: + SGLANG_BUILD_RUST_EXTS: ${{ steps.rust_ext.outcome == 'success' && 'none' || '' }} run: | CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion @@ -293,8 +330,19 @@ jobs: merge-multiple: true pattern: wheel-python3.10-cuda* + - name: Download prebuilt Rust extensions + id: rust_ext + if: ${{ inputs.rust_ext_artifact != '' }} + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.rust_ext_artifact }} + path: python/sglang/srt/ + - name: Install dependencies timeout-minutes: 20 + env: + SGLANG_BUILD_RUST_EXTS: ${{ steps.rust_ext.outcome == 'success' && 'none' || '' }} run: | CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion @@ -362,8 +410,19 @@ jobs: merge-multiple: true pattern: wheel-python3.10-cuda* + - name: Download prebuilt Rust extensions + id: rust_ext + if: ${{ inputs.rust_ext_artifact != '' }} + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.rust_ext_artifact }} + path: python/sglang/srt/ + - name: Install dependencies timeout-minutes: 20 + env: + SGLANG_BUILD_RUST_EXTS: ${{ steps.rust_ext.outcome == 'success' && 'none' || '' }} run: | CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion @@ -407,10 +466,20 @@ jobs: merge-multiple: true pattern: wheel-python3.10-cuda* + - name: Download prebuilt Rust extensions + id: rust_ext + if: ${{ inputs.rust_ext_artifact != '' }} + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.rust_ext_artifact }} + path: python/sglang/srt/ + - name: Install dependencies timeout-minutes: 20 env: SGLANG_CI_EARLY_LD_LIBRARY_PATH: "1" + SGLANG_BUILD_RUST_EXTS: ${{ steps.rust_ext.outcome == 'success' && 'none' || '' }} run: | CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion @@ -461,8 +530,19 @@ jobs: merge-multiple: true pattern: wheel-python3.10-cuda* + - name: Download prebuilt Rust extensions + id: rust_ext + if: ${{ inputs.rust_ext_artifact != '' }} + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.rust_ext_artifact }} + path: python/sglang/srt/ + - name: Install dependencies timeout-minutes: 20 + env: + SGLANG_BUILD_RUST_EXTS: ${{ steps.rust_ext.outcome == 'success' && 'none' || '' }} run: | CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion diff --git a/.github/workflows/pr-test-sgl-kernel.yml b/.github/workflows/pr-test-sgl-kernel.yml index dd0ccd7d1..8c3872716 100644 --- a/.github/workflows/pr-test-sgl-kernel.yml +++ b/.github/workflows/pr-test-sgl-kernel.yml @@ -6,6 +6,10 @@ on: sgl_kernel: required: true type: string + rust_ext_artifact: + description: 'Artifact of prebuilt Rust extension modules, from rust-ext-build. Empty, or a download that fails, means this job compiles them during install.' + type: string + default: '' runner_config: required: true type: string @@ -54,8 +58,19 @@ jobs: merge-multiple: true pattern: wheel-python3.10-cuda* + - name: Download prebuilt Rust extensions + id: rust_ext + if: ${{ inputs.rust_ext_artifact != '' }} + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.rust_ext_artifact }} + path: python/sglang/srt/ + - name: Install dependencies timeout-minutes: 20 + env: + SGLANG_BUILD_RUST_EXTS: ${{ steps.rust_ext.outcome == 'success' && 'none' || '' }} run: | CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion @@ -89,8 +104,19 @@ jobs: merge-multiple: true pattern: wheel-python3.10-cuda* + - name: Download prebuilt Rust extensions + id: rust_ext + if: ${{ inputs.rust_ext_artifact != '' }} + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.rust_ext_artifact }} + path: python/sglang/srt/ + - name: Install dependencies timeout-minutes: 20 + env: + SGLANG_BUILD_RUST_EXTS: ${{ steps.rust_ext.outcome == 'success' && 'none' || '' }} run: | CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh @@ -136,8 +162,19 @@ jobs: merge-multiple: true pattern: wheel-python3.10-cuda* + - name: Download prebuilt Rust extensions + id: rust_ext + if: ${{ inputs.rust_ext_artifact != '' }} + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.rust_ext_artifact }} + path: python/sglang/srt/ + - name: Install dependencies timeout-minutes: 20 + env: + SGLANG_BUILD_RUST_EXTS: ${{ steps.rust_ext.outcome == 'success' && 'none' || '' }} run: | CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 65a8b8d42..ab7bf10b6 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -208,8 +208,31 @@ jobs: skip_pr_test_health_check: ${{ inputs.skip_pr_test_health_check == true }} secrets: inherit + # =============================================== Rust extensions ==================================================== + + # Unlike sgl-kernel, not gated on source changes: the cache key is the source + # hash, so the job always runs but only compiles on a miss. + rust-ext-build: + needs: [check-changes, call-gate] + # Ungated on the event, unlike the jobs above: a cache entry is visible to + # every PR only if it was written from the default branch's ref, so every + # main-ref run - schedule, dispatch, workflow_call - has to seed it. + # + # !cancelled() so a skipped call-gate does not take this job with it, since a + # docs-only PR still installs sglang in every stage. + if: | + !cancelled() && + needs.check-changes.result == 'success' && + (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped') + uses: ./.github/workflows/_pr-test-rust-ext-build.yml + with: + runs_on: x64-kernel-build-node + git_ref: ${{ inputs.git_ref || '' }} + skip_pr_test_health_check: ${{ inputs.skip_pr_test_health_check == true }} + secrets: inherit + call-sgl-kernel-tests: - needs: [check-changes, call-gate, sgl-kernel-build-wheels] + needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build] if: | github.event_name != 'schedule' && inputs.test_parallel_dispatch != true && @@ -221,12 +244,13 @@ jobs: sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }} git_ref: ${{ inputs.git_ref || '' }} skip_pr_test_health_check: ${{ inputs.skip_pr_test_health_check == true }} + rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit # =============================================== jit-kernel ==================================================== call-jit-kernel-tests: - needs: [check-changes, call-gate, sgl-kernel-build-wheels] + needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build] # Run on scheduled/parallel-dispatch runs (same pattern as the base-* stages) so the # jit_kernel suite is exercised on main 3x daily, not only on PRs that touch kernels/**. # check-changes already forces jit_kernel='true' on scheduled runs (run_all_tests). @@ -246,13 +270,14 @@ jobs: git_ref: ${{ inputs.git_ref || '' }} test_parallel_dispatch: ${{ inputs.test_parallel_dispatch == true && 'true' || 'false' }} skip_pr_test_health_check: ${{ inputs.skip_pr_test_health_check == true }} + rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit # =============================================== primary ==================================================== # Runs on 5090 (32GB, SM120) base-a-test-1-gpu-small: - needs: [check-changes, call-gate, sgl-kernel-build-wheels] + needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/_pr-test-stage.yml with: @@ -262,10 +287,11 @@ jobs: caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} run_timeout_minutes: '10' + rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit base-a-test-cpu: - needs: [check-changes, call-gate] + needs: [check-changes, call-gate, rust-ext-build] if: | always() && needs.check-changes.result == 'success' && @@ -303,13 +329,29 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v5 - # Needed by setuptools-rust to build the bundled native gRPC extension - # (rust/sglang-grpc) when installing the main `sglang` wheel from source. + # This stage compiled the workspace too - 7+ minutes per partition on + # billable hosted minutes. rust-ext-build's modules need an older glibc than + # this runner has, which is the safe direction, and both pin Python 3.10. + - name: Download prebuilt Rust extensions + id: rust_ext + if: ${{ needs.rust-ext-build.outputs.artifact_name != '' }} + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: ${{ needs.rust-ext-build.outputs.artifact_name }} + path: python/sglang/srt/ + + # Both only serve the fallback where this stage compiles the extensions + # itself, so they follow the download's outcome, not the job output: an + # expired artifact still needs cargo and a warm target dir here. Otherwise + # rust-cache restores ~1 GB per partition for nothing, on an over-quota cache. - name: Install protoc + Rust toolchain + if: ${{ steps.rust_ext.outcome != 'success' }} timeout-minutes: 10 run: bash scripts/ci/utils/install_rust_protoc.sh - name: Rust cache (rust/ workspace) + if: ${{ steps.rust_ext.outcome != 'success' }} uses: Swatinem/rust-cache@v2 with: workspaces: rust @@ -321,6 +363,7 @@ jobs: timeout-minutes: 20 env: UV_SYSTEM_PYTHON: "1" + SGLANG_BUILD_RUST_EXTS: ${{ steps.rust_ext.outcome == 'success' && 'none' || '' }} run: | uv pip install -e "python[dev]" --index-strategy unsafe-best-match --prerelease allow @@ -357,7 +400,7 @@ jobs: # Runs on 5090 (32GB, SM120) base-b-test-1-gpu-small: - needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels] + needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels, rust-ext-build] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/_pr-test-stage.yml with: @@ -367,11 +410,12 @@ jobs: caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} run_timeout_minutes: '30' + rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit # Runs on H100 (80GB, SM90) - tests that don't pass on 5090 (FA3, FP8, high VRAM, etc.) base-b-test-1-gpu-large: - needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels] + needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels, rust-ext-build] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/_pr-test-stage.yml with: @@ -382,10 +426,11 @@ jobs: partitions: ${{ needs.check-changes.outputs.partitions }} run_timeout_minutes: '30' timeout_per_file: '1800' + rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit base-b-test-2-gpu-large: - needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels] + needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels, rust-ext-build] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/_pr-test-stage.yml with: @@ -395,10 +440,11 @@ jobs: caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} run_timeout_minutes: '30' + rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit base-b-test-4-gpu-b200: - needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels] + needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels, rust-ext-build] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/_pr-test-stage.yml with: @@ -408,10 +454,11 @@ jobs: caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} run_timeout_minutes: '40' + rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit call-multimodal-gen-tests: - needs: [check-changes, call-gate, sgl-kernel-build-wheels] + needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build] if: | always() && !cancelled() && @@ -428,10 +475,11 @@ jobs: test_parallel_dispatch: ${{ inputs.test_parallel_dispatch == true && 'true' || 'false' }} caller_needs_failure: ${{ (needs.call-gate.result == 'failure' || needs.sgl-kernel-build-wheels.result == 'failure' || needs.check-changes.result == 'failure') && 'true' || 'false' }} skip_pr_test_health_check: ${{ inputs.skip_pr_test_health_check == true && 'true' || 'false' }} + rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit base-c-test-4-gpu-h100: - needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels] + needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/_pr-test-stage.yml with: @@ -441,10 +489,11 @@ jobs: caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} run_timeout_minutes: '30' + rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit base-c-test-8-gpu-h200: - needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels] + needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/_pr-test-stage.yml with: @@ -461,10 +510,11 @@ jobs: # Server CUDA Graph warmup is combined into this step (warmup_server_models unset). warmup_deep_gemm_models: 'deepseek-ai/DeepSeek-V3-0324:8 deepseek-ai/DeepSeek-V3.2:8 zai-org/GLM-5-FP8:8 XiaomiMiMo/MiMo-V2-Flash:4 XiaomiMiMo/MiMo-V2.5:8' warmup_timeout_minutes: '60' + rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit base-c-test-8-gpu-h20: - needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels] + needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/_pr-test-stage.yml with: @@ -474,10 +524,11 @@ jobs: caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} run_timeout_minutes: '30' + rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit base-c-test-deepep-4-gpu-h100: - needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels] + needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/_pr-test-stage.yml with: @@ -489,10 +540,11 @@ jobs: run_timeout_minutes: '30' warmup_deep_gemm_models: 'lmsys/sglang-ci-dsv3-test:4' warmup_server_models: 'lmsys/sglang-ci-dsv3-test:4' + rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit base-c-test-deepep-4-gpu-b200: - needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels] + needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/_pr-test-stage.yml with: @@ -503,10 +555,11 @@ jobs: partitions: ${{ needs.check-changes.outputs.partitions }} run_timeout_minutes: '30' timeout_per_file: '1800' + rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit base-c-test-deepep-8-gpu-h200: - needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels] + needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/_pr-test-stage.yml with: @@ -517,10 +570,11 @@ jobs: partitions: ${{ needs.check-changes.outputs.partitions }} run_timeout_minutes: '30' timeout_per_file: '1800' + rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit base-c-test-4-gpu-b200: - needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels] + needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/_pr-test-stage.yml with: @@ -531,10 +585,11 @@ jobs: partitions: ${{ needs.check-changes.outputs.partitions }} run_timeout_minutes: '30' timeout_per_file: '1800' + rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }} secrets: inherit base-c-test-4-gpu-gb300: - needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels] + needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build] if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/_pr-test-stage.yml with: @@ -545,8 +600,13 @@ jobs: partitions: ${{ needs.check-changes.outputs.partitions }} run_timeout_minutes: '30' timeout_per_file: '1800' + # aarch64 has no prebuild, so this stage compiles during install - which also + # makes it the only stage still covering that path. Keep it that way. + rust_ext_artifact: '' secrets: inherit + # List every build and test job: `skipped` passes here, so an omission turns that + # job's failure into a green run with no tests. pr-test-finish: needs: [ @@ -557,6 +617,8 @@ jobs: sgl-kernel-build-wheels-arm, call-sgl-kernel-tests, + rust-ext-build, + wait-for-base-a, wait-for-base-b, diff --git a/scripts/ci/cuda/ci_install_dependency.sh b/scripts/ci/cuda/ci_install_dependency.sh index fa08dd3fa..4d02d1d3d 100755 --- a/scripts/ci/cuda/ci_install_dependency.sh +++ b/scripts/ci/cuda/ci_install_dependency.sh @@ -326,6 +326,40 @@ uninstall_stale_flashinfer() { mark_step_done "${FUNCNAME[0]}" } +require_prebuilt_rust_exts() { + # Stages whose download succeeded set this to none. Runs before + # setup_pip_toolchain uninstalls sglang, so clearing it here still reaches + # install_sglang below - setup.py reads it from the environment at build time. + if [ "${SGLANG_BUILD_RUST_EXTS:-}" != "none" ]; then + mark_step_done "${FUNCNAME[0]}" + return + fi + + # Exact EXT_SUFFIX rather than a _core*.so glob: no crate sets abi3, so a module + # built for another minor version satisfies the glob while the import system + # ignores it, leaving is_rust_server_built() false and the Rust-server tests + # silently skipped. Stages have no setup-python, so the interpreter is whatever + # the image ships, and the pools are not on one version (h20 is 3.12 while + # h100 is 3.10) - a mismatch is drift to route around, not a failure. + local suffix + suffix=$(python3 -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX"))') + local missing=() + local pkg + for pkg in server grpc multimodal; do + [ -f "python/sglang/srt/${pkg}/_core${suffix}" ] || missing+=("${pkg}") + done + if [ ${#missing[@]} -gt 0 ]; then + echo "::warning::no prebuilt _core${suffix} for: ${missing[*]}; building from source" + ls -l python/sglang/srt/*/_core*.so 2>/dev/null || echo "(no extension modules at all)" + export SGLANG_BUILD_RUST_EXTS= + mark_step_done "${FUNCNAME[0]}" + return + fi + echo "Using prebuilt Rust extension modules; skipping the cargo build." + + mark_step_done "${FUNCNAME[0]}" +} + install_sglang() { EXTRAS="dev,runai,tracing" if [ -n "$OPTIONAL_DEPS" ]; then @@ -654,6 +688,7 @@ main() { install_apt_packages clean_site_packages setup_cargo_cache + require_prebuilt_rust_exts setup_pip_toolchain remove_stale_cuda12_nvidia_wheels uninstall_stale_flashinfer diff --git a/scripts/ci/utils/stage_rust_ext_modules.sh b/scripts/ci/utils/stage_rust_ext_modules.sh new file mode 100755 index 000000000..27b64bf54 --- /dev/null +++ b/scripts/ci/utils/stage_rust_ext_modules.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Copy the built PyO3 extension modules into rust-ext-staging// for +# upload-artifact. Shared by both jobs of _pr-test-rust-ext-build.yml, so the +# archive layout and the module-count check cannot drift between them. +# +# MAX_GLIBC (optional): also reject a module requiring a newer GLIBC symbol +# version than the test runners have. Only set where the modules were just +# compiled - on a cache hit these are the same bytes that already passed. +set -euo pipefail +shopt -s nullglob +# upload-artifact strips the longest common prefix it matched, so a missing +# module would silently shift the archive layout. +rm -rf rust-ext-staging +built=() +for pkg in server grpc multimodal; do + found=(python/sglang/srt/"${pkg}"/_core*.so) + if [ ${#found[@]} -ne 1 ]; then + echo "::error::expected exactly one extension module for ${pkg}, found ${#found[@]}" + exit 1 + fi + mkdir -p "rust-ext-staging/${pkg}" + cp "${found[0]}" "rust-ext-staging/${pkg}/" + built+=("${found[0]}") +done +max_allowed="${MAX_GLIBC:-}" +[ -n "${max_allowed}" ] || exit 0 + +# Newer glibc than the test runners fails at import: "GLIBC_2.xx not found". +status=0 +for so in "${built[@]}"; do + # grep exits 1 with no match; without `|| true` pipefail kills the script. + needed=$(objdump -T "$so" \ + | grep -oE 'GLIBC_2\.[0-9]+' \ + | sed 's/GLIBC_//' \ + | sort -V | tail -1 || true) + echo "${so}: requires glibc <= ${needed:-none}" + if [ -n "${needed}" ] \ + && [ "$(printf '%s\n%s\n' "${max_allowed}" "${needed}" | sort -V | tail -1)" != "${max_allowed}" ]; then + echo "::error::${so} requires glibc ${needed} > ${max_allowed} supported by the test runners; build on an older image" + status=1 + fi +done +exit $status