minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[router] support arm, windows, mac, linux, reduce wheel size and number (#12285)

Browse Source
This commit is contained in:
Simo Lin
2025-10-28 12:18:51 -07:00
committed by GitHub
parent 737b58d6dc
commit 84a9d0eab2
3 changed files with 168 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/nightly-release-router.yml
+83 -34
View File
@@ -10,14 +10,40 @@ on:
jobs: jobs:
build: build:
name: Build on ${{ matrix.os }} (${{ matrix.target }}) name: build on ${{ matrix.platform || matrix.os }} (${{ matrix.target }} - ${{ matrix.manylinux || 'auto' }})
runs-on: ${{ matrix.os }}-latest runs-on: ${{ matrix.os }}-latest
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
os: [ubuntu, macos, windows]
target: [x86_64, aarch64]
manylinux: [auto]
include: include:
- os: ubuntu - os: ubuntu
platform: linux
- os: windows
ls: dir
target: x86_64 target: x86_64
python-architecture: x64
interpreter: 3.8 3.9 3.10 3.11 3.12 3.13
- os: macos
target: aarch64
interpreter: 3.8 3.9 3.10 3.11 3.12 3.13
- os: ubuntu
platform: linux
target: aarch64
# musllinux
- os: ubuntu
platform: linux
target: x86_64
manylinux: musllinux_1_1
- os: ubuntu
platform: linux
target: aarch64
manylinux: musllinux_1_1
exclude:
- os: windows
target: aarch64
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
@@ -29,56 +55,79 @@ jobs:
mv sglang-repo/sgl-router/* . mv sglang-repo/sgl-router/* .
rm -rf sglang-repo rm -rf sglang-repo
ls -alt ls -alt
shell: bash
- name: Set up Python - name: Set up Python
uses: actions/setup-python@v5 uses: actions/setup-python@v5
with: with:
python-version: "3.11" python-version: "3.13"
architecture: ${{ matrix.python-architecture || 'x64' }}
- name: Modify version for nightly release - name: Modify version for nightly release
run: | run: |
# Get current version from pyproject.toml # Get current version from pyproject.toml
CURRENT_VERSION=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])" 2>/dev/null || python -c "import tomli; print(tomli.load(open('pyproject.toml', 'rb'))['project']['version'])") CURRENT_VERSION=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])" 2>/dev/null || python -c "import tomli; print(tomli.load(open('pyproject.toml', 'rb'))['project']['version'])")
# Create nightly version with date: e.g., 0.1.9.dev20250112 # Create nightly version with date: e.g., 0.2.1.dev20250128
NIGHTLY_VERSION="${CURRENT_VERSION}.dev$(date +%Y%m%d)" NIGHTLY_VERSION="${CURRENT_VERSION}.dev$(date +%Y%m%d)"
echo "Nightly version: $NIGHTLY_VERSION" echo "Nightly version: $NIGHTLY_VERSION"
# Update pyproject.toml with nightly version (temporary, not committed) # Update pyproject.toml with nightly version (temporary, not committed)
sed -i "s/version = \"${CURRENT_VERSION}\"/version = \"${NIGHTLY_VERSION}\"/" pyproject.toml sed -i.bak "s/version = \"${CURRENT_VERSION}\"/version = \"${NIGHTLY_VERSION}\"/" pyproject.toml
# Verify the change # Verify the change
cat pyproject.toml | grep "^version" cat pyproject.toml | grep "^version"
shell: bash
- name: Install build dependencies - name: Install twine and tomli
run: | run: pip install -U twine tomli
python -m pip install -U pip
python -m pip install build twine auditwheel tomli
- name: Build package - name: Install protoc (macOS)
uses: pypa/cibuildwheel@v2.21.3 if: matrix.os == 'macos'
env: run: brew install protobuf
CIBW_BUILD: "cp38-manylinux_x86_64 cp39-manylinux_x86_64 cp310-manylinux_x86_64 cp311-manylinux_x86_64 cp312-manylinux_x86_64 cp313-manylinux_x86_64 cp314-manylinux_x86_64"
CIBW_BEFORE_ALL: | - name: Install protoc (Windows)
yum update -y && yum install -y openssl-devel wget unzip && \ if: matrix.os == 'windows'
# Install latest protoc (v32.0) that supports proto3 run: choco install protoc -y
cd /tmp && \
wget https://github.com/protocolbuffers/protobuf/releases/download/v32.0/protoc-32.0-linux-x86_64.zip && \ - name: Build wheels
unzip protoc-32.0-linux-x86_64.zip -d /usr/local && \ uses: PyO3/maturin-action@v1
rm protoc-32.0-linux-x86_64.zip && \ with:
# Install Rust target: ${{ matrix.target }}
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y manylinux: ${{ matrix.manylinux || 'auto' }}
CIBW_ENVIRONMENT: "PATH=$HOME/.cargo/bin:$PATH" args: --release --out dist --interpreter ${{ matrix.interpreter || '3.8 3.9 3.10 3.11 3.12 3.13' }}
rust-toolchain: stable
docker-options: -e CI -e CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc -e CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++
before-script-linux: |
# Install build dependencies (perl/make for vendored OpenSSL, protoc for gRPC)
if command -v yum &> /dev/null; then
yum update -y && yum install -y wget unzip gcc gcc-c++ perl-core make
# Install cross-compilation toolchain for aarch64 if needed
if [ "${{ matrix.target }}" = "aarch64" ]; then
yum install -y gcc-aarch64-linux-gnu gcc-c++-aarch64-linux-gnu || true
fi
elif command -v apt-get &> /dev/null; then
apt-get update && apt-get install -y wget unzip gcc g++ perl make
# Install cross-compilation toolchain for aarch64 if needed
if [ "${{ matrix.target }}" = "aarch64" ]; then
apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu || true
fi
fi
(cd /tmp && \
wget https://github.com/protocolbuffers/protobuf/releases/download/v32.0/protoc-32.0-linux-x86_64.zip && \
unzip protoc-32.0-linux-x86_64.zip -d /usr/local && \
rm protoc-32.0-linux-x86_64.zip)
protoc --version
- name: List built packages - name: List built packages
run: ls -lh wheelhouse/ run: ${{ matrix.ls || 'ls -lh' }} dist/
- name: Check packages - name: Check packages
run: twine check --strict wheelhouse/* run: twine check --strict dist/*
- uses: actions/upload-artifact@v4 - uses: actions/upload-artifact@v4
with: with:
name: packages-${{ matrix.os }}-${{ matrix.target }} name: packages-${{ matrix.os }}-${{ matrix.target }}-${{ matrix.manylinux || 'auto' }}
path: wheelhouse/ path: dist/
build-sdist: build-sdist:
name: Build SDist name: Build SDist
@@ -98,28 +147,28 @@ jobs:
- name: Set up Python - name: Set up Python
uses: actions/setup-python@v5 uses: actions/setup-python@v5
with: with:
python-version: "3.11" python-version: "3.13"
- name: Modify version for nightly release - name: Modify version for nightly release
run: | run: |
# Get current version from pyproject.toml # Get current version from pyproject.toml
CURRENT_VERSION=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])" 2>/dev/null || python -c "import tomli; print(tomli.load(open('pyproject.toml', 'rb'))['project']['version'])") CURRENT_VERSION=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])" 2>/dev/null || python -c "import tomli; print(tomli.load(open('pyproject.toml', 'rb'))['project']['version'])")
# Create nightly version with date: e.g., 0.1.9.dev20250112 # Create nightly version with date: e.g., 0.2.1.dev20250128
NIGHTLY_VERSION="${CURRENT_VERSION}.dev$(date +%Y%m%d)" NIGHTLY_VERSION="${CURRENT_VERSION}.dev$(date +%Y%m%d)"
echo "Nightly version: $NIGHTLY_VERSION" echo "Nightly version: $NIGHTLY_VERSION"
# Update pyproject.toml with nightly version (temporary, not committed) # Update pyproject.toml with nightly version (temporary, not committed)
sed -i "s/version = \"${CURRENT_VERSION}\"/version = \"${CURRENT_VERSION}\"/g" pyproject.toml sed -i "s/version = \"${CURRENT_VERSION}\"/version = \"${NIGHTLY_VERSION}\"/" pyproject.toml
sed -i "0,/version = \"${CURRENT_VERSION}\"/s//version = \"${NIGHTLY_VERSION}\"/" pyproject.toml
# Verify the change # Verify the change
cat pyproject.toml | grep "^version" cat pyproject.toml | grep "^version"
- name: Build SDist - name: Build SDist
run: | uses: PyO3/maturin-action@v1
pip install build with:
python -m pip install -U packaging tomli command: sdist
python -m build --sdist args: --out dist
rust-toolchain: stable
- uses: actions/upload-artifact@v4 - uses: actions/upload-artifact@v4
with: with:
.github/workflows/release-pypi-router.yml
+79 -32
View File
@@ -1,6 +1,4 @@
# Reference: https://github.com/openai/tiktoken/blob/63527649963def8c759b0f91f2eb69a40934e468/.github/workflows/build_wheels.yml#L1 name: Release SGLang Model Gateway to PyPI
name: Release SGLang Router to PyPI
on: on:
push: push:
@@ -12,14 +10,40 @@ on:
jobs: jobs:
build: build:
name: Build on ${{ matrix.os }} (${{ matrix.target }}) name: build on ${{ matrix.platform || matrix.os }} (${{ matrix.target }} - ${{ matrix.manylinux || 'auto' }})
runs-on: ${{ matrix.os }}-latest runs-on: ${{ matrix.os }}-latest
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
os: [ubuntu, macos, windows]
target: [x86_64, aarch64]
manylinux: [auto]
include: include:
- os: ubuntu - os: ubuntu
platform: linux
- os: windows
ls: dir
target: x86_64 target: x86_64
python-architecture: x64
interpreter: 3.8 3.9 3.10 3.11 3.12 3.13
- os: macos
target: aarch64
interpreter: 3.8 3.9 3.10 3.11 3.12 3.13
- os: ubuntu
platform: linux
target: aarch64
# musllinux
- os: ubuntu
platform: linux
target: x86_64
manylinux: musllinux_1_1
- os: ubuntu
platform: linux
target: aarch64
manylinux: musllinux_1_1
exclude:
- os: windows
target: aarch64
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
@@ -31,42 +55,64 @@ jobs:
mv sglang-repo/sgl-router/* . mv sglang-repo/sgl-router/* .
rm -rf sglang-repo rm -rf sglang-repo
ls -alt ls -alt
shell: bash
- name: Set up Python - name: Set up Python
uses: actions/setup-python@v5 uses: actions/setup-python@v5
with: with:
python-version: "3.11" python-version: "3.13"
architecture: ${{ matrix.python-architecture || 'x64' }}
- name: Install build dependencies - name: Install twine
run: | run: pip install -U twine
python -m pip install -U pip
python -m pip install build twine auditwheel
- name: Build package - name: Install protoc (macOS)
uses: pypa/cibuildwheel@v2.21.3 if: matrix.os == 'macos'
env: run: brew install protobuf
CIBW_BUILD: "cp38-manylinux_x86_64 cp39-manylinux_x86_64 cp310-manylinux_x86_64 cp311-manylinux_x86_64 cp312-manylinux_x86_64 cp313-manylinux_x86_64 cp314-manylinux_x86_64"
CIBW_BEFORE_ALL: | - name: Install protoc (Windows)
yum update -y && yum install -y openssl-devel wget unzip && \ if: matrix.os == 'windows'
# Install latest protoc (v32.0) that supports proto3 run: choco install protoc -y
cd /tmp && \
wget https://github.com/protocolbuffers/protobuf/releases/download/v32.0/protoc-32.0-linux-x86_64.zip && \ - name: Build wheels
unzip protoc-32.0-linux-x86_64.zip -d /usr/local && \ uses: PyO3/maturin-action@v1
rm protoc-32.0-linux-x86_64.zip && \ with:
# Install Rust target: ${{ matrix.target }}
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y manylinux: ${{ matrix.manylinux || 'auto' }}
CIBW_ENVIRONMENT: "PATH=$HOME/.cargo/bin:$PATH" args: --release --out dist --interpreter ${{ matrix.interpreter || '3.8 3.9 3.10 3.11 3.12 3.13' }}
rust-toolchain: stable
docker-options: -e CI -e CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc -e CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++
before-script-linux: |
# Install build dependencies (perl/make for vendored OpenSSL, protoc for gRPC)
if command -v yum &> /dev/null; then
yum update -y && yum install -y wget unzip gcc gcc-c++ perl-core make
# Install cross-compilation toolchain for aarch64 if needed
if [ "${{ matrix.target }}" = "aarch64" ]; then
yum install -y gcc-aarch64-linux-gnu gcc-c++-aarch64-linux-gnu || true
fi
elif command -v apt-get &> /dev/null; then
apt-get update && apt-get install -y wget unzip gcc g++ perl make
# Install cross-compilation toolchain for aarch64 if needed
if [ "${{ matrix.target }}" = "aarch64" ]; then
apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu || true
fi
fi
(cd /tmp && \
wget https://github.com/protocolbuffers/protobuf/releases/download/v32.0/protoc-32.0-linux-x86_64.zip && \
unzip protoc-32.0-linux-x86_64.zip -d /usr/local && \
rm protoc-32.0-linux-x86_64.zip)
protoc --version
- name: List built packages - name: List built packages
run: ls -lh wheelhouse/ run: ${{ matrix.ls || 'ls -lh' }} dist/
- name: Check packages - name: Check packages
run: twine check --strict wheelhouse/* run: twine check --strict dist/*
- uses: actions/upload-artifact@v4 - uses: actions/upload-artifact@v4
with: with:
name: packages-${{ matrix.os }}-${{ matrix.target }} name: packages-${{ matrix.os }}-${{ matrix.target }}-${{ matrix.manylinux || 'auto' }}
path: wheelhouse/ path: dist/
build-sdist: build-sdist:
name: Build SDist name: Build SDist
@@ -86,13 +132,14 @@ jobs:
- name: Set up Python - name: Set up Python
uses: actions/setup-python@v5 uses: actions/setup-python@v5
with: with:
python-version: "3.11" python-version: "3.13"
- name: Build SDist - name: Build SDist
run: | uses: PyO3/maturin-action@v1
pip install build with:
python -m pip install -U packaging command: sdist
python -m build --sdist args: --out dist
rust-toolchain: stable
- uses: actions/upload-artifact@v4 - uses: actions/upload-artifact@v4
with: with:
sgl-router/Cargo.toml
+6 -3
View File
@@ -36,7 +36,7 @@ rand = "0.9.2"
reqwest = { version = "0.12.8", features = ["stream", "blocking", "json", "rustls-tls"], default-features = false } reqwest = { version = "0.12.8", features = ["stream", "blocking", "json", "rustls-tls"], default-features = false }
futures-util = "0.3" futures-util = "0.3"
futures = "0.3" futures = "0.3"
pyo3 = { version = "0.27.1", features = ["extension-module"] } pyo3 = { version = "0.27.1", features = ["extension-module", "abi3-py38"] }
dashmap = "6.1.0" dashmap = "6.1.0"
blake3 = "1.5" blake3 = "1.5"
http = "1.1.0" http = "1.1.0"
@@ -66,6 +66,7 @@ tiktoken-rs = { version = "0.7.0" }
minijinja = { version = "2.0", features = ["unstable_machinery", "json", "builtins"] } minijinja = { version = "2.0", features = ["unstable_machinery", "json", "builtins"] }
minijinja-contrib = { version = "2.0", features = ["pycompat"] } minijinja-contrib = { version = "2.0", features = ["pycompat"] }
rustls = { version = "0.23", default-features = false, features = ["ring", "std"] } rustls = { version = "0.23", default-features = false, features = ["ring", "std"] }
openssl = { version = "0.10.73", features = ["vendored"] }
hf-hub = { version = "0.4.3", features = ["tokio"] } hf-hub = { version = "0.4.3", features = ["tokio"] }
rmcp = { version = "0.8.3", features = ["client", "server", rmcp = { version = "0.8.3", features = ["client", "server",
"transport-child-process", "transport-child-process",
@@ -119,8 +120,10 @@ harness = false
path = "benches/tool_parser_benchmark.rs" path = "benches/tool_parser_benchmark.rs"
[profile.release] [profile.release]
lto = "thin" opt-level = "z" # Optimize for size
codegen-units = 1 lto = "fat" # Full LTO for smaller binaries
codegen-units = 1 # Better optimization, slower compile
strip = true # Strip debug symbols
[profile.dev] [profile.dev]
opt-level = 0 opt-level = 0