minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Feature] Add process-local in-memory KV indexer and Router integration (#33370)

Browse Source 
Co-authored-by: Wu, Yutong <yutong.wu@amd.com>
Co-authored-by: TianDi101 <ditian12@amd.com>
Co-authored-by: Zhangheng <hzh0425@apache.org>
This commit is contained in:
wuyl1
2026-08-20 10:45:35 +08:00
committed by GitHub
co-authored by Wu, Yutong TianDi101 Zhangheng
parent 238ba40c27
commit 360d10d6bc
42 changed files with 6603 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker/sgl-router.Dockerfile
+25 -14
View File
@@ -34,21 +34,28 @@ ARG DEBIAN_VERSION=bookworm
######################## STAGE 1 — chef recipe ##########################
FROM rust:${RUST_VERSION}-${DEBIAN_VERSION} AS chef
RUN cargo install cargo-chef --locked --version ^0.1
WORKDIR /work
WORKDIR /work/sgl-router
COPY experimental/sgl-router/Cargo.toml ./
COPY experimental/sgl-router/rust-toolchain.toml ./
COPY experimental/sgl-router/sgl-kv-indexer/Cargo.toml sgl-kv-indexer/Cargo.toml
# Stub a minimal src tree so cargo can resolve the workspace, generate
# the lockfile (gitignored upstream), then prepare the chef recipe.
RUN mkdir -p src && echo "fn main() {}" > src/main.rs \
RUN mkdir -p src sgl-kv-indexer/src/bin \
&& echo "fn main() {}" > src/main.rs \
&& echo "" > src/lib.rs \
&& echo "" > sgl-kv-indexer/src/lib.rs \
&& echo "fn main() {}" > sgl-kv-indexer/src/bin/kv-indexer-server.rs \
&& echo "fn main() {}" > sgl-kv-indexer/src/bin/kv-indexer-bridge.rs \
&& cargo generate-lockfile \
&& cargo chef prepare --recipe-path recipe.json \
&& rm -rf src
&& rm -rf src sgl-kv-indexer/src
######################## STAGE 2 — builder ##############################
FROM rust:${RUST_VERSION}-${DEBIAN_VERSION} AS builder
RUN cargo install cargo-chef --locked --version ^0.1
WORKDIR /work
RUN apt-get update \
&& apt-get install -y --no-install-recommends protobuf-compiler \
&& rm -rf /var/lib/apt/lists/* \
&& cargo install cargo-chef --locked --version ^0.1
WORKDIR /work/sgl-router
# `dynamo-tokenizers` pulls in `pcre2-sys`, whose build.rs links the SYSTEM
# libpcre2-8 whenever pkg-config finds it (it does here — the rust:bookworm
@@ -58,29 +65,33 @@ WORKDIR /work
# it statically, keeping the runtime self-contained.
ENV PCRE2_SYS_STATIC=1
COPY --from=chef /work/recipe.json ./recipe.json
COPY --from=chef /work/Cargo.lock ./Cargo.lock
COPY experimental/sgl-router/rust-toolchain.toml ./
COPY --from=chef /work/sgl-router/recipe.json ./recipe.json
COPY --from=chef /work/sgl-router/Cargo.lock ./Cargo.lock
COPY experimental/sgl-router/sgl-kv-indexer/Cargo.toml sgl-kv-indexer/Cargo.toml
# Cook (compile + cache) the dep graph from the recipe. This layer's
# inputs are recipe.json + the toolchain — code changes in src/ do NOT
# invalidate it.
# Cook (compile + cache) the dep graph from the recipe. The recipe carries every
# workspace member's manifest, so chef recreates the Indexer's source stubs itself.
RUN cargo chef cook --release --recipe-path recipe.json
# Now bring in the real sources and the manifest they need.
COPY experimental/sgl-router/Cargo.toml ./
COPY experimental/sgl-router/src ./src
COPY experimental/sgl-router/sgl-kv-indexer/Cargo.toml sgl-kv-indexer/Cargo.toml
COPY experimental/sgl-router/sgl-kv-indexer/build.rs sgl-kv-indexer/build.rs
COPY experimental/sgl-router/sgl-kv-indexer/proto sgl-kv-indexer/proto
COPY experimental/sgl-router/sgl-kv-indexer/src sgl-kv-indexer/src
# --locked is intentionally omitted: the lockfile is generated in-container
# (gitignored upstream) and `cargo chef cook` may have mutated it during the
# dep-cook step, so a strict --locked check would spuriously fail.
RUN cargo build --release --bin sgl-router \
RUN touch sgl-kv-indexer/build.rs \
&& cargo build --release --bin sgl-router \
&& strip target/release/sgl-router
######################## STAGE 3 — runtime ##############################
FROM gcr.io/distroless/cc-debian12:nonroot AS runtime
COPY --from=builder /work/target/release/sgl-router /usr/local/bin/sgl-router
COPY --from=builder /work/sgl-router/target/release/sgl-router /usr/local/bin/sgl-router
# Default config path; mount your own via `-v <host-path>:/etc/sgl-router`.
ENV SGL_ROUTER_CONFIG=/etc/sgl-router/sgl-router.yaml