143 lines
5.4 KiB
TOML
143 lines
5.4 KiB
TOML
[workspace]
|
|
resolver = "2"
|
|
members = [".", "sgl-kv-indexer"]
|
|
|
|
[package]
|
|
name = "sgl-router"
|
|
version = "0.1.0"
|
|
edition = "2021"
|
|
license = "Apache-2.0"
|
|
description = "Slim KV-aware OpenAI-compatible router for SGLang workers"
|
|
publish = false # binary-only; not published to crates.io
|
|
|
|
[lib]
|
|
name = "sgl_router"
|
|
crate-type = ["rlib"]
|
|
|
|
[[bin]]
|
|
name = "sgl-router"
|
|
path = "src/main.rs"
|
|
|
|
[lints.rust]
|
|
unused_qualifications = "warn"
|
|
|
|
[dependencies]
|
|
# Pin Dynamo versions and commit Cargo.lock so builds are reproducible.
|
|
dynamo-tokenizers = "=1.8.1"
|
|
# Renders the model's chat template (HF Jinja, or Dynamo's built-in encoder for
|
|
# template-less models like DeepSeek-V4) so cache-aware routing hashes the same
|
|
# tokens the engine caches.
|
|
dynamo-renderer = "=5.2.0"
|
|
# `OAIChatLikeRequest` speaks minijinja values.
|
|
minijinja = "2.24"
|
|
|
|
# Async runtime + http
|
|
tokio = { version = "1.42", features = ["full"] }
|
|
# http2 on both sides: axum serves cleartext h2c on the inbound listener, reqwest
|
|
# dials it outbound.
|
|
#
|
|
# axum's `http2` looks redundant and is not — please don't drop it. `axum::serve`
|
|
# runs on hyper-util's auto Builder, whose HTTP/2 branch is compiled by
|
|
# `hyper-util/http2`, and that arrives today only transitively, via
|
|
# sgl-kv-indexer -> tonic "transport" -> tonic/server -> hyper-util/server-auto
|
|
# (which is server + http1 + http2). So the listener speaks h2c whether or not
|
|
# this line exists. Declaring it forwards `hyper-util/http2` from axum directly,
|
|
# so inbound h2c no longer rides on a gRPC dependency it has nothing to do with,
|
|
# and it enables HTTP/2 extended CONNECT for h2 websockets.
|
|
axum = { version = "0.8", features = ["macros", "tracing", "http2"] }
|
|
# Named directly (not just via axum) to implement a response-body wrapper:
|
|
# `Frame`/`SizeHint` are not re-exported through `axum::body`.
|
|
http-body = "1"
|
|
tower = { version = "0.5", features = ["full"] }
|
|
tower-http = { version = "0.6", features = ["trace", "compression-gzip", "cors", "timeout", "request-id", "catch-panic"] }
|
|
reqwest = { version = "0.12", features = ["stream", "json", "rustls-tls", "http2"], default-features = false }
|
|
sgl-kv-indexer = { path = "sgl-kv-indexer" }
|
|
|
|
# Serialization
|
|
serde = { version = "1", features = ["derive"] }
|
|
serde_json = { version = "1", features = ["preserve_order"] }
|
|
|
|
# Tokenizer auto-download from HuggingFace when --tokenizer-path is omitted.
|
|
# Sync (`ureq`) API only — it runs once at startup; `ureq` is on rustls, so
|
|
# this pulls no openssl/native-tls (matching reqwest's rustls-tls above).
|
|
hf-hub = { version = "0.4", default-features = false, features = ["ureq"] }
|
|
|
|
# Utilities
|
|
anyhow = "1"
|
|
thiserror = "2"
|
|
clap = { version = "4", features = ["derive", "env"] }
|
|
tracing = "0.1"
|
|
tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
|
|
futures = "0.3"
|
|
bytes = "1"
|
|
rand = "0.8"
|
|
tokio-stream = { version = "0.1", features = ["net"] }
|
|
dashmap = "6"
|
|
kube = { version = "0.96", features = ["runtime", "derive"] }
|
|
k8s-openapi = { version = "0.23", features = ["v1_31"] }
|
|
tokio-util = "0.7"
|
|
uuid = { version = "1", features = ["v4"] }
|
|
|
|
# KV-event subsystem — msgpack-encoded events over ZMQ and sha256-based
|
|
# block hashing matching SGLang's `radix_cache`. Wire format authority is
|
|
# `python/sglang/srt/disaggregation/kv_events.py`.
|
|
parking_lot = "0.12"
|
|
rmp-serde = "1"
|
|
# FxHash for the KV-event radix tree's integer-keyed maps (i64 block
|
|
# hashes, u64 node ids), off the routing hot path's SipHash cost. The
|
|
# keys are client-derivable, not trusted; what bounds a collision attack
|
|
# is that planting a key costs a real request — see the DoS note in
|
|
# src/policies/kv_events/tree.rs.
|
|
rustc-hash = "2"
|
|
sha2 = "0.10"
|
|
url = "2"
|
|
zeromq = { version = "0.6", default-features = false, features = ["tokio-runtime", "tcp-transport"] }
|
|
|
|
[dev-dependencies]
|
|
base64 = "0.22"
|
|
dirs = "5"
|
|
http-body-util = "0.1"
|
|
# HTTP/2-only mock server for the proxy h2c forwarding tests (tests/proxy/h2c_forward.rs).
|
|
# `http1` as well as `http2`: tests/proxy/pd_protocol_binding.rs drives both
|
|
# `hyper::server::conn::http1` and `::http2` to build single-protocol mocks.
|
|
# `http1` arrives transitively via axum's defaults today; declaring it here
|
|
# means an axum `default-features = false` later fails at the dependency
|
|
# rather than as an unresolved import in a test.
|
|
hyper = { version = "1", features = ["server", "http1", "http2"] }
|
|
hyper-util = { version = "0.1", features = ["tokio"] }
|
|
# Capture the router's `tracing` access log in tests. Also a runtime dep, but
|
|
# integration tests are a separate crate and need it declared here too.
|
|
tracing-subscriber = { version = "0.3", features = ["fmt"] }
|
|
serde = { version = "1", features = ["derive"] }
|
|
serde_json = "1"
|
|
tempfile = "3"
|
|
tower = { version = "0.5", features = ["util"] }
|
|
tokio = { version = "1.42", features = ["test-util"] }
|
|
tonic = { version = "0.14.6", features = ["transport"] }
|
|
# Low-level msgpack encoder used to hand-construct wire bytes in
|
|
# kv_events golden-bytes tests (decode-only path uses rmp-serde).
|
|
rmp = "0.8"
|
|
# Criterion benches that mirror the SMG `radix_tree_benchmark` +
|
|
# `manual_policy_benchmark` so routing-decision latency can be compared
|
|
# apples-to-apples against the gateway being deprecated.
|
|
criterion = { version = "0.5", features = ["html_reports"] }
|
|
rand = "0.8"
|
|
|
|
[[test]]
|
|
name = "component"
|
|
path = "tests/component/main.rs"
|
|
|
|
[[test]]
|
|
name = "proxy"
|
|
path = "tests/proxy/main.rs"
|
|
|
|
[[bench]]
|
|
name = "tree_lookup"
|
|
harness = false
|
|
path = "benches/tree_lookup.rs"
|
|
|
|
[[bench]]
|
|
name = "policy_select"
|
|
harness = false
|
|
path = "benches/policy_select.rs"
|