The sgalng api specs (#41585)

This commit is contained in:
Rain Jiang
2026-10-07 01:55:01 -07:00
committed by GitHub
parent ba77d30256
commit c52dcc5a01
95 changed files with 16648 additions and 4458 deletions
+1 -1
View File
@@ -1,3 +1,3 @@
[codespell]
ignore-words-list = ans,als,hel,boostrap,childs,te,vas,hsa,ment,cann,thi,makro,wil,rouge,PRIS,ather,MIS,medias,allready,inout,nd,fo,visibles,nothink,renderD,ond,tbe,CopyIn,notin,subtile,subtiles,dout,IST,kInf,datas,abl,valu
ignore-words-list = ans,als,hel,boostrap,childs,te,vas,hsa,ment,cann,thi,makro,wil,rouge,PRIS,ather,MIS,medias,allready,inout,nd,fo,visibles,nothink,renderD,ond,tbe,CopyIn,notin,subtile,subtiles,dout,IST,kInf,datas,abl,valu,ser
skip = *.json, *.jsonl, *.patch, *.txt, *.lock
+3 -2
View File
@@ -29,7 +29,7 @@ repos:
rev: 7.0.0
hooks:
- id: isort
exclude: '^python/sglang/srt/grpc/.*_pb2\.py$|^python/sglang/srt/grpc/.*_pb2_grpc\.py$|^python/sglang/srt/grpc/.*_pb2\.pyi$|^python/sglang/srt/grpc/.*_pb2_grpc\.pyi$'
exclude: '^python/sglang/srt/grpc/.*_pb2\.py$|^python/sglang/srt/grpc/.*_pb2_grpc\.py$|^python/sglang/srt/grpc/.*_pb2\.pyi$|^python/sglang/srt/grpc/.*_pb2_grpc\.pyi$|^python/sglang/(api|json)/v1/.*\.py$'
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.15.1
hooks:
@@ -47,9 +47,10 @@ repos:
python/sglang/srt/grpc/.*_pb2_grpc\.py$|
python/sglang/srt/grpc/.*_pb2\.pyi$|
python/sglang/srt/grpc/.*_pb2_grpc\.pyi$|
python/sglang/(api|json)/v1/.*\.py$|
)$
- id: ruff-format
exclude: '^python/sglang/srt/grpc/.*_pb2\.py$|^python/sglang/srt/grpc/.*_pb2_grpc\.py$|^python/sglang/srt/grpc/.*_pb2\.pyi$|^python/sglang/srt/grpc/.*_pb2_grpc\.pyi$'
exclude: '^python/sglang/srt/grpc/.*_pb2\.py$|^python/sglang/srt/grpc/.*_pb2_grpc\.py$|^python/sglang/srt/grpc/.*_pb2\.pyi$|^python/sglang/srt/grpc/.*_pb2_grpc\.pyi$|^python/sglang/(api|json)/v1/.*\.py$'
- repo: https://github.com/codespell-project/codespell
rev: v2.4.1
hooks:
+3 -3
View File
@@ -202,13 +202,13 @@ RUN --mount=type=cache,target=/root/.cache/uv \
# --no-deps prevents uv from pulling torch from default PyPI
&& uv pip install sglang-kernel==${SGL_KERNEL_VERSION} --reinstall --no-deps
# Copy the dependency spec, Rust extension sources, and shared protocol crate.
# Split from the dependency install so changes to these build inputs invalidate the
# Copy the dependency spec, Rust extension sources, and shared API types.
# Split from the pip install so changes to these build inputs invalidate the
# dependency layer, while unrelated Python source changes do not.
COPY python/pyproject.toml /tmp/sglang_deps/python/pyproject.toml
COPY rust/sglang-grpc /tmp/sglang_deps/rust/sglang-grpc
COPY rust/sglang-mm /tmp/sglang_deps/rust/sglang-mm
COPY proto /tmp/sglang_deps/proto
COPY rust/sglang-api-types /tmp/sglang_deps/rust/sglang-api-types
# Install sglang dependencies (torch, transformers, etc.).
# Generate constraints.txt to prevent reinstalling these deps in later stages.
+2 -2
View File
@@ -422,13 +422,13 @@ RUN --mount=type=cache,target=/root/.cache/pip \
; \
fi
# Copy the dependency spec, Rust extension sources, and shared protocol crate.
# Copy the dependency spec, Rust extension sources, and shared API types.
# Split from the pip install so changes to these build inputs invalidate the
# dependency layer, while unrelated Python source changes do not.
COPY python/pyproject.toml /tmp/sglang_deps/python/pyproject.toml
COPY rust/sglang-grpc /tmp/sglang_deps/rust/sglang-grpc
COPY rust/sglang-mm /tmp/sglang_deps/rust/sglang-mm
COPY proto /tmp/sglang_deps/proto
COPY rust/sglang-api-types /tmp/sglang_deps/rust/sglang-api-types
# Install sglang dependencies (torch, transformers, etc.).
# Generate constraints.txt to prevent reinstalling these deps in later stages.
-26
View File
@@ -1,26 +0,0 @@
[package]
name = "sglang-grpc-types"
description = "Generated Rust bindings for the SGLang runtime.v1 API"
version = "0.1.0"
edition.workspace = true
rust-version = "1.92"
license.workspace = true
repository = "https://github.com/sgl-project/sglang"
readme = "README.md"
workspace = "../rust"
include = [
"build.rs",
"README.md",
"src/**",
"sglang/**/*.proto",
]
[dependencies]
prost = { workspace = true }
prost-types = { workspace = true }
tonic = { workspace = true }
[build-dependencies]
prost-build = { workspace = true }
protoc-bin-vendored = { workspace = true }
tonic-build = { workspace = true }
-15
View File
@@ -1,15 +0,0 @@
# SGLang gRPC types
This crate provides the generated Prost messages and Tonic client/server
interfaces for SGLang's canonical `sglang.runtime.v1` API.
The language-independent contract remains in
`sglang/runtime/v1/sglang.proto`. Rust bindings are generated from that schema
at build time with a vendored `protoc`, so consumers do not need a system
protobuf compiler.
The generated API currently uses Tonic 0.12 to remain compatible with
SGLang's existing gRPC server.
This crate contains protocol types only. SGLang's request handling and server
implementation remain in their respective frontend crates.
+134
View File
@@ -0,0 +1,134 @@
// Shared carriers for the native SGLang API. The ground truth for the JSON
// shapes is the sglang.json.v1 options on each field; the protobuf shapes are
// the explicit oneofs/messages themselves.
syntax = "proto3";
package sglang.api.v1;
import "sglang/json/v1/options.proto";
// A token-id sequence. int64 matches the servers' token id width (Rust
// `TokenIds = Vec<i64>`, Python `array("q")`).
message TokenIds {
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST;
repeated int64 ids = 1;
}
// Scalar-or-list carriers: `/generate` accepts every per-request field as one
// value (single request) or a list (batch). JSON flattens via the one_or_many
// option; gRPC clients pick the variant explicitly.
message StringOrList {
oneof value {
option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_ONE_OR_MANY;
string one = 1;
StringList many = 2;
}
}
message StringList {
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST;
repeated string items = 1;
}
// A flat string list for one request, or one list per batch entry. JSON:
// `["a", "b"]` is `one`; `[["a"], ["b", "c"]]` is `many` (first element
// decides, `[]` is an empty `one`).
message StringListOrList {
oneof value {
option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_ONE_OR_MANY;
StringList one = 1;
StringListList many = 2;
}
}
message StringListList {
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST;
repeated StringList items = 1;
}
message BoolOrList {
oneof value {
option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_ONE_OR_MANY;
bool one = 1;
BoolList many = 2;
}
}
message BoolList {
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST;
repeated bool items = 1;
}
message Int64OrList {
oneof value {
option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_ONE_OR_MANY;
int64 one = 1;
Int64List many = 2;
}
}
message Int64List {
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST;
repeated int64 items = 1;
}
message TokenIdsOrList {
oneof value {
option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_ONE_OR_MANY;
TokenIds one = 1;
TokenIdsList many = 2;
}
}
message TokenIdsList {
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST;
repeated TokenIds items = 1;
}
// List variants whose ELEMENTS may be JSON null (PD bootstrap columns: a
// batch mixes bootstrapped and plain requests). Proto carries element
// presence as a per-element oneof-free wrapper.
message OptionalString {
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_TRANSPARENT;
optional string value = 1;
}
message OptionalStringOrList {
oneof value {
option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_ONE_OR_MANY;
OptionalString one = 1;
OptionalStringList many = 2;
}
}
message OptionalStringList {
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST;
repeated OptionalString items = 1;
}
message OptionalInt64 {
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_TRANSPARENT;
optional int64 value = 1;
}
message OptionalInt64OrList {
oneof value {
option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_ONE_OR_MANY;
OptionalInt64 one = 1;
OptionalInt64List many = 2;
}
}
message OptionalInt64List {
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST;
repeated OptionalInt64 items = 1;
}
// The native error body: `{"error": {"message", "code"}}` (Python
// `http_server.generate_request` parity). Transports frame it themselves
// (unary status + body, SSE error frame, gRPC status detail).
message ErrorBody {
string message = 1;
uint32 code = 2;
}
+60
View File
@@ -0,0 +1,60 @@
// The terminal finish reason — Python's `FinishReasonDict`
// (`BaseFinishReason.to_json()` in schedule_batch.py), echoed verbatim as
// `meta_info.finish_reason`. JSON is internally tagged by "type" with the
// variant payload inlined; a type this build doesn't know round-trips
// unchanged (unknown_variant_passthrough) so a Python-side addition never
// fails a frame decode.
syntax = "proto3";
package sglang.api.v1;
import "sglang/json/v1/options.proto";
// Python's `matched` key, `Union[str, int, List[int]]`: the three wire shapes
// are disjoint, so shape alone picks the arm (untagged).
message Matched {
oneof value {
option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_UNTAGGED;
int64 token = 1;
string str = 2;
MatchedTokens tokens = 3;
}
}
message MatchedTokens {
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST;
repeated int64 ids = 1;
}
// `FINISH_MATCHED_TOKEN` / `FINISH_MATCHED_STR` / `FINISHED_MATCHED_REGEX` —
// all three report `type: "stop"`, matching the OpenAI API's value.
message FinishStop {
optional Matched matched = 1;
}
// `FINISH_LENGTH` — hit `max_new_tokens` (or the context limit).
message FinishLength {
optional uint64 length = 1;
}
// The `FINISH_ABORT` payload. Python emits all three keys, nulls included.
message FinishAbort {
optional string message = 1 [(sglang.json.v1.field_json).emit_null_when_absent = true];
optional uint32 status_code = 2 [(sglang.json.v1.field_json).emit_null_when_absent = true];
optional string err_type = 3 [(sglang.json.v1.field_json).emit_null_when_absent = true];
}
message FinishReason {
oneof kind {
option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_TAGGED;
option (sglang.json.v1.oneof_json).tag = "type";
option (sglang.json.v1.oneof_json).unknown_variant_passthrough = true;
FinishStop stop = 1;
FinishLength length = 2;
FinishAbort abort = 3;
// The passthrough arm: an unrecognized `type`, kept as its raw map
// (JSON text — Struct would coerce integer values to doubles).
string unknown = 4 [(sglang.json.v1.field_json).raw_json = true];
}
}
+235
View File
@@ -0,0 +1,235 @@
// The native `/generate` surface — the ground truth for
// rust/sglang-server/src/message/request.rs's GenerateBody (request) and
// api_core/frame.rs's frame shape (response). Unknown JSON keys on the
// request are IGNORED: `deny_unknown_fields` broke real clients that send
// Python-only `GenerateReqInput` fields (priority, extra_key, session_id,
// ...), pinned by request.rs's unported-fields test.
syntax = "proto3";
package sglang.api.v1;
import "sglang/api/v1/common.proto";
import "sglang/api/v1/finish.proto";
import "sglang/api/v1/sampling.proto";
import "sglang/json/v1/options.proto";
message GenerateRequest {
// (message_json).unknown_fields defaults to IGNORE — stated here as schema
// intent, load-bearing for client compat.
StringOrList rid = 1;
StringOrList text = 2;
TokenIdsOrList input_ids = 3;
optional bool stream = 4 [
(sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT,
(sglang.json.v1.field_json).default_bool = false
];
// One set of params, or one per batch item. JSON errors keep serde's
// field-level texts (the hand-written SamplingParamsInput contract).
SamplingParamsOrList sampling_params = 5;
BoolOrList return_logprob = 6;
Int64OrList logprob_start_len = 7;
Int64OrList top_logprobs_num = 8;
TokenIdsOrList token_ids_logprob = 9;
BoolOrList return_hidden_states = 10;
optional bool return_text_in_logprobs = 11;
// PD bootstrap columns: a batch mixes bootstrapped and plain requests, so
// elements are individually nullable.
OptionalStringOrList bootstrap_host = 12;
OptionalInt64OrList bootstrap_port = 13;
OptionalInt64OrList bootstrap_room = 14;
OptionalStringOrList bootstrap_pair_key = 15;
OptionalInt64OrList decode_tp_size = 16;
optional int64 routed_dp_rank = 17;
optional int64 disagg_prefill_dp_rank = 18;
// Multimodal payloads pass through untyped (data URIs, URLs, file paths, or
// nested batch lists) to the MM pipeline, which owns their validation.
// Media inputs, one container per modality (see MediaInput). The item
// hints a modality does not support are rejected at the processor.
optional MediaInput image_data = 19;
// Per-item content hashes overriding the computed ones, in item order; a
// flat list for one request (the batch form is rejected downstream).
optional StringListOrList mm_hashes = 20;
optional MediaInput video_data = 21;
optional MediaInput audio_data = 22;
}
message SamplingParamsOrList {
oneof value {
option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_ONE_OR_MANY;
SamplingParams one = 1;
SamplingParamsList many = 2;
}
}
message SamplingParamsList {
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST;
repeated SamplingParams items = 1;
}
// One `[logprob, token_id, text|null]` entry (text present when
// `return_text_in_logprobs`, decoded on the detok shard). The logprob slot is
// optional: Python emits null for a position with no logprob (the first
// prefill token; Rust maps the scheduler's NaN to the same null).
message LogprobEntry {
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_TUPLE;
optional double logprob = 1;
int64 token_id = 2;
optional string text = 3;
}
message LogprobEntries {
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST;
repeated LogprobEntry entries = 1;
}
// One decode position's top-k alternatives.
message TopLogprobRow {
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST;
repeated LogprobEntry entries = 1;
}
// A position's row, or null: a position that recorded no alternatives is
// null on the wire (Python parity), never [].
message NullableTopLogprobs {
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_TRANSPARENT;
TopLogprobRow row = 1;
}
message HiddenStateRow {
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST;
repeated double values = 1;
}
message GenerateMetaInfo {
// The client's rid, byte-for-byte (`Rid::client_facing`).
string id = 1;
uint32 prompt_tokens = 2;
uint64 completion_tokens = 3;
// Always present in JSON, null until a terminal frame.
FinishReason finish_reason = 4 [(sglang.json.v1.field_json).emit_null_when_absent = true];
// input+output token logprobs are set together, empty lists included: a PD
// decode node never receives input logprobs, but the PD router keys its
// merge of prefill's on the key's presence.
LogprobEntries output_token_logprobs = 5;
LogprobEntries input_token_logprobs = 6;
repeated NullableTopLogprobs output_top_logprobs = 7;
repeated NullableTopLogprobs input_top_logprobs = 8;
repeated NullableTopLogprobs output_token_ids_logprobs = 9;
repeated NullableTopLogprobs input_token_ids_logprobs = 10;
repeated HiddenStateRow hidden_states = 11;
// Seconds from normalized-request acceptance to the terminal output; only
// on the terminal frame.
optional double e2e_latency = 12;
}
message GenerateResponse {
// JSON key order is FIELD-NUMBER order: text, meta_info, then the
// conditionally-present output_ids and index — matching frame_value's
// insertion order byte-for-byte.
string text = 1;
GenerateMetaInfo meta_info = 2;
// Omitted while empty (a text-only frame).
TokenIds output_ids = 3;
// Batch position; only on batch responses/frames.
optional uint32 index = 4;
}
// One request's failure inside a Generate stream; carries its batch position
// (the HTTP SSE shape puts `index` beside the error object the same way).
message GenerateStreamError {
ErrorBody error = 1;
optional uint32 index = 2;
}
// One item of the gRPC Generate stream: a frame, or one request's failure
// (the stream continues — other batch items may still be running). On HTTP
// the same failure is an in-stream SSE error frame.
message GenerateStreamItem {
oneof item {
GenerateResponse frame = 1;
GenerateStreamError error = 2;
}
}
// ---- Media inputs ----------------------------------------------------------
//
// Every form Python's `/generate` accepts, typed. Preprocessed inputs
// (`format: "processor_output"` / `"precomputed_embedding"`) are NOT part of
// the HTTP contract: their values are tensors, which JSON cannot carry; they
// exist for the in-process Engine only.
// One media item's source and per-item hints. Unknown keys are an error: a
// hint this schema does not name would otherwise be silently dropped.
message MediaRef {
option (sglang.json.v1.message_json).unknown_fields = UNKNOWN_FIELDS_DENY;
// URL, `file://` / absolute path, `data:` URI, or bare base64.
string url = 1;
// Image: "auto" | "low" | "high" (the OpenAI detail hint).
optional string detail = 2;
// Image/video: dynamic-patch budget (InternVL-style processors).
optional int64 max_dynamic_patch = 3;
// Image: a caller-supplied content hash that overrides the computed one.
optional string content_hash = 4;
// Video/image budgets read by the GLM and MiMo processors.
optional int64 min_pixels = 5;
optional int64 max_pixels = 6;
optional int64 total_max_pixels = 7;
optional double fps = 8;
optional int64 num_frames = 9;
optional int64 min_frames = 10;
optional int64 max_frames = 11;
optional int64 max_tokens_per_frame = 12;
optional int64 max_image_tokens = 13;
// Video: also decode the audio track.
optional bool use_audio = 14;
}
// One media item. JSON: a bare string is the source; an object is a MediaRef.
message MediaItem {
oneof value {
option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_UNTAGGED;
string source = 1;
MediaRef ref = 2;
}
}
// A list element that may be JSON null (Python's `[img, None]`).
message OptionalMediaItem {
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_TRANSPARENT;
optional MediaItem item = 1;
}
// One request's items (`[a, null, b]`), or one batch entry's items.
message MediaItems {
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST;
repeated OptionalMediaItem items = 1;
}
// A batch entry whose item list may be JSON null.
message OptionalMediaItems {
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_TRANSPARENT;
optional MediaItems items = 1;
}
// One item list per batch entry (`[[a, b], null, []]`).
message MediaItemsList {
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST;
repeated OptionalMediaItems items = 1;
}
// The three container forms Python's `_normalize_image_data` distinguishes.
// JSON: a string or object is `item` (one item; a broadcast in a batch); an
// array whose first element is not an array is `many` (a flat list; `[]` is
// an empty `many`); an array whose first element is an array is `nested`
// (one list per batch entry). Which forms a request may use depends on
// whether it is a batch, and is checked downstream, not here.
message MediaInput {
oneof value {
option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_UNTAGGED;
MediaItem item = 1;
MediaItems many = 2;
MediaItemsList nested = 3;
}
}
+100
View File
@@ -0,0 +1,100 @@
// The API face of SamplingParams — the ground truth for
// rust/sglang-server/src/message/sampling.rs's 25 API fields (its 5
// normalize-internal fields and the msgpack scheduler encoding are IPC, not
// API, and stay hand-written there). Unknown JSON keys are DENIED: parity with
// Python's `SamplingParams(**kwargs)` TypeError.
syntax = "proto3";
package sglang.api.v1;
import "sglang/api/v1/common.proto";
import "sglang/json/v1/options.proto";
message SamplingParams {
option (sglang.json.v1.message_json).unknown_fields = UNKNOWN_FIELDS_DENY;
// Defaulted fields are `optional`: proto3 gives plain scalars no presence,
// so an unset double would arrive as 0 instead of its schema default (the
// top_p=0 bug the first live gRPC call found). JSON deserialization bakes
// the default on absent/null; protobuf arrival reads it through the
// generated *_or_default accessors.
// JSON: absent = 128, explicit null = no limit (Python `Optional[int] =
// 128`). Protobuf has no null: unset = 128 (apply_absent_defaults at the
// gRPC entry); no limit needs an explicit large value.
optional int64 max_new_tokens = 1 [
(sglang.json.v1.field_json).null_policy = NULL_POLICY_IS_NONE,
(sglang.json.v1.field_json).default_i64 = 128
];
StringOrList stop = 2;
Int64List stop_token_ids = 3;
StringOrList stop_regex = 4;
optional double temperature = 5 [
(sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT,
(sglang.json.v1.field_json).default_f64 = 1.0
];
optional double top_p = 6 [
(sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT,
(sglang.json.v1.field_json).default_f64 = 1.0
];
// Default 1 << 30 (TOP_K_ALL): "consider the whole vocabulary".
optional int64 top_k = 7 [
(sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT,
(sglang.json.v1.field_json).default_i64 = 1073741824
];
optional double min_p = 8 [
(sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT,
(sglang.json.v1.field_json).default_f64 = 0.0
];
optional double frequency_penalty = 9 [
(sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT,
(sglang.json.v1.field_json).default_f64 = 0.0
];
optional double presence_penalty = 10 [
(sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT,
(sglang.json.v1.field_json).default_f64 = 0.0
];
optional double repetition_penalty = 11 [
(sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT,
(sglang.json.v1.field_json).default_f64 = 1.0
];
optional int64 min_new_tokens = 12 [
(sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT,
(sglang.json.v1.field_json).default_i64 = 0
];
optional int64 n = 13 [
(sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT,
(sglang.json.v1.field_json).default_i64 = 1
];
optional string json_schema = 14;
optional string regex = 15;
optional string ebnf = 16;
optional string structural_tag = 17;
optional bool ignore_eos = 18 [
(sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT,
(sglang.json.v1.field_json).default_bool = false
];
optional bool skip_special_tokens = 19 [
(sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT,
(sglang.json.v1.field_json).default_bool = true
];
optional bool spaces_between_special_tokens = 20 [
(sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT,
(sglang.json.v1.field_json).default_bool = true
];
optional bool no_stop_trim = 21 [
(sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT,
(sglang.json.v1.field_json).default_bool = false
];
optional int64 stream_interval = 22;
// Token id (numeric-string key, Python parity) -> bias.
map<string, double> logit_bias = 23 [(sglang.json.v1.field_json).int_key_map = true];
optional int64 sampling_seed = 24;
// Opaque JSON forwarded to a custom logit processor; never inspected here.
optional string custom_params = 25 [(sglang.json.v1.field_json).raw_json = true];
// beam_width > 1 turns the request into a beam search request; `n` is then
// the number of returned sequences (n <= beam_width).
optional int64 beam_width = 26;
}
+72
View File
@@ -0,0 +1,72 @@
// The native SGLang gRPC service served by rust/sglang-server's tonic
// transport, over the same api_core the HTTP endpoints use. Covers exactly the
// surface the native server implements (the legacy sglang.runtime.v1 bridge's
// embed/classify/tokenize RPCs have no native backend here and are not
// carried over).
syntax = "proto3";
package sglang.api.v1;
import "sglang/api/v1/generate.proto";
import "sglang/json/v1/options.proto";
service SglangService {
// `/generate`: always server-streaming (a unary request is a one-frame
// stream); per-batch-item failures ride the stream as GenerateStreamItem
// errors, pre-submit failures end the RPC with a gRPC status.
rpc Generate(GenerateRequest) returns (stream GenerateStreamItem);
// `/health_generate`: fires a 1-token probe and watches the response
// heartbeat. UNAVAILABLE on a stalled pipeline.
rpc HealthCheck(HealthCheckRequest) returns (HealthCheckResponse);
// `/get_model_info`: static model metadata, no scheduler round-trip.
rpc GetModelInfo(GetModelInfoRequest) returns (GetModelInfoResponse);
// `/server_info`: one control round-trip, shaped through the runtime-metric
// allowlist (never the raw server-args dump, which embeds keys).
rpc GetServerInfo(GetServerInfoRequest) returns (GetServerInfoResponse);
}
message HealthCheckRequest {}
message HealthCheckResponse {
bool healthy = 1;
}
message GetModelInfoRequest {}
message GetModelInfoResponse {
string model_path = 1;
string served_model_name = 2;
string tokenizer_path = 3;
bool is_generation = 4;
// The launch-time `--preferred-sampling-params` object, or null when unset.
// JSON carries the object itself (the HTTP `/get_model_info` shape); the
// protobuf field holds it as JSON text.
optional string preferred_sampling_params = 5 [
(sglang.json.v1.field_json).raw_json = true,
(sglang.json.v1.field_json).emit_null_when_absent = true
];
optional string weight_version = 6 [(sglang.json.v1.field_json).emit_null_when_absent = true];
optional string load_format = 7 [(sglang.json.v1.field_json).emit_null_when_absent = true];
optional string reasoning_parser = 8 [(sglang.json.v1.field_json).emit_null_when_absent = true];
optional string tool_call_parser = 9 [(sglang.json.v1.field_json).emit_null_when_absent = true];
}
message GetServerInfoRequest {}
message GetServerInfoResponse {
string model_path = 1;
string served_model_name = 2;
string tokenizer_path = 3;
uint32 max_context_length = 4;
optional uint64 max_total_num_tokens = 5;
string version = 6;
// The allowlisted runtime metrics (last_gen_throughput, memory_usage,
// effective_max_running_requests_per_dp, avg_spec_accept_length,
// step_time_dict), untyped because the scheduler owns their shapes; JSON
// passes them through verbatim.
repeated string internal_states = 7 [(sglang.json.v1.field_json).raw_json = true];
}
+156
View File
@@ -0,0 +1,156 @@
// The JSON-fidelity option vocabulary: how a message, field, or oneof behaves
// on the HTTP JSON wire, declared in the schema itself. The sglang-api-codegen
// generator reads these options from the FileDescriptorSet and emits the serde
// implementations; proto3's canonical JSON mapping is NOT used. Each option is
// one generator template with its own unit tests.
//
// gRPC never sees these options: the protobuf wire format is untouched.
syntax = "proto3";
package sglang.json.v1;
import "google/protobuf/descriptor.proto";
// How a message treats unknown JSON keys on deserialization.
enum UnknownFields {
// Silently dropped (the default): parity with endpoints that accept and
// ignore unported fields, where rejecting would break existing clients.
UNKNOWN_FIELDS_IGNORE = 0;
// A 400 with serde's field-level message: parity with Python constructors
// that TypeError on unexpected kwargs.
UNKNOWN_FIELDS_DENY = 1;
}
// How a message is shaped on the JSON wire. One shape per message: an enum,
// not independent flags, so two cannot be set at once.
enum MessageShape {
// A plain JSON object keyed by field name (the default).
MESSAGE_SHAPE_OBJECT = 0;
// A single-`repeated`-field carrier (TokenIds, StringList, ...): its JSON
// is the bare array, not an {items: [...]} object, wherever it appears.
MESSAGE_SHAPE_BARE_LIST = 1;
// A positional tuple: the fields, in declaration order, form a bare array
// (e.g. a logprob entry's [logprob, token_id, text|null]). Optional members
// serialize as null.
MESSAGE_SHAPE_TUPLE = 2;
// A single-field message that is invisible in JSON: it serializes as its
// sole field's value (null when unset). Exists to give proto a presence
// wrapper for nullable LIST ELEMENTS (OptionalString, OptionalInt64).
MESSAGE_SHAPE_TRANSPARENT = 3;
}
// Attached to a message: `option (sglang.json.v1.message_json).<name> = ...;`
// in its body. Decides the JSON form of the message as a whole, for both
// directions of the HTTP wire: `shape` says what the serializer writes and
// what the deserializer expects (object, bare array, tuple, or the sole
// field's value); `unknown_fields` says what the deserializer does with keys
// the message does not declare. Serialization never emits unknown keys.
message MessageJson {
UnknownFields unknown_fields = 1;
MessageShape shape = 2;
}
// How an absent key or an explicit JSON null reads into a field. One policy
// per field: an enum, not independent flags, so two cannot be set at once.
// Deserialization only; `emit_null_when_absent` is the serialization side.
enum NullPolicy {
// Absent is unset; an explicit null is a type error (the default).
NULL_POLICY_DEFAULT = 0;
// Explicit null (and an absent key) yields the stated default instead of
// an error: parity with Python dataclasses whose __post_init__ maps None
// back to the default.
NULL_POLICY_RESETS_DEFAULT = 1;
// An ABSENT key yields the stated default, but an explicit null clears the
// field to no-value (e.g. max_new_tokens: absent = 128, null = no limit).
NULL_POLICY_IS_NONE = 2;
}
// Attached to one field: `type name = N [(sglang.json.v1.field_json).<name>
// = ...];`. Decides how that key behaves, with the read and write sides kept
// apart. Deserialization: `null_policy` (with a `default_*`) maps an absent
// key or an explicit null onto a value. Serialization: `emit_null_when_absent`
// writes the key as null instead of omitting it. Both directions: `raw_json`
// carries the value through as JSON text, and `int_key_map` spells numeric
// map keys as strings.
message FieldJson {
NullPolicy null_policy = 1;
// The default a non-DEFAULT policy yields; exactly one must accompany it.
optional double default_f64 = 2;
optional int64 default_i64 = 3;
optional bool default_bool = 4;
optional string default_string = 5;
// The key is always serialized, an unset value as an explicit null (e.g.
// reasoning_content in chat deltas: Python always emits the key).
bool emit_null_when_absent = 6;
// On a string (or repeated string) field: the string holds JSON text, and
// HTTP (de)serialization passes the VALUE through — integers, nesting, and
// element shapes intact (google.protobuf.Struct cannot do this: it coerces
// integers to doubles). gRPC carries the JSON text.
bool raw_json = 7;
// On a map<string, ...> field whose keys are numeric strings (Python
// parity, e.g. logit_bias token ids).
bool int_key_map = 8;
}
// How a oneof (a union) is spelled on the JSON wire. One shape per union.
enum OneofShape {
// Unset: the message owning the oneof is a plain object.
ONEOF_SHAPE_UNSPECIFIED = 0;
// Untagged: the value's shape picks the variant, and variant mismatch
// errors keep serde's field-level texts (never "data did not match any
// variant").
ONEOF_SHAPE_UNTAGGED = 1;
// On a oneof of exactly { T one; TList many; }: a JSON scalar deserializes
// as `one`, a JSON array as `many`; serialization flattens the same way.
ONEOF_SHAPE_ONE_OR_MANY = 2;
// Internally tagged by the key in `tag` (e.g. finish_reason's
// {"type": "stop", ...}); variant payload fields inline into the same
// object, with each variant's tag value being its field name.
ONEOF_SHAPE_TAGGED = 3;
}
// Attached to a oneof: `option (sglang.json.v1.oneof_json).<name> = ...;` in
// its body. A oneof is a tagged union on the protobuf wire (the field number
// is the tag); JSON has no such tag, so `shape` picks how the union is
// spelled there and how the deserializer recovers the variant: by value
// shape (untagged), scalar versus array (one_or_many), or a discriminator key
// named by `tag` (tagged). `unknown_variant_passthrough` lets a tagged union
// round-trip a variant this schema does not know.
message OneofJson {
OneofShape shape = 1;
// The discriminator key; required with ONEOF_SHAPE_TAGGED, meaningless
// otherwise.
string tag = 2;
// With ONEOF_SHAPE_TAGGED: JSON input matching no variant is captured
// verbatim by the designated catch-all variant (a raw_json Struct member)
// and round-trips unchanged -- the forward-compatibility escape hatch for
// values another component may extend (e.g. finish_reason).
bool unknown_variant_passthrough = 3;
}
extend google.protobuf.MessageOptions {
// Field numbers from the internal-use extension range (50000-99999).
MessageJson message_json = 50100;
}
extend google.protobuf.FieldOptions {
FieldJson field_json = 50100;
}
extend google.protobuf.OneofOptions {
OneofJson oneof_json = 50100;
}
-13
View File
@@ -1,13 +0,0 @@
//! Rust bindings for SGLang's canonical `sglang.runtime.v1` protobuf API.
//!
//! The generated messages and Tonic client/server interfaces intentionally
//! contain no frontend implementation or transport policy.
/// Protobuf packages exposed with the same namespace as the wire contract.
pub mod sglang {
pub mod runtime {
pub mod v1 {
tonic::include_proto!("sglang.runtime.v1");
}
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,18 @@
"""The HTTP JSON contract gate: `proto/sglang/api/v1` is the schema for both
the Rust and the Python server, so a `/generate` body the Python dataclass
parse admitted but the schema rejects is refused here with the schema's own
error text. The generated decoder is the single source of that judgment.
"""
from typing import Any, Optional
from sglang.api.v1.api_types import GenerateRequest, JsonContractError
def generate_contract_error(body: Any) -> Optional[str]:
"""The contract violation in a parsed `/generate` JSON body, or None."""
try:
GenerateRequest.from_json_value(body)
except JsonContractError as e:
return str(e)
return None
@@ -73,6 +73,7 @@ from sglang.srt.entrypoints.anthropic.protocol import (
AnthropicMessagesRequest,
)
from sglang.srt.entrypoints.anthropic.serving import AnthropicServing
from sglang.srt.entrypoints.api_contract import generate_contract_error
from sglang.srt.entrypoints.engine import (
Engine,
init_tokenizer_manager,
@@ -908,6 +909,11 @@ if os.environ.get("DUMPER_SERVER_PORT") == "reuse":
)
async def generate_request(obj: GenerateReqInput, request: Request):
"""Handle a generate request."""
# Starlette caches the parsed body, so this is the schema check, not a
# second decode.
contract_error = generate_contract_error(await request.json())
if contract_error is not None:
return ORJSONResponse(status_code=400, content={"error": contract_error})
if envs.SGLANG_ENABLE_REQUEST_HEADER_OVERRIDES.get():
apply_header_overrides(obj, request.headers)
if obj.stream:
+5
View File
@@ -0,0 +1,5 @@
# Workspace-local cargo aliases.
[alias]
# Regenerate the API types (Rust + Python) from proto/sglang; see
# sglang-api-codegen/src/main.rs.
gen-api = "run -p sglang-api-codegen"
+137 -133
View File
@@ -292,40 +292,13 @@ dependencies = [
"arrayvec",
]
[[package]]
name = "axum"
version = "0.7.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f"
dependencies = [
"async-trait",
"axum-core 0.4.5",
"bytes",
"futures-util",
"http",
"http-body",
"http-body-util",
"itoa",
"matchit 0.7.3",
"memchr",
"mime",
"percent-encoding",
"pin-project-lite",
"rustversion",
"serde",
"sync_wrapper",
"tower 0.5.3",
"tower-layer",
"tower-service",
]
[[package]]
name = "axum"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90"
dependencies = [
"axum-core 0.5.6",
"axum-core",
"bytes",
"form_urlencoded",
"futures-util",
@@ -335,7 +308,7 @@ dependencies = [
"hyper",
"hyper-util",
"itoa",
"matchit 0.8.4",
"matchit",
"memchr",
"mime",
"percent-encoding",
@@ -346,32 +319,12 @@ dependencies = [
"serde_urlencoded",
"sync_wrapper",
"tokio",
"tower 0.5.3",
"tower",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "axum-core"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199"
dependencies = [
"async-trait",
"bytes",
"futures-util",
"http",
"http-body",
"http-body-util",
"mime",
"pin-project-lite",
"rustversion",
"sync_wrapper",
"tower-layer",
"tower-service",
]
[[package]]
name = "axum-core"
version = "0.5.6"
@@ -1265,6 +1218,12 @@ version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "foldhash"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
[[package]]
name = "form_urlencoded"
version = "1.2.2"
@@ -1492,6 +1451,15 @@ version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
[[package]]
name = "hashbrown"
version = "0.15.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
dependencies = [
"foldhash",
]
[[package]]
name = "hashbrown"
version = "0.16.1"
@@ -1658,7 +1626,7 @@ dependencies = [
"libc",
"percent-encoding",
"pin-project-lite",
"socket2 0.5.10",
"socket2",
"system-configuration",
"tokio",
"tower-service",
@@ -2088,12 +2056,6 @@ dependencies = [
"regex-automata",
]
[[package]]
name = "matchit"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
[[package]]
name = "matchit"
version = "0.8.4"
@@ -2558,11 +2520,12 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
[[package]]
name = "petgraph"
version = "0.7.1"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772"
checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455"
dependencies = [
"fixedbitset",
"hashbrown 0.15.5",
"indexmap 2.14.0",
]
@@ -2739,9 +2702,9 @@ dependencies = [
[[package]]
name = "prost"
version = "0.13.5"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5"
checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1"
dependencies = [
"bytes",
"prost-derive",
@@ -2749,19 +2712,20 @@ dependencies = [
[[package]]
name = "prost-build"
version = "0.13.5"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
dependencies = [
"heck",
"itertools",
"log",
"multimap",
"once_cell",
"petgraph",
"prettyplease",
"prost",
"prost-types",
"pulldown-cmark",
"pulldown-cmark-to-cmark",
"regex",
"syn 2.0.119",
"tempfile",
@@ -2769,9 +2733,9 @@ dependencies = [
[[package]]
name = "prost-derive"
version = "0.13.5"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
dependencies = [
"anyhow",
"itertools",
@@ -2781,10 +2745,20 @@ dependencies = [
]
[[package]]
name = "prost-types"
version = "0.13.5"
name = "prost-reflect"
version = "0.16.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16"
checksum = "01b80ea363c31af2de2b92e3c07ed1156628f7838c4afb4df75ee78a37fedbd1"
dependencies = [
"prost",
"prost-types",
]
[[package]]
name = "prost-types"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a"
dependencies = [
"prost",
]
@@ -2853,6 +2827,26 @@ version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95067976aca6421a523e491fce939a3e65249bac4b977adee0ee9771568e8aa3"
[[package]]
name = "pulldown-cmark"
version = "0.13.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e"
dependencies = [
"bitflags",
"memchr",
"unicase",
]
[[package]]
name = "pulldown-cmark-to-cmark"
version = "22.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "84bbb29c624230c4bd1047bbdb2aa47e41c860e9665ce62ba9504eebe91bf867"
dependencies = [
"pulldown-cmark",
]
[[package]]
name = "pulp"
version = "0.22.3"
@@ -2967,7 +2961,7 @@ dependencies = [
"quinn-udp",
"rustc-hash 2.1.3",
"rustls",
"socket2 0.5.10",
"socket2",
"thiserror",
"tokio",
"tracing",
@@ -3005,7 +2999,7 @@ dependencies = [
"cfg_aliases",
"libc",
"once_cell",
"socket2 0.5.10",
"socket2",
"tracing",
"windows-sys 0.61.2",
]
@@ -3344,7 +3338,7 @@ dependencies = [
"tokio",
"tokio-rustls",
"tokio-util",
"tower 0.5.3",
"tower",
"tower-http",
"tower-service",
"url",
@@ -3688,6 +3682,30 @@ dependencies = [
"syn 2.0.119",
]
[[package]]
name = "sglang-api-codegen"
version = "0.1.0"
dependencies = [
"prost",
"prost-build",
"prost-reflect",
"protoc-bin-vendored",
"tonic-prost-build",
]
[[package]]
name = "sglang-api-types"
version = "0.1.0"
dependencies = [
"bytes",
"prost",
"prost-types",
"serde",
"serde_json",
"tonic",
"tonic-prost",
]
[[package]]
name = "sglang-grpc"
version = "0.1.0"
@@ -3696,28 +3714,17 @@ dependencies = [
"prost-types",
"pyo3",
"serde_json",
"sglang-grpc-types",
"sglang-api-types",
"tokenizers",
"tokio",
"tokio-stream",
"tonic",
"tonic-prost",
"tracing",
"tracing-subscriber",
"uuid",
]
[[package]]
name = "sglang-grpc-types"
version = "0.1.0"
dependencies = [
"prost",
"prost-build",
"prost-types",
"protoc-bin-vendored",
"tonic",
"tonic-build",
]
[[package]]
name = "sglang-mm"
version = "0.1.0"
@@ -3757,7 +3764,7 @@ name = "sglang-renderer"
version = "0.1.0"
dependencies = [
"async-stream",
"axum 0.8.9",
"axum",
"clap",
"dynamo-parsers",
"dynamo-protocols",
@@ -3776,7 +3783,7 @@ dependencies = [
"sglang-processor",
"thiserror",
"tokio",
"tower 0.5.3",
"tower",
"tracing",
"tracing-subscriber",
"uuid",
@@ -3788,7 +3795,7 @@ version = "0.1.0"
dependencies = [
"arc-swap",
"async-stream",
"axum 0.8.9",
"axum",
"bytemuck",
"bytes",
"core_affinity",
@@ -3799,6 +3806,7 @@ dependencies = [
"flume",
"futures",
"hf-hub",
"http",
"itertools",
"minijinja",
"numpy",
@@ -3809,14 +3817,14 @@ dependencies = [
"rustix",
"serde",
"serde_json",
"sglang-grpc-types",
"sglang-api-types",
"sglang-mm",
"socket2 0.6.5",
"socket2",
"thiserror",
"tokio",
"tokio-stream",
"tonic",
"tower 0.5.3",
"tower",
"tracing",
"tracing-appender",
"tracing-subscriber",
@@ -3903,16 +3911,6 @@ version = "1.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
[[package]]
name = "socket2"
version = "0.5.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678"
dependencies = [
"libc",
"windows-sys 0.52.0",
]
[[package]]
name = "socket2"
version = "0.6.5"
@@ -4253,7 +4251,7 @@ dependencies = [
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
"socket2 0.6.5",
"socket2",
"tokio-macros",
"windows-sys 0.61.2",
]
@@ -4306,16 +4304,14 @@ dependencies = [
[[package]]
name = "tonic"
version = "0.12.3"
version = "0.14.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52"
checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef"
dependencies = [
"async-stream",
"async-trait",
"axum 0.7.9",
"axum",
"base64 0.22.1",
"bytes",
"flate2",
"h2",
"http",
"http-body",
@@ -4325,11 +4321,11 @@ dependencies = [
"hyper-util",
"percent-encoding",
"pin-project",
"prost",
"socket2 0.5.10",
"socket2",
"sync_wrapper",
"tokio",
"tokio-stream",
"tower 0.4.13",
"tower",
"tower-layer",
"tower-service",
"tracing",
@@ -4337,9 +4333,32 @@ dependencies = [
[[package]]
name = "tonic-build"
version = "0.12.3"
version = "0.14.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9557ce109ea773b399c9b9e5dca39294110b74f1f342cb347a80d1fce8c26a11"
checksum = "c68f61875ac5293cf72e6c8cf0158086428c82c37229e98c840878f1706b0322"
dependencies = [
"prettyplease",
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "tonic-prost"
version = "0.14.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0"
dependencies = [
"bytes",
"prost",
"tonic",
]
[[package]]
name = "tonic-prost-build"
version = "0.14.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "654e5643eff75d7f8c99197ce1440ed19a3474eada74c12bbac488b2cafdae27"
dependencies = [
"prettyplease",
"proc-macro2",
@@ -4347,26 +4366,8 @@ dependencies = [
"prost-types",
"quote",
"syn 2.0.119",
]
[[package]]
name = "tower"
version = "0.4.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
dependencies = [
"futures-core",
"futures-util",
"indexmap 1.9.3",
"pin-project",
"pin-project-lite",
"rand 0.8.7",
"slab",
"tokio",
"tokio-util",
"tower-layer",
"tower-service",
"tracing",
"tempfile",
"tonic-build",
]
[[package]]
@@ -4377,9 +4378,12 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4"
dependencies = [
"futures-core",
"futures-util",
"indexmap 2.14.0",
"pin-project-lite",
"slab",
"sync_wrapper",
"tokio",
"tokio-util",
"tower-layer",
"tower-service",
"tracing",
@@ -4397,7 +4401,7 @@ dependencies = [
"http",
"http-body",
"pin-project-lite",
"tower 0.5.3",
"tower",
"tower-layer",
"tower-service",
"url",
+9 -6
View File
@@ -1,7 +1,8 @@
[workspace]
resolver = "3"
members = [
"../proto",
"sglang-api-codegen",
"sglang-api-types",
"sglang-grpc",
"sglang-mm",
"sglang-processor",
@@ -30,9 +31,9 @@ futures = "0.3"
pyo3 = { version = "0.29.0", features = ["extension-module"] }
# These versions define the generated public gRPC API. Update them together
# and deliberately rather than changing bindings during an unrelated build.
prost = "=0.13.5"
prost-types = "=0.13.5"
prost-build = "=0.13.5"
prost = "0.14"
prost-types = "0.14"
prost-build = "0.14"
protoc-bin-vendored = "=3.2.0"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
@@ -40,11 +41,13 @@ socket2 = "0.6"
thiserror = "2"
tokio = { version = "1", features = ["full"] }
tokio-stream = { version = "0.1", features = ["net"] }
tonic = "0.14"
tonic-build = "0.14"
tonic-prost = "0.14"
tower = { version = "0.5", features = ["util"] }
tracing = "0.1"
tracing-appender = "0.2"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
tonic = { version = "=0.12.3", features = ["gzip", "transport"] }
tonic-build = "=0.12.3"
uuid = { version = "1", features = ["v4"] }
# Profiles only take effect at the workspace root; per-member profile sections
+19
View File
@@ -0,0 +1,19 @@
[package]
name = "sglang-api-codegen"
description = "Generator: proto/sglang -> checked-in bindings (prost structs, tonic services, schema-driven serde, Python types)"
version.workspace = true
edition.workspace = true
license.workspace = true
# No [package.metadata.sglang]: not a Python extension; wheel builds skip it.
[[bin]]
name = "sglang-api-codegen"
path = "src/main.rs"
[dependencies]
prost = { workspace = true }
prost-build = "0.14"
prost-reflect = "0.16"
protoc-bin-vendored = "3"
tonic-prost-build = "0.14"
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+179
View File
@@ -0,0 +1,179 @@
//! The generator: every checked-in binding of `proto/sglang/`, from one protoc.
//!
//! Targets (see [`TARGETS`]):
//! Both targets land in `rust/sglang-api-types/src/generated/`:
//! - `sglang.api.v1`: prost structs, tonic services, schema-driven serde, plus
//! `python/sglang/api/v1/types.py`.
//! - `sglang.runtime.v1`: prost structs and tonic services only.
//!
//! Pass 1 (prost-build + tonic-prost-build) emits the prost structs and tonic
//! service code, plus a `FileDescriptorSet`. For targets with a JSON contract
//! pass 2 reads the `sglang.json.v1` options from that set through
//! prost-reflect and emits, from one model (`model`), the Rust serde
//! implementations (`emit`) and the Python interface definition (`emit_py`);
//! proto3's canonical JSON mapping is NOT used, the options are the contract.
//!
//! Regeneration is manual. From anywhere in the `rust/` workspace:
//!
//! ```text
//! cargo gen-api
//! ```
//!
//! Each target has a single root .proto: every API must be defined (or
//! imported) there, and protoc's import closure supplies the rest of the
//! package; a file the root does not reach gets no generated code. All output
//! is checked in, so commit it together with the .proto change. The vendored
//! protoc (pinned by Cargo.lock) is used unless `PROTOC` is set, so the output
//! is byte-stable across machines.
mod emit;
mod emit_py;
mod model;
use std::path::{Path, PathBuf};
/// One generated binding set. Paths are relative to the repository root.
struct Target {
/// The root .proto under `proto/`; its import closure is the API surface.
root: &'static str,
package: &'static str,
rust_out: &'static str,
/// Set when the schema carries `sglang.json.v1` options: emits the serde
/// implementations next to the structs and the Python module here.
python_out: Option<&'static str>,
/// prost `boxed` paths (message fields stored as `Box<T>`).
boxed: &'static [&'static str],
/// Emit every service method with an UNIMPLEMENTED default body, so a
/// server may implement a subset (the runtime.v1 follower metadata
/// service). Off for a contract every server implements in full.
default_stubs: bool,
}
const TARGETS: &[Target] = &[
Target {
root: "sglang/api/v1/service.proto",
package: "sglang.api.v1",
rust_out: "rust/sglang-api-types/src/generated",
python_out: Some("python/sglang/api/v1/api_types.py"),
// Boxed abort payload won't set the size of every ChunkEvent.
boxed: &[".sglang.api.v1.FinishReason.kind.abort"],
default_stubs: false,
},
Target {
root: "sglang/runtime/v1/sglang.proto",
package: "sglang.runtime.v1",
rust_out: "rust/sglang-api-types/src/generated",
python_out: None,
boxed: &[],
default_stubs: true,
},
];
fn main() {
let repo_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../..");
let proto_root = repo_root.join("proto");
// Vendored protoc first: the checked-in output must not depend on
// whichever protoc a machine has. `PROTOC` still overrides for platforms
// the vendored crate lacks.
if std::env::var_os("PROTOC").is_none()
&& let Ok(vendored) = protoc_bin_vendored::protoc_bin_path()
{
// SAFETY: nothing else is running yet; the generator is single-threaded.
unsafe { std::env::set_var("PROTOC", vendored) };
}
// The descriptor set is only pass 2's input, not part of any crate; it is
// a protoc-version-dependent binary, so keep it out of the tree.
let scratch_dir = std::env::temp_dir().join("sglang-api-codegen");
std::fs::create_dir_all(&scratch_dir).expect("create scratch dir");
for target in TARGETS {
generate(target, &repo_root, &proto_root, &scratch_dir);
}
}
fn generate(target: &Target, repo_root: &Path, proto_root: &Path, scratch_dir: &Path) {
let package = target.package;
let out_dir = repo_root.join(target.rust_out);
std::fs::create_dir_all(&out_dir).expect("create generated dir");
let descriptor_path = scratch_dir.join(format!("{package}.descriptor.bin"));
// Pass 1: structs + tonic services (+ the descriptor set pass 2 reads).
// Everything the root imports comes in transitively via the include path,
// in topological order, so the output is stable.
let mut config = prost_build::Config::new();
config
.out_dir(&out_dir)
.file_descriptor_set_path(&descriptor_path)
.service_generator(
tonic_prost_build::configure()
.generate_default_stubs(target.default_stubs)
.service_generator(),
)
.protoc_arg("--experimental_allow_proto3_optional");
for path in target.boxed {
config.boxed(path);
}
config
.compile_protos(&[proto_root.join(target.root)], &[proto_root])
.unwrap_or_else(|e| panic!("pass 1 ({package}): {e}"));
// prost writes structs + services into one file; split the tonic modules
// out so the include! layout stays stable.
split_tonic(&out_dir, package);
if let Some(python_out) = target.python_out {
let bytes = std::fs::read(&descriptor_path).expect("descriptor set");
// Pass 2: schema-driven serde from the options in the descriptor set.
let serde_src = emit::emit_serde(&bytes, package);
std::fs::write(out_dir.join(format!("{package}.serde.rs")), serde_src)
.expect("write serde");
// Pass 2 (Python): the same model rendered as Structs + codecs.
let py_out = repo_root.join(python_out);
std::fs::create_dir_all(py_out.parent().expect("py out dir")).expect("create py dir");
std::fs::write(&py_out, emit_py::emit_python(&bytes, package)).expect("write python types");
println!("generated {}", py_out.display());
// The options package itself needs no Rust types (it exists for the
// generator); drop pass-1 output for it if prost emitted one.
let _ = std::fs::remove_file(out_dir.join("sglang.json.v1.rs"));
}
rustfmt(&out_dir);
println!("generated into {}", out_dir.display());
}
/// Move the tonic `pub mod *_client/_server` blocks from the prost output
/// `<package>.rs` into `<package>.tonic.rs`.
fn split_tonic(out_dir: &Path, package: &str) {
let prost_file = out_dir.join(format!("{package}.rs"));
let src = std::fs::read_to_string(&prost_file).expect("read pass-1 output");
let marker = "/// Generated client implementations.";
let (structs, tonic) = match src.find(marker) {
Some(pos) => src.split_at(pos),
None => (src.as_str(), ""),
};
std::fs::write(&prost_file, structs.trim_end().to_string() + "\n").expect("write structs");
std::fs::write(
out_dir.join(format!("{package}.tonic.rs")),
tonic.trim_start(),
)
.expect("write tonic");
}
fn rustfmt(out_dir: &Path) {
for entry in std::fs::read_dir(out_dir).expect("read generated dir") {
let path = entry.expect("dir entry").path();
if path.extension().is_some_and(|e| e == "rs") {
// Best effort: the output is already well-formed; rustfmt only
// normalizes.
let _ = std::process::Command::new("rustfmt")
.arg("--edition=2024")
.arg(&path)
.status();
}
}
}
+550
View File
@@ -0,0 +1,550 @@
//! Pass 2's input model: each message classified by the `sglang.json.v1`
//! options into the JSON shape the emitter renders. The options are read from
//! the FileDescriptorSet through prost-reflect, so the schema file is the
//! single source of truth.
use prost_reflect::{
DescriptorPool, ExtensionDescriptor, FieldDescriptor, Kind, MessageDescriptor,
};
pub struct Extensions {
message_json: ExtensionDescriptor,
field_json: ExtensionDescriptor,
oneof_json: ExtensionDescriptor,
}
impl Extensions {
pub fn resolve(pool: &DescriptorPool) -> Extensions {
let get = |name: &str| {
pool.get_extension_by_name(name)
.unwrap_or_else(|| panic!("options.proto extension missing: {name}"))
};
Extensions {
message_json: get("sglang.json.v1.message_json"),
field_json: get("sglang.json.v1.field_json"),
oneof_json: get("sglang.json.v1.oneof_json"),
}
}
}
fn opt_bool(options: &prost_reflect::DynamicMessage, name: &str) -> bool {
options
.get_field_by_name(name)
.and_then(|v| v.as_bool())
.unwrap_or(false)
}
/// An enum-typed option as its number; 0 (the unset/default variant) when absent.
fn opt_enum(options: &prost_reflect::DynamicMessage, name: &str) -> i32 {
options
.get_field_by_name(name)
.and_then(|v| v.as_enum_number())
.unwrap_or(0)
}
fn opt_str(options: &prost_reflect::DynamicMessage, name: &str) -> Option<String> {
options
.get_field_by_name(name)
.and_then(|v| v.as_str().map(str::to_owned))
.filter(|s| !s.is_empty())
}
#[derive(Debug, Clone, PartialEq)]
pub enum ScalarTy {
F64,
I32,
I64,
U32,
U64,
Bool,
Str,
}
impl ScalarTy {
pub fn rust(&self) -> &'static str {
match self {
ScalarTy::F64 => "f64",
ScalarTy::I32 => "i32",
ScalarTy::I64 => "i64",
ScalarTy::U32 => "u32",
ScalarTy::U64 => "u64",
ScalarTy::Bool => "bool",
ScalarTy::Str => "::prost::alloc::string::String",
}
}
}
#[derive(Debug, Clone)]
pub enum DefaultLit {
F64(f64),
I64(i64),
Bool(bool),
Str(String),
}
impl DefaultLit {
pub fn rust(&self) -> String {
match self {
DefaultLit::F64(v) => format!("{v:?}f64"),
DefaultLit::I64(v) => format!("{v}i64"),
DefaultLit::Bool(v) => format!("{v}"),
DefaultLit::Str(v) => format!("{v:?}.to_string()"),
}
}
}
#[derive(Debug, Clone)]
pub enum FieldKind {
/// A plain proto3 scalar (no presence). `null_default` set =
/// null_resets_default: absent AND explicit null yield the default.
Scalar {
ty: ScalarTy,
null_default: Option<DefaultLit>,
},
/// proto3 `optional` scalar. `null_is_none_default` set: absent yields the
/// default, explicit null clears to None. `null_default` set
/// (null_resets_default): absent AND null yield Some(default), and an
/// `*_or_default` accessor covers protobuf arrival (unset = None).
OptScalar {
ty: ScalarTy,
emit_null: bool,
null_is_none_default: Option<DefaultLit>,
null_default: Option<DefaultLit>,
},
/// A message-typed field: `Option<T>` presence; the value's JSON shape is
/// the message's own impl.
Message {
rust_path: String,
emit_null: bool,
},
RepeatedScalar {
ty: ScalarTy,
},
RepeatedMessage {
rust_path: String,
},
/// repeated string with raw_json: Vec<String> of JSON texts passing
/// through as an array of raw values.
RepeatedRawJson,
/// map<string, scalar>; int_key means numeric-string keys (no transform
/// needed on the JSON side, documented parity).
Map {
value: ScalarTy,
},
/// A string field holding JSON text (raw_json): the value passes through
/// verbatim on the HTTP side.
RawJson {
emit_null: bool,
},
}
#[derive(Debug, Clone)]
pub struct FieldModel {
/// The JSON key (= the proto field name).
pub json_name: String,
/// The prost struct field ident (keyword-escaped).
pub rust_ident: String,
pub kind: FieldKind,
}
#[derive(Debug, Clone)]
pub struct VariantModel {
/// The JSON discriminator value / shape source (= the proto field name).
pub json_name: String,
/// The prost oneof enum variant (PascalCase).
pub rust_variant: String,
pub kind: FieldKind,
/// For tagged unions: the payload message's fields, inlined into the
/// tagged object.
pub payload_fields: Vec<FieldModel>,
}
#[derive(Debug)]
pub enum MessageKind {
/// Single-repeated-field carrier: JSON is the bare array.
BareList { item: FieldModel },
/// Positional tuple: fields in declaration order form a bare array.
Tuple { fields: Vec<FieldModel> },
/// Single-field wrapper serializing as its sole field's value.
Transparent { field: FieldModel },
/// Single oneof { one, many }: scalar-or-list flattening.
OneOrMany {
oneof_mod: String,
one: VariantModel,
many: VariantModel,
},
/// Single oneof, internally tagged by `tag`; `passthrough` catches
/// unknown tags as a raw map (and any known-tag payload that fails to
/// parse, matching the hand-written FinishReason).
TaggedUnion {
oneof_mod: String,
oneof_ident: String,
tag: String,
variants: Vec<VariantModel>,
passthrough: Option<VariantModel>,
},
/// Single oneof, untagged: the value's shape picks the variant.
Untagged {
oneof_mod: String,
oneof_ident: String,
variants: Vec<VariantModel>,
},
/// A plain object.
Struct {
deny_unknown: bool,
fields: Vec<FieldModel>,
},
/// gRPC-only (a plain oneof container like GenerateStreamItem): no HTTP
/// JSON contract, no serde emitted.
Skip,
}
#[derive(Debug)]
pub struct MessageModel {
/// Rust type name (= proto message name).
pub rust_name: String,
pub kind: MessageKind,
}
pub fn build_models(pool: &DescriptorPool, package: &str) -> Vec<MessageModel> {
let ext = Extensions::resolve(pool);
pool.all_messages()
.filter(|m| m.full_name().starts_with(package))
// Map-entry synthetic messages have no JSON identity of their own.
.filter(|m| !m.is_map_entry())
.map(|m| message_model(&m, &ext))
.collect()
}
fn message_model(msg: &MessageDescriptor, ext: &Extensions) -> MessageModel {
let rust_name = msg.name().to_string();
let options = msg.options();
let mopts = options.get_extension(&ext.message_json);
let mopts = mopts.as_message();
// `MessageShape`: one shape per message, by construction of the enum.
let shape = mopts.map(|m| opt_enum(m, "shape")).unwrap_or(0);
let (bare_list, tuple, transparent) = (shape == 1, shape == 2, shape == 3);
let deny_unknown = mopts
.and_then(|m| m.get_field_by_name("unknown_fields"))
.and_then(|v| v.as_enum_number())
.unwrap_or(0)
== 1;
// proto3 `optional` is implemented as a synthetic oneof: those fields are
// plain fields with presence, not union members.
let oneofs: Vec<_> = msg.oneofs().filter(|o| !o.is_synthetic()).collect();
let plain_fields: Vec<FieldDescriptor> = msg
.fields()
.filter(|f| f.containing_oneof().is_none_or(|o| o.is_synthetic()))
.collect();
if bare_list {
assert_eq!(
plain_fields.len(),
1,
"{}: bare_list needs exactly one repeated field",
msg.full_name()
);
return MessageModel {
rust_name,
kind: MessageKind::BareList {
item: field_model(&plain_fields[0], ext),
},
};
}
if transparent {
assert_eq!(
plain_fields.len(),
1,
"{}: transparent needs exactly one field",
msg.full_name()
);
return MessageModel {
rust_name,
kind: MessageKind::Transparent {
field: field_model(&plain_fields[0], ext),
},
};
}
if tuple {
return MessageModel {
rust_name,
kind: MessageKind::Tuple {
fields: plain_fields.iter().map(|f| field_model(f, ext)).collect(),
},
};
}
if oneofs.len() == 1 && plain_fields.is_empty() {
let oneof = &oneofs[0];
let oopts = oneof.options();
let oopts = oopts.get_extension(&ext.oneof_json);
let oopts = oopts.as_message();
// `OneofShape`: one spelling per union, by construction of the enum.
let shape = oopts.map(|m| opt_enum(m, "shape")).unwrap_or(0);
let (untagged, one_or_many, tagged) = (shape == 1, shape == 2, shape == 3);
let tag = oopts.and_then(|m| opt_str(m, "tag"));
assert_eq!(
tagged,
tag.is_some(),
"{}: ONEOF_SHAPE_TAGGED and `tag` go together",
msg.full_name()
);
let passthrough_on = oopts
.map(|m| opt_bool(m, "unknown_variant_passthrough"))
.unwrap_or(false);
let oneof_mod = snake(&rust_name);
let oneof_ident = pascal(oneof.name());
let mut variants: Vec<VariantModel> = oneof
.fields()
.map(|f| VariantModel {
json_name: f.name().to_string(),
rust_variant: pascal(f.name()),
kind: field_model(&f, ext).kind,
payload_fields: match f.kind() {
// The raw_json passthrough arm carries a well-known type;
// its innards are never inlined.
Kind::Message(payload)
if tag.is_some() && !payload.full_name().starts_with("google.protobuf") =>
{
payload
.fields()
.map(|pf| field_model(&pf, ext))
.collect::<Vec<_>>()
}
_ => vec![],
},
})
.collect();
if one_or_many {
assert_eq!(variants.len(), 2, "{}: one_or_many", msg.full_name());
let many = variants.pop().expect("many variant");
let one = variants.pop().expect("one variant");
return MessageModel {
rust_name,
kind: MessageKind::OneOrMany {
oneof_mod,
one,
many,
},
};
}
if let Some(tag) = tag {
let passthrough = passthrough_on.then(|| {
let idx = variants
.iter()
.position(|v| matches!(v.kind, FieldKind::RawJson { .. }))
.expect("passthrough needs a raw_json string variant");
variants.remove(idx)
});
return MessageModel {
rust_name,
kind: MessageKind::TaggedUnion {
oneof_mod,
oneof_ident,
tag,
variants,
passthrough,
},
};
}
if untagged {
return MessageModel {
rust_name,
kind: MessageKind::Untagged {
oneof_mod,
oneof_ident,
variants,
},
};
}
// A plain oneof container (e.g. GenerateStreamItem) has no HTTP JSON
// contract of its own — it is gRPC-only. Emit nothing.
return MessageModel {
rust_name,
kind: MessageKind::Skip,
};
}
MessageModel {
rust_name,
kind: MessageKind::Struct {
deny_unknown,
fields: plain_fields.iter().map(|f| field_model(f, ext)).collect(),
},
}
}
fn field_model(f: &FieldDescriptor, ext: &Extensions) -> FieldModel {
let options = f.options();
let fopts = options.get_extension(&ext.field_json);
let fopts = fopts.as_message();
let get_bool = |name: &str| fopts.map(|m| opt_bool(m, name)).unwrap_or(false);
let default_lit = |m: &prost_reflect::DynamicMessage| -> Option<DefaultLit> {
// has_field_by_name: the default_* members are proto3 optional, so
// presence is meaningful.
if m.has_field_by_name("default_f64") {
return m
.get_field_by_name("default_f64")
.and_then(|v| v.as_f64())
.map(DefaultLit::F64);
}
if m.has_field_by_name("default_i64") {
return m
.get_field_by_name("default_i64")
.and_then(|v| v.as_i64())
.map(DefaultLit::I64);
}
if m.has_field_by_name("default_bool") {
return m
.get_field_by_name("default_bool")
.and_then(|v| v.as_bool())
.map(DefaultLit::Bool);
}
if m.has_field_by_name("default_string") {
return m
.get_field_by_name("default_string")
.and_then(|v| v.as_str().map(str::to_owned))
.map(DefaultLit::Str);
}
None
};
let raw_json = get_bool("raw_json");
let emit_null = get_bool("emit_null_when_absent");
// `NullPolicy`: one policy per field, by construction of the enum.
let null_policy = fopts.map(|m| opt_enum(m, "null_policy")).unwrap_or(0);
let (null_resets, null_is_none) = (null_policy == 1, null_policy == 2);
let default = fopts.and_then(default_lit);
let kind = if raw_json {
if f.is_list() {
FieldKind::RepeatedRawJson
} else {
assert!(
matches!(f.kind(), Kind::String),
"{}: raw_json requires a string field",
f.full_name()
);
FieldKind::RawJson { emit_null }
}
} else if f.is_map() {
let value_ty = match f.kind() {
Kind::Message(entry) => scalar_ty(
&entry
.get_field_by_name("value")
.expect("map value field")
.kind(),
),
_ => unreachable!("map fields are message-kinded"),
};
FieldKind::Map { value: value_ty }
} else if f.is_list() {
match f.kind() {
Kind::Message(m) => FieldKind::RepeatedMessage {
rust_path: m.name().to_string(),
},
other => FieldKind::RepeatedScalar {
ty: scalar_ty(&other),
},
}
} else {
match f.kind() {
Kind::Message(m) => FieldKind::Message {
rust_path: m.name().to_string(),
emit_null,
},
other => {
let ty = scalar_ty(&other);
if f.supports_presence() {
FieldKind::OptScalar {
ty,
emit_null,
null_is_none_default: null_is_none.then(|| {
default
.clone()
.expect("NULL_POLICY_IS_NONE requires a default_*")
}),
null_default: null_resets.then(|| {
default
.clone()
.expect("NULL_POLICY_RESETS_DEFAULT requires a default_*")
}),
}
} else {
FieldKind::Scalar {
ty,
null_default: null_resets.then(|| {
default
.clone()
.expect("NULL_POLICY_RESETS_DEFAULT requires a default_*")
}),
}
}
}
}
};
FieldModel {
json_name: f.name().to_string(),
rust_ident: escape_ident(f.name()),
kind,
}
}
fn scalar_ty(kind: &Kind) -> ScalarTy {
match kind {
Kind::Double | Kind::Float => ScalarTy::F64,
Kind::Int64 | Kind::Sint64 | Kind::Sfixed64 => ScalarTy::I64,
Kind::Int32 | Kind::Sint32 | Kind::Sfixed32 => ScalarTy::I32,
Kind::Uint32 | Kind::Fixed32 => ScalarTy::U32,
Kind::Uint64 | Kind::Fixed64 => ScalarTy::U64,
Kind::Bool => ScalarTy::Bool,
Kind::String => ScalarTy::Str,
other => panic!("unsupported scalar kind {other:?}"),
}
}
/// prost's keyword escaping for field idents.
fn escape_ident(name: &str) -> String {
const KEYWORDS: &[&str] = &[
"as", "break", "const", "continue", "crate", "dyn", "else", "enum", "extern", "false",
"fn", "for", "if", "impl", "in", "let", "loop", "match", "mod", "move", "mut", "pub",
"ref", "return", "self", "static", "struct", "super", "trait", "true", "type", "unsafe",
"use", "where", "while",
];
if KEYWORDS.contains(&name) {
format!("r#{name}")
} else {
name.to_string()
}
}
pub fn pascal(name: &str) -> String {
name.split('_')
.filter(|s| !s.is_empty())
.map(|s| {
let mut chars = s.chars();
match chars.next() {
Some(first) => first.to_ascii_uppercase().to_string() + chars.as_str(),
None => String::new(),
}
})
.collect()
}
pub fn snake(name: &str) -> String {
let mut out = String::new();
for (i, ch) in name.chars().enumerate() {
if ch.is_ascii_uppercase() {
if i > 0 {
out.push('_');
}
out.push(ch.to_ascii_lowercase());
} else {
out.push(ch);
}
}
out
}
+25
View File
@@ -0,0 +1,25 @@
[package]
name = "sglang-api-types"
description = "Generated SGLang API types: prost structs, tonic services, schema-driven serde"
version.workspace = true
edition.workspace = true
license.workspace = true
# No [package.metadata.sglang]: not a Python extension; wheel builds skip it.
[lib]
name = "sglang_api_types"
crate-type = ["rlib"]
[dependencies]
bytes = { workspace = true }
prost = { workspace = true }
prost-types = { workspace = true }
serde = { workspace = true }
# preserve_order: raw_json passthrough (custom_params, internal_states, an
# unknown finish reason) re-serializes through serde_json::Value, and the
# contract is "verbatim": key order must not depend on what else links the
# crate.
serde_json = { workspace = true, features = ["preserve_order"] }
tonic = { workspace = true }
tonic-prost = { workspace = true }
+63
View File
@@ -0,0 +1,63 @@
//! Hand-written extensions on the generated types: the accessors and
//! constructors server logic needs beyond the wire shapes. Grows with each
//! migration stage.
use std::borrow::Cow;
use crate::api::v1::{FinishReason, FinishStop, Matched, MatchedTokens, finish_reason, matched};
impl FinishReason {
/// A stop reason (`type: "stop"`), optionally carrying what matched.
pub fn stop(matched: Option<matched::Value>) -> Self {
FinishReason {
kind: Some(finish_reason::Kind::Stop(FinishStop {
matched: matched.map(|value| Matched { value: Some(value) }),
})),
}
}
/// Returns the wire `type` without reserializing the finish reason (an
/// unknown reason's is read out of its raw JSON, hence the `Cow`).
pub fn kind_name(&self) -> Option<Cow<'_, str>> {
match self.kind.as_ref()? {
finish_reason::Kind::Stop(_) => Some(Cow::Borrowed("stop")),
finish_reason::Kind::Length(_) => Some(Cow::Borrowed("length")),
finish_reason::Kind::Abort(_) => Some(Cow::Borrowed("abort")),
finish_reason::Kind::Unknown(raw) => {
let parsed: serde_json::Value = serde_json::from_str(raw).ok()?;
parsed
.get("type")
.and_then(serde_json::Value::as_str)
.map(|s| Cow::Owned(s.to_owned()))
}
}
}
/// The stop this request matched, if it stopped on one. `None` for
/// length/abort and for an unknown type.
pub fn matched(&self) -> Option<&matched::Value> {
match self.kind.as_ref()? {
finish_reason::Kind::Stop(stop) => stop.matched.as_ref()?.value.as_ref(),
_ => None,
}
}
/// `Some((status, message))` when this is an abort carrying a `status_code` —
/// a scheduler-side request error the API surfaces as that HTTP status instead
/// of as a normal completion. A plain abort (no code) reads as `None`.
pub fn abort_status(&self) -> Option<(u16, &str)> {
match self.kind.as_ref()? {
finish_reason::Kind::Abort(abort) => Some((
u16::try_from(abort.status_code?).ok()?,
abort.message.as_deref().unwrap_or("request aborted"),
)),
_ => None,
}
}
}
impl From<Vec<i64>> for MatchedTokens {
fn from(ids: Vec<i64>) -> Self {
MatchedTokens { ids }
}
}
@@ -0,0 +1,657 @@
// This file is @generated by prost-build.
/// A token-id sequence. int64 matches the servers' token id width (Rust
/// `TokenIds = Vec<i64>`, Python `array("q")`).
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct TokenIds {
#[prost(int64, repeated, tag = "1")]
pub ids: ::prost::alloc::vec::Vec<i64>,
}
/// Scalar-or-list carriers: `/generate` accepts every per-request field as one
/// value (single request) or a list (batch). JSON flattens via the one_or_many
/// option; gRPC clients pick the variant explicitly.
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct StringOrList {
#[prost(oneof = "string_or_list::Value", tags = "1, 2")]
pub value: ::core::option::Option<string_or_list::Value>,
}
/// Nested message and enum types in `StringOrList`.
pub mod string_or_list {
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)]
pub enum Value {
#[prost(string, tag = "1")]
One(::prost::alloc::string::String),
#[prost(message, tag = "2")]
Many(super::StringList),
}
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct StringList {
#[prost(string, repeated, tag = "1")]
pub items: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
}
/// A flat string list for one request, or one list per batch entry. JSON:
/// `\["a", "b"\]` is `one`; `\[["a"\], \["b", "c"]\]` is `many` (first element
/// decides, `\[\]` is an empty `one`).
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct StringListOrList {
#[prost(oneof = "string_list_or_list::Value", tags = "1, 2")]
pub value: ::core::option::Option<string_list_or_list::Value>,
}
/// Nested message and enum types in `StringListOrList`.
pub mod string_list_or_list {
#[derive(Clone, PartialEq, ::prost::Oneof)]
pub enum Value {
#[prost(message, tag = "1")]
One(super::StringList),
#[prost(message, tag = "2")]
Many(super::StringListList),
}
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct StringListList {
#[prost(message, repeated, tag = "1")]
pub items: ::prost::alloc::vec::Vec<StringList>,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct BoolOrList {
#[prost(oneof = "bool_or_list::Value", tags = "1, 2")]
pub value: ::core::option::Option<bool_or_list::Value>,
}
/// Nested message and enum types in `BoolOrList`.
pub mod bool_or_list {
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)]
pub enum Value {
#[prost(bool, tag = "1")]
One(bool),
#[prost(message, tag = "2")]
Many(super::BoolList),
}
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct BoolList {
#[prost(bool, repeated, tag = "1")]
pub items: ::prost::alloc::vec::Vec<bool>,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct Int64OrList {
#[prost(oneof = "int64_or_list::Value", tags = "1, 2")]
pub value: ::core::option::Option<int64_or_list::Value>,
}
/// Nested message and enum types in `Int64OrList`.
pub mod int64_or_list {
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)]
pub enum Value {
#[prost(int64, tag = "1")]
One(i64),
#[prost(message, tag = "2")]
Many(super::Int64List),
}
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct Int64List {
#[prost(int64, repeated, tag = "1")]
pub items: ::prost::alloc::vec::Vec<i64>,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct TokenIdsOrList {
#[prost(oneof = "token_ids_or_list::Value", tags = "1, 2")]
pub value: ::core::option::Option<token_ids_or_list::Value>,
}
/// Nested message and enum types in `TokenIdsOrList`.
pub mod token_ids_or_list {
#[derive(Clone, PartialEq, ::prost::Oneof)]
pub enum Value {
#[prost(message, tag = "1")]
One(super::TokenIds),
#[prost(message, tag = "2")]
Many(super::TokenIdsList),
}
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct TokenIdsList {
#[prost(message, repeated, tag = "1")]
pub items: ::prost::alloc::vec::Vec<TokenIds>,
}
/// List variants whose ELEMENTS may be JSON null (PD bootstrap columns: a
/// batch mixes bootstrapped and plain requests). Proto carries element
/// presence as a per-element oneof-free wrapper.
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct OptionalString {
#[prost(string, optional, tag = "1")]
pub value: ::core::option::Option<::prost::alloc::string::String>,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct OptionalStringOrList {
#[prost(oneof = "optional_string_or_list::Value", tags = "1, 2")]
pub value: ::core::option::Option<optional_string_or_list::Value>,
}
/// Nested message and enum types in `OptionalStringOrList`.
pub mod optional_string_or_list {
#[derive(Clone, PartialEq, ::prost::Oneof)]
pub enum Value {
#[prost(message, tag = "1")]
One(super::OptionalString),
#[prost(message, tag = "2")]
Many(super::OptionalStringList),
}
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct OptionalStringList {
#[prost(message, repeated, tag = "1")]
pub items: ::prost::alloc::vec::Vec<OptionalString>,
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct OptionalInt64 {
#[prost(int64, optional, tag = "1")]
pub value: ::core::option::Option<i64>,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct OptionalInt64OrList {
#[prost(oneof = "optional_int64_or_list::Value", tags = "1, 2")]
pub value: ::core::option::Option<optional_int64_or_list::Value>,
}
/// Nested message and enum types in `OptionalInt64OrList`.
pub mod optional_int64_or_list {
#[derive(Clone, PartialEq, ::prost::Oneof)]
pub enum Value {
#[prost(message, tag = "1")]
One(super::OptionalInt64),
#[prost(message, tag = "2")]
Many(super::OptionalInt64List),
}
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct OptionalInt64List {
#[prost(message, repeated, tag = "1")]
pub items: ::prost::alloc::vec::Vec<OptionalInt64>,
}
/// The native error body: `{"error": {"message", "code"}}` (Python
/// `http_server.generate_request` parity). Transports frame it themselves
/// (unary status + body, SSE error frame, gRPC status detail).
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ErrorBody {
#[prost(string, tag = "1")]
pub message: ::prost::alloc::string::String,
#[prost(uint32, tag = "2")]
pub code: u32,
}
/// Python's `matched` key, `Union\[str, int, List[int]\]`: the three wire shapes
/// are disjoint, so shape alone picks the arm (untagged).
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct Matched {
#[prost(oneof = "matched::Value", tags = "1, 2, 3")]
pub value: ::core::option::Option<matched::Value>,
}
/// Nested message and enum types in `Matched`.
pub mod matched {
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)]
pub enum Value {
#[prost(int64, tag = "1")]
Token(i64),
#[prost(string, tag = "2")]
Str(::prost::alloc::string::String),
#[prost(message, tag = "3")]
Tokens(super::MatchedTokens),
}
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct MatchedTokens {
#[prost(int64, repeated, tag = "1")]
pub ids: ::prost::alloc::vec::Vec<i64>,
}
/// `FINISH_MATCHED_TOKEN` / `FINISH_MATCHED_STR` / `FINISHED_MATCHED_REGEX` —
/// all three report `type: "stop"`, matching the OpenAI API's value.
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct FinishStop {
#[prost(message, optional, tag = "1")]
pub matched: ::core::option::Option<Matched>,
}
/// `FINISH_LENGTH` — hit `max_new_tokens` (or the context limit).
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct FinishLength {
#[prost(uint64, optional, tag = "1")]
pub length: ::core::option::Option<u64>,
}
/// The `FINISH_ABORT` payload. Python emits all three keys, nulls included.
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct FinishAbort {
#[prost(string, optional, tag = "1")]
pub message: ::core::option::Option<::prost::alloc::string::String>,
#[prost(uint32, optional, tag = "2")]
pub status_code: ::core::option::Option<u32>,
#[prost(string, optional, tag = "3")]
pub err_type: ::core::option::Option<::prost::alloc::string::String>,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct FinishReason {
#[prost(oneof = "finish_reason::Kind", tags = "1, 2, 3, 4")]
pub kind: ::core::option::Option<finish_reason::Kind>,
}
/// Nested message and enum types in `FinishReason`.
pub mod finish_reason {
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)]
pub enum Kind {
#[prost(message, tag = "1")]
Stop(super::FinishStop),
#[prost(message, tag = "2")]
Length(super::FinishLength),
#[prost(message, tag = "3")]
Abort(::prost::alloc::boxed::Box<super::FinishAbort>),
/// The passthrough arm: an unrecognized `type`, kept as its raw map
/// (JSON text — Struct would coerce integer values to doubles).
#[prost(string, tag = "4")]
Unknown(::prost::alloc::string::String),
}
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct SamplingParams {
/// JSON: absent = 128, explicit null = no limit (Python `Optional\[int\] = 128`). Protobuf has no null: unset = 128 (apply_absent_defaults at the
/// gRPC entry); no limit needs an explicit large value.
#[prost(int64, optional, tag = "1")]
pub max_new_tokens: ::core::option::Option<i64>,
#[prost(message, optional, tag = "2")]
pub stop: ::core::option::Option<StringOrList>,
#[prost(message, optional, tag = "3")]
pub stop_token_ids: ::core::option::Option<Int64List>,
#[prost(message, optional, tag = "4")]
pub stop_regex: ::core::option::Option<StringOrList>,
#[prost(double, optional, tag = "5")]
pub temperature: ::core::option::Option<f64>,
#[prost(double, optional, tag = "6")]
pub top_p: ::core::option::Option<f64>,
/// Default 1 \<\< 30 (TOP_K_ALL): "consider the whole vocabulary".
#[prost(int64, optional, tag = "7")]
pub top_k: ::core::option::Option<i64>,
#[prost(double, optional, tag = "8")]
pub min_p: ::core::option::Option<f64>,
#[prost(double, optional, tag = "9")]
pub frequency_penalty: ::core::option::Option<f64>,
#[prost(double, optional, tag = "10")]
pub presence_penalty: ::core::option::Option<f64>,
#[prost(double, optional, tag = "11")]
pub repetition_penalty: ::core::option::Option<f64>,
#[prost(int64, optional, tag = "12")]
pub min_new_tokens: ::core::option::Option<i64>,
#[prost(int64, optional, tag = "13")]
pub n: ::core::option::Option<i64>,
#[prost(string, optional, tag = "14")]
pub json_schema: ::core::option::Option<::prost::alloc::string::String>,
#[prost(string, optional, tag = "15")]
pub regex: ::core::option::Option<::prost::alloc::string::String>,
#[prost(string, optional, tag = "16")]
pub ebnf: ::core::option::Option<::prost::alloc::string::String>,
#[prost(string, optional, tag = "17")]
pub structural_tag: ::core::option::Option<::prost::alloc::string::String>,
#[prost(bool, optional, tag = "18")]
pub ignore_eos: ::core::option::Option<bool>,
#[prost(bool, optional, tag = "19")]
pub skip_special_tokens: ::core::option::Option<bool>,
#[prost(bool, optional, tag = "20")]
pub spaces_between_special_tokens: ::core::option::Option<bool>,
#[prost(bool, optional, tag = "21")]
pub no_stop_trim: ::core::option::Option<bool>,
#[prost(int64, optional, tag = "22")]
pub stream_interval: ::core::option::Option<i64>,
/// Token id (numeric-string key, Python parity) -> bias.
#[prost(map = "string, double", tag = "23")]
pub logit_bias: ::std::collections::HashMap<::prost::alloc::string::String, f64>,
#[prost(int64, optional, tag = "24")]
pub sampling_seed: ::core::option::Option<i64>,
/// Opaque JSON forwarded to a custom logit processor; never inspected here.
#[prost(string, optional, tag = "25")]
pub custom_params: ::core::option::Option<::prost::alloc::string::String>,
/// beam_width > 1 turns the request into a beam search request; `n` is then
/// the number of returned sequences (n \<= beam_width).
#[prost(int64, optional, tag = "26")]
pub beam_width: ::core::option::Option<i64>,
}
/// (message_json).unknown_fields defaults to IGNORE — stated here as schema
/// intent, load-bearing for client compat.
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct GenerateRequest {
#[prost(message, optional, tag = "1")]
pub rid: ::core::option::Option<StringOrList>,
#[prost(message, optional, tag = "2")]
pub text: ::core::option::Option<StringOrList>,
#[prost(message, optional, tag = "3")]
pub input_ids: ::core::option::Option<TokenIdsOrList>,
#[prost(bool, optional, tag = "4")]
pub stream: ::core::option::Option<bool>,
/// One set of params, or one per batch item. JSON errors keep serde's
/// field-level texts (the hand-written SamplingParamsInput contract).
#[prost(message, optional, tag = "5")]
pub sampling_params: ::core::option::Option<SamplingParamsOrList>,
#[prost(message, optional, tag = "6")]
pub return_logprob: ::core::option::Option<BoolOrList>,
#[prost(message, optional, tag = "7")]
pub logprob_start_len: ::core::option::Option<Int64OrList>,
#[prost(message, optional, tag = "8")]
pub top_logprobs_num: ::core::option::Option<Int64OrList>,
#[prost(message, optional, tag = "9")]
pub token_ids_logprob: ::core::option::Option<TokenIdsOrList>,
#[prost(message, optional, tag = "10")]
pub return_hidden_states: ::core::option::Option<BoolOrList>,
#[prost(bool, optional, tag = "11")]
pub return_text_in_logprobs: ::core::option::Option<bool>,
/// PD bootstrap columns: a batch mixes bootstrapped and plain requests, so
/// elements are individually nullable.
#[prost(message, optional, tag = "12")]
pub bootstrap_host: ::core::option::Option<OptionalStringOrList>,
#[prost(message, optional, tag = "13")]
pub bootstrap_port: ::core::option::Option<OptionalInt64OrList>,
#[prost(message, optional, tag = "14")]
pub bootstrap_room: ::core::option::Option<OptionalInt64OrList>,
#[prost(message, optional, tag = "15")]
pub bootstrap_pair_key: ::core::option::Option<OptionalStringOrList>,
#[prost(message, optional, tag = "16")]
pub decode_tp_size: ::core::option::Option<OptionalInt64OrList>,
#[prost(int64, optional, tag = "17")]
pub routed_dp_rank: ::core::option::Option<i64>,
#[prost(int64, optional, tag = "18")]
pub disagg_prefill_dp_rank: ::core::option::Option<i64>,
/// Multimodal payloads pass through untyped (data URIs, URLs, file paths, or
/// nested batch lists) to the MM pipeline, which owns their validation.
/// Media inputs, one container per modality (see MediaInput). The item
/// hints a modality does not support are rejected at the processor.
#[prost(message, optional, tag = "19")]
pub image_data: ::core::option::Option<MediaInput>,
/// Per-item content hashes overriding the computed ones, in item order; a
/// flat list for one request (the batch form is rejected downstream).
#[prost(message, optional, tag = "20")]
pub mm_hashes: ::core::option::Option<StringListOrList>,
#[prost(message, optional, tag = "21")]
pub video_data: ::core::option::Option<MediaInput>,
#[prost(message, optional, tag = "22")]
pub audio_data: ::core::option::Option<MediaInput>,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct SamplingParamsOrList {
#[prost(oneof = "sampling_params_or_list::Value", tags = "1, 2")]
pub value: ::core::option::Option<sampling_params_or_list::Value>,
}
/// Nested message and enum types in `SamplingParamsOrList`.
pub mod sampling_params_or_list {
#[derive(Clone, PartialEq, ::prost::Oneof)]
pub enum Value {
#[prost(message, tag = "1")]
One(super::SamplingParams),
#[prost(message, tag = "2")]
Many(super::SamplingParamsList),
}
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct SamplingParamsList {
#[prost(message, repeated, tag = "1")]
pub items: ::prost::alloc::vec::Vec<SamplingParams>,
}
/// One `\[logprob, token_id, text|null\]` entry (text present when
/// `return_text_in_logprobs`, decoded on the detok shard). The logprob slot is
/// optional: Python emits null for a position with no logprob (the first
/// prefill token; Rust maps the scheduler's NaN to the same null).
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct LogprobEntry {
#[prost(double, optional, tag = "1")]
pub logprob: ::core::option::Option<f64>,
#[prost(int64, tag = "2")]
pub token_id: i64,
#[prost(string, optional, tag = "3")]
pub text: ::core::option::Option<::prost::alloc::string::String>,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct LogprobEntries {
#[prost(message, repeated, tag = "1")]
pub entries: ::prost::alloc::vec::Vec<LogprobEntry>,
}
/// One decode position's top-k alternatives.
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct TopLogprobRow {
#[prost(message, repeated, tag = "1")]
pub entries: ::prost::alloc::vec::Vec<LogprobEntry>,
}
/// A position's row, or null: a position that recorded no alternatives is
/// null on the wire (Python parity), never \[\].
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct NullableTopLogprobs {
#[prost(message, optional, tag = "1")]
pub row: ::core::option::Option<TopLogprobRow>,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct HiddenStateRow {
#[prost(double, repeated, tag = "1")]
pub values: ::prost::alloc::vec::Vec<f64>,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct GenerateMetaInfo {
/// The client's rid, byte-for-byte (`Rid::client_facing`).
#[prost(string, tag = "1")]
pub id: ::prost::alloc::string::String,
#[prost(uint32, tag = "2")]
pub prompt_tokens: u32,
#[prost(uint64, tag = "3")]
pub completion_tokens: u64,
/// Always present in JSON, null until a terminal frame.
#[prost(message, optional, tag = "4")]
pub finish_reason: ::core::option::Option<FinishReason>,
/// input+output token logprobs are set together, empty lists included: a PD
/// decode node never receives input logprobs, but the PD router keys its
/// merge of prefill's on the key's presence.
#[prost(message, optional, tag = "5")]
pub output_token_logprobs: ::core::option::Option<LogprobEntries>,
#[prost(message, optional, tag = "6")]
pub input_token_logprobs: ::core::option::Option<LogprobEntries>,
#[prost(message, repeated, tag = "7")]
pub output_top_logprobs: ::prost::alloc::vec::Vec<NullableTopLogprobs>,
#[prost(message, repeated, tag = "8")]
pub input_top_logprobs: ::prost::alloc::vec::Vec<NullableTopLogprobs>,
#[prost(message, repeated, tag = "9")]
pub output_token_ids_logprobs: ::prost::alloc::vec::Vec<NullableTopLogprobs>,
#[prost(message, repeated, tag = "10")]
pub input_token_ids_logprobs: ::prost::alloc::vec::Vec<NullableTopLogprobs>,
#[prost(message, repeated, tag = "11")]
pub hidden_states: ::prost::alloc::vec::Vec<HiddenStateRow>,
/// Seconds from normalized-request acceptance to the terminal output; only
/// on the terminal frame.
#[prost(double, optional, tag = "12")]
pub e2e_latency: ::core::option::Option<f64>,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct GenerateResponse {
/// JSON key order is FIELD-NUMBER order: text, meta_info, then the
/// conditionally-present output_ids and index — matching frame_value's
/// insertion order byte-for-byte.
#[prost(string, tag = "1")]
pub text: ::prost::alloc::string::String,
#[prost(message, optional, tag = "2")]
pub meta_info: ::core::option::Option<GenerateMetaInfo>,
/// Omitted while empty (a text-only frame).
#[prost(message, optional, tag = "3")]
pub output_ids: ::core::option::Option<TokenIds>,
/// Batch position; only on batch responses/frames.
#[prost(uint32, optional, tag = "4")]
pub index: ::core::option::Option<u32>,
}
/// One request's failure inside a Generate stream; carries its batch position
/// (the HTTP SSE shape puts `index` beside the error object the same way).
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct GenerateStreamError {
#[prost(message, optional, tag = "1")]
pub error: ::core::option::Option<ErrorBody>,
#[prost(uint32, optional, tag = "2")]
pub index: ::core::option::Option<u32>,
}
/// One item of the gRPC Generate stream: a frame, or one request's failure
/// (the stream continues — other batch items may still be running). On HTTP
/// the same failure is an in-stream SSE error frame.
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct GenerateStreamItem {
#[prost(oneof = "generate_stream_item::Item", tags = "1, 2")]
pub item: ::core::option::Option<generate_stream_item::Item>,
}
/// Nested message and enum types in `GenerateStreamItem`.
pub mod generate_stream_item {
#[derive(Clone, PartialEq, ::prost::Oneof)]
pub enum Item {
#[prost(message, tag = "1")]
Frame(super::GenerateResponse),
#[prost(message, tag = "2")]
Error(super::GenerateStreamError),
}
}
/// One media item's source and per-item hints. Unknown keys are an error: a
/// hint this schema does not name would otherwise be silently dropped.
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct MediaRef {
/// URL, `file://` / absolute path, `data:` URI, or bare base64.
#[prost(string, tag = "1")]
pub url: ::prost::alloc::string::String,
/// Image: "auto" | "low" | "high" (the OpenAI detail hint).
#[prost(string, optional, tag = "2")]
pub detail: ::core::option::Option<::prost::alloc::string::String>,
/// Image/video: dynamic-patch budget (InternVL-style processors).
#[prost(int64, optional, tag = "3")]
pub max_dynamic_patch: ::core::option::Option<i64>,
/// Image: a caller-supplied content hash that overrides the computed one.
#[prost(string, optional, tag = "4")]
pub content_hash: ::core::option::Option<::prost::alloc::string::String>,
/// Video/image budgets read by the GLM and MiMo processors.
#[prost(int64, optional, tag = "5")]
pub min_pixels: ::core::option::Option<i64>,
#[prost(int64, optional, tag = "6")]
pub max_pixels: ::core::option::Option<i64>,
#[prost(int64, optional, tag = "7")]
pub total_max_pixels: ::core::option::Option<i64>,
#[prost(double, optional, tag = "8")]
pub fps: ::core::option::Option<f64>,
#[prost(int64, optional, tag = "9")]
pub num_frames: ::core::option::Option<i64>,
#[prost(int64, optional, tag = "10")]
pub min_frames: ::core::option::Option<i64>,
#[prost(int64, optional, tag = "11")]
pub max_frames: ::core::option::Option<i64>,
#[prost(int64, optional, tag = "12")]
pub max_tokens_per_frame: ::core::option::Option<i64>,
#[prost(int64, optional, tag = "13")]
pub max_image_tokens: ::core::option::Option<i64>,
/// Video: also decode the audio track.
#[prost(bool, optional, tag = "14")]
pub use_audio: ::core::option::Option<bool>,
}
/// One media item. JSON: a bare string is the source; an object is a MediaRef.
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct MediaItem {
#[prost(oneof = "media_item::Value", tags = "1, 2")]
pub value: ::core::option::Option<media_item::Value>,
}
/// Nested message and enum types in `MediaItem`.
pub mod media_item {
#[derive(Clone, PartialEq, ::prost::Oneof)]
pub enum Value {
#[prost(string, tag = "1")]
Source(::prost::alloc::string::String),
#[prost(message, tag = "2")]
Ref(super::MediaRef),
}
}
/// A list element that may be JSON null (Python's `\[img, None\]`).
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct OptionalMediaItem {
#[prost(message, optional, tag = "1")]
pub item: ::core::option::Option<MediaItem>,
}
/// One request's items (`\[a, null, b\]`), or one batch entry's items.
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct MediaItems {
#[prost(message, repeated, tag = "1")]
pub items: ::prost::alloc::vec::Vec<OptionalMediaItem>,
}
/// A batch entry whose item list may be JSON null.
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct OptionalMediaItems {
#[prost(message, optional, tag = "1")]
pub items: ::core::option::Option<MediaItems>,
}
/// One item list per batch entry (`\[[a, b\], null, \[]\]`).
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct MediaItemsList {
#[prost(message, repeated, tag = "1")]
pub items: ::prost::alloc::vec::Vec<OptionalMediaItems>,
}
/// The three container forms Python's `_normalize_image_data` distinguishes.
/// JSON: a string or object is `item` (one item; a broadcast in a batch); an
/// array whose first element is not an array is `many` (a flat list; `\[\]` is
/// an empty `many`); an array whose first element is an array is `nested`
/// (one list per batch entry). Which forms a request may use depends on
/// whether it is a batch, and is checked downstream, not here.
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct MediaInput {
#[prost(oneof = "media_input::Value", tags = "1, 2, 3")]
pub value: ::core::option::Option<media_input::Value>,
}
/// Nested message and enum types in `MediaInput`.
pub mod media_input {
#[derive(Clone, PartialEq, ::prost::Oneof)]
pub enum Value {
#[prost(message, tag = "1")]
Item(super::MediaItem),
#[prost(message, tag = "2")]
Many(super::MediaItems),
#[prost(message, tag = "3")]
Nested(super::MediaItemsList),
}
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct HealthCheckRequest {}
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct HealthCheckResponse {
#[prost(bool, tag = "1")]
pub healthy: bool,
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct GetModelInfoRequest {}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct GetModelInfoResponse {
#[prost(string, tag = "1")]
pub model_path: ::prost::alloc::string::String,
#[prost(string, tag = "2")]
pub served_model_name: ::prost::alloc::string::String,
#[prost(string, tag = "3")]
pub tokenizer_path: ::prost::alloc::string::String,
#[prost(bool, tag = "4")]
pub is_generation: bool,
/// The launch-time `--preferred-sampling-params` object, or null when unset.
/// JSON carries the object itself (the HTTP `/get_model_info` shape); the
/// protobuf field holds it as JSON text.
#[prost(string, optional, tag = "5")]
pub preferred_sampling_params: ::core::option::Option<::prost::alloc::string::String>,
#[prost(string, optional, tag = "6")]
pub weight_version: ::core::option::Option<::prost::alloc::string::String>,
#[prost(string, optional, tag = "7")]
pub load_format: ::core::option::Option<::prost::alloc::string::String>,
#[prost(string, optional, tag = "8")]
pub reasoning_parser: ::core::option::Option<::prost::alloc::string::String>,
#[prost(string, optional, tag = "9")]
pub tool_call_parser: ::core::option::Option<::prost::alloc::string::String>,
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct GetServerInfoRequest {}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct GetServerInfoResponse {
#[prost(string, tag = "1")]
pub model_path: ::prost::alloc::string::String,
#[prost(string, tag = "2")]
pub served_model_name: ::prost::alloc::string::String,
#[prost(string, tag = "3")]
pub tokenizer_path: ::prost::alloc::string::String,
#[prost(uint32, tag = "4")]
pub max_context_length: u32,
#[prost(uint64, optional, tag = "5")]
pub max_total_num_tokens: ::core::option::Option<u64>,
#[prost(string, tag = "6")]
pub version: ::prost::alloc::string::String,
/// The allowlisted runtime metrics (last_gen_throughput, memory_usage,
/// effective_max_running_requests_per_dp, avg_spec_accept_length,
/// step_time_dict), untyped because the scheduler owns their shapes; JSON
/// passes them through verbatim.
#[prost(string, repeated, tag = "7")]
pub internal_states: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,489 @@
/// Generated client implementations.
pub mod sglang_service_client {
#![allow(
unused_variables,
dead_code,
missing_docs,
clippy::wildcard_imports,
clippy::let_unit_value
)]
use tonic::codegen::http::Uri;
use tonic::codegen::*;
#[derive(Debug, Clone)]
pub struct SglangServiceClient<T> {
inner: tonic::client::Grpc<T>,
}
impl SglangServiceClient<tonic::transport::Channel> {
/// Attempt to create a new client by connecting to a given endpoint.
pub async fn connect<D>(dst: D) -> Result<Self, tonic::transport::Error>
where
D: TryInto<tonic::transport::Endpoint>,
D::Error: Into<StdError>,
{
let conn = tonic::transport::Endpoint::new(dst)?.connect().await?;
Ok(Self::new(conn))
}
}
impl<T> SglangServiceClient<T>
where
T: tonic::client::GrpcService<tonic::body::Body>,
T::Error: Into<StdError>,
T::ResponseBody: Body<Data = Bytes> + std::marker::Send + 'static,
<T::ResponseBody as Body>::Error: Into<StdError> + std::marker::Send,
{
pub fn new(inner: T) -> Self {
let inner = tonic::client::Grpc::new(inner);
Self { inner }
}
pub fn with_origin(inner: T, origin: Uri) -> Self {
let inner = tonic::client::Grpc::with_origin(inner, origin);
Self { inner }
}
pub fn with_interceptor<F>(
inner: T,
interceptor: F,
) -> SglangServiceClient<InterceptedService<T, F>>
where
F: tonic::service::Interceptor,
T::ResponseBody: Default,
T: tonic::codegen::Service<
http::Request<tonic::body::Body>,
Response = http::Response<
<T as tonic::client::GrpcService<tonic::body::Body>>::ResponseBody,
>,
>,
<T as tonic::codegen::Service<http::Request<tonic::body::Body>>>::Error:
Into<StdError> + std::marker::Send + std::marker::Sync,
{
SglangServiceClient::new(InterceptedService::new(inner, interceptor))
}
/// Compress requests with the given encoding.
///
/// This requires the server to support it otherwise it might respond with an
/// error.
#[must_use]
pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self {
self.inner = self.inner.send_compressed(encoding);
self
}
/// Enable decompressing responses.
#[must_use]
pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self {
self.inner = self.inner.accept_compressed(encoding);
self
}
/// Limits the maximum size of a decoded message.
///
/// Default: `4MB`
#[must_use]
pub fn max_decoding_message_size(mut self, limit: usize) -> Self {
self.inner = self.inner.max_decoding_message_size(limit);
self
}
/// Limits the maximum size of an encoded message.
///
/// Default: `usize::MAX`
#[must_use]
pub fn max_encoding_message_size(mut self, limit: usize) -> Self {
self.inner = self.inner.max_encoding_message_size(limit);
self
}
/// `/generate`: always server-streaming (a unary request is a one-frame
/// stream); per-batch-item failures ride the stream as GenerateStreamItem
/// errors, pre-submit failures end the RPC with a gRPC status.
pub async fn generate(
&mut self,
request: impl tonic::IntoRequest<super::GenerateRequest>,
) -> std::result::Result<
tonic::Response<tonic::codec::Streaming<super::GenerateStreamItem>>,
tonic::Status,
> {
self.inner.ready().await.map_err(|e| {
tonic::Status::unknown(format!("Service was not ready: {}", e.into()))
})?;
let codec = tonic_prost::ProstCodec::default();
let path =
http::uri::PathAndQuery::from_static("/sglang.api.v1.SglangService/Generate");
let mut req = request.into_request();
req.extensions_mut()
.insert(GrpcMethod::new("sglang.api.v1.SglangService", "Generate"));
self.inner.server_streaming(req, path, codec).await
}
/// `/health_generate`: fires a 1-token probe and watches the response
/// heartbeat. UNAVAILABLE on a stalled pipeline.
pub async fn health_check(
&mut self,
request: impl tonic::IntoRequest<super::HealthCheckRequest>,
) -> std::result::Result<tonic::Response<super::HealthCheckResponse>, tonic::Status>
{
self.inner.ready().await.map_err(|e| {
tonic::Status::unknown(format!("Service was not ready: {}", e.into()))
})?;
let codec = tonic_prost::ProstCodec::default();
let path =
http::uri::PathAndQuery::from_static("/sglang.api.v1.SglangService/HealthCheck");
let mut req = request.into_request();
req.extensions_mut().insert(GrpcMethod::new(
"sglang.api.v1.SglangService",
"HealthCheck",
));
self.inner.unary(req, path, codec).await
}
/// `/get_model_info`: static model metadata, no scheduler round-trip.
pub async fn get_model_info(
&mut self,
request: impl tonic::IntoRequest<super::GetModelInfoRequest>,
) -> std::result::Result<tonic::Response<super::GetModelInfoResponse>, tonic::Status>
{
self.inner.ready().await.map_err(|e| {
tonic::Status::unknown(format!("Service was not ready: {}", e.into()))
})?;
let codec = tonic_prost::ProstCodec::default();
let path =
http::uri::PathAndQuery::from_static("/sglang.api.v1.SglangService/GetModelInfo");
let mut req = request.into_request();
req.extensions_mut().insert(GrpcMethod::new(
"sglang.api.v1.SglangService",
"GetModelInfo",
));
self.inner.unary(req, path, codec).await
}
/// `/server_info`: one control round-trip, shaped through the runtime-metric
/// allowlist (never the raw server-args dump, which embeds keys).
pub async fn get_server_info(
&mut self,
request: impl tonic::IntoRequest<super::GetServerInfoRequest>,
) -> std::result::Result<tonic::Response<super::GetServerInfoResponse>, tonic::Status>
{
self.inner.ready().await.map_err(|e| {
tonic::Status::unknown(format!("Service was not ready: {}", e.into()))
})?;
let codec = tonic_prost::ProstCodec::default();
let path =
http::uri::PathAndQuery::from_static("/sglang.api.v1.SglangService/GetServerInfo");
let mut req = request.into_request();
req.extensions_mut().insert(GrpcMethod::new(
"sglang.api.v1.SglangService",
"GetServerInfo",
));
self.inner.unary(req, path, codec).await
}
}
}
/// Generated server implementations.
pub mod sglang_service_server {
#![allow(
unused_variables,
dead_code,
missing_docs,
clippy::wildcard_imports,
clippy::let_unit_value
)]
use tonic::codegen::*;
/// Generated trait containing gRPC methods that should be implemented for use with SglangServiceServer.
#[async_trait]
pub trait SglangService: std::marker::Send + std::marker::Sync + 'static {
/// Server streaming response type for the Generate method.
type GenerateStream: tonic::codegen::tokio_stream::Stream<
Item = std::result::Result<super::GenerateStreamItem, tonic::Status>,
> + std::marker::Send
+ 'static;
/// `/generate`: always server-streaming (a unary request is a one-frame
/// stream); per-batch-item failures ride the stream as GenerateStreamItem
/// errors, pre-submit failures end the RPC with a gRPC status.
async fn generate(
&self,
request: tonic::Request<super::GenerateRequest>,
) -> std::result::Result<tonic::Response<Self::GenerateStream>, tonic::Status>;
/// `/health_generate`: fires a 1-token probe and watches the response
/// heartbeat. UNAVAILABLE on a stalled pipeline.
async fn health_check(
&self,
request: tonic::Request<super::HealthCheckRequest>,
) -> std::result::Result<tonic::Response<super::HealthCheckResponse>, tonic::Status>;
/// `/get_model_info`: static model metadata, no scheduler round-trip.
async fn get_model_info(
&self,
request: tonic::Request<super::GetModelInfoRequest>,
) -> std::result::Result<tonic::Response<super::GetModelInfoResponse>, tonic::Status>;
/// `/server_info`: one control round-trip, shaped through the runtime-metric
/// allowlist (never the raw server-args dump, which embeds keys).
async fn get_server_info(
&self,
request: tonic::Request<super::GetServerInfoRequest>,
) -> std::result::Result<tonic::Response<super::GetServerInfoResponse>, tonic::Status>;
}
#[derive(Debug)]
pub struct SglangServiceServer<T> {
inner: Arc<T>,
accept_compression_encodings: EnabledCompressionEncodings,
send_compression_encodings: EnabledCompressionEncodings,
max_decoding_message_size: Option<usize>,
max_encoding_message_size: Option<usize>,
}
impl<T> SglangServiceServer<T> {
pub fn new(inner: T) -> Self {
Self::from_arc(Arc::new(inner))
}
pub fn from_arc(inner: Arc<T>) -> Self {
Self {
inner,
accept_compression_encodings: Default::default(),
send_compression_encodings: Default::default(),
max_decoding_message_size: None,
max_encoding_message_size: None,
}
}
pub fn with_interceptor<F>(inner: T, interceptor: F) -> InterceptedService<Self, F>
where
F: tonic::service::Interceptor,
{
InterceptedService::new(Self::new(inner), interceptor)
}
/// Enable decompressing requests with the given encoding.
#[must_use]
pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self {
self.accept_compression_encodings.enable(encoding);
self
}
/// Compress responses with the given encoding, if the client supports it.
#[must_use]
pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self {
self.send_compression_encodings.enable(encoding);
self
}
/// Limits the maximum size of a decoded message.
///
/// Default: `4MB`
#[must_use]
pub fn max_decoding_message_size(mut self, limit: usize) -> Self {
self.max_decoding_message_size = Some(limit);
self
}
/// Limits the maximum size of an encoded message.
///
/// Default: `usize::MAX`
#[must_use]
pub fn max_encoding_message_size(mut self, limit: usize) -> Self {
self.max_encoding_message_size = Some(limit);
self
}
}
impl<T, B> tonic::codegen::Service<http::Request<B>> for SglangServiceServer<T>
where
T: SglangService,
B: Body + std::marker::Send + 'static,
B::Error: Into<StdError> + std::marker::Send + 'static,
{
type Response = http::Response<tonic::body::Body>;
type Error = std::convert::Infallible;
type Future = BoxFuture<Self::Response, Self::Error>;
fn poll_ready(
&mut self,
_cx: &mut Context<'_>,
) -> Poll<std::result::Result<(), Self::Error>> {
Poll::Ready(Ok(()))
}
fn call(&mut self, req: http::Request<B>) -> Self::Future {
match req.uri().path() {
"/sglang.api.v1.SglangService/Generate" => {
#[allow(non_camel_case_types)]
struct GenerateSvc<T: SglangService>(pub Arc<T>);
impl<T: SglangService>
tonic::server::ServerStreamingService<super::GenerateRequest>
for GenerateSvc<T>
{
type Response = super::GenerateStreamItem;
type ResponseStream = T::GenerateStream;
type Future =
BoxFuture<tonic::Response<Self::ResponseStream>, tonic::Status>;
fn call(
&mut self,
request: tonic::Request<super::GenerateRequest>,
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move {
<T as SglangService>::generate(&inner, request).await
};
Box::pin(fut)
}
}
let accept_compression_encodings = self.accept_compression_encodings;
let send_compression_encodings = self.send_compression_encodings;
let max_decoding_message_size = self.max_decoding_message_size;
let max_encoding_message_size = self.max_encoding_message_size;
let inner = self.inner.clone();
let fut = async move {
let method = GenerateSvc(inner);
let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(
accept_compression_encodings,
send_compression_encodings,
)
.apply_max_message_size_config(
max_decoding_message_size,
max_encoding_message_size,
);
let res = grpc.server_streaming(method, req).await;
Ok(res)
};
Box::pin(fut)
}
"/sglang.api.v1.SglangService/HealthCheck" => {
#[allow(non_camel_case_types)]
struct HealthCheckSvc<T: SglangService>(pub Arc<T>);
impl<T: SglangService> tonic::server::UnaryService<super::HealthCheckRequest>
for HealthCheckSvc<T>
{
type Response = super::HealthCheckResponse;
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
fn call(
&mut self,
request: tonic::Request<super::HealthCheckRequest>,
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move {
<T as SglangService>::health_check(&inner, request).await
};
Box::pin(fut)
}
}
let accept_compression_encodings = self.accept_compression_encodings;
let send_compression_encodings = self.send_compression_encodings;
let max_decoding_message_size = self.max_decoding_message_size;
let max_encoding_message_size = self.max_encoding_message_size;
let inner = self.inner.clone();
let fut = async move {
let method = HealthCheckSvc(inner);
let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(
accept_compression_encodings,
send_compression_encodings,
)
.apply_max_message_size_config(
max_decoding_message_size,
max_encoding_message_size,
);
let res = grpc.unary(method, req).await;
Ok(res)
};
Box::pin(fut)
}
"/sglang.api.v1.SglangService/GetModelInfo" => {
#[allow(non_camel_case_types)]
struct GetModelInfoSvc<T: SglangService>(pub Arc<T>);
impl<T: SglangService> tonic::server::UnaryService<super::GetModelInfoRequest>
for GetModelInfoSvc<T>
{
type Response = super::GetModelInfoResponse;
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
fn call(
&mut self,
request: tonic::Request<super::GetModelInfoRequest>,
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move {
<T as SglangService>::get_model_info(&inner, request).await
};
Box::pin(fut)
}
}
let accept_compression_encodings = self.accept_compression_encodings;
let send_compression_encodings = self.send_compression_encodings;
let max_decoding_message_size = self.max_decoding_message_size;
let max_encoding_message_size = self.max_encoding_message_size;
let inner = self.inner.clone();
let fut = async move {
let method = GetModelInfoSvc(inner);
let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(
accept_compression_encodings,
send_compression_encodings,
)
.apply_max_message_size_config(
max_decoding_message_size,
max_encoding_message_size,
);
let res = grpc.unary(method, req).await;
Ok(res)
};
Box::pin(fut)
}
"/sglang.api.v1.SglangService/GetServerInfo" => {
#[allow(non_camel_case_types)]
struct GetServerInfoSvc<T: SglangService>(pub Arc<T>);
impl<T: SglangService> tonic::server::UnaryService<super::GetServerInfoRequest>
for GetServerInfoSvc<T>
{
type Response = super::GetServerInfoResponse;
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
fn call(
&mut self,
request: tonic::Request<super::GetServerInfoRequest>,
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move {
<T as SglangService>::get_server_info(&inner, request).await
};
Box::pin(fut)
}
}
let accept_compression_encodings = self.accept_compression_encodings;
let send_compression_encodings = self.send_compression_encodings;
let max_decoding_message_size = self.max_decoding_message_size;
let max_encoding_message_size = self.max_encoding_message_size;
let inner = self.inner.clone();
let fut = async move {
let method = GetServerInfoSvc(inner);
let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(
accept_compression_encodings,
send_compression_encodings,
)
.apply_max_message_size_config(
max_decoding_message_size,
max_encoding_message_size,
);
let res = grpc.unary(method, req).await;
Ok(res)
};
Box::pin(fut)
}
_ => Box::pin(async move {
let mut response = http::Response::new(tonic::body::Body::default());
let headers = response.headers_mut();
headers.insert(
tonic::Status::GRPC_STATUS,
(tonic::Code::Unimplemented as i32).into(),
);
headers.insert(
http::header::CONTENT_TYPE,
tonic::metadata::GRPC_CONTENT_TYPE,
);
Ok(response)
}),
}
}
}
impl<T> Clone for SglangServiceServer<T> {
fn clone(&self) -> Self {
let inner = self.inner.clone();
Self {
inner,
accept_compression_encodings: self.accept_compression_encodings,
send_compression_encodings: self.send_compression_encodings,
max_decoding_message_size: self.max_decoding_message_size,
max_encoding_message_size: self.max_encoding_message_size,
}
}
}
/// Generated gRPC service name
pub const SERVICE_NAME: &str = "sglang.api.v1.SglangService";
impl<T> tonic::server::NamedService for SglangServiceServer<T> {
const NAME: &'static str = SERVICE_NAME;
}
}
@@ -0,0 +1,457 @@
// This file is @generated by prost-build.
/// Disaggregated prefill/decode rendezvous parameters.
///
/// Carried on Generate/TextGenerate requests when the worker is part of a
/// disaggregated deployment. The prefill leg writes its KV cache to the
/// bootstrap server identified by (bootstrap_host, bootstrap_port) under
/// the room id; the decode leg fetches it from the same room. Both legs
/// receive the same `bootstrap_room` from the router so they rendezvous
/// on the KV transfer.
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct DisaggregatedParams {
#[prost(string, tag = "1")]
pub bootstrap_host: ::prost::alloc::string::String,
#[prost(int32, tag = "2")]
pub bootstrap_port: i32,
#[prost(int64, tag = "3")]
pub bootstrap_room: i64,
}
/// Sampling parameters shared across text and tokenized RPCs.
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct SamplingParams {
#[prost(float, optional, tag = "1")]
pub temperature: ::core::option::Option<f32>,
#[prost(float, optional, tag = "2")]
pub top_p: ::core::option::Option<f32>,
#[prost(int32, optional, tag = "3")]
pub top_k: ::core::option::Option<i32>,
#[prost(float, optional, tag = "4")]
pub min_p: ::core::option::Option<f32>,
#[prost(float, optional, tag = "5")]
pub frequency_penalty: ::core::option::Option<f32>,
#[prost(float, optional, tag = "6")]
pub presence_penalty: ::core::option::Option<f32>,
#[prost(float, optional, tag = "7")]
pub repetition_penalty: ::core::option::Option<f32>,
#[prost(int32, optional, tag = "8")]
pub max_new_tokens: ::core::option::Option<i32>,
#[prost(int32, optional, tag = "9")]
pub min_new_tokens: ::core::option::Option<i32>,
#[prost(string, repeated, tag = "10")]
pub stop: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
#[prost(int32, repeated, tag = "11")]
pub stop_token_ids: ::prost::alloc::vec::Vec<i32>,
#[prost(bool, optional, tag = "12")]
pub ignore_eos: ::core::option::Option<bool>,
#[prost(int32, optional, tag = "13")]
pub n: ::core::option::Option<i32>,
#[deprecated]
#[prost(string, optional, tag = "14")]
pub json_schema: ::core::option::Option<::prost::alloc::string::String>,
#[deprecated]
#[prost(string, optional, tag = "15")]
pub regex: ::core::option::Option<::prost::alloc::string::String>,
#[prost(int64, optional, tag = "16")]
pub seed: ::core::option::Option<i64>,
#[prost(message, optional, tag = "17")]
pub guided_decoding: ::core::option::Option<GuidedDecoding>,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct GuidedDecoding {
#[prost(oneof = "guided_decoding::Constraint", tags = "1, 2, 3, 4, 5")]
pub constraint: ::core::option::Option<guided_decoding::Constraint>,
}
/// Nested message and enum types in `GuidedDecoding`.
pub mod guided_decoding {
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)]
pub enum Constraint {
#[prost(string, tag = "1")]
JsonSchema(::prost::alloc::string::String),
#[prost(string, tag = "2")]
Regex(::prost::alloc::string::String),
#[prost(string, tag = "3")]
Ebnf(::prost::alloc::string::String),
#[prost(message, tag = "4")]
Choice(super::ChoiceConstraint),
#[prost(string, tag = "5")]
StructuralTag(::prost::alloc::string::String),
}
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ChoiceConstraint {
#[prost(string, repeated, tag = "1")]
pub values: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct TextGenerateRequest {
#[prost(string, tag = "1")]
pub text: ::prost::alloc::string::String,
#[prost(message, optional, tag = "2")]
pub sampling_params: ::core::option::Option<SamplingParams>,
#[prost(bool, optional, tag = "3")]
pub stream: ::core::option::Option<bool>,
#[prost(bool, optional, tag = "4")]
pub return_logprob: ::core::option::Option<bool>,
#[prost(int32, optional, tag = "5")]
pub top_logprobs_num: ::core::option::Option<i32>,
#[prost(int32, optional, tag = "6")]
pub logprob_start_len: ::core::option::Option<i32>,
#[prost(bool, optional, tag = "7")]
pub return_text_in_logprobs: ::core::option::Option<bool>,
#[prost(string, optional, tag = "8")]
pub rid: ::core::option::Option<::prost::alloc::string::String>,
#[prost(string, optional, tag = "9")]
pub lora_path: ::core::option::Option<::prost::alloc::string::String>,
#[prost(string, optional, tag = "10")]
pub routing_key: ::core::option::Option<::prost::alloc::string::String>,
#[prost(int32, optional, tag = "11")]
pub routed_dp_rank: ::core::option::Option<i32>,
#[prost(map = "string, string", tag = "12")]
pub trace_headers:
::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>,
#[prost(string, optional, tag = "13")]
pub session_id: ::core::option::Option<::prost::alloc::string::String>,
#[prost(message, optional, tag = "14")]
pub disaggregated_params: ::core::option::Option<DisaggregatedParams>,
#[prost(int32, optional, tag = "15")]
pub priority: ::core::option::Option<i32>,
#[prost(bool, optional, tag = "16")]
pub require_reasoning: ::core::option::Option<bool>,
#[prost(uint32, optional, tag = "17")]
pub max_thinking_tokens: ::core::option::Option<u32>,
#[prost(message, optional, tag = "18")]
pub kv_hints: ::core::option::Option<KvHintsEnvelope>,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct TextGenerateResponse {
#[prost(string, tag = "1")]
pub text: ::prost::alloc::string::String,
#[prost(map = "string, string", tag = "2")]
pub meta_info:
::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>,
#[prost(bool, tag = "3")]
pub finished: bool,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct GenerateRequest {
#[prost(int32, repeated, tag = "1")]
pub input_ids: ::prost::alloc::vec::Vec<i32>,
#[prost(message, optional, tag = "2")]
pub sampling_params: ::core::option::Option<SamplingParams>,
#[prost(bool, optional, tag = "3")]
pub stream: ::core::option::Option<bool>,
#[prost(bool, optional, tag = "4")]
pub return_logprob: ::core::option::Option<bool>,
#[prost(int32, optional, tag = "5")]
pub top_logprobs_num: ::core::option::Option<i32>,
#[prost(int32, optional, tag = "6")]
pub logprob_start_len: ::core::option::Option<i32>,
#[prost(string, optional, tag = "7")]
pub rid: ::core::option::Option<::prost::alloc::string::String>,
#[prost(string, optional, tag = "8")]
pub lora_path: ::core::option::Option<::prost::alloc::string::String>,
#[prost(string, optional, tag = "9")]
pub routing_key: ::core::option::Option<::prost::alloc::string::String>,
#[prost(int32, optional, tag = "10")]
pub routed_dp_rank: ::core::option::Option<i32>,
#[prost(map = "string, string", tag = "11")]
pub trace_headers:
::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>,
#[prost(string, optional, tag = "12")]
pub session_id: ::core::option::Option<::prost::alloc::string::String>,
#[prost(message, optional, tag = "13")]
pub disaggregated_params: ::core::option::Option<DisaggregatedParams>,
#[prost(int32, optional, tag = "14")]
pub priority: ::core::option::Option<i32>,
#[prost(bool, optional, tag = "15")]
pub require_reasoning: ::core::option::Option<bool>,
#[prost(uint32, optional, tag = "16")]
pub max_thinking_tokens: ::core::option::Option<u32>,
#[prost(message, optional, tag = "17")]
pub kv_hints: ::core::option::Option<KvHintsEnvelope>,
}
/// Versioned KV-hint envelope forwarded from a trusted orchestrator.
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct KvHintsEnvelope {
#[prost(string, tag = "1")]
pub protocol_version: ::prost::alloc::string::String,
#[prost(string, tag = "2")]
pub message_id: ::prost::alloc::string::String,
#[prost(message, repeated, tag = "3")]
pub actions: ::prost::alloc::vec::Vec<KvHintAction>,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct KvHintAction {
#[prost(string, tag = "1")]
pub action_id: ::prost::alloc::string::String,
#[prost(string, tag = "2")]
pub action_type: ::prost::alloc::string::String,
#[prost(string, tag = "3")]
pub action_version: ::prost::alloc::string::String,
/// Action-specific; validated by the component implementing action_type.
#[prost(message, optional, tag = "4")]
pub payload: ::core::option::Option<::prost_types::Struct>,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct GenerateResponse {
#[prost(int32, repeated, tag = "1")]
pub output_ids: ::prost::alloc::vec::Vec<i32>,
#[prost(map = "string, string", tag = "2")]
pub meta_info:
::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>,
#[prost(bool, tag = "3")]
pub finished: bool,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct TextEmbedRequest {
#[prost(string, tag = "1")]
pub text: ::prost::alloc::string::String,
#[prost(string, optional, tag = "2")]
pub rid: ::core::option::Option<::prost::alloc::string::String>,
#[prost(string, optional, tag = "3")]
pub routing_key: ::core::option::Option<::prost::alloc::string::String>,
#[prost(map = "string, string", tag = "4")]
pub trace_headers:
::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct TextEmbedResponse {
#[prost(float, repeated, tag = "1")]
pub embedding: ::prost::alloc::vec::Vec<f32>,
#[prost(map = "string, string", tag = "2")]
pub meta_info:
::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct EmbedRequest {
#[prost(int32, repeated, tag = "1")]
pub input_ids: ::prost::alloc::vec::Vec<i32>,
#[prost(string, optional, tag = "2")]
pub rid: ::core::option::Option<::prost::alloc::string::String>,
#[prost(string, optional, tag = "3")]
pub routing_key: ::core::option::Option<::prost::alloc::string::String>,
#[prost(map = "string, string", tag = "4")]
pub trace_headers:
::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct EmbedResponse {
#[prost(float, repeated, tag = "1")]
pub embedding: ::prost::alloc::vec::Vec<f32>,
#[prost(map = "string, string", tag = "2")]
pub meta_info:
::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>,
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct HealthCheckRequest {}
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct HealthCheckResponse {
#[prost(bool, tag = "1")]
pub healthy: bool,
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct WatchEngineStateRequest {}
/// A complete discovery and lifecycle snapshot. The JSON discovery payloads
/// intentionally retain the same wire shape as GetModelInfo/GetServerInfo.
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct EngineStateSnapshot {
/// Unix time in nanoseconds, captured once for this engine process.
#[prost(uint64, tag = "1")]
pub instance_id: u64,
/// Starts at one and increases for each snapshot from this instance.
#[prost(uint64, tag = "2")]
pub revision: u64,
#[prost(bool, tag = "3")]
pub healthy: bool,
#[prost(bool, tag = "4")]
pub is_pause: bool,
#[prost(message, optional, tag = "5")]
pub model_info: ::core::option::Option<GetModelInfoResponse>,
#[prost(message, optional, tag = "6")]
pub server_info: ::core::option::Option<GetServerInfoResponse>,
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct GetModelInfoRequest {}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct GetModelInfoResponse {
#[prost(string, tag = "1")]
pub model_path: ::prost::alloc::string::String,
#[prost(string, tag = "2")]
pub json_info: ::prost::alloc::string::String,
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct GetServerInfoRequest {}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct GetServerInfoResponse {
#[prost(string, tag = "1")]
pub json_info: ::prost::alloc::string::String,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct AbortRequest {
#[prost(string, tag = "1")]
pub rid: ::prost::alloc::string::String,
#[prost(bool, tag = "2")]
pub abort_all: bool,
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct AbortResponse {
#[prost(bool, tag = "1")]
pub success: bool,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ClassifyRequest {
#[prost(string, tag = "1")]
pub text: ::prost::alloc::string::String,
#[prost(int32, repeated, tag = "2")]
pub input_ids: ::prost::alloc::vec::Vec<i32>,
#[prost(string, optional, tag = "3")]
pub rid: ::core::option::Option<::prost::alloc::string::String>,
#[prost(string, optional, tag = "4")]
pub routing_key: ::core::option::Option<::prost::alloc::string::String>,
#[prost(map = "string, string", tag = "5")]
pub trace_headers:
::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ClassifyResponse {
#[prost(float, repeated, tag = "1")]
pub embedding: ::prost::alloc::vec::Vec<f32>,
#[prost(map = "string, string", tag = "2")]
pub meta_info:
::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct TokenizeRequest {
#[prost(string, tag = "1")]
pub text: ::prost::alloc::string::String,
#[prost(bool, optional, tag = "2")]
pub add_special_tokens: ::core::option::Option<bool>,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct TokenizeResponse {
#[prost(int32, repeated, tag = "1")]
pub tokens: ::prost::alloc::vec::Vec<i32>,
#[prost(int32, tag = "2")]
pub count: i32,
#[prost(int32, tag = "3")]
pub max_model_len: i32,
#[prost(string, tag = "4")]
pub input_text: ::prost::alloc::string::String,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct DetokenizeRequest {
#[prost(int32, repeated, tag = "1")]
pub tokens: ::prost::alloc::vec::Vec<i32>,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct DetokenizeResponse {
#[prost(string, tag = "1")]
pub text: ::prost::alloc::string::String,
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ListModelsRequest {}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct ListModelsResponse {
#[prost(message, repeated, tag = "1")]
pub models: ::prost::alloc::vec::Vec<ModelCard>,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ModelCard {
#[prost(string, tag = "1")]
pub id: ::prost::alloc::string::String,
#[prost(string, tag = "2")]
pub root: ::prost::alloc::string::String,
#[prost(string, optional, tag = "3")]
pub parent: ::core::option::Option<::prost::alloc::string::String>,
#[prost(int32, optional, tag = "4")]
pub max_model_len: ::core::option::Option<i32>,
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct GetLoadRequest {
#[prost(int32, optional, tag = "1")]
pub dp_rank: ::core::option::Option<i32>,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct GetLoadResponse {
#[prost(string, tag = "1")]
pub json_info: ::prost::alloc::string::String,
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct FlushCacheRequest {}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct FlushCacheResponse {
#[prost(bool, tag = "1")]
pub success: bool,
#[prost(string, tag = "2")]
pub message: ::prost::alloc::string::String,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct PauseGenerationRequest {
#[prost(string, tag = "1")]
pub mode: ::prost::alloc::string::String,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct PauseGenerationResponse {
#[prost(string, tag = "1")]
pub message: ::prost::alloc::string::String,
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ContinueGenerationRequest {}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ContinueGenerationResponse {
#[prost(string, tag = "1")]
pub message: ::prost::alloc::string::String,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct OpenAiRequest {
#[prost(bytes = "vec", tag = "1")]
pub json_body: ::prost::alloc::vec::Vec<u8>,
#[prost(map = "string, string", tag = "2")]
pub trace_headers:
::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct OpenAiStreamChunk {
#[prost(bytes = "vec", tag = "1")]
pub json_chunk: ::prost::alloc::vec::Vec<u8>,
#[prost(bool, tag = "2")]
pub finished: bool,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct OpenAiResponse {
#[prost(bytes = "vec", tag = "1")]
pub json_body: ::prost::alloc::vec::Vec<u8>,
#[prost(int32, tag = "2")]
pub status_code: i32,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct StartProfileRequest {
#[prost(string, optional, tag = "1")]
pub output_dir: ::core::option::Option<::prost::alloc::string::String>,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct StartProfileResponse {
#[prost(string, tag = "1")]
pub message: ::prost::alloc::string::String,
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct StopProfileRequest {}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct StopProfileResponse {
#[prost(string, tag = "1")]
pub message: ::prost::alloc::string::String,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct UpdateWeightsRequest {
#[prost(string, tag = "1")]
pub model_path: ::prost::alloc::string::String,
#[prost(string, optional, tag = "2")]
pub load_format: ::core::option::Option<::prost::alloc::string::String>,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct UpdateWeightsResponse {
#[prost(bool, tag = "1")]
pub success: bool,
#[prost(string, tag = "2")]
pub message: ::prost::alloc::string::String,
}
File diff suppressed because it is too large Load Diff
+28
View File
@@ -0,0 +1,28 @@
//! Generated SGLang API types. `src/generated/` is written by
//! `cargo gen-api` from `proto/sglang/` -- the schema is the ground truth;
//! edit the .proto files and regenerate, never these files, and commit the
//! regenerated output with the .proto change.
//!
//! Module paths mirror the proto packages: `api::v1` is `sglang.api.v1` (the
//! HTTP/gRPC contract, with schema-driven JSON serde), `runtime::v1` is
//! `sglang.runtime.v1` (the gRPC bridge's protocol types, prost + tonic only).
mod ext;
pub mod api {
// Generated code optimizes for template uniformity, not lint cleanliness.
#[allow(clippy::all)]
pub mod v1 {
include!("generated/sglang.api.v1.rs");
include!("generated/sglang.api.v1.serde.rs");
include!("generated/sglang.api.v1.tonic.rs");
}
}
pub mod runtime {
#[allow(clippy::all)]
pub mod v1 {
include!("generated/sglang.runtime.v1.rs");
include!("generated/sglang.runtime.v1.tonic.rs");
}
}
+373
View File
@@ -0,0 +1,373 @@
{
"_": "Cross-language JSON parity fixtures for sglang.api.v1. Each case: decoding `input` and re-encoding yields `canonical` byte for byte, and `canonical` re-encodes to itself. Checked by rust/sglang-api-types/tests/json_parity.rs and test/registered/unit/api/v1/test_parity.py.",
"cases": [
{
"type": "GenerateRequest",
"name": "single text prompt with a sampling object; stream absent",
"input": {
"text": "hi",
"sampling_params": {
"temperature": 0.5,
"max_new_tokens": 8
}
},
"canonical": "{\"text\":\"hi\",\"stream\":false,\"sampling_params\":{\"max_new_tokens\":8,\"temperature\":0.5,\"top_p\":1.0,\"top_k\":1073741824,\"min_p\":0.0,\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"repetition_penalty\":1.0,\"min_new_tokens\":0,\"n\":1,\"ignore_eos\":false,\"skip_special_tokens\":true,\"spaces_between_special_tokens\":true,\"no_stop_trim\":false}}"
},
{
"type": "GenerateRequest",
"name": "batch form: per-item sampling, broadcast flag, nullable PD columns",
"input": {
"text": [
"a",
"b"
],
"stream": true,
"sampling_params": [
{
"top_p": 0.9
},
{
"temperature": 0.5
}
],
"return_logprob": true,
"bootstrap_host": [
null,
"h"
],
"bootstrap_port": [
8998,
null
]
},
"canonical": "{\"text\":[\"a\",\"b\"],\"stream\":true,\"sampling_params\":[{\"max_new_tokens\":128,\"temperature\":1.0,\"top_p\":0.9,\"top_k\":1073741824,\"min_p\":0.0,\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"repetition_penalty\":1.0,\"min_new_tokens\":0,\"n\":1,\"ignore_eos\":false,\"skip_special_tokens\":true,\"spaces_between_special_tokens\":true,\"no_stop_trim\":false},{\"max_new_tokens\":128,\"temperature\":0.5,\"top_p\":1.0,\"top_k\":1073741824,\"min_p\":0.0,\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"repetition_penalty\":1.0,\"min_new_tokens\":0,\"n\":1,\"ignore_eos\":false,\"skip_special_tokens\":true,\"spaces_between_special_tokens\":true,\"no_stop_trim\":false}],\"return_logprob\":true,\"bootstrap_host\":[null,\"h\"],\"bootstrap_port\":[8998,null]}"
},
{
"type": "GenerateRequest",
"name": "token prompt with every logprob knob; stream null resets to false",
"input": {
"input_ids": [
1,
2,
3
],
"rid": "r-1",
"stream": null,
"return_logprob": true,
"logprob_start_len": 0,
"top_logprobs_num": 2,
"token_ids_logprob": [
5,
6
],
"return_text_in_logprobs": true,
"return_hidden_states": true,
"routed_dp_rank": 1
},
"canonical": "{\"rid\":\"r-1\",\"input_ids\":[1,2,3],\"stream\":false,\"return_logprob\":true,\"logprob_start_len\":0,\"top_logprobs_num\":2,\"token_ids_logprob\":[5,6],\"return_hidden_states\":true,\"return_text_in_logprobs\":true,\"routed_dp_rank\":1}"
},
{
"type": "GenerateRequest",
"name": "media containers: nested, ref object, bare source; unknown keys ignored",
"input": {
"text": "see",
"image_data": [
[
"a",
null
],
null,
[]
],
"video_data": {
"url": "v",
"fps": 2.0,
"use_audio": true
},
"audio_data": "data:audio/wav;base64,AA==",
"mm_hashes": [
"h1",
"h2"
],
"priority": 3,
"session_id": "ignored"
},
"canonical": "{\"text\":\"see\",\"stream\":false,\"image_data\":[[\"a\",null],null,[]],\"mm_hashes\":[\"h1\",\"h2\"],\"video_data\":{\"url\":\"v\",\"fps\":2.0,\"use_audio\":true},\"audio_data\":\"data:audio/wav;base64,AA==\"}"
},
{
"type": "SamplingParams",
"name": "empty object: every schema default is emitted",
"input": {},
"canonical": "{\"max_new_tokens\":128,\"temperature\":1.0,\"top_p\":1.0,\"top_k\":1073741824,\"min_p\":0.0,\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"repetition_penalty\":1.0,\"min_new_tokens\":0,\"n\":1,\"ignore_eos\":false,\"skip_special_tokens\":true,\"spaces_between_special_tokens\":true,\"no_stop_trim\":false}"
},
{
"type": "SamplingParams",
"name": "explicit null resets a default; lists, maps and raw custom_params round-trip",
"input": {
"temperature": null,
"stop": [
"a",
"b"
],
"stop_token_ids": [
1,
2
],
"logit_bias": {
"7": -1.5
},
"custom_params": {
"k": [
1,
"x",
null
]
},
"json_schema": "{}",
"beam_width": 2
},
"canonical": "{\"max_new_tokens\":128,\"stop\":[\"a\",\"b\"],\"stop_token_ids\":[1,2],\"temperature\":1.0,\"top_p\":1.0,\"top_k\":1073741824,\"min_p\":0.0,\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"repetition_penalty\":1.0,\"min_new_tokens\":0,\"n\":1,\"json_schema\":\"{}\",\"ignore_eos\":false,\"skip_special_tokens\":true,\"spaces_between_special_tokens\":true,\"no_stop_trim\":false,\"logit_bias\":{\"7\":-1.5},\"custom_params\":{\"k\":[1,\"x\",null]},\"beam_width\":2}"
},
{
"type": "GenerateResponse",
"name": "plain frame: finish_reason null pre-terminal, output_ids present",
"input": {
"text": "ok",
"meta_info": {
"id": "client-rid",
"prompt_tokens": 5,
"completion_tokens": 2,
"finish_reason": null
},
"output_ids": [
7,
8
]
},
"canonical": "{\"text\":\"ok\",\"meta_info\":{\"id\":\"client-rid\",\"prompt_tokens\":5,\"completion_tokens\":2,\"finish_reason\":null},\"output_ids\":[7,8]}"
},
{
"type": "GenerateResponse",
"name": "terminal batch frame with every logprob family, null rows, hidden states, e2e_latency, index",
"input": {
"text": "ok",
"meta_info": {
"id": "r",
"prompt_tokens": 1,
"completion_tokens": 1,
"finish_reason": {
"type": "stop",
"matched": [
1,
2
]
},
"output_token_logprobs": [
[
-0.5,
5,
"x"
],
[
null,
6,
null
]
],
"input_token_logprobs": [],
"output_top_logprobs": [
null,
[
[
-0.25,
6,
null
]
]
],
"input_top_logprobs": [
[
[
-0.125,
1,
"<s>"
]
]
],
"output_token_ids_logprobs": [
[
[
-0.5,
9,
null
]
]
],
"input_token_ids_logprobs": [
null
],
"hidden_states": [
[
0.5,
0.25
],
[]
],
"e2e_latency": 0.01
},
"output_ids": [
5
],
"index": 0
},
"canonical": "{\"text\":\"ok\",\"meta_info\":{\"id\":\"r\",\"prompt_tokens\":1,\"completion_tokens\":1,\"finish_reason\":{\"type\":\"stop\",\"matched\":[1,2]},\"output_token_logprobs\":[[-0.5,5,\"x\"],[null,6,null]],\"input_token_logprobs\":[],\"output_top_logprobs\":[null,[[-0.25,6,null]]],\"input_top_logprobs\":[[[-0.125,1,\"<s>\"]]],\"output_token_ids_logprobs\":[[[-0.5,9,null]]],\"input_token_ids_logprobs\":[null],\"hidden_states\":[[0.5,0.25],[]],\"e2e_latency\":0.01},\"output_ids\":[5],\"index\":0}"
},
{
"type": "GenerateResponse",
"name": "finish length",
"input": {
"text": "",
"meta_info": {
"id": "r",
"prompt_tokens": 1,
"completion_tokens": 9,
"finish_reason": {
"type": "length",
"length": 9
}
}
},
"canonical": "{\"text\":\"\",\"meta_info\":{\"id\":\"r\",\"prompt_tokens\":1,\"completion_tokens\":9,\"finish_reason\":{\"type\":\"length\",\"length\":9}}}"
},
{
"type": "GenerateResponse",
"name": "finish abort emits its null keys",
"input": {
"text": "",
"meta_info": {
"id": "r",
"prompt_tokens": 1,
"completion_tokens": 0,
"finish_reason": {
"type": "abort",
"message": "over the limit",
"status_code": 400,
"err_type": null
}
}
},
"canonical": "{\"text\":\"\",\"meta_info\":{\"id\":\"r\",\"prompt_tokens\":1,\"completion_tokens\":0,\"finish_reason\":{\"type\":\"abort\",\"message\":\"over the limit\",\"status_code\":400,\"err_type\":null}}}"
},
{
"type": "GenerateResponse",
"name": "unknown finish type passes through unchanged",
"input": {
"text": "",
"meta_info": {
"id": "r",
"prompt_tokens": 1,
"completion_tokens": 0,
"finish_reason": {
"detail": 3,
"type": "novel"
}
}
},
"canonical": "{\"text\":\"\",\"meta_info\":{\"id\":\"r\",\"prompt_tokens\":1,\"completion_tokens\":0,\"finish_reason\":{\"detail\":3,\"type\":\"novel\"}}}"
},
{
"type": "GenerateStreamError",
"name": "in-stream error item with batch index",
"input": {
"error": {
"message": "bad sampling",
"code": 400
},
"index": 1
},
"canonical": "{\"error\":{\"message\":\"bad sampling\",\"code\":400},\"index\":1}"
},
{
"type": "GetModelInfoResponse",
"name": "absent optionals are emitted as null",
"input": {
"model_path": "/m",
"served_model_name": "m",
"tokenizer_path": "/t",
"is_generation": true,
"weight_version": "v1"
},
"canonical": "{\"model_path\":\"/m\",\"served_model_name\":\"m\",\"tokenizer_path\":\"/t\",\"is_generation\":true,\"preferred_sampling_params\":null,\"weight_version\":\"v1\",\"load_format\":null,\"reasoning_parser\":null,\"tool_call_parser\":null}"
},
{
"type": "GetServerInfoResponse",
"name": "internal_states is a list of raw JSON documents",
"input": {
"model_path": "/m",
"served_model_name": "m",
"tokenizer_path": "/t",
"max_context_length": 4096,
"max_total_num_tokens": 8192,
"version": "0.5.0",
"internal_states": [
{
"last_gen_throughput": 1.5,
"memory_usage": {
"weight": 2.0
}
}
]
},
"canonical": "{\"model_path\":\"/m\",\"served_model_name\":\"m\",\"tokenizer_path\":\"/t\",\"max_context_length\":4096,\"max_total_num_tokens\":8192,\"version\":\"0.5.0\",\"internal_states\":[{\"last_gen_throughput\":1.5,\"memory_usage\":{\"weight\":2.0}}]}"
},
{
"type": "SamplingParams",
"name": "null_is_none: an explicit max_new_tokens null (unbounded) survives the round trip as null",
"input": {
"max_new_tokens": null,
"n": 2
},
"canonical": "{\"max_new_tokens\":null,\"temperature\":1.0,\"top_p\":1.0,\"top_k\":1073741824,\"min_p\":0.0,\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"repetition_penalty\":1.0,\"min_new_tokens\":0,\"n\":2,\"ignore_eos\":false,\"skip_special_tokens\":true,\"spaces_between_special_tokens\":true,\"no_stop_trim\":false}"
},
{
"type": "GetModelInfoResponse",
"name": "preferred_sampling_params is the launch object itself (raw JSON)",
"input": {
"model_path": "/m",
"served_model_name": "m",
"tokenizer_path": "/t",
"is_generation": true,
"preferred_sampling_params": {
"temperature": 0.5,
"max_new_tokens": 64
},
"weight_version": "v1"
},
"canonical": "{\"model_path\":\"/m\",\"served_model_name\":\"m\",\"tokenizer_path\":\"/t\",\"is_generation\":true,\"preferred_sampling_params\":{\"temperature\":0.5,\"max_new_tokens\":64},\"weight_version\":\"v1\",\"load_format\":null,\"reasoning_parser\":null,\"tool_call_parser\":null}"
},
{
"type": "GenerateResponse",
"name": "unknown finish type passes through with its key order",
"input": {
"text": "",
"meta_info": {
"id": "r",
"prompt_tokens": 1,
"completion_tokens": 0,
"finish_reason": {
"type": "novel",
"detail": 3,
"again": [
1,
{
"z": 1,
"a": 2
}
]
}
}
},
"canonical": "{\"text\":\"\",\"meta_info\":{\"id\":\"r\",\"prompt_tokens\":1,\"completion_tokens\":0,\"finish_reason\":{\"type\":\"novel\",\"detail\":3,\"again\":[1,{\"z\":1,\"a\":2}]}}}"
}
]
}
@@ -0,0 +1,346 @@
//! Golden tests for the schema-driven JSON contract: each sglang.json.v1
//! option template, pinned against the behavior of the hand-written types it
//! replaces (message/sampling.rs, message/request.rs, message/finish_reason.rs,
//! api_server/core/frame.rs). These fixtures are the contract's only gate,
//! so they keep the generator honest on their own.
use sglang_api_types::api::v1::*;
fn sp(json: &str) -> Result<SamplingParams, serde_json::Error> {
serde_json::from_str(json)
}
/// Absent keys yield the schema defaults (sampling.rs `defaulted!` values).
#[test]
fn sampling_defaults_match_hand_written() {
let p = sp("{}").unwrap();
assert_eq!(p.max_new_tokens, Some(128));
assert_eq!(p.temperature, Some(1.0));
assert_eq!(p.top_p, Some(1.0));
assert_eq!(p.top_k, Some(1 << 30));
assert_eq!(p.min_p, Some(0.0));
assert_eq!(p.repetition_penalty, Some(1.0));
assert_eq!(p.min_new_tokens, Some(0));
assert_eq!(p.n, Some(1));
assert_eq!(p.ignore_eos, Some(false));
assert_eq!(p.skip_special_tokens, Some(true));
assert_eq!(p.spaces_between_special_tokens, Some(true));
assert_eq!(p.no_stop_trim, Some(false));
}
/// Protobuf arrival: a prost-decoded (all-unset) SamplingParams reads its
/// schema defaults through the accessors — the top_p=0 bug the first live
/// gRPC call found.
#[test]
fn proto_unset_reads_schema_defaults() {
let p = SamplingParams::default();
assert_eq!(p.temperature_or_default(), 1.0);
assert_eq!(p.top_p_or_default(), 1.0);
assert_eq!(p.top_k_or_default(), 1 << 30);
assert_eq!(p.n_or_default(), 1);
assert!(p.skip_special_tokens_or_default());
let mut p = SamplingParams::default();
p.apply_absent_defaults();
assert_eq!(p.max_new_tokens, Some(128));
assert_eq!(p.top_p, Some(1.0));
}
/// null_resets_default: an explicit null is the default, not an error
/// (Python `__post_init__` parity).
#[test]
fn sampling_null_resets_default() {
let p = sp(r#"{"temperature": null, "top_k": null, "ignore_eos": null}"#).unwrap();
assert_eq!(p.temperature, Some(1.0));
assert_eq!(p.top_k, Some(1 << 30));
assert_eq!(p.ignore_eos, Some(false));
}
/// max_new_tokens is the null_is_none field: absent = 128, null = unbounded.
#[test]
fn max_new_tokens_absent_vs_null() {
assert_eq!(sp("{}").unwrap().max_new_tokens, Some(128));
assert_eq!(
sp(r#"{"max_new_tokens": null}"#).unwrap().max_new_tokens,
None
);
assert_eq!(
sp(r#"{"max_new_tokens": 7}"#).unwrap().max_new_tokens,
Some(7)
);
}
/// SamplingParams DENIES unknown keys with serde's own error text (parity
/// with the hand-written `deny_unknown_fields`).
#[test]
fn sampling_denies_unknown_keys() {
let err = sp(r#"{"temperatur": 0.5}"#).unwrap_err().to_string(); // codespell:ignore temperatur
assert!(
err.starts_with("unknown field `temperatur`, expected one of"), // codespell:ignore temperatur
"{err}"
);
}
/// GenerateRequest IGNORES unknown keys (unported Python fields must not 400;
/// pinned in message/request.rs).
#[test]
fn generate_request_ignores_unknown_keys() {
let req: GenerateRequest = serde_json::from_str(
r#"{"text": "hi", "priority": 3, "session_id": "s", "custom_logit_processor": "x"}"#,
)
.unwrap();
assert!(matches!(
req.text,
Some(StringOrList {
value: Some(string_or_list::Value::One(ref s))
}) if s == "hi"
));
}
/// one_or_many: a scalar and a list both parse; serialization flattens back.
#[test]
fn one_or_many_flattens() {
let one: StringOrList = serde_json::from_str(r#""a""#).unwrap();
assert_eq!(serde_json::to_string(&one).unwrap(), r#""a""#);
let many: StringOrList = serde_json::from_str(r#"["a", "b"]"#).unwrap();
assert_eq!(serde_json::to_string(&many).unwrap(), r#"["a","b"]"#);
}
/// The TokenIds ambiguity: [1,2] is one id list, [[1],[2]] is a batch, [] is
/// one empty list (the hand-written untagged first-arm rule).
#[test]
fn token_ids_or_list_shape_dispatch() {
let one: TokenIdsOrList = serde_json::from_str("[1, 2]").unwrap();
assert!(matches!(
one.value,
Some(token_ids_or_list::Value::One(ref t)) if t.ids == vec![1, 2]
));
let many: TokenIdsOrList = serde_json::from_str("[[1], [2, 3]]").unwrap();
assert!(matches!(
many.value,
Some(token_ids_or_list::Value::Many(ref l)) if l.items.len() == 2
));
let empty: TokenIdsOrList = serde_json::from_str("[]").unwrap();
assert!(matches!(
empty.value,
Some(token_ids_or_list::Value::One(ref t)) if t.ids.is_empty()
));
}
/// Nullable list elements (PD bootstrap columns): [null, "h"] keeps per-item
/// presence; a bare scalar is the one-arm.
#[test]
fn optional_carriers_keep_element_nulls() {
let v: OptionalStringOrList = serde_json::from_str(r#"[null, "h"]"#).unwrap();
assert_eq!(serde_json::to_string(&v).unwrap(), r#"[null,"h"]"#);
match v.value {
Some(optional_string_or_list::Value::Many(l)) => {
assert_eq!(l.items[0].value, None);
assert_eq!(l.items[1].value.as_deref(), Some("h"));
}
other => panic!("expected many, got {other:?}"),
}
let one: OptionalInt64OrList = serde_json::from_str("17000").unwrap();
assert!(matches!(
one.value,
Some(optional_int64_or_list::Value::One(OptionalInt64 {
value: Some(17000)
}))
));
}
/// SamplingParams one-or-many keeps serde's field-level error texts (the
/// hand-written SamplingParamsInput contract: never "data did not match any
/// variant").
#[test]
fn sampling_or_list_keeps_field_errors() {
let err = serde_json::from_str::<SamplingParamsOrList>(r#"{"temperatur": 1}"#) // codespell:ignore temperatur
.unwrap_err()
.to_string();
assert!(
err.starts_with("unknown field `temperatur`"), // codespell:ignore temperatur
"field-level text lost: {err}"
);
}
/// finish_reason round-trips key-for-key, tagged by "type"; matched keeps its
/// wire shape (str / int / int list).
#[test]
fn finish_reason_round_trips() {
for json in [
r#"{"type":"stop","matched":"</s>"}"#,
r#"{"type":"stop","matched":7}"#,
r#"{"type":"stop","matched":[1,2]}"#,
r#"{"type":"length","length":2}"#,
] {
let parsed: FinishReason = serde_json::from_str(json).unwrap();
let back: serde_json::Value = serde_json::to_value(&parsed).unwrap();
let want: serde_json::Value = serde_json::from_str(json).unwrap();
assert_eq!(back, want, "round trip of {json}");
}
}
/// An abort carries all three keys, nulls included (Python emits them).
#[test]
fn finish_abort_emits_null_keys() {
let parsed: FinishReason = serde_json::from_str(r#"{"type":"abort","message":"m"}"#).unwrap();
let back = serde_json::to_string(&parsed).unwrap();
let v: serde_json::Value = serde_json::from_str(&back).unwrap();
assert_eq!(v["type"], "abort");
assert_eq!(v["message"], "m");
assert!(v.as_object().unwrap().contains_key("status_code"));
assert!(v["status_code"].is_null());
assert!(v.as_object().unwrap().contains_key("err_type"));
}
/// An unknown finish type round-trips unchanged (the forward-compat escape
/// hatch: a Python-side addition must not fail frame decode).
#[test]
fn unknown_finish_reason_passes_through() {
let json = r#"{"type":"paused","step":3}"#;
let parsed: FinishReason = serde_json::from_str(json).unwrap();
assert!(matches!(parsed.kind, Some(finish_reason::Kind::Unknown(_))));
let back: serde_json::Value = serde_json::to_value(&parsed).unwrap();
let want: serde_json::Value = serde_json::from_str(json).unwrap();
assert_eq!(back, want);
}
/// A known tag with a malformed payload also falls back to passthrough (the
/// hand-written FinishReason's untagged-outer behavior).
#[test]
fn malformed_known_finish_reason_passes_through() {
let json = r#"{"type":"stop","matched":{"bad":"shape"}}"#;
let parsed: FinishReason = serde_json::from_str(json).unwrap();
assert!(matches!(parsed.kind, Some(finish_reason::Kind::Unknown(_))));
}
/// Logprob entries are [logprob, token_id, text|null] tuples; the logprob
/// slot is nullable (Python emits null for the first prefill position).
#[test]
fn logprob_tuple_shape() {
let e = LogprobEntry {
logprob: Some(-0.25),
token_id: 7,
text: None,
};
assert_eq!(serde_json::to_string(&e).unwrap(), "[-0.25,7,null]");
let parsed: LogprobEntry = serde_json::from_str(r#"[-0.5, 9, "x"]"#).unwrap();
assert_eq!(parsed.token_id, 9);
assert_eq!(parsed.text.as_deref(), Some("x"));
let first_prefill: LogprobEntry = serde_json::from_str(r#"[null, 10, null]"#).unwrap();
assert_eq!(first_prefill.logprob, None);
assert_eq!(
serde_json::to_string(&first_prefill).unwrap(),
"[null,10,null]"
);
}
/// The /generate frame shape: key order (text, meta_info, output_ids),
/// conditional keys, finish_reason always present (null pre-terminal) --
/// matching api_core/frame.rs's frame_value.
#[test]
fn generate_response_matches_frame_value_shape() {
let frame = GenerateResponse {
text: "ok".into(),
output_ids: Some(TokenIds { ids: vec![7, 8] }),
meta_info: Some(GenerateMetaInfo {
id: "client-rid".into(),
prompt_tokens: 5,
completion_tokens: 2,
finish_reason: None,
..Default::default()
}),
index: None,
};
assert_eq!(
serde_json::to_string(&frame).unwrap(),
r#"{"text":"ok","meta_info":{"id":"client-rid","prompt_tokens":5,"completion_tokens":2,"finish_reason":null},"output_ids":[7,8]}"#
);
}
/// Typed media input: every container form Python's `/generate` accepts, with
/// the value's shape picking the arm (string/object = one item, array = flat
/// list, array of arrays = per batch entry) and JSON null kept per element.
#[test]
fn media_input_shapes_dispatch() {
use media_input::Value;
let parse = |json: &str| -> MediaInput { serde_json::from_str(json).unwrap() };
assert!(matches!(parse(r#""u""#).value, Some(Value::Item(_))));
assert!(matches!(
parse(r#"{"url": "u", "detail": "high"}"#).value,
Some(Value::Item(_))
));
let flat = parse(r#"["a", null, {"url": "b"}]"#);
let Some(Value::Many(items)) = flat.value else {
panic!("flat list")
};
assert_eq!(items.items.len(), 3);
assert!(items.items[1].item.is_none(), "null element kept");
assert!(
matches!(parse("[]").value, Some(Value::Many(_))),
"[] is an empty flat list"
);
let nested = parse(r#"[["a", null], null, []]"#);
let Some(Value::Nested(lists)) = nested.value else {
panic!("nested list")
};
assert_eq!(lists.items.len(), 3);
assert!(lists.items[1].items.is_none(), "null entry kept");
assert_eq!(lists.items[2].items.as_ref().unwrap().items.len(), 0);
// Round trip keeps the client's spelling.
for json in [
r#""u""#,
r#"["a",null,{"url":"b"}]"#,
r#"[["a",null],null,[]]"#,
] {
assert_eq!(serde_json::to_string(&parse(json)).unwrap(), json);
}
}
/// A media object is a MediaRef with named hints; an unknown hint is an error
/// (it used to be silently dropped), and a preprocessed-input dict is not an
/// HTTP shape at all.
#[test]
fn media_ref_is_typed_and_strict() {
let r: MediaRef =
serde_json::from_str(r#"{"url": "u", "max_dynamic_patch": 6, "fps": 2.5}"#).unwrap();
assert_eq!(
(r.url.as_str(), r.max_dynamic_patch, r.fps),
("u", Some(6), Some(2.5))
);
let err = serde_json::from_str::<MediaInput>(r#"{"url": "u", "max_dynamic_patc": 6}"#)
.unwrap_err()
.to_string();
assert!(err.contains("unknown field `max_dynamic_patc`"), "{err}");
let err = serde_json::from_str::<MediaInput>(r#"[{"format": "processor_output"}]"#)
.unwrap_err()
.to_string();
assert!(err.contains("unknown field `format`"), "{err}");
let err = serde_json::from_str::<MediaInput>(r#"["a", ["b"]]"#)
.unwrap_err()
.to_string();
assert!(!err.is_empty(), "mixed flat/nested is rejected");
}
/// mm_hashes: a flat string list for one request, nested for a batch; the
/// first element decides (the bare-scalar-list rule, generalized past ids).
#[test]
fn string_list_or_list_shape_dispatch() {
use string_list_or_list::Value;
let parse = |json: &str| -> StringListOrList { serde_json::from_str(json).unwrap() };
assert!(matches!(parse(r#"["a", "b"]"#).value, Some(Value::One(_))));
assert!(matches!(
parse(r#"[["a"], ["b", "c"]]"#).value,
Some(Value::Many(_))
));
assert!(matches!(parse("[]").value, Some(Value::One(_))));
let req: GenerateRequest =
serde_json::from_str(r#"{"mm_hashes": ["a1b2", "0xff"], "image_data": "u"}"#).unwrap();
assert!(req.mm_hashes.is_some() && req.image_data.is_some());
}
/// custom_params is the one remaining raw_json passthrough on a request.
#[test]
fn custom_params_raw_json_passthrough() {
let p: SamplingParams = serde_json::from_str(r#"{"custom_params": {"k": 3}}"#).unwrap();
assert_eq!(p.custom_params.as_deref(), Some(r#"{"k":3}"#));
}
@@ -0,0 +1,50 @@
//! Cross-language parity: the fixtures in `testdata/json_parity.json` are
//! checked here against the generated Rust types and, case for case, against
//! the generated Python types by `test/registered/unit/api/v1/test_parity.py`.
//! Both emitters must decode each `input` to the same `canonical` bytes (field
//! set, key order, presence and null rules) and keep `canonical` fixed under
//! a second round trip.
use serde::{Serialize, de::DeserializeOwned};
use sglang_api_types::api::v1 as api;
const FIXTURES: &str = include_str!("../testdata/json_parity.json");
fn check<T: DeserializeOwned + Serialize>(name: &str, input: &serde_json::Value, canonical: &str) {
let decoded: T = serde_json::from_value(input.clone())
.unwrap_or_else(|error| panic!("{name}: input does not decode: {error}"));
assert_eq!(
serde_json::to_string(&decoded).unwrap(),
canonical,
"{name}: input -> canonical"
);
let again: T = serde_json::from_str(canonical)
.unwrap_or_else(|error| panic!("{name}: canonical does not decode: {error}"));
assert_eq!(
serde_json::to_string(&again).unwrap(),
canonical,
"{name}: canonical is not a fixed point"
);
}
#[test]
fn fixtures_decode_to_their_canonical_bytes() {
let fixtures: serde_json::Value = serde_json::from_str(FIXTURES).unwrap();
let cases = fixtures["cases"].as_array().expect("cases array");
assert!(!cases.is_empty());
for case in cases {
let ty = case["type"].as_str().expect("type");
let name = format!("{ty}: {}", case["name"].as_str().expect("name"));
let input = &case["input"];
let canonical = case["canonical"].as_str().expect("canonical");
match ty {
"GenerateRequest" => check::<api::GenerateRequest>(&name, input, canonical),
"SamplingParams" => check::<api::SamplingParams>(&name, input, canonical),
"GenerateResponse" => check::<api::GenerateResponse>(&name, input, canonical),
"GenerateStreamError" => check::<api::GenerateStreamError>(&name, input, canonical),
"GetModelInfoResponse" => check::<api::GetModelInfoResponse>(&name, input, canonical),
"GetServerInfoResponse" => check::<api::GetServerInfoResponse>(&name, input, canonical),
other => panic!("fixture type {other} has no Rust mapping; add it here"),
}
}
}
+5 -4
View File
@@ -9,9 +9,6 @@ license.workspace = true
# of the main sglang wheel at the given import path.
[package.metadata.sglang]
python-module = "sglang.srt.rust_extensions._grpc"
# The shared protocol dependency lives outside rust/; include it in the
# source-loader artifact fingerprint.
source-inputs = ["../../proto"]
# Always build optimized, even for an editable install.
debug = false
@@ -26,7 +23,7 @@ crate-type = ["cdylib"]
[dependencies]
async-stream = { workspace = true }
pyo3 = { workspace = true }
sglang-grpc-types = { path = "../../proto" }
sglang-api-types = { path = "../sglang-api-types" }
serde_json = { workspace = true }
tokio = { workspace = true }
tokio-stream = { workspace = true }
@@ -38,5 +35,9 @@ uuid = { workspace = true }
prost-types = { workspace = true }
tokenizers = { version = "0.21", default-features = false, features = ["onig"] }
[dev-dependencies]
# `ProstCodec` for the hand-rolled unary call in the metadata-service test.
tonic-prost = { workspace = true }
[features]
default = ["pyo3/extension-module"]
+3 -3
View File
@@ -1,9 +1,9 @@
# Standalone dev builds only (`maturin build` / `maturin develop`); the
# extension ships to users inside the main sglang wheel via setuptools-rust
# (python/pyproject.toml, target sglang.srt.rust_extensions._grpc). Its shared
# protocol dependency lives in ../../proto, so standalone builds require a
# repository checkout. A system protoc is unnecessary; that crate uses a
# vendored binary.
# protocol dependency is the sibling sglang-api-types crate, so standalone
# builds require a repository checkout. No protoc is needed: the bindings are
# generated ahead of time and checked in.
[build-system]
requires = ["maturin>=1.5,<2"]
build-backend = "maturin"
+1 -1
View File
@@ -17,7 +17,7 @@ use crate::utils::{
build_classify_dict, build_embed_dict, build_generate_dict, build_text_embed_dict,
build_text_generate_dict, extract_model_path,
};
use sglang_grpc_types::sglang::runtime::v1 as proto;
use sglang_api_types::runtime::v1 as proto;
pub struct SglangServiceImpl {
pub bridge: Arc<PyBridge>,
+2 -2
View File
@@ -81,8 +81,8 @@ fn resolve_max_message_size_honors_env_var() {
#[tokio::test]
async fn follower_metadata_server_exposes_only_server_info_and_shuts_down() {
use sglang_grpc_types::sglang::runtime::v1::{GetServerInfoRequest, GetServerInfoResponse};
use tonic::codec::ProstCodec;
use sglang_api_types::runtime::v1::{GetServerInfoRequest, GetServerInfoResponse};
use tonic_prost::ProstCodec;
let _guard = PAYLOAD_ENV_LOCK.lock().await;
+1 -1
View File
@@ -1,6 +1,6 @@
use std::collections::HashMap;
use sglang_grpc_types::sglang::runtime::v1 as proto;
use sglang_api_types::runtime::v1 as proto;
fn regex_escape_literal(value: &str) -> String {
let mut escaped = String::with_capacity(value.len());
+2 -4
View File
@@ -10,9 +10,6 @@ license.workspace = true
python-module = "sglang.srt.rust_extensions._server"
# Always build optimized, even for an editable install.
debug = false
# The shared runtime.v1 crate lives outside rust/; include it in extension
# source fingerprints so protobuf changes cannot reuse a stale _server build.
source-inputs = ["../../proto"]
[lib]
name = "sglang_server"
@@ -51,6 +48,7 @@ dynamo-renderer = "5.1.2"
minijinja = "2.24"
flume = "0.12.0"
hf-hub = { version = "0.4", default-features = false }
http = "1.5.0"
itertools = "0.14"
# POSIX shm (`shm_open` family) for the MM feature fan-out (`utils::shm`).
rustix = { version = "1", features = ["fs", "mm", "shm"] }
@@ -63,13 +61,13 @@ numpy = "0.29.0"
regex-syntax = "=0.8.11"
rmp-serde = "1"
rmpv = { version = "1", features = ["with-serde"] }
sglang-grpc-types = { path = "../../proto" }
# The pure-Rust core of the MM pipeline. `default-features = false` drops the
# pyo3 bindings so it links as a plain rlib; renamed so `use sglang_mm::…` reads
# naturally while the crate keeps its own artifact name in the shared target/.
sglang_mm = { package = "sglang-mm", path = "../sglang-mm", default-features = false }
tonic = { workspace = true }
tokio-stream = { workspace = true }
sglang_api_types = { package = "sglang-api-types", path = "../sglang-api-types" }
[dev-dependencies]
# `Router::oneshot` for handler-level router tests.
+8 -81
View File
@@ -1,83 +1,10 @@
//! API server (axum / tokio). I/O-bound; own pinned multi-thread runtime. Only
//! this module knows HTTP. Its handlers use the transport-neutral
//! [`crate::frontend::FrontendHandle`] to enter the shared runtime pipeline.
//! Generation handlers render semantic frontend events as unary JSON or SSE;
//! control handlers serialize typed frontend results such as server metadata.
pub mod app;
mod common;
//! API server (axum / tokio). I/O-bound; own pinned multi-thread runtime. The
//! transport-neutral [`core::CoreHandle`] is the shared entry into the runtime
//! pipeline; `http` and `grpc` are the wire adapters on top of it. Generation
//! handlers render semantic core events as unary JSON, SSE, or gRPC frames;
//! control handlers serialize typed core results such as server metadata.
pub(crate) mod core;
mod disaggregation;
mod frame;
pub(crate) mod grpc;
pub(crate) mod http;
mod log;
mod native_api;
mod openai;
use axum::http::StatusCode;
use crate::frontend::{FrontendError, FrontendErrorKind};
/// Map transport-neutral failures into HTTP status codes. Response bodies and
/// stream framing remain the responsibility of each HTTP API surface.
pub(crate) fn frontend_error_status(error: &FrontendError) -> StatusCode {
if let FrontendError::RuntimeRejected {
legacy_http_status, ..
} = error
{
return StatusCode::from_u16(*legacy_http_status)
.unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
}
match error.kind() {
FrontendErrorKind::InvalidArgument => StatusCode::BAD_REQUEST,
FrontendErrorKind::NotFound => StatusCode::NOT_FOUND,
FrontendErrorKind::FailedPrecondition => StatusCode::PRECONDITION_FAILED,
// Existing intake backpressure has historically surfaced as 503.
FrontendErrorKind::ResourceExhausted | FrontendErrorKind::Unavailable => {
StatusCode::SERVICE_UNAVAILABLE
}
FrontendErrorKind::Cancelled => StatusCode::from_u16(499).expect("499 is valid"),
FrontendErrorKind::DeadlineExceeded => StatusCode::GATEWAY_TIMEOUT,
FrontendErrorKind::Internal => StatusCode::INTERNAL_SERVER_ERROR,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn frontend_error_categories_map_at_the_http_boundary() {
assert_eq!(
frontend_error_status(&FrontendError::InvalidArgument("bad".into())),
StatusCode::BAD_REQUEST
);
assert_eq!(
frontend_error_status(&FrontendError::Unavailable),
StatusCode::SERVICE_UNAVAILABLE
);
assert_eq!(
frontend_error_status(&FrontendError::Overloaded("full".into())),
StatusCode::SERVICE_UNAVAILABLE
);
assert_eq!(
frontend_error_status(&FrontendError::Cancelled("gone".into())).as_u16(),
499
);
assert_eq!(
frontend_error_status(&FrontendError::RuntimeRejected {
kind: FrontendErrorKind::InvalidArgument,
message: "runtime rejected request".into(),
legacy_http_status: 432,
})
.as_u16(),
432
);
assert_eq!(
frontend_error_status(&FrontendError::InvalidResponse("bad reply".into())),
StatusCode::INTERNAL_SERVER_ERROR
);
assert_eq!(
frontend_error_status(&FrontendError::Internal("bug".into())),
StatusCode::INTERNAL_SERVER_ERROR
);
}
}
+18
View File
@@ -0,0 +1,18 @@
//! Transport-neutral core shared by the API adapters: the entry point into the Rust request pipeline.
//!
//! Wire adapters normalize their protocol into the in-process
//! [`GenerateRequest`](crate::message::request::GenerateRequest), submit it
//! through [`CoreHandle`], and render semantic [`CoreEvent`]s for their
//! own transport. This module owns shared preprocessing, runtime translation,
//! capabilities, request lifetime, and the rendering of outputs into the
//! generated `api.v1` response types that both adapters put on the wire; it
//! deliberately knows nothing about Axum, SSE framing, or Tonic.
mod error;
mod frame;
mod handle;
mod prefetch;
pub(crate) use error::{CoreError, CoreErrorKind};
pub(crate) use frame::{CoreEvent, CoreOutput, HealthStatus};
pub(crate) use handle::{CoreCall, CoreConfig, CoreHandle, CoreMetadata, recv_indexed};
@@ -0,0 +1,186 @@
//! Transport-neutral failure categories, and how each wire adapter maps them
//! into its own status space.
use http::StatusCode;
use sglang_api_types::api::v1 as api;
/// Protocol-independent error category used by transport adapters.
///
/// The set intentionally follows operation semantics rather than HTTP or gRPC
/// status spaces. An adapter maps this category into its own wire status.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum CoreErrorKind {
InvalidArgument,
NotFound,
FailedPrecondition,
ResourceExhausted,
Cancelled,
DeadlineExceeded,
Unavailable,
Internal,
}
/// A transport-neutral failure category.
///
/// HTTP and gRPC adapters map these categories into their own status spaces;
/// neither adapter needs to understand errors emitted by runtime stages.
#[derive(Clone, Debug, thiserror::Error)]
pub(crate) enum CoreError {
#[error("{0}")]
InvalidArgument(String),
/// The intake loop has shut down. A caller may retry elsewhere.
#[error("service unavailable")]
Unavailable,
/// Runtime capacity was exhausted before the request could make progress.
#[error("{0}")]
Overloaded(String),
#[error("{0}")]
Cancelled(String),
/// The internal producer disappeared before returning a terminal result.
///
/// This is a server-side failure, not evidence that the caller cancelled.
#[error("response truncated before completion")]
ResponseTruncated,
/// A scheduler-side abort that carried an error status.
///
/// The current Python scheduler communicates this category as an HTTP
/// status code inside its finish reason. The semantic event is still
/// `Failed`; retaining the legacy code only lets the HTTP adapter preserve
/// existing behavior while another adapter maps it into its own status
/// space. New core failures should use the semantic variants above.
#[error("{message}")]
RuntimeRejected {
kind: CoreErrorKind,
message: String,
legacy_http_status: u16,
},
/// The runtime replied with an unexpected or malformed result.
#[error("{0}")]
InvalidResponse(String),
#[error("{0}")]
Internal(String),
}
impl CoreError {
/// Semantic status for adapters that do not use the scheduler's legacy
/// HTTP code space (notably the future gRPC adapter).
pub(crate) fn kind(&self) -> CoreErrorKind {
match self {
Self::InvalidArgument(_) => CoreErrorKind::InvalidArgument,
Self::Unavailable => CoreErrorKind::Unavailable,
Self::Overloaded(_) => CoreErrorKind::ResourceExhausted,
Self::Cancelled(_) => CoreErrorKind::Cancelled,
Self::ResponseTruncated => CoreErrorKind::Internal,
Self::RuntimeRejected { kind, .. } => *kind,
Self::InvalidResponse(_) | Self::Internal(_) => CoreErrorKind::Internal,
}
}
/// Translate the scheduler's inherited HTTP-coded failure into a semantic
/// category once, at the runtime boundary. HTTP can retain the exact legacy
/// status while other adapters consume [`Self::kind`].
pub(super) fn from_runtime_rejection(message: String, legacy_http_status: u16) -> Self {
let kind = match legacy_http_status {
404 => CoreErrorKind::NotFound,
408 | 504 => CoreErrorKind::DeadlineExceeded,
412 => CoreErrorKind::FailedPrecondition,
413 | 429 => CoreErrorKind::ResourceExhausted,
499 => CoreErrorKind::Cancelled,
502 | 503 => CoreErrorKind::Unavailable,
400..=499 => CoreErrorKind::InvalidArgument,
_ => CoreErrorKind::Internal,
};
Self::RuntimeRejected {
kind,
message,
legacy_http_status,
}
}
/// The HTTP status for this failure. Response bodies and stream framing
/// remain the responsibility of each HTTP API surface.
/// This failure as one `/generate` item: the native error body with the
/// HTTP status as `code`, plus the batch position where there is one.
pub(crate) fn stream_error(&self, index: Option<u32>) -> api::GenerateStreamError {
api::GenerateStreamError {
error: Some(api::ErrorBody {
message: self.to_string(),
code: u32::from(self.http_status().as_u16()),
}),
index,
}
}
pub(crate) fn http_status(&self) -> StatusCode {
if let CoreError::RuntimeRejected {
legacy_http_status, ..
} = self
{
return StatusCode::from_u16(*legacy_http_status)
.unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
}
match self.kind() {
CoreErrorKind::InvalidArgument => StatusCode::BAD_REQUEST,
CoreErrorKind::NotFound => StatusCode::NOT_FOUND,
CoreErrorKind::FailedPrecondition => StatusCode::PRECONDITION_FAILED,
// Existing intake backpressure has historically surfaced as 503.
CoreErrorKind::ResourceExhausted | CoreErrorKind::Unavailable => {
StatusCode::SERVICE_UNAVAILABLE
}
CoreErrorKind::Cancelled => StatusCode::from_u16(499).expect("499 is valid"),
CoreErrorKind::DeadlineExceeded => StatusCode::GATEWAY_TIMEOUT,
CoreErrorKind::Internal => StatusCode::INTERNAL_SERVER_ERROR,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn core_error_categories_map_at_the_http_boundary() {
assert_eq!(
CoreError::InvalidArgument("bad".into()).http_status(),
StatusCode::BAD_REQUEST
);
assert_eq!(
CoreError::Unavailable.http_status(),
StatusCode::SERVICE_UNAVAILABLE
);
assert_eq!(
CoreError::Overloaded("full".into()).http_status(),
StatusCode::SERVICE_UNAVAILABLE
);
assert_eq!(
CoreError::Cancelled("gone".into()).http_status().as_u16(),
499
);
assert_eq!(
CoreError::RuntimeRejected {
kind: CoreErrorKind::InvalidArgument,
message: "runtime rejected request".into(),
legacy_http_status: 432,
}
.http_status()
.as_u16(),
432
);
assert_eq!(
CoreError::InvalidResponse("bad reply".into()).http_status(),
StatusCode::INTERNAL_SERVER_ERROR
);
assert_eq!(
CoreError::Internal("bug".into()).http_status(),
StatusCode::INTERNAL_SERVER_ERROR
);
}
}
@@ -0,0 +1,457 @@
//! The semantic events a call yields (incremental and terminal outputs, a
//! request-scoped failure, the health probe's result) and the rendering of an
//! output into the `api.v1` response types. The generated schema is the wire
//! shape for both adapters: HTTP serializes it through serde, gRPC encodes it
//! as protobuf, so a column added here reaches both the same way. No Axum,
//! SSE, Tonic, or scheduler-channel details.
use sglang_api_types::api::v1 as api;
use super::CoreError;
use crate::message::finish_reason::{FinishKind, FinishReason, Matched};
use crate::message::response::{ChunkEvent, ChunkExtras};
use crate::message::types::TokenIds;
/// Canonical transport-neutral output payload for generation events.
///
/// The runtime correlation ID is deliberately absent: it is an implementation
/// detail used for scheduler routing, while adapters obtain the client-visible
/// identity from [`crate::api_server::core::CoreCall::public_id`]. Runtime-only
/// response variants such as raw control bytes are likewise hidden by
/// [`crate::api_server::core::CoreCall`].
#[derive(Clone, Debug, Default)]
pub(crate) struct CoreOutput {
pub(crate) token_ids: TokenIds,
pub(crate) finish_reason: Option<FinishReason>,
pub(crate) prompt_tokens: u32,
pub(crate) text: String,
pub(crate) completion_tokens: u64,
pub(crate) extras: Option<Box<ChunkExtras>>,
}
impl From<ChunkEvent> for CoreOutput {
fn from(output: ChunkEvent) -> Self {
let ChunkEvent {
rid: _,
token_ids,
finish_reason,
prompt_tokens,
text,
completion_tokens,
extras,
} = output;
Self {
token_ids,
finish_reason,
prompt_tokens,
text,
completion_tokens,
extras,
}
}
}
impl CoreOutput {
/// Fold one runtime delta into this cumulative native-generation output.
///
/// Runtime events are always incremental. Native HTTP and `api.v1`
/// gRPC independently decide whether to expose those deltas or the
/// cumulative result, so the fold itself lives at their shared semantic
/// boundary rather than in either wire adapter.
pub(crate) fn append_delta(&mut self, delta: &Self) {
self.text.push_str(&delta.text);
self.token_ids.extend_from_slice(&delta.token_ids);
self.completion_tokens += delta.completion_tokens;
self.prompt_tokens = delta.prompt_tokens;
if delta.finish_reason.is_some() {
self.finish_reason = delta.finish_reason.clone();
}
let Some(delta_extras) = delta.extras.as_deref() else {
return;
};
let extras = self
.extras
.get_or_insert_with(|| Box::new(ChunkExtras::default()));
extras
.out_lp_val
.extend_from_slice(&delta_extras.out_lp_val);
extras
.out_lp_idx
.extend_from_slice(&delta_extras.out_lp_idx);
extras
.out_top_val
.extend_from_slice(&delta_extras.out_top_val);
extras
.out_top_idx
.extend_from_slice(&delta_extras.out_top_idx);
extras
.out_top_lens
.extend_from_slice(&delta_extras.out_top_lens);
extras
.out_tid_val
.extend_from_slice(&delta_extras.out_tid_val);
extras
.out_tid_idx
.extend_from_slice(&delta_extras.out_tid_idx);
extras
.out_tid_lens
.extend_from_slice(&delta_extras.out_tid_lens);
extras
.out_lp_txt
.extend_from_slice(&delta_extras.out_lp_txt);
extras
.out_top_txt
.extend_from_slice(&delta_extras.out_top_txt);
extras
.out_tid_txt
.extend_from_slice(&delta_extras.out_tid_txt);
// Input logprobs arrive once (during prefill). A later non-empty value
// replaces the earlier one, matching the existing native HTTP fold.
if !delta_extras.in_lp_val.is_empty() {
extras.in_lp_val = delta_extras.in_lp_val.clone();
extras.in_lp_idx = delta_extras.in_lp_idx.clone();
extras.in_lp_txt = delta_extras.in_lp_txt.clone();
}
if !delta_extras.in_top_lens.is_empty() {
extras.in_top_val = delta_extras.in_top_val.clone();
extras.in_top_idx = delta_extras.in_top_idx.clone();
extras.in_top_lens = delta_extras.in_top_lens.clone();
extras.in_top_txt = delta_extras.in_top_txt.clone();
}
if !delta_extras.in_tid_lens.is_empty() {
extras.in_tid_val = delta_extras.in_tid_val.clone();
extras.in_tid_idx = delta_extras.in_tid_idx.clone();
extras.in_tid_lens = delta_extras.in_tid_lens.clone();
extras.in_tid_txt = delta_extras.in_tid_txt.clone();
}
// Hidden states are non-cumulative: the latest complete set wins.
if !delta_extras.hidden_lens.is_empty() {
extras.hidden_val = delta_extras.hidden_val.clone();
extras.hidden_lens = delta_extras.hidden_lens.clone();
}
}
}
/// Semantic result of a deep core health check.
///
/// Expected lifecycle states are values rather than transport errors so each
/// adapter can render them according to its own protocol.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum HealthStatus {
Healthy,
NotReady,
Stalled,
}
/// A semantic generation event before any transport-specific framing.
#[derive(Debug)]
pub(crate) enum CoreEvent {
/// One incremental model-output update.
Delta(CoreOutput),
/// The final model-output update for this request.
Finished(CoreOutput),
/// A request-scoped failure. Other requests in a batch may continue.
Failed(CoreError),
}
impl CoreEvent {
pub(crate) fn is_terminal(&self) -> bool {
matches!(self, Self::Finished(_) | Self::Failed(_))
}
}
impl CoreOutput {
/// One `/generate` frame. `index` is the batch position (batch form only);
/// `e2e_latency` rides the terminal frame only.
///
/// Consumes the output so `text` and `output_ids` move onto the wire: a
/// delta or terminal frame never copies them. A cumulative intermediate
/// frame is the one caller that must clone first, since the accumulated
/// output has to stay behind for the next frame.
pub(crate) fn frame(
self,
id: &str,
index: Option<u32>,
e2e_latency: Option<f64>,
) -> api::GenerateResponse {
let meta_info = Some(self.meta_info(id, e2e_latency));
api::GenerateResponse {
text: self.text,
meta_info,
// Omitted while empty (a text-only frame).
output_ids: (!self.token_ids.is_empty()).then_some(api::TokenIds {
ids: self.token_ids,
}),
index,
}
}
pub(crate) fn meta_info(&self, id: &str, e2e_latency: Option<f64>) -> api::GenerateMetaInfo {
let mut meta = api::GenerateMetaInfo {
id: id.to_owned(),
prompt_tokens: self.prompt_tokens,
completion_tokens: self.completion_tokens,
finish_reason: self.finish_reason.as_ref().map(finish_reason),
e2e_latency,
..Default::default()
};
let Some(extras) = self.extras.as_deref() else {
return meta;
};
// Python always sets input and output token logprobs together, including
// the empty input list emitted by a disaggregated decode node.
if !extras.out_lp_val.is_empty() || !extras.in_lp_val.is_empty() {
meta.output_token_logprobs = Some(logprob_entries(
&extras.out_lp_val,
&extras.out_lp_idx,
opt_texts(&extras.out_lp_txt),
));
meta.input_token_logprobs = Some(logprob_entries(
&extras.in_lp_val,
&extras.in_lp_idx,
opt_texts(&extras.in_lp_txt),
));
}
if !extras.out_top_lens.is_empty() {
meta.output_top_logprobs = ragged_logprob_rows(
&extras.out_top_val,
&extras.out_top_idx,
&extras.out_top_lens,
opt_texts(&extras.out_top_txt),
);
}
if !extras.in_top_lens.is_empty() {
meta.input_top_logprobs = ragged_logprob_rows(
&extras.in_top_val,
&extras.in_top_idx,
&extras.in_top_lens,
opt_texts(&extras.in_top_txt),
);
}
if !extras.out_tid_lens.is_empty() {
meta.output_token_ids_logprobs = ragged_logprob_rows(
&extras.out_tid_val,
&extras.out_tid_idx,
&extras.out_tid_lens,
opt_texts(&extras.out_tid_txt),
);
}
if !extras.in_tid_lens.is_empty() {
meta.input_token_ids_logprobs = ragged_logprob_rows(
&extras.in_tid_val,
&extras.in_tid_idx,
&extras.in_tid_lens,
opt_texts(&extras.in_tid_txt),
);
}
if !extras.hidden_lens.is_empty() {
meta.hidden_states = hidden_state_rows(&extras.hidden_val, &extras.hidden_lens);
}
meta
}
}
/// A decoded-text column becomes the entries' text source only when populated.
fn opt_texts(texts: &[String]) -> Option<&[String]> {
(!texts.is_empty()).then_some(texts)
}
/// One `[logprob, token_id, text]` entry.
fn logprob_entry(value: f32, token_id: i32, text: Option<&String>) -> api::LogprobEntry {
api::LogprobEntry {
// The scheduler's NaN sentinel means "no logprob at this position".
logprob: (!value.is_nan()).then(|| f64::from(value)),
token_id: i64::from(token_id),
text: text.cloned(),
}
}
fn logprob_entries(
values: &[f32],
token_ids: &[i32],
texts: Option<&[String]>,
) -> api::LogprobEntries {
let entries = values
.iter()
.zip(token_ids)
.enumerate()
.map(|(j, (&value, &token_id))| {
logprob_entry(value, token_id, texts.and_then(|texts| texts.get(j)))
})
.collect();
api::LogprobEntries { entries }
}
/// Ragged top-k / requested-token rows: one per position, null where that
/// position's length is zero.
fn ragged_logprob_rows(
values: &[f32],
token_ids: &[i32],
lengths: &[u32],
texts: Option<&[String]>,
) -> Vec<api::NullableTopLogprobs> {
let mut rows = Vec::with_capacity(lengths.len());
let mut offset = 0usize;
for &length in lengths {
let length = length as usize;
let row = (length > 0).then(|| {
// Bounds checks turn malformed parallel columns into a shortened
// row instead of panicking a transport worker.
let entries = (offset..offset + length)
.filter_map(|j| {
Some(logprob_entry(
*values.get(j)?,
*token_ids.get(j)?,
texts.and_then(|texts| texts.get(j)),
))
})
.collect();
api::TopLogprobRow { entries }
});
rows.push(api::NullableTopLogprobs { row });
offset += length;
}
rows
}
/// Reshape flat hidden-state values and per-row lengths into rows.
fn hidden_state_rows(values: &[f32], lengths: &[u32]) -> Vec<api::HiddenStateRow> {
let mut rows = Vec::with_capacity(lengths.len());
let mut offset = 0usize;
for &length in lengths {
let length = length as usize;
let values = values
.get(offset..offset + length)
.unwrap_or(&[])
.iter()
.copied()
.map(f64::from)
.collect();
rows.push(api::HiddenStateRow { values });
offset += length;
}
rows
}
fn finish_reason(reason: &FinishReason) -> api::FinishReason {
use api::finish_reason::Kind;
let kind = match reason {
FinishReason::Known(FinishKind::Stop { matched }) => Kind::Stop(api::FinishStop {
matched: matched.as_ref().map(matched_value),
}),
FinishReason::Known(FinishKind::Length { length }) => {
Kind::Length(api::FinishLength { length: *length })
}
FinishReason::Known(FinishKind::Abort(abort)) => Kind::Abort(Box::new(api::FinishAbort {
message: abort.message.clone(),
status_code: abort.status_code.map(u32::from),
err_type: abort.err_type.clone(),
})),
// The passthrough arm keeps the raw map as JSON text, the proto's own
// encoding for a `type` this build does not know.
FinishReason::Unknown(fields) => {
Kind::Unknown(serde_json::to_string(fields.as_ref()).unwrap_or_default())
}
};
api::FinishReason { kind: Some(kind) }
}
fn matched_value(matched: &Matched) -> api::Matched {
use api::matched::Value;
let value = match matched {
Matched::Token(token) => Value::Token(*token),
Matched::Str(text) => Value::Str(text.clone()),
Matched::Tokens(ids) => Value::Tokens(api::MatchedTokens { ids: ids.clone() }),
};
api::Matched { value: Some(value) }
}
#[cfg(test)]
mod tests {
use super::*;
use crate::message::response::ChunkExtras;
fn json<T: serde::Serialize>(value: &T) -> serde_json::Value {
serde_json::to_value(value).unwrap()
}
/// Python parity pinned on the typed shape: a NaN logprob is a null slot,
/// input logprobs are set together with output ones, and a zero-length
/// top-k position is a null row rather than an empty one.
#[test]
fn meta_info_keeps_the_native_null_conventions() {
let extras = ChunkExtras {
out_lp_val: vec![f32::NAN, -0.5],
out_lp_idx: vec![7, 8],
out_top_val: vec![-0.1],
out_top_idx: vec![9],
out_top_lens: vec![0, 1],
..Default::default()
};
let output = CoreOutput {
extras: Some(Box::new(extras)),
..Default::default()
};
let meta = output.meta_info("rid", None);
let entries = meta.output_token_logprobs.unwrap().entries;
assert_eq!(entries[0].logprob, None);
assert_eq!(entries[1].logprob, Some(f64::from(-0.5_f32)));
assert_eq!(entries[1].token_id, 8);
assert_eq!(meta.input_token_logprobs.unwrap().entries, []);
assert!(meta.output_top_logprobs[0].row.is_none());
let row = meta.output_top_logprobs[1].row.as_ref().unwrap();
assert_eq!(row.entries[0].token_id, 9);
assert!(meta.input_top_logprobs.is_empty());
}
/// The JSON the schema renders is SGLang's `[logprob, token_id, text]`
/// tuple list, with the text slot null unless `return_text_in_logprobs`
/// supplied a text column, and null positions in the ragged families.
#[test]
fn logprob_json_keeps_the_tuple_shape() {
let texts = vec!["a".to_string(), "b".to_string()];
assert_eq!(
json(&logprob_entries(&[-0.5, -1.5], &[10, 20], None)),
serde_json::json!([[-0.5f32, 10, null], [-1.5f32, 20, null]])
);
assert_eq!(
json(&logprob_entries(&[-0.5, -1.5], &[10, 20], Some(&texts))),
serde_json::json!([[-0.5f32, 10, "a"], [-1.5f32, 20, "b"]])
);
assert_eq!(
json(&ragged_logprob_rows(&[-0.3], &[9], &[0, 1], None)),
serde_json::json!([null, [[-0.3f32, 9, null]]])
);
// The NaN sentinel (the Python `None` logprob for the first prompt
// token) is a null logprob with its token id preserved.
assert_eq!(
json(&ragged_logprob_rows(&[f32::NAN], &[7], &[1], None)),
serde_json::json!([[[null, 7, null]]])
);
assert!(opt_texts(&[]).is_none());
assert_eq!(opt_texts(&texts), Some(texts.as_slice()));
}
/// A finish type this build does not know still reaches the client, as
/// the JSON text the proto's passthrough arm carries.
#[test]
fn unknown_finish_reason_passes_through_as_json_text() {
let reason: FinishReason =
serde_json::from_value(serde_json::json!({"type": "novel", "detail": 3})).unwrap();
let api::FinishReason {
kind: Some(api::finish_reason::Kind::Unknown(text)),
} = finish_reason(&reason)
else {
panic!("unknown type takes the passthrough arm");
};
assert_eq!(
serde_json::from_str::<serde_json::Value>(&text).unwrap(),
serde_json::json!({"type": "novel", "detail": 3})
);
}
}
@@ -1,10 +1,6 @@
//! Transport-neutral entry point into the Rust frontend pipeline.
//!
//! Wire adapters normalize their protocol into [`FrontendRequest`], submit it
//! through [`FrontendHandle`], and render semantic [`FrontendEvent`]s for their
//! own transport. This module owns shared preprocessing, runtime translation,
//! capabilities, and request lifetime; it deliberately knows nothing about
//! Axum, HTTP response shapes, Tonic, or protobuf.
//! The shared handle: submission with admission and cancellation ownership,
//! runtime-to-semantic event translation, and the typed control operations
//! (model info, server info, detokenize, health probe).
use std::collections::BTreeMap;
use std::sync::{
@@ -18,7 +14,7 @@ use tokio::sync::mpsc;
use crate::message::config::{DisaggregationMode, ServerArgs};
use crate::message::ids::Rid;
use crate::message::io_struct::{ControlRequest, GetInternalStateReq};
use crate::message::request::{Request, RequestKind};
use crate::message::request::{GenerateRequest, Request, RequestKind};
use crate::message::response::{ResponseItem, ResponseSink};
use crate::message::sampling::SamplingParams;
use crate::message::types::TokenIds;
@@ -27,32 +23,27 @@ use crate::tokenizer_manager::wiring::{AbortSource, RequestAdmission, TmEvent};
use crate::utils::error::Error;
use crate::utils::fsm::RequestState;
mod contract;
mod prefetch;
pub(crate) use contract::{
FrontendError, FrontendErrorKind, FrontendEvent, FrontendOutput, FrontendRequest, HealthStatus,
InternalState, ModelInfo, ServerInfo,
};
use super::{CoreError, CoreEvent, HealthStatus, prefetch};
use crate::message::info::{InternalState, ModelInfo, ServerInfo};
/// Sentinel host that makes the KV connector no-op. Parity with
/// `sglang.srt.disaggregation.utils.FAKE_BOOTSTRAP_HOST`.
const FAKE_BOOTSTRAP_HOST: &str = "2.2.2.2";
pub(crate) struct FrontendConfig {
pub(crate) struct CoreConfig {
pub(crate) response_capacity: usize,
pub(crate) response_activity: ActivityCounter,
pub(crate) startup_ready: bool,
pub(crate) is_disaggregation: bool,
pub(crate) mm_limits: BTreeMap<String, usize>,
pub(crate) metadata: FrontendMetadata,
pub(crate) metadata: CoreMetadata,
}
/// Immutable, transport-independent metadata retained by the frontend.
/// Immutable, transport-independent metadata retained by the core.
/// Keeping this snapshot narrow avoids making adapters or the shared handle
/// depend on the full launch-configuration object.
#[derive(Clone, Debug, Default)]
pub(crate) struct FrontendMetadata {
pub(crate) struct CoreMetadata {
model_path: String,
served_model_name: String,
tokenizer_path: String,
@@ -67,7 +58,7 @@ pub(crate) struct FrontendMetadata {
version: String,
}
impl From<&ServerArgs> for FrontendMetadata {
impl From<&ServerArgs> for CoreMetadata {
fn from(args: &ServerArgs) -> Self {
Self {
model_path: args.model_path.clone(),
@@ -86,16 +77,16 @@ impl From<&ServerArgs> for FrontendMetadata {
}
}
/// Cloneable capability for submitting work to the existing frontend runtime.
/// Cloneable capability for submitting work to the runtime pipeline.
///
/// Only the runtime constructs this handle. Protocol adapters receive clones;
/// they cannot reach the raw stage wiring directly.
#[derive(Clone)]
pub(crate) struct FrontendHandle {
inner: Arc<FrontendInner>,
pub(crate) struct CoreHandle {
inner: Arc<CoreInner>,
}
struct FrontendInner {
struct CoreInner {
intake_tx: flume::Sender<TmEvent>,
abort_tx: flume::Sender<AbortSource>,
response_capacity: usize,
@@ -103,17 +94,17 @@ struct FrontendInner {
startup_ready: AtomicBool,
is_disaggregation: bool,
mm_limits: BTreeMap<String, usize>,
metadata: FrontendMetadata,
metadata: CoreMetadata,
}
impl FrontendHandle {
impl CoreHandle {
pub(crate) fn new(
intake_tx: flume::Sender<TmEvent>,
abort_tx: flume::Sender<AbortSource>,
config: FrontendConfig,
config: CoreConfig,
) -> Self {
Self {
inner: Arc::new(FrontendInner {
inner: Arc::new(CoreInner {
intake_tx,
abort_tx,
response_capacity: config.response_capacity,
@@ -143,13 +134,13 @@ impl FrontendHandle {
pub(crate) async fn generate(
&self,
mut request: FrontendRequest,
) -> Result<FrontendCall, FrontendError> {
mut request: GenerateRequest,
) -> Result<CoreCall, CoreError> {
self.prepare_generations(std::slice::from_mut(&mut request))
.await?;
self.submit(RequestKind::Generate(Box::new(request)))
.await
.map(FrontendCall::new)
.map(CoreCall::new)
}
/// Preprocess and submit a group atomically from the adapter's perspective:
@@ -158,8 +149,8 @@ impl FrontendHandle {
/// the already-created calls aborts those accepted requests.
pub(crate) async fn generate_batch(
&self,
mut requests: Vec<FrontendRequest>,
) -> Result<Vec<FrontendCall>, FrontendError> {
mut requests: Vec<GenerateRequest>,
) -> Result<Vec<CoreCall>, CoreError> {
self.prepare_generations(&mut requests).await?;
let mut calls = Vec::with_capacity(requests.len());
@@ -167,19 +158,16 @@ impl FrontendHandle {
calls.push(
self.submit(RequestKind::Generate(Box::new(request)))
.await
.map(FrontendCall::new)?,
.map(CoreCall::new)?,
);
}
Ok(calls)
}
async fn prepare_generations(
&self,
requests: &mut [FrontendRequest],
) -> Result<(), FrontendError> {
async fn prepare_generations(&self, requests: &mut [GenerateRequest]) -> Result<(), CoreError> {
prefetch::prefetch_all(requests, &self.inner.mm_limits)
.await
.map_err(FrontendError::InvalidArgument)
.map_err(CoreError::InvalidArgument)
}
/// Return static model metadata without exposing the full launch config.
@@ -201,8 +189,8 @@ impl FrontendHandle {
/// Return public server metadata with the current scheduler metrics.
/// Raw control bytes and the scheduler's full launch-argument dump never
/// cross the frontend boundary.
pub(crate) async fn server_info(&self) -> Result<ServerInfo, FrontendError> {
/// cross the core boundary.
pub(crate) async fn server_info(&self) -> Result<ServerInfo, CoreError> {
let internal_state = self.internal_state().await?;
let metadata = &self.inner.metadata;
Ok(ServerInfo {
@@ -217,7 +205,7 @@ impl FrontendHandle {
})
}
async fn internal_state(&self) -> Result<InternalState, FrontendError> {
async fn internal_state(&self) -> Result<InternalState, CoreError> {
let bytes = self
.control(ControlRequest::GetInternalStateReq(
GetInternalStateReq::new(Rid::new().to_string()),
@@ -226,58 +214,54 @@ impl FrontendHandle {
rmp_serde::from_slice::<InternalStateEnvelope>(&bytes)
.map(|response| response.internal_state)
.map_err(|error| {
FrontendError::InvalidResponse(format!("invalid internal-state response: {error}"))
CoreError::InvalidResponse(format!("invalid internal-state response: {error}"))
})
}
async fn control(&self, request: ControlRequest) -> Result<bytes::Bytes, FrontendError> {
async fn control(&self, request: ControlRequest) -> Result<bytes::Bytes, CoreError> {
let mut call = self.submit(RequestKind::Control(Box::new(request))).await?;
match call.recv().await {
Some(ResponseItem::Control(bytes)) => Ok(bytes),
Some(ResponseItem::Error(error)) => Err(translate_runtime_error(error)),
Some(_) => Err(FrontendError::InvalidResponse(
Some(_) => Err(CoreError::InvalidResponse(
"unexpected generation output for control request".into(),
)),
None => Err(FrontendError::ResponseTruncated),
None => Err(CoreError::ResponseTruncated),
}
}
/// Decode token IDs into text without exposing the runtime's byte payload.
pub(crate) async fn detokenize(&self, token_ids: TokenIds) -> Result<String, FrontendError> {
pub(crate) async fn detokenize(&self, token_ids: TokenIds) -> Result<String, CoreError> {
let mut call = self.submit(RequestKind::Detokenize { token_ids }).await?;
let bytes = match call.recv().await {
Some(ResponseItem::Data(bytes)) => bytes,
Some(ResponseItem::Error(Error::Validation(message))) => {
return Err(FrontendError::InvalidArgument(message));
return Err(CoreError::InvalidArgument(message));
}
Some(ResponseItem::Error(error)) => return Err(translate_runtime_error(error)),
Some(_) => {
return Err(FrontendError::InvalidResponse(
return Err(CoreError::InvalidResponse(
"unexpected output for detokenize request".into(),
));
}
None => {
return Err(FrontendError::Internal("reply channel closed".into()));
return Err(CoreError::Internal("reply channel closed".into()));
}
};
String::from_utf8(bytes.to_vec()).map_err(|_| {
FrontendError::InvalidResponse("detokenized prompt is not valid UTF-8".into())
})
String::from_utf8(bytes.to_vec())
.map_err(|_| CoreError::InvalidResponse("detokenized prompt is not valid UTF-8".into()))
}
/// Confirm that scheduler output is moving. The response heartbeat is the
/// signal rather than this probe's own result, matching Python's
/// `last_receive_tstamp` behavior on a busy server.
pub(crate) async fn probe_health(
&self,
timeout: Duration,
) -> Result<HealthStatus, FrontendError> {
pub(crate) async fn probe_health(&self, timeout: Duration) -> Result<HealthStatus, CoreError> {
if !self.is_ready() {
return Ok(HealthStatus::NotReady);
}
let baseline = self.response_activity();
let probe = FrontendRequest {
let probe = GenerateRequest {
rid: Rid::new_health_check(),
input_ids: Some(vec![0]),
sampling_params: SamplingParams {
@@ -315,7 +299,7 @@ impl FrontendHandle {
/// The async send preserves TokenizerManager inbox backpressure. Once the
/// request is accepted, the returned call owns cancellation until it
/// observes a terminal response.
async fn submit(&self, kind: RequestKind) -> Result<RuntimeCall, FrontendError> {
async fn submit(&self, kind: RequestKind) -> Result<RuntimeCall, CoreError> {
let rid = request_rid(&kind);
let expected_response = ExpectedResponse::for_request(&kind);
let (response_tx, response_rx) = mpsc::channel(self.inner.response_capacity);
@@ -349,21 +333,21 @@ impl FrontendHandle {
// Disarm before returning the transport-neutral rejection.
call.in_flight = false;
tracing::error!(%rid, "tm inbox closed; request rejected");
return Err(FrontendError::Unavailable);
return Err(CoreError::Unavailable);
}
Ok(call)
}
}
fn translate_runtime_error(error: Error) -> FrontendError {
fn translate_runtime_error(error: Error) -> CoreError {
match error {
Error::Validation(message) => {
FrontendError::InvalidArgument(format!("validation failed: {message}"))
CoreError::InvalidArgument(format!("validation failed: {message}"))
}
Error::QueueFull => FrontendError::Overloaded("to_scheduler channel full".into()),
Error::Disconnected => FrontendError::Cancelled("client disconnected".into()),
error => FrontendError::Internal(error.to_string()),
Error::QueueFull => CoreError::Overloaded("to_scheduler channel full".into()),
Error::Disconnected => CoreError::Cancelled("client disconnected".into()),
error => CoreError::Internal(error.to_string()),
}
}
@@ -372,12 +356,12 @@ fn translate_runtime_error(error: Error) -> FrontendError {
/// Runtime response variants are translated here so adapters cannot depend on
/// scheduler/control encodings. Dropping an unfinished call retains the
/// cancellation guarantees owned by [`RuntimeCall`].
pub(crate) struct FrontendCall {
pub(crate) struct CoreCall {
runtime: RuntimeCall,
events_finished: bool,
}
impl FrontendCall {
impl CoreCall {
fn new(runtime: RuntimeCall) -> Self {
Self {
runtime,
@@ -393,14 +377,14 @@ impl FrontendCall {
/// Receive the next semantic event. A runtime channel that closes before a
/// terminal response is converted into one terminal failure event; adapters
/// never need to infer whether an empty runtime channel means success.
pub(crate) async fn recv(&mut self) -> Option<FrontendEvent> {
pub(crate) async fn recv(&mut self) -> Option<CoreEvent> {
if self.events_finished {
return None;
}
let event = match self.runtime.recv().await {
Some(item) => generation_event(item),
None => FrontendEvent::Failed(FrontendError::Internal(
None => CoreEvent::Failed(CoreError::Internal(
"response truncated before completion".into(),
)),
};
@@ -410,7 +394,7 @@ impl FrontendCall {
/// Append every semantic event that is ready without waiting. Channel
/// emptiness and closure remain private runtime details.
pub(crate) fn drain_ready(&mut self, events: &mut Vec<FrontendEvent>) {
pub(crate) fn drain_ready(&mut self, events: &mut Vec<CoreEvent>) {
while !self.events_finished {
let event = match self.runtime.try_recv() {
Ok(item) => generation_event(item),
@@ -446,39 +430,56 @@ impl FrontendCall {
}
}
fn generation_event(item: ResponseItem) -> FrontendEvent {
/// Await the next event from `call`, then drain whatever queued behind it (so the
/// caller can coalesce a backlog, as Python's `state.out_list` does), handing the
/// call back for `FuturesUnordered` to re-poll. An empty result means the semantic
/// stream was already exhausted. Shared by the HTTP and gRPC batch multiplexers.
pub(crate) async fn recv_indexed(
index: usize,
mut call: CoreCall,
) -> (usize, CoreCall, Vec<CoreEvent>) {
let mut items = Vec::new();
match call.recv().await {
Some(item) => items.push(item),
None => return (index, call, items), // already exhausted
}
call.drain_ready(&mut items);
(index, call, items)
}
fn generation_event(item: ResponseItem) -> CoreEvent {
match item {
ResponseItem::Frame(output) => FrontendEvent::Delta(output.into()),
ResponseItem::Frame(output) => CoreEvent::Delta(output.into()),
ResponseItem::Done(output) => {
if let Some((legacy_http_status, message)) = output
.finish_reason
.as_ref()
.and_then(|reason| reason.abort_status())
{
FrontendEvent::Failed(FrontendError::from_runtime_rejection(
CoreEvent::Failed(CoreError::from_runtime_rejection(
message.to_owned(),
legacy_http_status,
))
} else {
FrontendEvent::Finished(output.into())
CoreEvent::Finished(output.into())
}
}
ResponseItem::Error(error) => FrontendEvent::Failed(translate_runtime_error(error)),
ResponseItem::Control(_) | ResponseItem::Data(_) => FrontendEvent::Failed(
FrontendError::InvalidResponse("unexpected non-generation response".into()),
ResponseItem::Error(error) => CoreEvent::Failed(translate_runtime_error(error)),
ResponseItem::Control(_) | ResponseItem::Data(_) => CoreEvent::Failed(
CoreError::InvalidResponse("unexpected non-generation response".into()),
),
}
}
/// Private scheduler response envelope. This MessagePack shape is decoded at
/// the runtime boundary and deliberately is not part of the frontend contract.
/// the runtime boundary and deliberately is not part of the core contract.
#[derive(serde::Deserialize)]
struct InternalStateEnvelope {
#[serde(default)]
internal_state: InternalState,
}
/// Runtime-facing request guard kept private behind [`FrontendCall`].
/// Runtime-facing request guard kept private behind [`CoreCall`].
///
/// This type is the only owner of the raw response channel and admission/abort
/// capabilities. It is also used directly by typed unary operations such as
@@ -569,13 +570,14 @@ mod tests {
use bytes::Bytes;
use super::*;
use crate::api_server::core::{CoreErrorKind, CoreOutput};
use crate::message::io_struct::GetInternalStateReq;
use crate::message::multimodal::MmItem;
use crate::message::request::{GenerateRequest, MmData};
use crate::message::request::MmData;
use crate::message::response::ChunkEvent;
struct Harness {
handle: FrontendHandle,
handle: CoreHandle,
intake_rx: flume::Receiver<TmEvent>,
abort_rx: flume::Receiver<AbortSource>,
}
@@ -596,7 +598,7 @@ mod tests {
intake_tx: flume::Sender<TmEvent>,
abort_tx: flume::Sender<AbortSource>,
response_capacity: usize,
) -> FrontendHandle {
) -> CoreHandle {
configured_handle(
intake_tx,
abort_tx,
@@ -616,17 +618,17 @@ mod tests {
startup_ready: bool,
is_disaggregation: bool,
mm_limits: BTreeMap<String, usize>,
) -> FrontendHandle {
FrontendHandle::new(
) -> CoreHandle {
CoreHandle::new(
intake_tx,
abort_tx,
FrontendConfig {
CoreConfig {
response_capacity,
response_activity,
startup_ready,
is_disaggregation,
mm_limits,
metadata: FrontendMetadata::default(),
metadata: CoreMetadata::default(),
},
)
}
@@ -700,44 +702,44 @@ mod tests {
fn runtime_errors_become_transport_neutral_categories() {
assert!(matches!(
translate_runtime_error(Error::Validation("bad input".into())),
FrontendError::InvalidArgument(message)
CoreError::InvalidArgument(message)
if message == "validation failed: bad input"
));
assert!(matches!(
translate_runtime_error(Error::QueueFull),
FrontendError::Overloaded(message) if message == "to_scheduler channel full"
CoreError::Overloaded(message) if message == "to_scheduler channel full"
));
assert!(matches!(
translate_runtime_error(Error::Disconnected),
FrontendError::Cancelled(message) if message == "client disconnected"
CoreError::Cancelled(message) if message == "client disconnected"
));
assert!(matches!(
translate_runtime_error(Error::Internal("bug".into())),
FrontendError::Internal(message) if message == "internal error: bug"
CoreError::Internal(message) if message == "internal error: bug"
));
}
#[test]
fn scheduler_status_is_classified_once_for_non_http_adapters() {
for (status, expected) in [
(400, FrontendErrorKind::InvalidArgument),
(404, FrontendErrorKind::NotFound),
(408, FrontendErrorKind::DeadlineExceeded),
(412, FrontendErrorKind::FailedPrecondition),
(413, FrontendErrorKind::ResourceExhausted),
(429, FrontendErrorKind::ResourceExhausted),
(432, FrontendErrorKind::InvalidArgument),
(499, FrontendErrorKind::Cancelled),
(500, FrontendErrorKind::Internal),
(503, FrontendErrorKind::Unavailable),
(504, FrontendErrorKind::DeadlineExceeded),
(400, CoreErrorKind::InvalidArgument),
(404, CoreErrorKind::NotFound),
(408, CoreErrorKind::DeadlineExceeded),
(412, CoreErrorKind::FailedPrecondition),
(413, CoreErrorKind::ResourceExhausted),
(429, CoreErrorKind::ResourceExhausted),
(432, CoreErrorKind::InvalidArgument),
(499, CoreErrorKind::Cancelled),
(500, CoreErrorKind::Internal),
(503, CoreErrorKind::Unavailable),
(504, CoreErrorKind::DeadlineExceeded),
] {
let error = FrontendError::from_runtime_rejection("rejected".into(), status);
let error = CoreError::from_runtime_rejection("rejected".into(), status);
assert_eq!(error.kind(), expected, "legacy status {status}");
assert!(matches!(
error,
FrontendError::RuntimeRejected {
CoreError::RuntimeRejected {
legacy_http_status,
..
} if legacy_http_status == status
@@ -777,11 +779,11 @@ mod tests {
assert!(matches!(
call.recv().await,
Some(FrontendEvent::Delta(FrontendOutput { text, .. })) if text == "delta"
Some(CoreEvent::Delta(CoreOutput { text, .. })) if text == "delta"
));
assert!(matches!(
call.recv().await,
Some(FrontendEvent::Finished(FrontendOutput { text, .. })) if text == "final"
Some(CoreEvent::Finished(CoreOutput { text, .. })) if text == "final"
));
assert!(call.recv().await.is_none());
drop(call);
@@ -816,7 +818,7 @@ mod tests {
assert!(matches!(
call.recv().await,
Some(FrontendEvent::Finished(FrontendOutput {
Some(CoreEvent::Finished(CoreOutput {
finish_reason: Some(_),
..
}))
@@ -855,7 +857,7 @@ mod tests {
drop(intake_rx);
assert!(matches!(
submitter.await.unwrap(),
Err(FrontendError::Unavailable)
Err(CoreError::Unavailable)
));
assert!(matches!(
@@ -891,7 +893,7 @@ mod tests {
assert!(matches!(
handle.generate(request).await,
Err(FrontendError::InvalidArgument(message))
Err(CoreError::InvalidArgument(message))
if message == "Image count 2 exceeds limit 1 per request."
));
assert!(intake_rx.try_recv().is_err());
@@ -1,4 +1,4 @@
//! Resolve I/O-backed media sources on the async frontend runtime, before MM dispatch.
//! Resolve I/O-backed media sources on the async API runtime, before MM dispatch.
//!
//! The MM worker pool is fixed, core-pinned CPU capacity: a slow image host — or
//! a file on a hanging network mount — must never occupy it, and a request's
-697
View File
@@ -1,697 +0,0 @@
//! Frame shaping for the native `/generate` protocol: the cumulative
//! [`OutputAccumulator`] plus the functions that render [`FrontendOutput`]s /
//! accumulated state into wire JSON (`meta_info`, logprob tuples, error and
//! abort frames). No HTTP here — the sibling `native_api` module owns the handlers
//! and streams; it calls these per frame.
use crate::frontend::FrontendOutput;
#[cfg(test)]
use crate::message::response::ChunkExtras;
use crate::native_generation::{
hidden_states_rows, lp_value, meta_info_value, opt_texts, text_slot,
};
#[cfg(test)]
use crate::native_generation::{logprob_tuples, ragged_logprob_tuples};
/// Append a flat family's `[logprob, token_id, text]` tuples to `dst`, comma
/// separated and WITHOUT the enclosing brackets, so a cumulative frame can
/// concatenate each delta instead of re-rendering every accumulated position.
///
/// Byte-identical to [`logprob_tuples`]'s serialization: `serde_json` writes an
/// array as `[`, elements joined by `,`, `]` with no spaces, and each element here
/// is rendered by the same `Value` Display.
fn push_logprob_tuples(dst: &mut String, vals: &[f32], idxs: &[i32], texts: Option<&[String]>) {
use std::fmt::Write;
for (j, (&v, &tid)) in vals.iter().zip(idxs.iter()).enumerate() {
if !dst.is_empty() {
dst.push(',');
}
let _ = write!(dst, "[{},{tid},{}]", lp_value(v), text_slot(texts, j));
}
}
/// Ragged counterpart of [`push_logprob_tuples`] — one entry per position, `null`
/// where `lens[p] == 0`. Mirrors [`ragged_logprob_tuples`] including its
/// bounds-checked skip, so a header whose `lens` run past the value buffer yields
/// the same (shortened) row rather than panicking.
fn push_ragged_tuples(
dst: &mut String,
vals: &[f32],
idxs: &[i32],
lens: &[u32],
texts: Option<&[String]>,
) {
use std::fmt::Write;
let mut off = 0usize;
for &l in lens {
let l = l as usize;
if !dst.is_empty() {
dst.push(',');
}
if l == 0 {
dst.push_str("null");
} else {
dst.push('[');
let mut first = true;
for j in off..off + l {
let (Some(&v), Some(&tid)) = (vals.get(j), idxs.get(j)) else {
continue;
};
if !first {
dst.push(',');
}
first = false;
let _ = write!(dst, "[{},{tid},{}]", lp_value(v), text_slot(texts, j));
}
dst.push(']');
}
off += l;
}
}
/// Render one set-once family (they ride the prefill or the final chunk) into a
/// standalone JSON array, so it is serialized when it arrives rather than on every
/// subsequent frame.
fn ragged_array_json(vals: &[f32], idxs: &[i32], lens: &[u32], texts: Option<&[String]>) -> String {
let mut body = String::new();
push_ragged_tuples(&mut body, vals, idxs, lens, texts);
format!("[{body}]")
}
/// Format a decoded [`FrontendOutput`] as one SGLang `/generate` frame's JSON. `rid`
/// (response `meta_info.id`) is passed as a string; the event's numeric `rid` is
/// just the shard routing key.
pub(super) fn frame_value(out: &FrontendOutput, rid: &str) -> serde_json::Value {
let mut v = serde_json::json!({
"text": out.text,
"meta_info": meta_info_value(out, rid),
});
if !out.token_ids.is_empty() {
v["output_ids"] = serde_json::json!(out.token_ids);
}
v
}
/// Cumulative frame JSON from the accumulator's memoized parts — O(1) in the
/// accumulated length, where rebuilding the `Value` is O(T) per frame and so O(T²)
/// per request.
///
/// Encodes the same JSON document as `frame_value(..).to_string()` — same keys,
/// same values, same escaping — pinned for both the plain and the logprob shapes
/// by `cumulative_frame_json_matches_serde`.
pub(super) fn cumulative_frame_json(
acc: &OutputAccumulator,
rid: &str,
index: Option<usize>,
) -> Option<String> {
use std::fmt::Write;
if acc.extras_memo_broken {
return None;
}
let o = acc.snapshot();
// Through `Value` rather than `to_string` on the struct: it is the same
// encoder the slow path runs the finish reason through, so any representation
// quirk is reproduced instead of re-derived.
let finish = serde_json::to_value(&o.finish_reason).ok()?.to_string();
// Alphabetical by convention only — a stable order that is easy to extend and
// diff.
let mut m = String::new();
let _ = write!(m, "{{\"completion_tokens\":{}", o.completion_tokens);
let _ = write!(m, ",\"finish_reason\":{finish}");
if let Some(h) = &acc.hidden_json {
let _ = write!(m, ",\"hidden_states\":{h}");
}
let _ = write!(m, ",\"id\":{}", serde_json::Value::String(rid.to_string()));
if let Some(v) = &acc.in_tid_json {
let _ = write!(m, ",\"input_token_ids_logprobs\":{v}");
}
// Input+output token logprobs are emitted as a PAIR whenever either side has
// data (empty list included), matching `frame_value` byte for byte — see the
// PD-router rationale there.
let lp_pair = acc.in_lp_json.is_some() || !acc.out_lp_json.is_empty();
if lp_pair {
let v = acc.in_lp_json.as_deref().unwrap_or("[]");
let _ = write!(m, ",\"input_token_logprobs\":{v}");
}
if let Some(v) = &acc.in_top_json {
let _ = write!(m, ",\"input_top_logprobs\":{v}");
}
// The `Value` path keys these off the source columns being non-empty; an empty
// source renders to an empty body, so the two guards coincide.
if !acc.out_tid_json.is_empty() {
let _ = write!(m, ",\"output_token_ids_logprobs\":[{}]", acc.out_tid_json);
}
if lp_pair {
let _ = write!(m, ",\"output_token_logprobs\":[{}]", acc.out_lp_json);
}
if !acc.out_top_json.is_empty() {
let _ = write!(m, ",\"output_top_logprobs\":[{}]", acc.out_top_json);
}
let _ = write!(m, ",\"prompt_tokens\":{}}}", o.prompt_tokens);
let mut s = String::with_capacity(acc.text_json.len() + acc.ids_json.len() + m.len() + 40);
s.push('{');
if let Some(i) = index {
let _ = write!(s, "\"index\":{i},");
}
s.push_str("\"meta_info\":");
s.push_str(&m);
if !acc.ids_json.is_empty() {
s.push_str(",\"output_ids\":[");
s.push_str(&acc.ids_json);
s.push(']');
}
s.push_str(",\"text\":\"");
s.push_str(&acc.text_json);
s.push_str("\"}");
Some(s)
}
/// Attach the batch `index` (batch streams only) and render to the SSE `data` text.
pub(super) fn tag_value(mut v: serde_json::Value, index: Option<usize>) -> String {
if let Some(i) = index {
v["index"] = serde_json::json!(i);
}
v.to_string()
}
/// One streaming frame's JSON: cumulative ignores `delta`, incremental ships it.
pub(super) fn stream_frame_string(
delta: FrontendOutput,
acc: &OutputAccumulator,
incremental: bool,
rid_str: &str,
index: Option<usize>,
) -> String {
if !incremental {
return cumulative_frame_string(acc, rid_str, index);
}
tag_value(stream_frame_value(delta, acc, true, rid_str), index)
}
/// A cumulative frame's JSON, built purely from the accumulator (which is why a
/// backlog can coalesce to its last); falls back to the `Value` builder on extras.
pub(super) fn cumulative_frame_string(
acc: &OutputAccumulator,
rid_str: &str,
index: Option<usize>,
) -> String {
cumulative_frame_json(acc, rid_str, index)
.unwrap_or_else(|| tag_value(frame_value(acc.snapshot(), rid_str), index))
}
/// Format one streaming frame: the accumulator's cumulative view (default), or this
/// step's delta with the cumulative token count in `meta_info` (matching Python).
pub(super) fn stream_frame_value(
delta: FrontendOutput,
acc: &OutputAccumulator,
incremental: bool,
rid_str: &str,
) -> serde_json::Value {
if incremental {
let mut d = delta;
d.completion_tokens = acc.snapshot().completion_tokens;
frame_value(&d, rid_str)
} else {
frame_value(acc.snapshot(), rid_str)
}
}
/// Folds per-chunk [`FrontendOutput`] deltas into a cumulative view — used by the drain
/// loops needing cumulative output (every unary response + the cumulative SGLang
/// stream; OpenAI streaming forwards deltas and skips this). Holds a single
/// [`FrontendOutput`] so `snapshot` hands back a **borrow** per frame — no per-frame
/// clone of the growing buffers (that added O(T²) atop the wire's inherent O(T²)).
/// Shared with the [`openai`] submodule.
#[derive(Default)]
pub(super) struct OutputAccumulator {
out: FrontendOutput,
/// Serialized cumulative `output_ids` body (`"1,2,3"`, no brackets), appended per
/// delta so a frame memcpy's it instead of rebuilding the array — O(T), not O(T²).
ids_json: String,
/// JSON-escaped cumulative text, without the surrounding quotes. Escaping is
/// per-character, so `escape(a + b) == escape(a) + escape(b)` and deltas append.
text_json: String,
/// Memoized bodies (no enclosing brackets) of the three CUMULATIVE logprob
/// families, appended per delta — the same O(T) trick `ids_json` uses, extended
/// to the families that made a cumulative stream with logprobs O(T²). Cumulative
/// is SGLang's default, so that path re-rendered every accumulated position on
/// every frame: measured 117 ms for one 500-token top-5 request, versus 1.2 ms
/// incremental.
out_lp_json: String,
out_top_json: String,
out_tid_json: String,
/// Set-once families — they ride the prefill or the final chunk, so they are
/// rendered when they arrive rather than on every frame after.
in_lp_json: Option<String>,
in_top_json: Option<String>,
in_tid_json: Option<String>,
hidden_json: Option<String>,
/// Set once a family's text column falls out of lockstep with its values, at
/// which point the memo is abandoned for the `Value` path.
///
/// Appending per delta assumes `text_slot(accumulated, global_j)` equals
/// `text_slot(delta, local_j)`, which holds only while every delta supplies
/// either a text per value or none at all. That is what a real request does —
/// `return_text_in_logprobs` is per-request, so the detok shard fills `*_txt`
/// for all deltas or none — but a mixed sequence would silently diverge from
/// `frame_value`, so it is detected rather than assumed.
extras_memo_broken: bool,
}
/// Append `s` JSON-escaped (no surrounding quotes) — `serde_json` quotes it, and the
/// quotes are the first and last bytes of a string encoding.
fn push_escaped(dst: &mut String, s: &str) {
if s.is_empty() {
return;
}
let quoted = serde_json::to_string(s).expect("str-to-json should never fail");
dst.push_str(&quoted[1..quoted.len() - 1]);
}
impl OutputAccumulator {
/// Fold one delta frame in. Output families concatenate; input families and
/// hidden states are set-once / last-writer-wins (they ride the prefill/final
/// chunk), matching the Python `meta_info` assignment.
pub(super) fn fold(&mut self, d: &FrontendOutput) {
use std::fmt::Write;
// Grow the memoized serializations alongside the raw cumulative buffers.
push_escaped(&mut self.text_json, &d.text);
for &id in &d.token_ids {
if !self.ids_json.is_empty() {
self.ids_json.push(',');
}
let _ = write!(self.ids_json, "{id}");
}
let Some(de) = d.extras.as_deref() else {
self.out.append_delta(d);
return;
};
// Append THIS delta's tuples, indexed within the delta — equivalent to
// indexing the accumulated arrays only while texts stay in lockstep, which
// the guard below verifies.
push_logprob_tuples(
&mut self.out_lp_json,
&de.out_lp_val,
&de.out_lp_idx,
opt_texts(&de.out_lp_txt),
);
push_ragged_tuples(
&mut self.out_top_json,
&de.out_top_val,
&de.out_top_idx,
&de.out_top_lens,
opt_texts(&de.out_top_txt),
);
push_ragged_tuples(
&mut self.out_tid_json,
&de.out_tid_val,
&de.out_tid_idx,
&de.out_tid_lens,
opt_texts(&de.out_tid_txt),
);
if !de.in_lp_val.is_empty() {
let mut body = String::new();
push_logprob_tuples(
&mut body,
&de.in_lp_val,
&de.in_lp_idx,
opt_texts(&de.in_lp_txt),
);
self.in_lp_json = Some(format!("[{body}]"));
}
// Input families ride once (prefill); `lens` non-empty marks their arrival.
if !de.in_top_lens.is_empty() {
self.in_top_json = Some(ragged_array_json(
&de.in_top_val,
&de.in_top_idx,
&de.in_top_lens,
opt_texts(&de.in_top_txt),
));
}
if !de.in_tid_lens.is_empty() {
self.in_tid_json = Some(ragged_array_json(
&de.in_tid_val,
&de.in_tid_idx,
&de.in_tid_lens,
opt_texts(&de.in_tid_txt),
));
}
// Hidden states are non-cumulative: the latest non-empty set wins.
if !de.hidden_lens.is_empty() {
self.hidden_json =
Some(hidden_states_rows(&de.hidden_val, &de.hidden_lens).to_string());
}
self.out.append_delta(d);
let oe = self
.out
.extras
.as_deref()
.expect("a delta with extras creates cumulative extras");
let lockstep = |txt: &Vec<String>, val: &Vec<f32>| txt.is_empty() || txt.len() == val.len();
if !lockstep(&oe.out_lp_txt, &oe.out_lp_val)
|| !lockstep(&oe.out_top_txt, &oe.out_top_val)
|| !lockstep(&oe.out_tid_txt, &oe.out_tid_val)
{
self.extras_memo_broken = true;
}
}
/// Borrow the cumulative output for an intermediate streaming frame.
pub(super) fn snapshot(&self) -> &FrontendOutput {
&self.out
}
/// Consume into the final cumulative output.
pub(super) fn into_output(self) -> FrontendOutput {
self.out
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn flat_logprob_tuples_shape() {
let v = logprob_tuples(&[-0.5, -1.5], &[10, 20], None);
assert_eq!(
v,
serde_json::json!([
[-0.5f32, 10, serde_json::Value::Null],
[-1.5f32, 20, serde_json::Value::Null]
])
);
}
/// With a text buffer, the tuple's third slot carries the decoded token.
#[test]
fn flat_logprob_tuples_with_text() {
let texts = vec!["a".to_string(), "b".to_string()];
let v = logprob_tuples(&[-0.5, -1.5], &[10, 20], Some(&texts));
assert_eq!(
v,
serde_json::json!([[-0.5f32, 10, "a"], [-1.5f32, 20, "b"]])
);
}
/// Ragged reshape restores null positions (len 0) — mirrors
/// detokenize_top_logprobs_tokens emitting None for empty positions.
#[test]
fn ragged_logprob_tuples_restores_null_positions() {
// 2 positions: first null (len 0), second k=1.
let v = ragged_logprob_tuples(&[-0.3], &[9], &[0, 1], None);
assert_eq!(
v,
serde_json::json!([
serde_json::Value::Null,
[[-0.3f32, 9, serde_json::Value::Null]]
])
);
}
/// The `NaN` sentinel (the Python `None` logprob for the first prompt token)
/// becomes a JSON `null` logprob, while its token id in the parallel `idx`
/// column is preserved. Guards the scheduler-killing prompt-logprob crash.
#[test]
fn nan_sentinel_becomes_null_logprob() {
// Flat (input/output logprobs): first value absent, second present.
let flat = logprob_tuples(&[f32::NAN, -0.5], &[10, 20], None);
assert_eq!(
flat,
serde_json::json!([
[serde_json::Value::Null, 10, serde_json::Value::Null],
[-0.5f32, 20, serde_json::Value::Null],
])
);
// Ragged (top-k / token-ids logprobs): a NaN inside a position → null.
let ragged = ragged_logprob_tuples(&[f32::NAN], &[7], &[1], None);
assert_eq!(
ragged,
serde_json::json!([[[serde_json::Value::Null, 7, serde_json::Value::Null]]])
);
}
/// End-to-end: a `FrontendOutput` carrying a prompt-logprob request (first input
/// logprob is the `NaN` sentinel) formats without panicking and emits
/// `input_token_logprobs` with a leading `[null, token_id, text]`.
#[test]
fn prompt_logprob_frame_emits_null_first() {
let out = FrontendOutput {
extras: Some(Box::new(ChunkExtras {
in_lp_val: vec![f32::NAN, -0.5],
in_lp_idx: vec![10, 20],
in_lp_txt: vec!["<s>".into(), "hi".into()],
..Default::default()
})),
..Default::default()
};
let frame = frame_value(&out, "1");
assert_eq!(
frame["meta_info"]["input_token_logprobs"],
serde_json::json!([[serde_json::Value::Null, 10, "<s>"], [-0.5f32, 20, "hi"]])
);
}
/// The accumulator folds deltas cumulatively and `snapshot` borrows the
/// running state (no per-frame clone); `into_output` moves the same state.
#[test]
fn accumulator_snapshot_is_cumulative() {
let mut acc = OutputAccumulator::default();
acc.fold(&FrontendOutput {
text: "he".into(),
token_ids: vec![1, 2],
completion_tokens: 2,
..Default::default()
});
{
let s = acc.snapshot();
assert_eq!(s.text, "he");
assert_eq!(s.token_ids, vec![1, 2]);
}
acc.fold(&FrontendOutput {
text: "llo".into(),
token_ids: vec![3],
completion_tokens: 1,
..Default::default()
});
{
let s = acc.snapshot();
assert_eq!(s.text, "hello"); // cumulative
assert_eq!(s.token_ids, vec![1, 2, 3]);
assert_eq!(s.completion_tokens, 3);
}
let out = acc.into_output();
assert_eq!(out.text, "hello");
}
/// A populated text column (decoded on the detok shard) → `Some`; empty
/// (`return_text_in_logprobs` off) → `None` → null text slots.
#[test]
fn opt_texts_gates_on_population() {
assert!(opt_texts(&[]).is_none());
let t = vec!["x".to_string()];
assert_eq!(opt_texts(&t), Some(t.as_slice()));
}
/// Parse a frame so the fast and slow paths can be compared as documents.
fn as_json(frame: &str) -> serde_json::Value {
serde_json::from_str(frame).expect("a frame must be valid JSON")
}
/// The memoized cumulative fast path must emit the **same JSON document** as the
/// `serde_json::Value` builder it replaces — same keys, same values, same
/// escaping. Covers unicode and control chars, an empty-ids first frame, a
/// finish_reason, and the batch `index`. Guards the O(T) rewrite of the O(T²)
/// `output_ids` serialization.
#[test]
fn cumulative_frame_json_matches_serde() {
let deltas = [
FrontendOutput {
text: String::new(),
token_ids: vec![],
completion_tokens: 0,
prompt_tokens: 128,
..Default::default()
},
FrontendOutput {
text: "He\"llo\n\t".into(),
token_ids: vec![1000],
completion_tokens: 1,
prompt_tokens: 128,
..Default::default()
},
FrontendOutput {
text: " 世界 🌍 \\".into(),
token_ids: vec![-2, 3],
completion_tokens: 2,
prompt_tokens: 128,
..Default::default()
},
FrontendOutput {
text: "!".into(),
token_ids: vec![9],
completion_tokens: 1,
prompt_tokens: 128,
finish_reason: serde_json::from_value(
serde_json::json!({"type": "stop", "matched": 9}),
)
.expect("finish reason must parse"),
..Default::default()
},
];
for index in [None, Some(3usize)] {
let mut acc = OutputAccumulator::default();
for d in &deltas {
acc.fold(d);
let fast = cumulative_frame_json(&acc, "7", index).expect("no extras → fast path");
let slow = tag_value(frame_value(acc.snapshot(), "7"), index);
println!("fast={fast:?}");
println!("slow={slow:?}");
assert_eq!(
as_json(&fast),
as_json(&slow),
"index={index:?} text={:?}",
acc.snapshot().text
);
}
}
}
/// The same equivalence, for the shape that made cumulative streaming O(T²):
/// every logprob family at once, across several deltas, with and without
/// `return_text_in_logprobs` texts and with a null ragged position.
#[test]
fn cumulative_frame_json_matches_serde_with_logprobs() {
for with_texts in [false, true] {
let txt = |v: &[&str]| -> Vec<String> {
if with_texts {
v.iter().map(|s| (*s).to_string()).collect()
} else {
Vec::new()
}
};
let deltas = [
// Prefill: the set-once input families and a null top-k position.
FrontendOutput {
prompt_tokens: 4,
extras: Some(Box::new(ChunkExtras {
in_lp_val: vec![f32::NAN, -1.5],
in_lp_idx: vec![10, 11],
in_lp_txt: txt(&["a", "b"]),
in_top_val: vec![-0.25],
in_top_idx: vec![12],
in_top_lens: vec![0, 1],
in_top_txt: txt(&["c"]),
in_tid_val: vec![-2.0],
in_tid_idx: vec![13],
in_tid_lens: vec![1],
in_tid_txt: txt(&["d"]),
..Default::default()
})),
..Default::default()
},
FrontendOutput {
text: "He\"llo".into(),
token_ids: vec![100],
completion_tokens: 1,
prompt_tokens: 4,
extras: Some(Box::new(ChunkExtras {
out_lp_val: vec![-0.5],
out_lp_idx: vec![100],
out_lp_txt: txt(&["He\"llo"]),
out_top_val: vec![-0.5, -3.0],
out_top_idx: vec![100, 7],
out_top_lens: vec![2],
out_top_txt: txt(&["He\"llo", "x"]),
out_tid_val: vec![-0.5],
out_tid_idx: vec![100],
out_tid_lens: vec![1],
out_tid_txt: txt(&["He\"llo"]),
..Default::default()
})),
..Default::default()
},
FrontendOutput {
text: " 世界".into(),
token_ids: vec![-2, 3],
completion_tokens: 2,
prompt_tokens: 4,
finish_reason: serde_json::from_value(
serde_json::json!({"type": "stop", "matched": 3}),
)
.expect("finish reason must parse"),
extras: Some(Box::new(ChunkExtras {
out_lp_val: vec![f32::NAN, -0.125],
out_lp_idx: vec![-2, 3],
out_lp_txt: txt(&[" 世", "界"]),
// A zero-length position must render as `null`, not `[]`.
out_top_val: vec![-0.125],
out_top_idx: vec![3],
out_top_lens: vec![0, 1],
out_top_txt: txt(&["界"]),
out_tid_val: vec![],
out_tid_idx: vec![],
out_tid_lens: vec![0, 0],
out_tid_txt: txt(&[]),
hidden_val: vec![0.5, -0.25, 1.0],
hidden_lens: vec![2, 1],
..Default::default()
})),
},
];
for index in [None, Some(2usize)] {
let mut acc = OutputAccumulator::default();
for d in &deltas {
acc.fold(d);
let fast = cumulative_frame_json(&acc, "9", index)
.expect("the extras memo must stay valid for a well-formed request");
let slow = tag_value(frame_value(acc.snapshot(), "9"), index);
assert_eq!(
as_json(&fast),
as_json(&slow),
"with_texts={with_texts} index={index:?}"
);
}
}
}
}
/// A delta sequence that supplies texts for some values and not others breaks
/// the append-equivalence the memo rests on (`text_slot` is indexed globally,
/// so a gap shifts every later text). The accumulator must notice and defer to
/// the `Value` builder rather than emit a frame that disagrees with it.
#[test]
fn mismatched_logprob_texts_fall_back_to_the_value_path() {
let mut acc = OutputAccumulator::default();
acc.fold(&FrontendOutput {
extras: Some(Box::new(ChunkExtras {
out_lp_val: vec![-0.5],
out_lp_idx: vec![5],
..Default::default() // no texts
})),
..Default::default()
});
assert!(cumulative_frame_json(&acc, "1", None).is_some());
acc.fold(&FrontendOutput {
extras: Some(Box::new(ChunkExtras {
out_lp_val: vec![-0.25],
out_lp_idx: vec![6],
out_lp_txt: vec!["b".into()], // …now texts: out of lockstep
..Default::default()
})),
..Default::default()
});
assert!(
cumulative_frame_json(&acc, "1", None).is_none(),
"a text column out of lockstep must invalidate the memo"
);
}
}
+15
View File
@@ -0,0 +1,15 @@
//! Tonic adapter for the native `sglang.api.v1` service.
//!
//! The service is the gRPC rendering of the contract the native HTTP endpoints
//! serve: `Generate` is `/generate`, `HealthCheck` is `/health_generate`, and
//! `GetModelInfo` / `GetServerInfo` are their HTTP namesakes. Requests decode
//! through the shared `/generate` fan-out and responses stream through the
//! shared [`CoreHandle`], which owns preprocessing, admission, runtime
//! communication, and request cancellation.
//!
//! The thin Tonic-service structure, streamed response approach.
pub(crate) mod app;
mod info;
mod native_api;
pub(crate) mod service;
@@ -1,18 +1,17 @@
//! Tonic listener lifecycle for the native Rust frontend.
//!
//! The pre-bound socket, shared API runtime, and shutdown shape build on Rain
//! Jiang's multi-protocol prototype in `sgl-project/sglang#36923`. Unlike that
//! prototype, the listener mounts the canonical `runtime.v1` adapter on the
//! transport-neutral [`crate::frontend::FrontendHandle`] contract.
//! Jiang's multi-protocol prototype in `sgl-project/sglang#36923`. The listener
//! mounts the `api.v1` adapter on the transport-neutral
//! [`crate::api_server::core::CoreHandle`] contract.
use sglang_grpc_types::sglang::runtime::v1::sglang_service_server::SglangServiceServer;
use sglang_api_types::api::v1::sglang_service_server::SglangServiceServer;
use tokio_stream::wrappers::TcpListenerStream;
use super::GrpcService;
use super::service::GrpcService;
/// Keep the native Rust listener aligned with the existing Python-backed
/// runtime.v1 server. Tonic's 4 MiB default is too small for large token or
/// multimodal request bodies.
/// Tonic's 4 MiB default is too small for large token or multimodal request
/// bodies; keep the limit the Python-backed gRPC server established.
const DEFAULT_GRPC_MAX_MESSAGE_SIZE: usize = 64 * 1024 * 1024;
fn max_message_size() -> usize {
@@ -0,0 +1,68 @@
//! Frontend metadata to the `api.v1` info responses.
//!
//! Both protos carry the JSON-typed members (`preferred_sampling_params`, the
//! allowlisted `internal_states`) as JSON text, so the gRPC reply holds the
//! same documents the HTTP reply embeds.
use sglang_api_types::api::v1 as api;
use crate::message::info::{ModelInfo, ServerInfo};
pub(super) fn model_info(info: ModelInfo) -> Result<api::GetModelInfoResponse, String> {
let ModelInfo {
model_path,
served_model_name,
tokenizer_path,
is_generation,
preferred_sampling_params,
weight_version,
load_format,
reasoning_parser,
tool_call_parser,
disaggregation_mode: _,
} = info;
let preferred_sampling_params = preferred_sampling_params
.map(|params| serde_json::to_string(&params.0))
.transpose()
.map_err(|error| format!("preferred_sampling_params is not JSON: {error}"))?;
Ok(api::GetModelInfoResponse {
model_path,
served_model_name,
tokenizer_path,
is_generation,
preferred_sampling_params,
weight_version,
load_format,
reasoning_parser,
tool_call_parser,
})
}
pub(super) fn server_info(info: ServerInfo) -> Result<api::GetServerInfoResponse, String> {
let ServerInfo {
model_path,
served_model_name,
tokenizer_path,
max_context_length,
max_total_num_tokens,
version,
frontend: _,
internal_states,
} = info;
let max_context_length = u32::try_from(max_context_length)
.map_err(|_| format!("max_context_length {max_context_length} exceeds uint32"))?;
let internal_states = internal_states
.iter()
.map(serde_json::to_string)
.collect::<Result<Vec<_>, _>>()
.map_err(|error| format!("internal_states are not JSON: {error}"))?;
Ok(api::GetServerInfoResponse {
model_path,
served_model_name,
tokenizer_path,
max_context_length,
max_total_num_tokens: Some(max_total_num_tokens),
version,
internal_states,
})
}
@@ -0,0 +1,215 @@
//! Core events to `api.v1` stream items and statuses.
use std::time::{Duration, Instant};
use futures::StreamExt;
use futures::stream::FuturesUnordered;
use sglang_api_types::api::v1 as api;
use tonic::{Code, Status};
use super::service::ResponseStream;
use crate::api_server::core::{
CoreCall, CoreError, CoreErrorKind, CoreEvent, CoreOutput, recv_indexed,
};
pub(super) struct StreamOptions {
/// `stream` as sent: false folds each item to its one terminal frame.
pub stream: bool,
/// Streamed frames are deltas (true) or cumulative (false, the default).
pub incremental: bool,
/// Batch form: every item carries its position.
pub with_index: bool,
pub response_timeout: Duration,
pub created_at: Instant,
}
/// Multiplex the admitted calls into one `GenerateStreamItem` stream, the
/// gRPC twin of the native SSE multiplexer. Each call aborts itself if the
/// stream is dropped while it is unfinished.
pub(super) fn generate_stream(
calls: Vec<CoreCall>,
options: StreamOptions,
) -> ResponseStream<api::GenerateStreamItem> {
let stream = async_stream::stream! {
let public_ids: Vec<String> = calls
.iter()
.map(|call| call.public_id().to_owned())
.collect();
let mut cumulative: Vec<CoreOutput> =
calls.iter().map(|_| CoreOutput::default()).collect();
let index = |i: usize| options.with_index.then_some(i as u32);
// Poll all receivers concurrently; re-arm a receiver's future after each
// non-terminal event so its stream keeps flowing.
let mut pending = FuturesUnordered::new();
for (i, call) in calls.into_iter().enumerate() {
pending.push(recv_indexed(i, call));
}
loop {
let next = tokio::time::timeout(options.response_timeout, pending.next()).await;
let (i, call, events) = match next {
Ok(Some(next)) => next,
// Every call has reached its terminal event.
Ok(None) => break,
Err(_) => {
yield Err(Status::deadline_exceeded("Stream chunk timed out"));
break;
}
};
if events.is_empty() {
// Defensive fallback: CoreCall normally turns a premature
// runtime close into a Failed event.
yield Ok(error_item(&CoreError::ResponseTruncated, index(i)));
continue;
}
let acc = &mut cumulative[i];
// Cumulative frames supersede one another, so a drained backlog
// collapses to its last (Python's `out_list[-1]`); deltas can't be
// dropped.
let mut coalesced = false;
let mut terminal = None;
let mut failed = None;
for event in events {
match event {
CoreEvent::Delta(mut delta) => {
acc.append_delta(&delta);
if !options.stream {
continue;
}
if options.incremental {
// Python's incremental stream still reports the
// cumulative completion-token count in every frame.
delta.completion_tokens = acc.completion_tokens;
yield Ok(frame_item(delta, &public_ids[i], index(i), None));
} else {
coalesced = true;
}
}
CoreEvent::Finished(delta) => {
acc.append_delta(&delta);
terminal = Some(delta);
}
CoreEvent::Failed(error) => failed = Some(error),
}
}
if let Some(error) = failed {
yield Ok(error_item(&error, index(i)));
} else if let Some(mut delta) = terminal {
let e2e_latency = Some(options.created_at.elapsed().as_secs_f64());
// A unary reply is the cumulative result whatever the stream
// policy. The terminal frame is the accumulated output's last
// use, so it moves out instead of cloning.
let output = if options.stream && options.incremental {
delta.completion_tokens = acc.completion_tokens;
delta
} else {
std::mem::take(acc)
};
yield Ok(frame_item(output, &public_ids[i], index(i), e2e_latency));
} else {
if coalesced {
// An intermediate cumulative frame has to clone: the
// accumulated output stays behind for the next one.
yield Ok(frame_item(acc.clone(), &public_ids[i], index(i), None));
}
pending.push(recv_indexed(i, call));
}
}
// Unfinished calls are still owned by `pending`: dropping the stream
// drops them and uses the core abort lane; a terminal event has
// already disarmed its call.
};
Box::pin(stream)
}
fn frame_item(
output: CoreOutput,
public_id: &str,
index: Option<u32>,
e2e_latency: Option<f64>,
) -> api::GenerateStreamItem {
api::GenerateStreamItem {
item: Some(api::generate_stream_item::Item::Frame(output.frame(
public_id,
index,
e2e_latency,
))),
}
}
/// One item's failure inside the stream. `code` is the native error body's
/// HTTP status, as on the SSE error frame.
fn error_item(error: &CoreError, index: Option<u32>) -> api::GenerateStreamItem {
api::GenerateStreamItem {
item: Some(api::generate_stream_item::Item::Error(
error.stream_error(index),
)),
}
}
pub(super) fn status(error: CoreError) -> Status {
let code = match error.kind() {
CoreErrorKind::InvalidArgument => Code::InvalidArgument,
CoreErrorKind::NotFound => Code::NotFound,
CoreErrorKind::FailedPrecondition => Code::FailedPrecondition,
CoreErrorKind::ResourceExhausted => Code::ResourceExhausted,
CoreErrorKind::Cancelled => Code::Cancelled,
CoreErrorKind::DeadlineExceeded => Code::DeadlineExceeded,
CoreErrorKind::Unavailable => Code::Unavailable,
CoreErrorKind::Internal => Code::Internal,
};
Status::new(code, error.to_string())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn semantic_errors_map_to_canonical_grpc_codes() {
let cases = [
(
CoreError::InvalidArgument("bad".into()),
Code::InvalidArgument,
),
(
CoreError::RuntimeRejected {
kind: CoreErrorKind::NotFound,
message: "missing".into(),
legacy_http_status: 404,
},
Code::NotFound,
),
(
CoreError::RuntimeRejected {
kind: CoreErrorKind::FailedPrecondition,
message: "not ready".into(),
legacy_http_status: 412,
},
Code::FailedPrecondition,
),
(
CoreError::Overloaded("full".into()),
Code::ResourceExhausted,
),
(CoreError::Cancelled("gone".into()), Code::Cancelled),
(
CoreError::RuntimeRejected {
kind: CoreErrorKind::DeadlineExceeded,
message: "late".into(),
legacy_http_status: 504,
},
Code::DeadlineExceeded,
),
(CoreError::Unavailable, Code::Unavailable),
(CoreError::Internal("bug".into()), Code::Internal),
];
for (error, expected) in cases {
assert_eq!(status(error).code(), expected);
}
}
}
@@ -0,0 +1,718 @@
//! The `SglangService` implementation: per-RPC translation between the
//! `api.v1` messages and the shared core.
use std::pin::Pin;
use std::time::{Duration, Instant};
use futures::Stream;
use sglang_api_types::api::v1 as api;
use sglang_api_types::api::v1::sglang_service_server::SglangService;
use tonic::{Request, Response, Status};
use super::{info, native_api};
use crate::api_server::core::{CoreHandle, HealthStatus};
use crate::message::config::{PreferredSamplingParams, ServerArgs};
use crate::message::request::into_requests;
use crate::message::wire::fill_preferred_sampling;
use crate::utils::environ;
const DEFAULT_RESPONSE_TIMEOUT: Duration = Duration::from_secs(300);
pub(super) type ResponseStream<T> = Pin<Box<dyn Stream<Item = Result<T, Status>> + Send + 'static>>;
/// Narrow snapshot of launch policy the adapter needs per call. It
/// intentionally excludes listener, authentication, TLS, and lifecycle
/// configuration.
#[derive(Clone)]
struct AdapterConfig {
preferred_sampling_params: Option<PreferredSamplingParams>,
incremental_streaming_output: bool,
/// Longest wait for the next event of any in-flight call before the RPC
/// ends with DEADLINE_EXCEEDED.
response_timeout: Duration,
/// `/health_generate`'s heartbeat window (`SGLANG_HEALTH_CHECK_TIMEOUT`).
health_timeout: Duration,
}
/// Tonic-facing implementation backed by the transport-neutral core.
pub(crate) struct GrpcService {
core: CoreHandle,
config: AdapterConfig,
}
impl GrpcService {
pub(crate) fn new(core: CoreHandle, server_args: &ServerArgs) -> Self {
Self {
core,
config: AdapterConfig {
preferred_sampling_params: server_args.preferred_sampling_params.clone(),
incremental_streaming_output: server_args.incremental_streaming_output,
response_timeout: DEFAULT_RESPONSE_TIMEOUT,
health_timeout: Duration::from_secs(
environ::env_i64("SGLANG_HEALTH_CHECK_TIMEOUT", 20).max(0) as u64,
),
},
}
}
#[cfg(test)]
pub(super) fn for_test(
core: CoreHandle,
preferred_sampling_params: Option<PreferredSamplingParams>,
incremental_streaming_output: bool,
response_timeout: Duration,
) -> Self {
Self {
core,
config: AdapterConfig {
preferred_sampling_params,
incremental_streaming_output,
response_timeout,
health_timeout: Duration::from_millis(20),
},
}
}
}
#[tonic::async_trait]
impl SglangService for GrpcService {
type GenerateStream = ResponseStream<api::GenerateStreamItem>;
/// `/generate` over gRPC: one schema, one fan-out, one admission path. A
/// body that fails normalization ends the RPC with INVALID_ARGUMENT before
/// anything reaches the scheduler, as the HTTP 400 does.
async fn generate(
&self,
request: Request<api::GenerateRequest>,
) -> Result<Response<Self::GenerateStream>, Status> {
let mut request = request.into_inner();
if let Some(preferred) = &self.config.preferred_sampling_params {
request.sampling_params =
fill_preferred_sampling(request.sampling_params.take(), &preferred.0)
.map_err(Status::internal)?;
}
// Protobuf has no null: a field the client left unset takes the
// schema default, as an absent JSON key does.
request.apply_absent_defaults();
let stream = request.stream.unwrap_or(false);
let (payloads, is_batch) =
into_requests(request).map_err(|error| Status::invalid_argument(error.to_string()))?;
// Python starts its request clock after normalization and before
// tokenization / multimodal preprocessing; `e2e_latency` measures from
// the same boundary here.
let created_at = Instant::now();
let calls = self
.core
.generate_batch(payloads)
.await
.map_err(native_api::status)?;
Ok(Response::new(native_api::generate_stream(
calls,
native_api::StreamOptions {
stream,
incremental: self.config.incremental_streaming_output,
with_index: is_batch,
response_timeout: self.config.response_timeout,
created_at,
},
)))
}
async fn health_check(
&self,
_request: Request<api::HealthCheckRequest>,
) -> Result<Response<api::HealthCheckResponse>, Status> {
match self.core.probe_health(self.config.health_timeout).await {
Ok(HealthStatus::Healthy) => {
Ok(Response::new(api::HealthCheckResponse { healthy: true }))
}
Ok(HealthStatus::NotReady) => Err(Status::unavailable(
"server is still completing its startup warmup",
)),
Ok(HealthStatus::Stalled) => Err(Status::unavailable(
"no scheduler output within the health-check timeout",
)),
Err(error) => Err(native_api::status(error)),
}
}
async fn get_model_info(
&self,
_request: Request<api::GetModelInfoRequest>,
) -> Result<Response<api::GetModelInfoResponse>, Status> {
info::model_info(self.core.model_info())
.map(Response::new)
.map_err(Status::internal)
}
async fn get_server_info(
&self,
_request: Request<api::GetServerInfoRequest>,
) -> Result<Response<api::GetServerInfoResponse>, Status> {
let server_info = self.core.server_info().await.map_err(native_api::status)?;
info::server_info(server_info)
.map(Response::new)
.map_err(Status::internal)
}
}
#[cfg(test)]
mod tests {
use std::time::Duration;
use futures::StreamExt;
use sglang_api_types::api::v1 as api;
use sglang_api_types::api::v1::generate_stream_item::Item;
use sglang_api_types::api::v1::sglang_service_server::SglangService;
use tonic::{Code, Request};
use super::GrpcService;
use crate::api_server::core::{CoreConfig, CoreHandle, CoreMetadata};
use crate::message::config::PreferredSamplingParams;
use crate::message::finish_reason::FinishReason;
use crate::message::ids::Rid;
use crate::message::request::{GenerateRequest, Request as RuntimeRequest, RequestKind};
use crate::message::response::{ChunkEvent, ResponseItem, ResponseSink};
use crate::tokenizer_manager::wiring::{AbortSource, RequestAdmission, TmEvent};
struct Harness {
service: GrpcService,
intake_rx: flume::Receiver<TmEvent>,
abort_rx: flume::Receiver<AbortSource>,
}
struct GenerationIntake {
request: Box<GenerateRequest>,
rid: Rid,
sink: ResponseSink,
admission: RequestAdmission,
}
impl Harness {
fn new(response_capacity: usize, incremental: bool, response_timeout: Duration) -> Self {
Self::build(response_capacity, incremental, response_timeout, true, None)
}
fn build(
response_capacity: usize,
incremental: bool,
response_timeout: Duration,
startup_ready: bool,
preferred: Option<PreferredSamplingParams>,
) -> Self {
let (intake_tx, intake_rx) = flume::unbounded();
let (abort_tx, abort_rx) = flume::unbounded();
let core = CoreHandle::new(
intake_tx,
abort_tx,
CoreConfig {
response_capacity,
response_activity: Default::default(),
startup_ready,
is_disaggregation: false,
mm_limits: Default::default(),
metadata: CoreMetadata::default(),
},
);
Self {
service: GrpcService::for_test(core, preferred, incremental, response_timeout),
intake_rx,
abort_rx,
}
}
async fn next_generation(&self) -> GenerationIntake {
let TmEvent::Intake { request, admission } = self
.intake_rx
.recv_async()
.await
.expect("request submitted")
else {
panic!("expected generation intake");
};
let RuntimeRequest {
rid,
sink,
kind,
state: _,
} = request;
let RequestKind::Generate(request) = kind else {
panic!("expected generation request");
};
GenerationIntake {
request,
rid,
sink,
admission,
}
}
}
fn chunk(rid: &Rid, text: &str, token_id: i64, finished: bool) -> ChunkEvent {
ChunkEvent {
rid: rid.clone(),
token_ids: vec![token_id],
finish_reason: finished.then(stop_reason),
prompt_tokens: 3,
text: text.into(),
completion_tokens: 1,
extras: None,
}
}
fn stop_reason() -> FinishReason {
serde_json::from_value(serde_json::json!({"type": "stop"})).unwrap()
}
fn one_string(value: &str) -> api::StringOrList {
api::StringOrList {
value: Some(api::string_or_list::Value::One(value.into())),
}
}
fn many_strings(values: &[&str]) -> api::StringOrList {
api::StringOrList {
value: Some(api::string_or_list::Value::Many(api::StringList {
items: values.iter().map(|value| (*value).to_owned()).collect(),
})),
}
}
fn one_token_ids(ids: &[i64]) -> api::TokenIdsOrList {
api::TokenIdsOrList {
value: Some(api::token_ids_or_list::Value::One(api::TokenIds {
ids: ids.to_vec(),
})),
}
}
fn text_request(text: &str, rid: &str) -> api::GenerateRequest {
api::GenerateRequest {
text: Some(one_string(text)),
rid: Some(one_string(rid)),
stream: Some(true),
..Default::default()
}
}
fn ids_request(ids: &[i64], rid: Option<&str>) -> api::GenerateRequest {
api::GenerateRequest {
input_ids: Some(one_token_ids(ids)),
rid: rid.map(one_string),
stream: Some(true),
..Default::default()
}
}
fn frame(item: api::GenerateStreamItem) -> api::GenerateResponse {
match item.item {
Some(Item::Frame(frame)) => frame,
other => panic!("expected a frame, got {other:?}"),
}
}
fn stream_error(item: api::GenerateStreamItem) -> api::GenerateStreamError {
match item.item {
Some(Item::Error(error)) => error,
other => panic!("expected an error item, got {other:?}"),
}
}
fn meta(frame: &api::GenerateResponse) -> &api::GenerateMetaInfo {
frame
.meta_info
.as_ref()
.expect("every frame carries meta_info")
}
fn output_ids(frame: &api::GenerateResponse) -> &[i64] {
frame
.output_ids
.as_ref()
.map_or(&[], |ids| ids.ids.as_slice())
}
#[tokio::test]
async fn generate_streams_cumulative_frames_for_a_text_prompt() {
let harness = Harness::new(2, false, Duration::from_secs(1));
let mut stream = harness
.service
.generate(Request::new(text_request("prompt", "request-1")))
.await
.unwrap()
.into_inner();
let intake = harness.next_generation().await;
assert!(intake.admission.try_accept());
assert_eq!(intake.request.text.as_deref(), Some("prompt"));
assert!(intake.request.input_ids.is_none());
assert!(intake.request.stream);
assert_eq!(intake.rid.client_facing(), "request-1");
// One chunk at a time: a queued backlog of cumulative frames collapses to
// its last, so sending both up front would yield only the terminal frame.
intake
.sink
.try_send(ResponseItem::Frame(chunk(&intake.rid, "Hel", 1, false)))
.unwrap();
let first = frame(stream.next().await.unwrap().unwrap());
assert_eq!(first.text, "Hel");
assert_eq!(output_ids(&first), [1]);
assert_eq!(first.index, None);
let first_meta = meta(&first);
assert_eq!(first_meta.id, "request-1");
assert_eq!(first_meta.prompt_tokens, 3);
assert_eq!(first_meta.completion_tokens, 1);
assert!(first_meta.finish_reason.is_none());
assert!(first_meta.e2e_latency.is_none());
intake
.sink
.try_send(ResponseItem::Done(chunk(&intake.rid, "lo", 2, true)))
.unwrap();
let finished = frame(stream.next().await.unwrap().unwrap());
assert_eq!(finished.text, "Hello");
assert_eq!(output_ids(&finished), [1, 2]);
let finished_meta = meta(&finished);
assert_eq!(finished_meta.completion_tokens, 2);
assert!(matches!(
finished_meta.finish_reason,
Some(api::FinishReason {
kind: Some(api::finish_reason::Kind::Stop(_))
})
));
assert!(finished_meta.e2e_latency.is_some());
assert!(stream.next().await.is_none());
assert!(harness.abort_rx.try_recv().is_err());
}
#[tokio::test]
async fn incremental_frames_carry_deltas_with_cumulative_count() {
let harness = Harness::new(2, true, Duration::from_secs(1));
let mut stream = harness
.service
.generate(Request::new(ids_request(&[4, 5], Some("tokens"))))
.await
.unwrap()
.into_inner();
let intake = harness.next_generation().await;
assert!(intake.admission.try_accept());
assert_eq!(intake.request.input_ids.as_deref(), Some(&[4, 5][..]));
assert!(intake.request.text.is_none());
intake
.sink
.try_send(ResponseItem::Frame(chunk(&intake.rid, "A", 10, false)))
.unwrap();
intake
.sink
.try_send(ResponseItem::Done(chunk(&intake.rid, "B", 11, true)))
.unwrap();
let first = frame(stream.next().await.unwrap().unwrap());
let finished = frame(stream.next().await.unwrap().unwrap());
assert_eq!(output_ids(&first), [10]);
assert_eq!(output_ids(&finished), [11]);
assert_eq!(finished.text, "B");
assert_eq!(meta(&finished).completion_tokens, 2);
assert!(stream.next().await.is_none());
}
/// `stream: false` is a one-frame stream: deltas fold into the terminal
/// result, and the server's incremental policy does not split a unary reply.
#[tokio::test]
async fn unary_request_yields_only_the_cumulative_terminal_frame() {
let harness = Harness::new(2, true, Duration::from_secs(1));
let mut request = text_request("prompt", "unary");
request.stream = Some(false);
let mut stream = harness
.service
.generate(Request::new(request))
.await
.unwrap()
.into_inner();
let intake = harness.next_generation().await;
assert!(intake.admission.try_accept());
assert!(!intake.request.stream);
intake
.sink
.try_send(ResponseItem::Frame(chunk(&intake.rid, "Hel", 1, false)))
.unwrap();
intake
.sink
.try_send(ResponseItem::Done(chunk(&intake.rid, "lo", 2, true)))
.unwrap();
let only = frame(stream.next().await.unwrap().unwrap());
assert_eq!(only.text, "Hello");
assert_eq!(output_ids(&only), [1, 2]);
assert_eq!(meta(&only).completion_tokens, 2);
assert!(stream.next().await.is_none());
}
#[tokio::test]
async fn stream_drop_aborts_admitted_request() {
let harness = Harness::new(1, false, Duration::from_secs(1));
let mut stream = harness
.service
.generate(Request::new(ids_request(&[1], Some("cancel-me"))))
.await
.unwrap()
.into_inner();
let intake = harness.next_generation().await;
assert!(intake.admission.try_accept());
intake
.sink
.try_send(ResponseItem::Frame(chunk(&intake.rid, "partial", 1, false)))
.unwrap();
assert!(stream.next().await.unwrap().is_ok());
drop(stream);
let abort = harness.abort_rx.recv_async().await.unwrap();
assert_eq!(abort.rid().client_facing(), "cancel-me");
assert!(harness.abort_rx.try_recv().is_err());
}
/// A runtime failure after admission is this request's in-stream error item
/// (the SSE error frame's twin), not an RPC status; the call is disarmed.
#[tokio::test]
async fn runtime_failure_rides_the_stream_as_an_error_item() {
let harness = Harness::new(1, false, Duration::from_secs(1));
let mut stream = harness
.service
.generate(Request::new(ids_request(&[1], None)))
.await
.unwrap()
.into_inner();
let intake = harness.next_generation().await;
assert!(intake.admission.try_accept());
intake
.sink
.try_send(ResponseItem::Error(crate::utils::error::Error::Validation(
"bad sampling".into(),
)))
.unwrap();
let error = stream_error(stream.next().await.unwrap().unwrap());
assert_eq!(error.index, None);
let body = error.error.unwrap();
assert_eq!(body.code, 400);
assert!(body.message.contains("bad sampling"));
assert!(stream.next().await.is_none());
drop(stream);
assert!(harness.abort_rx.try_recv().is_err());
}
#[tokio::test]
async fn per_chunk_timeout_after_progress_returns_deadline_exceeded_and_aborts() {
let harness = Harness::new(1, false, Duration::from_millis(10));
let mut stream = harness
.service
.generate(Request::new(ids_request(&[1], Some("too-slow"))))
.await
.unwrap()
.into_inner();
let intake = harness.next_generation().await;
assert!(intake.admission.try_accept());
intake
.sink
.try_send(ResponseItem::Frame(chunk(&intake.rid, "partial", 1, false)))
.unwrap();
assert!(stream.next().await.unwrap().is_ok());
let error = stream.next().await.unwrap().unwrap_err();
assert_eq!(error.code(), Code::DeadlineExceeded);
drop(stream);
let abort = harness.abort_rx.recv_async().await.unwrap();
assert_eq!(abort.rid().client_facing(), "too-slow");
}
#[tokio::test]
async fn closed_intake_is_a_top_level_unavailable_status() {
let (intake_tx, intake_rx) = flume::unbounded();
drop(intake_rx);
let (abort_tx, _abort_rx) = flume::unbounded();
let core = CoreHandle::new(
intake_tx,
abort_tx,
CoreConfig {
response_capacity: 1,
response_activity: Default::default(),
startup_ready: true,
is_disaggregation: false,
mm_limits: Default::default(),
metadata: CoreMetadata::default(),
},
);
let service = GrpcService::for_test(core, None, false, Duration::from_secs(1));
let result = service
.generate(Request::new(ids_request(&[1], None)))
.await;
let error = match result {
Ok(_) => panic!("closed intake must reject the RPC"),
Err(error) => error,
};
assert_eq!(error.code(), Code::Unavailable);
}
/// Normalization failures end the RPC before anything is submitted, with the
/// status that mirrors the HTTP 400.
#[tokio::test]
async fn malformed_request_is_rejected_before_submission() {
let harness = Harness::new(1, false, Duration::from_secs(1));
let mut both = text_request("prompt", "both");
both.input_ids = Some(one_token_ids(&[1]));
let error = harness
.service
.generate(Request::new(both))
.await
.err()
.expect("text and input_ids together must be rejected");
assert_eq!(error.code(), Code::InvalidArgument);
assert!(error.message().contains("text"));
let error = harness
.service
.generate(Request::new(ids_request(&[], None)))
.await
.err()
.expect("empty input_ids must be rejected");
assert_eq!(error.code(), Code::InvalidArgument);
assert!(harness.intake_rx.try_recv().is_err());
}
/// A list-form request fans out like the HTTP batch: every item is tagged
/// with its position, and one item's failure neither ends the stream nor
/// touches its siblings.
#[tokio::test]
async fn batch_items_are_indexed_and_fail_independently() {
let harness = Harness::new(2, false, Duration::from_secs(1));
let mut stream = harness
.service
.generate(Request::new(api::GenerateRequest {
text: Some(many_strings(&["a", "b"])),
stream: Some(true),
..Default::default()
}))
.await
.unwrap()
.into_inner();
let first = harness.next_generation().await;
let second = harness.next_generation().await;
assert!(first.admission.try_accept());
assert!(second.admission.try_accept());
assert_eq!(first.request.text.as_deref(), Some("a"));
assert_eq!(second.request.text.as_deref(), Some("b"));
first
.sink
.try_send(ResponseItem::Error(crate::utils::error::Error::Validation(
"first failed".into(),
)))
.unwrap();
second
.sink
.try_send(ResponseItem::Done(chunk(&second.rid, "ok", 5, true)))
.unwrap();
let mut items = Vec::new();
while let Some(item) = stream.next().await {
items.push(item.unwrap());
}
assert_eq!(items.len(), 2);
let (errors, frames): (Vec<_>, Vec<_>) = items
.into_iter()
.partition(|item| matches!(item.item, Some(Item::Error(_))));
let error = stream_error(errors.into_iter().next().unwrap());
assert_eq!(error.index, Some(0));
assert!(error.error.unwrap().message.contains("first failed"));
let ok = frame(frames.into_iter().next().unwrap());
assert_eq!(ok.index, Some(1));
assert_eq!(ok.text, "ok");
assert_eq!(meta(&ok).id, second.rid.client_facing());
assert!(harness.abort_rx.try_recv().is_err());
}
/// Launch-time preferred sampling params fill the fields a protobuf request
/// left unset, beneath the ones it set: the HTTP precedence at the gRPC entry.
#[tokio::test]
async fn preferred_sampling_params_fill_unset_fields() {
let preferred = PreferredSamplingParams(serde_json::json!({
"temperature": 0.25,
"max_new_tokens": 32,
}));
let harness = Harness::build(1, false, Duration::from_secs(1), true, Some(preferred));
let mut request = ids_request(&[1], None);
request.sampling_params = Some(api::SamplingParamsOrList {
value: Some(api::sampling_params_or_list::Value::One(
api::SamplingParams {
temperature: Some(0.8),
..Default::default()
},
)),
});
let _stream = harness
.service
.generate(Request::new(request))
.await
.unwrap()
.into_inner();
let intake = harness.next_generation().await;
assert_eq!(intake.request.sampling_params.temperature, 0.8);
assert_eq!(intake.request.sampling_params.max_new_tokens, Some(32));
}
/// Protobuf has no null: a sampling field the client left unset takes the
/// schema default, as an absent JSON key does. `max_new_tokens` is the one
/// field whose Rust `None` means "unbounded", so without the absent-defaults
/// pass an unset value would lift the limit instead of applying 128.
#[tokio::test]
async fn unset_protobuf_sampling_fields_take_schema_defaults() {
let harness = Harness::new(1, false, Duration::from_secs(1));
let mut request = ids_request(&[1], None);
request.sampling_params = Some(api::SamplingParamsOrList {
value: Some(api::sampling_params_or_list::Value::One(
api::SamplingParams {
temperature: Some(0.8),
..Default::default()
},
)),
});
let _stream = harness
.service
.generate(Request::new(request))
.await
.unwrap()
.into_inner();
let intake = harness.next_generation().await;
assert_eq!(intake.request.sampling_params.temperature, 0.8);
assert_eq!(intake.request.sampling_params.max_new_tokens, Some(128));
assert_eq!(intake.request.sampling_params.top_p, 1.0);
}
/// `HealthCheck` is `/health_generate`: not ready and stalled are both
/// UNAVAILABLE, and the stalled probe is a real generation submission.
#[tokio::test]
async fn health_check_reports_not_ready_and_stalled_as_unavailable() {
let not_ready = Harness::build(1, false, Duration::from_secs(1), false, None);
let error = not_ready
.service
.health_check(Request::new(api::HealthCheckRequest {}))
.await
.expect_err("warmup must not report healthy");
assert_eq!(error.code(), Code::Unavailable);
assert!(not_ready.intake_rx.try_recv().is_err());
let stalled = Harness::new(1, false, Duration::from_secs(1));
let error = stalled
.service
.health_check(Request::new(api::HealthCheckRequest {}))
.await
.expect_err("no heartbeat within the window must not report healthy");
assert_eq!(error.code(), Code::Unavailable);
let probe = stalled.next_generation().await;
assert_eq!(probe.request.input_ids.as_deref(), Some(&[0][..]));
}
}
@@ -0,0 +1,4 @@
pub(crate) mod app;
mod info;
mod native_api;
mod openai;
@@ -12,20 +12,21 @@ use axum::{
response::Response,
};
use super::disaggregation::bootstrap as pd_bootstrap;
use super::{common, log, native_api, openai};
use crate::frontend::FrontendHandle;
use super::{info, native_api, openai};
use crate::api_server::core::CoreHandle;
use crate::api_server::disaggregation::bootstrap as pd_bootstrap;
use crate::api_server::log;
use crate::message::config::ServerArgs;
/// HTTP adapter state: the shared frontend capability, immutable server
/// HTTP adapter state: the shared core capability, immutable server
/// configuration needed for HTTP request preparation, and the chat formatter.
///
/// axum clones the router state into **every** request, so it is mounted as
/// `Arc<AppState>` — one refcount bump per request instead of cloning the
/// frontend handle and chat formatter. Deliberately not `Clone`, so it
/// core handle and chat formatter. Deliberately not `Clone`, so it
/// can only be shared through that `Arc`.
pub(super) struct AppState {
pub(super) frontend: FrontendHandle,
pub(super) core: CoreHandle,
pub(super) server_args: Arc<ServerArgs>,
pub(super) chat_formatter: Option<openai::ChatFormatter>,
}
@@ -45,23 +46,19 @@ async fn mark_startup_ready(
"/generate" | "/encode" | "/v1/chat/completions"
);
let response = next.run(req).await;
record_startup_warmup_status(&state.frontend, is_startup_warmup, response.status());
record_startup_warmup_status(&state.core, is_startup_warmup, response.status());
response
}
fn record_startup_warmup_status(
frontend: &FrontendHandle,
is_startup_warmup: bool,
status: StatusCode,
) {
fn record_startup_warmup_status(core: &CoreHandle, is_startup_warmup: bool, status: StatusCode) {
if is_startup_warmup && status.is_success() {
frontend.mark_ready();
core.mark_ready();
}
}
pub async fn serve(
listener: std::net::TcpListener,
frontend: FrontendHandle,
core: CoreHandle,
server_args: Arc<ServerArgs>,
// The runtime's shutdown signal, shared with every worker stage: it fires
// (disconnects) when `Runtime::request_shutdown` drops the sender, at
@@ -71,13 +68,13 @@ pub async fn serve(
) {
let chat_formatter = openai::load_chat_support(&server_args);
let state = Arc::new(AppState {
frontend,
core,
server_args: server_args.clone(),
chat_formatter,
});
// Each endpoint module registers its own routes and merges here.
let router = Router::new()
.merge(common::routes())
.merge(info::routes())
.merge(native_api::routes())
.merge(openai::routes());
@@ -138,32 +135,32 @@ pub async fn serve(
mod tests {
use super::*;
fn frontend() -> FrontendHandle {
FrontendHandle::new(
fn core_handle() -> CoreHandle {
CoreHandle::new(
flume::unbounded().0,
flume::unbounded().0,
crate::frontend::FrontendConfig {
crate::api_server::core::CoreConfig {
response_capacity: 8,
response_activity: Default::default(),
startup_ready: false,
is_disaggregation: false,
mm_limits: Default::default(),
metadata: crate::frontend::FrontendMetadata::default(),
metadata: crate::api_server::core::CoreMetadata::default(),
},
)
}
#[test]
fn only_a_successful_recognized_warmup_marks_frontend_ready() {
let frontend = frontend();
let core = core_handle();
record_startup_warmup_status(&frontend, false, StatusCode::OK);
assert!(!frontend.is_ready());
record_startup_warmup_status(&core, false, StatusCode::OK);
assert!(!core.is_ready());
record_startup_warmup_status(&frontend, true, StatusCode::INTERNAL_SERVER_ERROR);
assert!(!frontend.is_ready());
record_startup_warmup_status(&core, true, StatusCode::INTERNAL_SERVER_ERROR);
assert!(!core.is_ready());
record_startup_warmup_status(&frontend, true, StatusCode::OK);
assert!(frontend.is_ready());
record_startup_warmup_status(&core, true, StatusCode::OK);
assert!(core.is_ready());
}
}
@@ -1,6 +1,6 @@
//! Common control-plane endpoints — `/server_info`, `/get_model_info`
//! (+ `/model_info` alias), backed by typed operations on
//! [`crate::frontend::FrontendHandle`]. Data-plane endpoints (incl. `/health*`,
//! [`crate::api_server::core::CoreHandle`]. Data-plane endpoints (incl. `/health*`,
//! which round-trips a generate probe) live in the sibling `native_api` and
//! `openai` modules; the shared `AppState` lives in the parent
//! `api_server` module.
@@ -15,14 +15,14 @@ use axum::{
use std::sync::Arc;
use super::app::AppState;
use super::frontend_error_status;
use super::native_api::native_error;
use crate::frontend::{FrontendError, ServerInfo};
use crate::api_server::core::CoreError;
use crate::message::info::ServerInfo;
/// The routes this module owns, mounted by `api_server::serve`.
pub(super) fn routes() -> Router<Arc<AppState>> {
Router::new()
// Control-plane: the frontend performs the typed operation and this
// Control-plane: the core performs the typed operation and this
// adapter renders its result as one non-streamed JSON response.
.route("/server_info", get(server_info))
// Static config, no scheduler round-trip. `/get_model_info` (+ `/model_info`
@@ -31,23 +31,23 @@ pub(super) fn routes() -> Router<Arc<AppState>> {
.route("/model_info", get(model_info))
}
/// Await the frontend's complete semantic server-info result. HTTP status and
/// Await the core's complete semantic server-info result. HTTP status and
/// serialization stay here; scheduler control bytes stay behind the contract.
async fn await_server_info(state: &AppState) -> Result<ServerInfo, Response> {
match state.frontend.server_info().await {
match state.core.server_info().await {
Ok(server_info) => Ok(server_info),
Err(FrontendError::Unavailable) => Err(native_error(
Err(CoreError::Unavailable) => Err(native_error(
StatusCode::SERVICE_UNAVAILABLE,
"service unavailable",
false,
)),
// Preserve the established HTTP behavior for a scheduler-side control
// stream that disappears, while non-HTTP adapters see Internal via
// FrontendError::kind().
Err(FrontendError::ResponseTruncated) => {
// CoreError::kind().
Err(CoreError::ResponseTruncated) => {
Err((StatusCode::from_u16(499).unwrap(), "request aborted").into_response())
}
Err(error @ FrontendError::InvalidResponse(_)) => {
Err(error @ CoreError::InvalidResponse(_)) => {
tracing::error!(%error, "server_info: invalid runtime response");
Err((
StatusCode::INTERNAL_SERVER_ERROR,
@@ -55,14 +55,14 @@ async fn await_server_info(state: &AppState) -> Result<ServerInfo, Response> {
)
.into_response())
}
Err(error) => Err((frontend_error_status(&error), error.to_string()).into_response()),
Err(error) => Err((error.http_status(), error.to_string()).into_response()),
}
}
/// `GET /get_model_info` (+ `/model_info` alias) — serialize the shared static
/// model metadata (no scheduler round-trip).
async fn model_info(State(state): State<Arc<AppState>>) -> Response {
let info = state.frontend.model_info();
let info = state.core.model_info();
(
StatusCode::OK,
[("content-type", "application/json")],
@@ -104,7 +104,7 @@ mod tests {
use tower::ServiceExt;
use super::*;
use crate::frontend::{FrontendConfig, FrontendHandle, FrontendMetadata};
use crate::api_server::core::{CoreConfig, CoreHandle, CoreMetadata};
use crate::message::config::{DisaggregationMode, ModelConfig, ServerArgs};
use crate::message::request::RequestKind;
use crate::message::response::ResponseItem;
@@ -120,21 +120,21 @@ mod tests {
let server_args = Arc::new(server_args);
let (intake_tx, intake_rx) = flume::unbounded();
let (abort_tx, abort_rx) = flume::unbounded();
let frontend = FrontendHandle::new(
let core = CoreHandle::new(
intake_tx,
abort_tx,
FrontendConfig {
CoreConfig {
response_capacity: 8,
response_activity: Default::default(),
startup_ready: true,
is_disaggregation: false,
mm_limits: Default::default(),
metadata: FrontendMetadata::from(server_args.as_ref()),
metadata: CoreMetadata::from(server_args.as_ref()),
},
);
(
Arc::new(AppState {
frontend,
core,
server_args,
chat_formatter: None,
}),
@@ -4,23 +4,20 @@
//! `http_server.generate_request`) and `/health` + `/health_generate` (which
//! round-trip a 1-token generate probe). Frame shaping (`meta_info`, logprob
//! tuples, cumulative vs incremental streams) lives here; submission and
//! request lifetime belong to the shared [`crate::frontend::FrontendHandle`].
//! request lifetime belong to the shared [`crate::api_server::core::CoreHandle`].
use std::convert::Infallible;
use std::sync::Arc;
use std::time::{Duration, Instant};
use super::app::AppState;
use super::frame::{
OutputAccumulator, cumulative_frame_string, frame_value, stream_frame_string, tag_value,
};
use super::frontend_error_status;
use crate::frontend::{
FrontendCall, FrontendError, FrontendEvent, FrontendOutput, FrontendRequest, HealthStatus,
use crate::api_server::core::{
CoreCall, CoreError, CoreEvent, CoreOutput, HealthStatus, recv_indexed,
};
#[cfg(test)]
use crate::message::ids::Rid;
use crate::message::request::GenerateBody;
use crate::message::request::{GenerateRequest, into_requests};
use crate::message::wire::merge_preferred_sampling;
use crate::utils::{
environ,
response::{error_response, error_value},
@@ -36,6 +33,7 @@ use axum::{
},
routing::{get, post},
};
use sglang_api_types::api::v1 as api;
/// API-local timing for one request.
///
@@ -89,8 +87,8 @@ pub(super) fn native_error(code: StatusCode, message: &str, stream: bool) -> Res
error_response(code, error_value(code.as_u16(), message), stream)
}
fn native_frontend_error(error: FrontendError, stream: bool) -> Response {
native_error(frontend_error_status(&error), &error.to_string(), stream)
fn native_core_error(error: CoreError, stream: bool) -> Response {
native_error(error.http_status(), &error.to_string(), stream)
}
/// `/health` + `/health_generate`. Both env knobs are resolved ONCE here, at
@@ -116,7 +114,7 @@ fn health_routes() -> Router<Arc<AppState>> {
}
async fn health_without_generation(State(state): State<Arc<AppState>>) -> Response {
if state.frontend.is_ready() {
if state.core.is_ready() {
StatusCode::OK.into_response()
} else {
StatusCode::SERVICE_UNAVAILABLE.into_response()
@@ -131,19 +129,19 @@ async fn health_without_generation(State(state): State<Arc<AppState>>) -> Respon
///
/// Fires a pre-tokenized 1-token probe (`input_ids = [0]`, skips the tokenizer) so
/// an idle pipeline produces a frame, then watches the *global*
/// frontend response-activity counter (not the probe's own rid) — so a busy
/// core response-activity counter (not the probe's own rid) — so a busy
/// server passes immediately and a backlog never false-503s (the analogue of
/// Python's `last_receive_tstamp`).
async fn health_generate(
State(state): State<Arc<AppState>>,
timeout: std::time::Duration,
) -> Response {
match state.frontend.probe_health(timeout).await {
match state.core.probe_health(timeout).await {
Ok(HealthStatus::Healthy) => StatusCode::OK.into_response(),
Ok(HealthStatus::NotReady | HealthStatus::Stalled) => {
StatusCode::SERVICE_UNAVAILABLE.into_response()
}
Err(FrontendError::Unavailable) => native_error(
Err(CoreError::Unavailable) => native_error(
StatusCode::SERVICE_UNAVAILABLE,
"service unavailable",
false,
@@ -162,30 +160,35 @@ async fn health_generate(
/// message, instead of axum's default 422.
async fn generate(
State(state): State<Arc<AppState>>,
body: Result<Json<GenerateBody>, JsonRejection>,
body: Result<Json<serde_json::Value>, JsonRejection>,
) -> Response {
let mut body = match body {
Ok(Json(body)) => body,
// A body that fails to parse has no readable `stream` flag, so this one
// can only answer unary — as Python's does (FastAPI rejects before its
// can only answer unary -- as Python's does (FastAPI rejects before its
// handler runs).
Err(rejection) => {
return native_error(StatusCode::BAD_REQUEST, &rejection.body_text(), false);
}
};
let stream = body.stream;
// Launch-time preferred sampling params fill in beneath the request's own
// keys before decoding, so an explicit request value (even a default or
// null) wins: Python TokenizerManager's precedence.
if let Some(preferred) = &state.server_args.preferred_sampling_params
&& let Err(error) = body.apply_preferred_sampling(&preferred.0)
&& let Err(error) = merge_preferred_sampling(&mut body, &preferred.0)
{
return native_error(
StatusCode::INTERNAL_SERVER_ERROR,
&error.to_string(),
stream,
);
return native_error(StatusCode::INTERNAL_SERVER_ERROR, &error, false);
}
// The schema (sglang_api_types) is the contract: decode errors carry
// serde's field-level text, as Python's 400s do.
let body: api::GenerateRequest = match serde_json::from_value(body) {
Ok(body) => body,
Err(error) => return native_error(StatusCode::BAD_REQUEST, &error.to_string(), false),
};
let stream = body.stream.unwrap_or(false);
// Fan `text`/`input_ids`/`sampling_params` (scalar or list) into per-request
// payloads. `is_batch` = list form → the response is a JSON array.
let (payloads, is_batch) = match body.into_requests() {
// payloads. `is_batch` = list form -> the response is a JSON array.
let (payloads, is_batch) = match into_requests(body) {
Ok(v) => v,
// Request normalization only returns validation failures.
Err(e) => {
@@ -197,7 +200,7 @@ async fn generate(
// equivalent boundary: into_requests() has normalized the body, while prefetch
// and every downstream stage are still ahead of us.
let timing = RequestTiming::new();
// Shared frontend admission performs media I/O (URL downloads, file reads)
// Shared core admission performs media I/O (URL downloads, file reads)
// on this async runtime before any request reaches the MM worker pool.
if !is_batch {
// `into_requests` guarantees exactly one payload for a non-batch body.
@@ -215,15 +218,15 @@ async fn generate(
/// SSE frames or fold to one unary response.
async fn generate_single(
state: &AppState,
req: FrontendRequest,
req: GenerateRequest,
stream: bool,
timing: RequestTiming,
) -> Response {
// `return_text_in_logprobs` is decoded on the detok shard into `*_txt`, so
// `frame_value` just reads them — no tokenizer needed here.
let mut call = match state.frontend.generate(req).await {
// the frame builder just reads them — no tokenizer needed here.
let mut call = match state.core.generate(req).await {
Ok(call) => call,
Err(error) => return native_frontend_error(error, stream),
Err(error) => return native_core_error(error, stream),
};
let rid = call.public_id().to_owned();
// Cumulative frames (SGLang default) vs per-step deltas.
@@ -237,47 +240,59 @@ async fn generate_single(
.map(|data| Ok::<_, Infallible>(Event::default().data(data)));
Sse::new(s).into_response()
} else {
// Unary: fold to the terminal and respond once. `FrontendCall` disarms
// Unary: fold to the terminal and respond once. `CoreCall` disarms
// itself on a terminal item; truncation leaves it armed for drop cleanup.
let (status, value) = drain_unary(&mut call, &rid, timing).await;
(status, Json(value)).into_response()
let (status, item) = drain_unary(&mut call, &rid, timing).await;
(status, Json(item)).into_response()
}
}
/// One unary `/generate` result: the frame, or the item's own error body (a
/// failed batch item is its own `{ "error": ... }` entry in the array).
#[derive(Debug, serde::Serialize)]
#[serde(untagged)]
enum GenerateItem {
// Boxed: a frame is ~300 bytes against the error's ~40, and a batch
// array holds one per item.
Frame(Box<api::GenerateResponse>),
Error(api::GenerateStreamError),
}
/// Fold a unary request to its terminal HTTP result. Shared by single + batch.
async fn drain_unary(
call: &mut FrontendCall,
call: &mut CoreCall,
rid_str: &str,
mut timing: RequestTiming,
) -> (StatusCode, serde_json::Value) {
let mut acc = OutputAccumulator::default();
) -> (StatusCode, GenerateItem) {
let mut accumulated = CoreOutput::default();
while let Some(item) = call.recv().await {
match item {
FrontendEvent::Delta(out) => {
CoreEvent::Delta(out) => {
timing.observe_first_output();
acc.fold(&out);
accumulated.append_delta(&out);
}
FrontendEvent::Finished(out) => {
CoreEvent::Finished(out) => {
timing.observe_first_output();
timing.finish();
acc.fold(&out);
let final_out = acc.into_output();
let mut value = frame_value(&final_out, rid_str);
add_e2e_latency(&mut value, &timing);
return (StatusCode::OK, value);
accumulated.append_delta(&out);
let frame = accumulated.frame(rid_str, None, Some(e2e_latency(&timing)));
return (StatusCode::OK, GenerateItem::Frame(Box::new(frame)));
}
FrontendEvent::Failed(error) => {
CoreEvent::Failed(error) => {
timing.finish();
let status = frontend_error_status(&error);
return (status, error_value(status.as_u16(), &error.to_string()));
return (
error.http_status(),
GenerateItem::Error(error.stream_error(None)),
);
}
}
}
// Defensive fallback: FrontendCall normally turns premature runtime
// Defensive fallback: CoreCall normally turns premature runtime
// termination into a terminal Failed event before exposing stream end.
let truncated = CoreError::ResponseTruncated;
(
StatusCode::INTERNAL_SERVER_ERROR,
error_value(500, "response truncated before completion"),
truncated.http_status(),
GenerateItem::Error(truncated.stream_error(None)),
)
}
@@ -289,19 +304,19 @@ async fn drain_unary(
/// own `{ "error": … }` entry; the batch response is 200.
async fn generate_batch(
state: &AppState,
requests: Vec<FrontendRequest>,
requests: Vec<GenerateRequest>,
stream: bool,
timing: RequestTiming,
) -> Response {
// No cross-item rid collision to worry about: `into_requests` rejected duplicate
// rids within this batch, and `Rid::from_client` made each one unique against
// every other in-flight request.
let calls = match state.frontend.generate_batch(requests).await {
let calls = match state.core.generate_batch(requests).await {
Ok(calls) => calls
.into_iter()
.map(|call| (call, timing.clone()))
.collect(),
Err(error) => return native_frontend_error(error, stream),
Err(error) => return native_core_error(error, stream),
};
if stream {
@@ -319,37 +334,20 @@ async fn generate_batch(
let drained = futures::future::join_all(calls.into_iter().map(
|(mut call, request_timing)| async move {
let client_rid = call.public_id().to_owned();
let (_status, value) = drain_unary(&mut call, &client_rid, request_timing).await;
value
let (_status, item) = drain_unary(&mut call, &client_rid, request_timing).await;
item
},
))
.await;
(StatusCode::OK, Json(serde_json::Value::Array(drained))).into_response()
(StatusCode::OK, Json(drained)).into_response()
}
}
/// Await the next event from `call`, then drain whatever queued behind it (so the
/// caller can coalesce a backlog, as Python's `state.out_list` does), handing the
/// call back for `FuturesUnordered` to re-poll. An empty result means the semantic
/// stream was already exhausted.
async fn recv_indexed(
index: usize,
mut call: FrontendCall,
) -> (usize, FrontendCall, Vec<FrontendEvent>) {
let mut items = Vec::new();
match call.recv().await {
Some(item) => items.push(item),
None => return (index, call, items), // already exhausted
}
call.drain_ready(&mut items);
(index, call, items)
}
/// Multiplex calls into SSE `data` strings + a final `[DONE]`; `with_index` tags
/// each frame (batch only), `incremental` = delta vs cumulative. Each call
/// aborts itself if the stream is dropped while it is unfinished.
fn generation_event_stream(
calls: Vec<(FrontendCall, RequestTiming)>,
calls: Vec<(CoreCall, RequestTiming)>,
incremental: bool,
with_index: bool,
) -> impl futures::Stream<Item = String> {
@@ -365,11 +363,10 @@ fn generation_event_stream(
.iter()
.map(|(_, timing)| timing.clone())
.collect();
let mut accs: Vec<OutputAccumulator> =
(0..n).map(|_| OutputAccumulator::default()).collect();
let mut accumulated: Vec<CoreOutput> = (0..n).map(|_| CoreOutput::default()).collect();
// Batch position, tagged onto every frame (a single request omits it).
let idx = |i: usize| with_index.then_some(i);
let idx = |i: usize| with_index.then_some(i as u32);
// Poll all receivers concurrently; re-arm a receiver's future after each
// non-terminal frame so its stream keeps flowing.
@@ -381,9 +378,9 @@ fn generation_event_stream(
while let Some((i, call, items)) = futs.next().await {
if items.is_empty() {
// Defensive fallback: a premature runtime close is normally a
// Failed event produced by FrontendCall. Keep this item's call
// Failed event produced by CoreCall. Keep this item's call
// armed if its semantic stream somehow ends unexpectedly.
yield tag_value(error_value(500, "response truncated before completion"), idx(i));
yield json_string(&CoreError::ResponseTruncated.stream_error(idx(i)));
continue;
}
@@ -395,22 +392,22 @@ fn generation_event_stream(
for item in items {
match item {
FrontendEvent::Delta(out) => {
CoreEvent::Delta(out) => {
timings[i].observe_first_output();
accs[i].fold(&out);
accumulated[i].append_delta(&out);
if incremental {
yield stream_frame_string(out, &accs[i], true, &rid_strs[i], idx(i));
yield json_string(&delta_frame(out, &accumulated[i], &rid_strs[i], idx(i), None));
} else {
coalesced = true;
}
}
FrontendEvent::Finished(out) => {
CoreEvent::Finished(out) => {
timings[i].observe_first_output();
timings[i].finish();
accs[i].fold(&out);
accumulated[i].append_delta(&out);
terminal = Some(out);
}
FrontendEvent::Failed(error) => {
CoreEvent::Failed(error) => {
timings[i].finish();
failed = Some(error);
}
@@ -418,22 +415,24 @@ fn generation_event_stream(
}
if let Some(error) = failed {
let status = frontend_error_status(&error);
yield tag_value(error_value(status.as_u16(), &error.to_string()), idx(i));
yield json_string(&error.stream_error(idx(i)));
} else if let Some(out) = terminal {
// The final frame carries the full cumulative state, so any
// coalesced non-terminal frames are moot.
yield terminal_stream_frame_string(
out,
&accs[i],
incremental,
&rid_strs[i],
idx(i),
&timings[i],
);
let e2e_latency = Some(e2e_latency(&timings[i]));
// The terminal frame is the accumulated output's last use, so
// the cumulative form moves it out instead of cloning.
let frame = if incremental {
delta_frame(out, &accumulated[i], &rid_strs[i], idx(i), e2e_latency)
} else {
std::mem::take(&mut accumulated[i]).frame(&rid_strs[i], idx(i), e2e_latency)
};
yield json_string(&frame);
} else {
if coalesced {
yield cumulative_frame_string(&accs[i], &rid_strs[i], idx(i));
// An intermediate cumulative frame has to clone: the accumulated
// output stays behind for the next one.
yield json_string(&accumulated[i].clone().frame(&rid_strs[i], idx(i), None));
}
futs.push(recv_indexed(i, call)); // keep this item flowing
}
@@ -444,30 +443,33 @@ fn generation_event_stream(
/// Python's `e2e_latency` is `finished_time - created_time`, in seconds, and is
/// attached only when the request finishes. The Rust native API owns the same
/// lifecycle boundary, so it adds the value while handling the terminal egress
/// item rather than putting API-only timing onto every shared frontend output.
fn add_e2e_latency(value: &mut serde_json::Value, timing: &RequestTiming) {
/// lifecycle boundary, so it sets the value while handling the terminal egress
/// item rather than putting API-only timing onto every shared core output.
fn e2e_latency(timing: &RequestTiming) -> f64 {
let (time_to_first_token, e2e_latency) = timing
.terminal_latencies()
.expect("a successful terminal output has complete request timing");
debug_assert!(time_to_first_token <= e2e_latency);
value["meta_info"]["e2e_latency"] = serde_json::json!(e2e_latency.as_secs_f64());
e2e_latency.as_secs_f64()
}
/// Render a terminal streaming frame. Intermediate cumulative frames keep the
/// memoized fast path; the one terminal frame uses the Value path so it can carry
/// the request-local `e2e_latency`, exactly as Python does.
fn terminal_stream_frame_string(
out: FrontendOutput,
acc: &OutputAccumulator,
incremental: bool,
/// One incremental streaming frame: this step's delta with the cumulative
/// token count in `meta_info` (matching Python). Only the terminal frame
/// carries `e2e_latency`.
fn delta_frame(
mut delta: CoreOutput,
accumulated: &CoreOutput,
rid_str: &str,
index: Option<usize>,
timing: &RequestTiming,
) -> String {
let mut value = super::frame::stream_frame_value(out, acc, incremental, rid_str);
add_e2e_latency(&mut value, timing);
tag_value(value, index)
index: Option<u32>,
e2e_latency: Option<f64>,
) -> api::GenerateResponse {
delta.completion_tokens = accumulated.completion_tokens;
delta.frame(rid_str, index, e2e_latency)
}
/// The SSE `data` text of one `api.v1` item (a frame or an error).
fn json_string<T: serde::Serialize>(value: &T) -> String {
serde_json::to_string(value).expect("api.v1 types serialize to JSON")
}
#[cfg(test)]
@@ -509,7 +511,7 @@ mod tests {
serde_json::from_str(s).expect("frame is JSON")
}
fn timed_call(rid: u64, rx: mpsc::Receiver<ResponseItem>) -> (FrontendCall, RequestTiming) {
fn timed_call(rid: u64, rx: mpsc::Receiver<ResponseItem>) -> (CoreCall, RequestTiming) {
timed_call_with_abort(rid, rx, flume::unbounded().0)
}
@@ -517,9 +519,9 @@ mod tests {
rid: u64,
rx: mpsc::Receiver<ResponseItem>,
abort_tx: flume::Sender<crate::tokenizer_manager::wiring::AbortSource>,
) -> (FrontendCall, RequestTiming) {
) -> (CoreCall, RequestTiming) {
(
FrontendCall::from_test_generation_parts(Rid::from(rid.to_string()), rx, abort_tx),
CoreCall::from_test_generation_parts(Rid::from(rid.to_string()), rx, abort_tx),
RequestTiming {
created_at: Instant::now() - Duration::from_millis(10),
time_to_first_token: None,
@@ -533,16 +535,16 @@ mod tests {
let (intake_tx, intake_rx) = flume::unbounded();
let (abort_tx, abort_rx) = flume::unbounded();
let state = Arc::new(AppState {
frontend: crate::frontend::FrontendHandle::new(
core: crate::api_server::core::CoreHandle::new(
intake_tx,
abort_tx,
crate::frontend::FrontendConfig {
crate::api_server::core::CoreConfig {
response_capacity: 8,
response_activity: Default::default(),
startup_ready: false,
is_disaggregation: false,
mm_limits: Default::default(),
metadata: crate::frontend::FrontendMetadata::default(),
metadata: crate::api_server::core::CoreMetadata::default(),
},
),
server_args: Arc::new(crate::message::config::ServerArgs::default()),
@@ -559,20 +561,20 @@ mod tests {
async fn unary_http_generate_round_trips_through_frontend_handle() {
let (intake_tx, intake_rx) = flume::unbounded();
let (abort_tx, abort_rx) = flume::unbounded();
let frontend = crate::frontend::FrontendHandle::new(
let core = crate::api_server::core::CoreHandle::new(
intake_tx,
abort_tx,
crate::frontend::FrontendConfig {
crate::api_server::core::CoreConfig {
response_capacity: 8,
response_activity: Default::default(),
startup_ready: true,
is_disaggregation: false,
mm_limits: Default::default(),
metadata: crate::frontend::FrontendMetadata::default(),
metadata: crate::api_server::core::CoreMetadata::default(),
},
);
let state = Arc::new(AppState {
frontend,
core,
server_args: Arc::new(crate::message::config::ServerArgs::default()),
chat_formatter: None,
});
@@ -580,7 +582,7 @@ mod tests {
let responder = tokio::spawn(async move {
let TmEvent::Intake { request, admission } = intake_rx.recv_async().await.unwrap()
else {
panic!("HTTP generation must enter through frontend intake");
panic!("HTTP generation must enter through core intake");
};
assert!(admission.try_accept());
let RequestKind::Generate(generate) = &request.kind else {
@@ -681,18 +683,16 @@ mod tests {
}))
.await
.unwrap();
let mut call = FrontendCall::from_test_generation_parts(
"internal-rid".into(),
rx,
flume::unbounded().0,
);
let mut call =
CoreCall::from_test_generation_parts("internal-rid".into(), rx, flume::unbounded().0);
let timing = RequestTiming {
created_at: Instant::now() - Duration::from_millis(20),
time_to_first_token: None,
e2e_latency: None,
};
let (status, value) = drain_unary(&mut call, "client-rid", timing).await;
let (status, item) = drain_unary(&mut call, "client-rid", timing).await;
let value = serde_json::to_value(&item).unwrap();
assert_eq!(status, StatusCode::OK);
assert_eq!(value["meta_info"]["id"], "client-rid");
assert_eq!(value["meta_info"]["prompt_tokens"], 5);
@@ -1,7 +1,7 @@
//! OpenAI-compatible generation endpoints.
//!
//! The HTTP adapter stays deliberately thin: Dynamo owns the standard OpenAI
//! request and response primitives. [`FrontendOutput`] remains the one backend
//! request and response primitives. [`CoreOutput`] remains the one backend
//! output type for both unary and streaming responses.
use axum::{Router, http::StatusCode, response::Response};
@@ -21,12 +21,9 @@ mod tools;
pub(super) use template::{ChatFormatter, ChatTemplateKwargs};
use super::app::AppState;
use super::frame::OutputAccumulator;
use super::frontend_error_status;
use crate::frontend::{
FrontendCall, FrontendError, FrontendEvent, FrontendOutput, FrontendRequest,
};
use crate::api_server::core::{CoreCall, CoreError, CoreEvent, CoreOutput};
use crate::message::config::ServerArgs;
use crate::message::request::GenerateRequest;
use crate::tokenizer_manager::tokenizer;
use crate::utils::response::error_response;
@@ -118,17 +115,17 @@ pub(super) fn openai_error(code: StatusCode, message: impl Into<String>, stream:
/// Drain one submitted request to its terminal output, fold frames, and map
/// semantic failures / truncation to `(status, message)` for the OpenAI error
/// shape. The call owns cancellation and disarms itself.
async fn collect_output(mut call: FrontendCall) -> Result<FrontendOutput, (StatusCode, String)> {
let mut accumulator = OutputAccumulator::default();
async fn collect_output(mut call: CoreCall) -> Result<CoreOutput, (StatusCode, String)> {
let mut accumulated = CoreOutput::default();
let output = loop {
match call.recv().await {
Some(FrontendEvent::Delta(output)) => accumulator.fold(&output),
Some(FrontendEvent::Finished(output)) => {
accumulator.fold(&output);
break accumulator.into_output();
Some(CoreEvent::Delta(output)) => accumulated.append_delta(&output),
Some(CoreEvent::Finished(output)) => {
accumulated.append_delta(&output);
break accumulated;
}
Some(FrontendEvent::Failed(error)) => {
let status = frontend_error_status(&error);
Some(CoreEvent::Failed(error)) => {
let status = error.http_status();
return Err((status, error.to_string()));
}
None => {
@@ -144,15 +141,15 @@ async fn collect_output(mut call: FrontendCall) -> Result<FrontendOutput, (Statu
async fn submit_generation(
state: &AppState,
request: FrontendRequest,
request: GenerateRequest,
stream: bool,
) -> Result<FrontendCall, Response> {
match state.frontend.generate(request).await {
) -> Result<CoreCall, Response> {
match state.core.generate(request).await {
Ok(call) => Ok(call),
// Same `error_response` rule: a committed stream gets 200 plus an
// SSE error frame + `[DONE]`, not a unary 503 — but with the OpenAI
// error shape, since this is the OpenAI frontend.
Err(FrontendError::Unavailable) => Err(openai_error(
Err(CoreError::Unavailable) => Err(openai_error(
StatusCode::SERVICE_UNAVAILABLE,
"service unavailable",
stream,
@@ -167,8 +164,8 @@ async fn submit_generation(
fn indexed_decode_stream(
index: usize,
call: FrontendCall,
) -> futures::stream::BoxStream<'static, (usize, FrontendEvent)> {
call: CoreCall,
) -> futures::stream::BoxStream<'static, (usize, CoreEvent)> {
futures::stream::unfold((call, false), move |(mut call, finished)| async move {
if finished {
return None;
@@ -36,10 +36,10 @@ use super::{
AppState, ChatFormatter, ChatTemplateKwargs, collect_output, contains_media, error_payload,
indexed_decode_stream, openai_error, submit_generation, unix_seconds_u32,
};
use crate::api_server::frontend_error_status;
use crate::frontend::{FrontendCall, FrontendEvent, FrontendRequest};
use crate::api_server::core::{CoreCall, CoreEvent};
use crate::message::config::{DefaultSamplingParams, ServerArgs};
use crate::message::ids::Rid;
use crate::message::request::GenerateRequest;
use crate::message::response::ChunkExtras;
use crate::message::sampling::SamplingParams;
use crate::message::types::OneOrMany;
@@ -204,7 +204,7 @@ async fn chat_completions(
.expect("chat prompt exists until the last choice")
.clone()
};
let native = FrontendRequest {
let native = GenerateRequest {
rid: rid.clone(),
text: Some(choice_prompt),
// Rendered templates own their special tokens — the pool must not
@@ -438,7 +438,7 @@ pub(super) fn chat_sampling_params(
#[allow(clippy::too_many_arguments)]
pub(super) async fn unary_chat(
submitted: Vec<(usize, FrontendCall)>,
submitted: Vec<(usize, CoreCall)>,
response_id: String,
model: String,
created: u32,
@@ -521,7 +521,7 @@ pub(super) async fn unary_chat(
#[allow(clippy::too_many_arguments)]
pub(super) fn chat_event_stream(
submitted: Vec<(usize, FrontendCall)>,
submitted: Vec<(usize, CoreCall)>,
response_id: String,
model: String,
created: u32,
@@ -582,14 +582,14 @@ pub(super) fn chat_event_stream(
let mut events = futures::stream::select_all(streams);
while let Some((index, event)) = events.next().await {
let output = match event {
FrontendEvent::Delta(output) | FrontendEvent::Finished(output) => output,
FrontendEvent::Failed(error) => {
CoreEvent::Delta(output) | CoreEvent::Finished(output) => output,
CoreEvent::Failed(error) => {
yield Annotated {
data: None,
id: None,
event: None,
comment: None,
error: Some(error_payload(frontend_error_status(&error), error.to_string()).to_string()),
error: Some(error_payload(error.http_status(), error.to_string()).to_string()),
};
continue;
}
@@ -24,12 +24,10 @@ use super::{
AppState, MAX_OPENAI_CHOICES, collect_output, error_payload, indexed_decode_stream,
openai_error, submit_generation, unix_seconds_u32,
};
use crate::api_server::frontend_error_status;
use crate::frontend::{
FrontendCall, FrontendError, FrontendEvent, FrontendOutput, FrontendRequest,
};
use crate::api_server::core::{CoreCall, CoreError, CoreEvent, CoreOutput};
use crate::message::finish_reason::Matched;
use crate::message::ids::Rid;
use crate::message::request::GenerateRequest;
use crate::message::response::ChunkExtras;
use crate::message::sampling::SamplingParams;
use crate::message::types::{OneOrMany, TokenIds};
@@ -48,7 +46,7 @@ pub(super) struct SubmittedChoice {
pub(super) index: usize,
pub(super) prompt_index: usize,
pub(super) echo: String,
pub(super) call: FrontendCall,
pub(super) call: CoreCall,
}
#[derive(Debug, Default)]
pub(super) struct ChoiceExtensions {
@@ -82,7 +80,7 @@ async fn completions(
if request.prompt_embeds.is_some() {
return openai_error(
StatusCode::BAD_REQUEST,
"prompt_embeds is not supported by the Rust frontend",
"prompt_embeds is not supported by the Rust server",
false,
);
}
@@ -164,7 +162,7 @@ async fn completions(
Err(response) => return response,
};
}
let native = FrontendRequest {
let native = GenerateRequest {
rid: rid.clone(),
text: text.clone(),
input_ids: input_ids.clone(),
@@ -230,29 +228,29 @@ async fn completions(
}
/// Decode a token-id prompt back to text for `echo=true`, via a
/// detokenize operation through the shared frontend handle — the
/// detokenize operation through the shared core handle — the
/// detok stage answers it with a single `Data` payload (the raw UTF-8 text),
/// or an `Error` (e.g. out-of-range ids → `Validation` → 400).
async fn decode_prompt_echo(state: &AppState, token_ids: TokenIds) -> Result<String, Response> {
match state.frontend.detokenize(token_ids).await {
match state.core.detokenize(token_ids).await {
Ok(text) => Ok(text),
// Same rule as `submit_generation`: build the refusal in the OpenAI
// error shape rather than forwarding the native-shaped response.
Err(FrontendError::Unavailable) => Err(openai_error(
Err(CoreError::Unavailable) => Err(openai_error(
StatusCode::SERVICE_UNAVAILABLE,
"service unavailable",
false,
)),
Err(FrontendError::InvalidArgument(message)) => {
Err(CoreError::InvalidArgument(message)) => {
Err(openai_error(StatusCode::BAD_REQUEST, &message, false))
}
Err(FrontendError::InvalidResponse(message)) => Err(openai_error(
Err(CoreError::InvalidResponse(message)) => Err(openai_error(
StatusCode::INTERNAL_SERVER_ERROR,
message,
false,
)),
Err(error) => {
let status = frontend_error_status(&error);
let status = error.http_status();
Err(openai_error(
status,
format!("failed to decode prompt for echo: {error}"),
@@ -406,7 +404,7 @@ pub(super) async fn unary_completion(
fn completion_choice(
index: usize,
text: String,
output: &FrontendOutput,
output: &CoreOutput,
want_logprobs: bool,
include_input_logprobs: bool,
) -> (Choice, ChoiceExtensions) {
@@ -521,9 +519,9 @@ pub(super) fn completion_event_stream(
while let Some((index, event)) = events.next().await {
let output = match event {
FrontendEvent::Delta(output) | FrontendEvent::Finished(output) => output,
FrontendEvent::Failed(error) => {
yield error_payload(frontend_error_status(&error), error.to_string()).to_string();
CoreEvent::Delta(output) | CoreEvent::Finished(output) => output,
CoreEvent::Failed(error) => {
yield error_payload(error.http_status(), error.to_string()).to_string();
continue;
}
};
@@ -3,7 +3,7 @@
//! Submodule tests live next to the code they cover: `chat`, `completions`,
//! `tools`, and `reasoning` each carry their own
//! `#[cfg(test)] mod tests`. This module keeps the fixtures they all share —
//! frontend/call fixtures (`frontend`, `chunk`, `submitted`, `chat_submitted`) and the
//! core/call fixtures (`core_handle`, `chunk`, `submitted`, `chat_submitted`) and the
//! full-router harness (`server_args`, `app_state`,
//! `oneshot`, `post_json`, `body_json`) — plus the handler-level tests that
//! exercise [`routes`] end to end. The helpers are `pub(super)` so sibling
@@ -20,20 +20,20 @@ use serde_json::json;
use tower::util::ServiceExt;
use super::{indexed_decode_stream, openai_error, routes};
use crate::frontend::{FrontendCall, FrontendEvent, FrontendHandle};
use crate::api_server::core::{CoreCall, CoreEvent, CoreHandle};
use crate::message::config::ServerArgs;
use crate::message::response::{ChunkEvent, ResponseItem};
pub(super) fn frontend() -> FrontendHandle {
FrontendHandle::new(
pub(super) fn core_handle() -> CoreHandle {
CoreHandle::new(
flume::unbounded().0,
flume::unbounded().0,
crate::frontend::FrontendConfig {
crate::api_server::core::CoreConfig {
response_capacity: 8,
response_activity: Default::default(),
startup_ready: false,
is_disaggregation: false,
mm_limits: Default::default(),
metadata: crate::frontend::FrontendMetadata::from(server_args().as_ref()),
metadata: crate::api_server::core::CoreMetadata::from(server_args().as_ref()),
},
)
}
@@ -76,7 +76,7 @@ pub(super) fn submitted(
index,
prompt_index,
echo: String::new(),
call: FrontendCall::from_test_generation_parts(rid.into(), rx, flume::unbounded().0),
call: CoreCall::from_test_generation_parts(rid.into(), rx, flume::unbounded().0),
},
tx,
)
@@ -86,15 +86,12 @@ pub(super) fn submitted(
pub(super) fn chat_submitted(
index: usize,
rid: &str,
) -> (
(usize, FrontendCall),
tokio::sync::mpsc::Sender<ResponseItem>,
) {
) -> ((usize, CoreCall), tokio::sync::mpsc::Sender<ResponseItem>) {
let (tx, rx) = tokio::sync::mpsc::channel(8);
(
(
index,
FrontendCall::from_test_generation_parts(rid.into(), rx, flume::unbounded().0),
CoreCall::from_test_generation_parts(rid.into(), rx, flume::unbounded().0),
),
tx,
)
@@ -107,31 +104,31 @@ pub(super) fn server_args() -> Arc<ServerArgs> {
})
}
pub(super) fn app_state(frontend: FrontendHandle) -> Arc<super::AppState> {
pub(super) fn app_state(core: CoreHandle) -> Arc<super::AppState> {
Arc::new(super::AppState {
frontend,
core,
server_args: server_args(),
chat_formatter: None,
})
}
pub(super) fn frontend_closed() -> FrontendHandle {
// Dropping the receivers makes frontend admission report the shutdown
pub(super) fn frontend_closed() -> CoreHandle {
// Dropping the receivers makes core admission report the shutdown
// state as a 503.
let (tm_tx, tm_rx) = flume::unbounded();
drop(tm_rx);
let (abort_tx, abort_rx) = flume::unbounded();
drop(abort_rx);
FrontendHandle::new(
CoreHandle::new(
tm_tx,
abort_tx,
crate::frontend::FrontendConfig {
crate::api_server::core::CoreConfig {
response_capacity: 8,
response_activity: Default::default(),
startup_ready: false,
is_disaggregation: false,
mm_limits: Default::default(),
metadata: crate::frontend::FrontendMetadata::from(server_args().as_ref()),
metadata: crate::api_server::core::CoreMetadata::from(server_args().as_ref()),
},
)
}
@@ -166,13 +163,13 @@ async fn dropping_indexed_stream_aborts_its_live_call() {
let (response_tx, response_rx) = tokio::sync::mpsc::channel(8);
let (abort_tx, abort_rx) = flume::unbounded();
let call = FrontendCall::from_test_generation_parts("live".into(), response_rx, abort_tx);
let call = CoreCall::from_test_generation_parts("live".into(), response_rx, abort_tx);
let mut stream = indexed_decode_stream(0, call);
response_tx.send(chunk("live", "x", false)).await.unwrap();
assert!(matches!(
stream.next().await,
Some((0, FrontendEvent::Delta(_)))
Some((0, CoreEvent::Delta(_)))
));
drop(stream);
@@ -217,7 +214,7 @@ async fn openai_error_response_covers_unary_and_sse() {
#[tokio::test]
async fn completions_handler_validates_before_submit() {
let app = routes().with_state(app_state(frontend()));
let app = routes().with_state(app_state(core_handle()));
let cases = [
(json!({"model": "other", "prompt": "hi"}), "unknown model"),
(json!({"model": "model", "prompt": "hi", "n": 0}), "n=0"),
@@ -265,7 +262,7 @@ async fn completions_handler_validates_before_submit() {
#[tokio::test]
async fn chat_handler_validates_before_submit() {
let app = routes().with_state(app_state(frontend()));
let app = routes().with_state(app_state(core_handle()));
let cases = [
(
json!({"model": "other", "messages": [{"role": "user", "content": "hi"}]}),
@@ -309,7 +306,7 @@ async fn chat_handler_validates_before_submit() {
#[tokio::test]
async fn basic_openai_router_excludes_responses_api() {
let app = routes().with_state(app_state(frontend()));
let app = routes().with_state(app_state(core_handle()));
let response = post_json(app, "/v1/responses", json!({"input": "hi"})).await;
assert_eq!(response.status(), StatusCode::NOT_FOUND);
}
@@ -36,7 +36,7 @@ use dynamo_protocols::types::{
Role,
};
use crate::frontend::FrontendOutput;
use crate::api_server::core::CoreOutput;
use crate::message::sampling::SamplingParams;
/// Canonicalize a tool-call parser name onto the dynamo-parsers registry keys.
@@ -246,7 +246,7 @@ pub(super) async fn parse_chat_tool_calls(
/// Map the scheduler's finish kind onto the OpenAI wire values. Length and
/// content-filter keep their names; everything else (including a bare abort)
/// reports as `stop`, matching Python's fallback.
pub(super) fn chat_finish_reason(output: &FrontendOutput) -> Option<OpenAIFinishReason> {
pub(super) fn chat_finish_reason(output: &CoreOutput) -> Option<OpenAIFinishReason> {
let kind = output
.finish_reason
.as_ref()
@@ -264,7 +264,7 @@ mod tests {
apply_tool_constraint, chat_delta, chat_finish_reason, dynamo_parser_name,
dynamo_tool_choice, parse_chat_tool_calls,
};
use crate::frontend::FrontendOutput;
use crate::api_server::core::CoreOutput;
use crate::message::sampling::SamplingParams;
use dynamo_parsers::tool_calling::jail::{Annotated, apply_tool_calling_jail};
use dynamo_parsers::{ToolChoice as DynamoToolChoice, ToolDefinition};
@@ -766,7 +766,7 @@ mod tests {
#[test]
fn chat_finish_reason_maps_scheduler_kinds() {
let output = |finish: serde_json::Value| FrontendOutput {
let output = |finish: serde_json::Value| CoreOutput {
text: "x".into(),
token_ids: vec![1],
prompt_tokens: 1,
@@ -794,7 +794,7 @@ mod tests {
Some(OpenAIFinishReason::Stop)
);
assert_eq!(
chat_finish_reason(&FrontendOutput {
chat_finish_reason(&CoreOutput {
finish_reason: None,
..Default::default()
}),
-356
View File
@@ -1,356 +0,0 @@
//! Transport-neutral types shared by frontend adapters.
//!
//! These types describe what an inference operation means after a wire adapter
//! has normalized its request. They intentionally contain no Axum, HTTP, SSE,
//! Tonic, protobuf, or scheduler-channel details.
use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
use crate::message::config::{DisaggregationMode, PreferredSamplingParams};
use crate::message::finish_reason::FinishReason;
use crate::message::request::GenerateRequest;
use crate::message::response::{ChunkEvent, ChunkExtras};
use crate::message::types::TokenIds;
/// Canonical transport-neutral input for one generation operation.
///
/// The existing in-process request already contains semantic generation fields
/// and has no Serde, Axum, Tonic, or protobuf representation attached to it.
/// Re-exporting it through the frontend contract gives every adapter one public
/// boundary without duplicating the runtime's request model.
pub(crate) type FrontendRequest = GenerateRequest;
/// Canonical transport-neutral output payload for generation events.
///
/// The runtime correlation ID is deliberately absent: it is an implementation
/// detail used for scheduler routing, while adapters obtain the client-visible
/// identity from [`crate::frontend::FrontendCall::public_id`]. Runtime-only
/// response variants such as raw control bytes are likewise hidden by
/// [`crate::frontend::FrontendCall`].
#[derive(Clone, Debug, Default)]
pub(crate) struct FrontendOutput {
pub(crate) token_ids: TokenIds,
pub(crate) finish_reason: Option<FinishReason>,
pub(crate) prompt_tokens: u32,
pub(crate) text: String,
pub(crate) completion_tokens: u64,
pub(crate) extras: Option<Box<ChunkExtras>>,
}
impl From<ChunkEvent> for FrontendOutput {
fn from(output: ChunkEvent) -> Self {
let ChunkEvent {
rid: _,
token_ids,
finish_reason,
prompt_tokens,
text,
completion_tokens,
extras,
} = output;
Self {
token_ids,
finish_reason,
prompt_tokens,
text,
completion_tokens,
extras,
}
}
}
impl FrontendOutput {
/// Fold one runtime delta into this cumulative native-generation output.
///
/// Runtime events are always incremental. Native HTTP and `runtime.v1`
/// gRPC independently decide whether to expose those deltas or the
/// cumulative result, so the fold itself lives at their shared semantic
/// boundary rather than in either wire adapter.
pub(crate) fn append_delta(&mut self, delta: &Self) {
self.text.push_str(&delta.text);
self.token_ids.extend_from_slice(&delta.token_ids);
self.completion_tokens += delta.completion_tokens;
self.prompt_tokens = delta.prompt_tokens;
if delta.finish_reason.is_some() {
self.finish_reason = delta.finish_reason.clone();
}
let Some(delta_extras) = delta.extras.as_deref() else {
return;
};
let extras = self
.extras
.get_or_insert_with(|| Box::new(ChunkExtras::default()));
extras
.out_lp_val
.extend_from_slice(&delta_extras.out_lp_val);
extras
.out_lp_idx
.extend_from_slice(&delta_extras.out_lp_idx);
extras
.out_top_val
.extend_from_slice(&delta_extras.out_top_val);
extras
.out_top_idx
.extend_from_slice(&delta_extras.out_top_idx);
extras
.out_top_lens
.extend_from_slice(&delta_extras.out_top_lens);
extras
.out_tid_val
.extend_from_slice(&delta_extras.out_tid_val);
extras
.out_tid_idx
.extend_from_slice(&delta_extras.out_tid_idx);
extras
.out_tid_lens
.extend_from_slice(&delta_extras.out_tid_lens);
extras
.out_lp_txt
.extend_from_slice(&delta_extras.out_lp_txt);
extras
.out_top_txt
.extend_from_slice(&delta_extras.out_top_txt);
extras
.out_tid_txt
.extend_from_slice(&delta_extras.out_tid_txt);
// Input logprobs arrive once (during prefill). A later non-empty value
// replaces the earlier one, matching the existing native HTTP fold.
if !delta_extras.in_lp_val.is_empty() {
extras.in_lp_val = delta_extras.in_lp_val.clone();
extras.in_lp_idx = delta_extras.in_lp_idx.clone();
extras.in_lp_txt = delta_extras.in_lp_txt.clone();
}
if !delta_extras.in_top_lens.is_empty() {
extras.in_top_val = delta_extras.in_top_val.clone();
extras.in_top_idx = delta_extras.in_top_idx.clone();
extras.in_top_lens = delta_extras.in_top_lens.clone();
extras.in_top_txt = delta_extras.in_top_txt.clone();
}
if !delta_extras.in_tid_lens.is_empty() {
extras.in_tid_val = delta_extras.in_tid_val.clone();
extras.in_tid_idx = delta_extras.in_tid_idx.clone();
extras.in_tid_lens = delta_extras.in_tid_lens.clone();
extras.in_tid_txt = delta_extras.in_tid_txt.clone();
}
// Hidden states are non-cumulative: the latest complete set wins.
if !delta_extras.hidden_lens.is_empty() {
extras.hidden_val = delta_extras.hidden_val.clone();
extras.hidden_lens = delta_extras.hidden_lens.clone();
}
}
}
/// Semantic result of a deep frontend health check.
///
/// Expected lifecycle states are values rather than transport errors so each
/// adapter can render them according to its own protocol.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum HealthStatus {
Healthy,
NotReady,
Stalled,
}
/// A semantic generation event before any transport-specific framing.
#[derive(Debug)]
pub(crate) enum FrontendEvent {
/// One incremental model-output update.
Delta(FrontendOutput),
/// The final model-output update for this request.
Finished(FrontendOutput),
/// A request-scoped failure. Other requests in a batch may continue.
Failed(FrontendError),
}
/// Protocol-independent error category used by transport adapters.
///
/// The set intentionally follows operation semantics rather than HTTP or gRPC
/// status spaces. An adapter maps this category into its own wire status.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum FrontendErrorKind {
InvalidArgument,
NotFound,
FailedPrecondition,
ResourceExhausted,
Cancelled,
DeadlineExceeded,
Unavailable,
Internal,
}
impl FrontendEvent {
pub(crate) fn is_terminal(&self) -> bool {
matches!(self, Self::Finished(_) | Self::Failed(_))
}
}
/// A transport-neutral failure category.
///
/// HTTP and gRPC adapters map these categories into their own status spaces;
/// neither adapter needs to understand errors emitted by runtime stages.
#[derive(Clone, Debug, thiserror::Error)]
pub(crate) enum FrontendError {
#[error("{0}")]
InvalidArgument(String),
/// The frontend intake loop has shut down. A caller may retry elsewhere.
#[error("service unavailable")]
Unavailable,
/// Runtime capacity was exhausted before the request could make progress.
#[error("{0}")]
Overloaded(String),
#[error("{0}")]
Cancelled(String),
/// The internal producer disappeared before returning a terminal result.
///
/// This is a server-side failure, not evidence that the caller cancelled.
#[error("response truncated before completion")]
ResponseTruncated,
/// A scheduler-side abort that carried an error status.
///
/// The current Python scheduler communicates this category as an HTTP
/// status code inside its finish reason. The semantic event is still
/// `Failed`; retaining the legacy code only lets the HTTP adapter preserve
/// existing behavior while another adapter maps it into its own status
/// space. New frontend failures should use the semantic variants above.
#[error("{message}")]
RuntimeRejected {
kind: FrontendErrorKind,
message: String,
legacy_http_status: u16,
},
/// The runtime replied with an unexpected or malformed result.
#[error("{0}")]
InvalidResponse(String),
#[error("{0}")]
Internal(String),
}
impl FrontendError {
/// Semantic status for adapters that do not use the scheduler's legacy
/// HTTP code space (notably the future gRPC adapter).
pub(crate) fn kind(&self) -> FrontendErrorKind {
match self {
Self::InvalidArgument(_) => FrontendErrorKind::InvalidArgument,
Self::Unavailable => FrontendErrorKind::Unavailable,
Self::Overloaded(_) => FrontendErrorKind::ResourceExhausted,
Self::Cancelled(_) => FrontendErrorKind::Cancelled,
Self::ResponseTruncated => FrontendErrorKind::Internal,
Self::RuntimeRejected { kind, .. } => *kind,
Self::InvalidResponse(_) | Self::Internal(_) => FrontendErrorKind::Internal,
}
}
/// Translate the scheduler's inherited HTTP-coded failure into a semantic
/// category once, at the runtime boundary. HTTP can retain the exact legacy
/// status while other adapters consume [`Self::kind`].
pub(super) fn from_runtime_rejection(message: String, legacy_http_status: u16) -> Self {
let kind = match legacy_http_status {
404 => FrontendErrorKind::NotFound,
408 | 504 => FrontendErrorKind::DeadlineExceeded,
412 => FrontendErrorKind::FailedPrecondition,
413 | 429 => FrontendErrorKind::ResourceExhausted,
499 => FrontendErrorKind::Cancelled,
502 | 503 => FrontendErrorKind::Unavailable,
400..=499 => FrontendErrorKind::InvalidArgument,
_ => FrontendErrorKind::Internal,
};
Self::RuntimeRejected {
kind,
message,
legacy_http_status,
}
}
}
/// Static model metadata shared by every public transport.
#[derive(Clone, Debug, Serialize)]
pub(crate) struct ModelInfo {
pub(crate) model_path: String,
pub(crate) served_model_name: String,
pub(crate) tokenizer_path: String,
pub(crate) is_generation: bool,
pub(crate) preferred_sampling_params: Option<PreferredSamplingParams>,
pub(crate) weight_version: Option<String>,
pub(crate) load_format: Option<String>,
pub(crate) reasoning_parser: Option<String>,
pub(crate) tool_call_parser: Option<String>,
pub(crate) disaggregation_mode: DisaggregationMode,
}
/// Public server metadata plus scheduler-owned runtime metrics.
#[derive(Clone, Debug, Serialize)]
pub(crate) struct ServerInfo {
pub(crate) model_path: String,
pub(crate) served_model_name: String,
pub(crate) tokenizer_path: String,
pub(crate) max_context_length: u64,
pub(crate) max_total_num_tokens: u64,
pub(crate) version: String,
pub(crate) frontend: &'static str,
pub(crate) internal_states: Vec<InternalState>,
}
/// Scheduler-owned runtime metrics used by the public server-info operation.
///
/// Deserializing into this allowlisted shape is intentional: the scheduler's
/// raw state also contains its full launch arguments, including credentials.
/// Unknown fields are discarded before any transport receives the result.
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
#[serde(default)]
pub(crate) struct InternalState {
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) last_gen_throughput: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) memory_usage: Option<MemoryUsage>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) effective_max_running_requests_per_dp: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) avg_spec_accept_length: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) step_time_dict: Option<BTreeMap<u64, Vec<f64>>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) rust_mm_transport: Option<BTreeMap<String, u64>>,
}
/// Public subset of scheduler memory metrics.
///
/// This nested allowlist is intentional: adding a scheduler field does not
/// silently add different public data to HTTP before the gRPC contract can add
/// the same field. New metrics should be promoted here deliberately.
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
#[serde(default)]
pub(crate) struct MemoryUsage {
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) weight: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) kvcache: Option<MemoryMeasurement>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) startup_available: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) token_capacity: Option<u64>,
pub(crate) token_capacity_swa: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) graph: Option<BTreeMap<String, f64>>,
}
/// The scheduler's KV-cache measurement can be a native float or a NumPy
/// scalar stringified by the MessagePack bridge. Preserve that representation
/// so extracting the typed frontend contract does not change public responses.
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[serde(untagged)]
pub(crate) enum MemoryMeasurement {
Number(f64),
String(String),
}
-531
View File
@@ -1,531 +0,0 @@
//! `runtime.v1` request conversion for the Rust frontend.
//!
//! The canonical protobuf is wider than the Rust frontend today. Fields with
//! no internal representation are rejected explicitly instead of being
//! silently ignored or expanding frontend behavior in this wiring change.
use sglang_grpc_types::sglang::runtime::v1 as proto;
use tonic::Status;
use crate::frontend::FrontendRequest;
use crate::message::config::PreferredSamplingParams;
use crate::message::ids::Rid;
use crate::message::sampling::SamplingParams;
type ConvertResult<T> = Result<T, ConvertError>;
#[derive(Debug, thiserror::Error)]
pub(super) enum ConvertError {
#[error("{0}")]
InvalidArgument(String),
#[error("{0}")]
Unsupported(String),
#[error("{0}")]
Internal(String),
}
impl ConvertError {
fn invalid_argument(message: impl Into<String>) -> Self {
Self::InvalidArgument(message.into())
}
fn internal(message: impl Into<String>) -> Self {
Self::Internal(message.into())
}
}
impl From<ConvertError> for Status {
fn from(error: ConvertError) -> Self {
match error {
ConvertError::InvalidArgument(message) => Self::invalid_argument(message),
ConvertError::Unsupported(message) => Self::unimplemented(message),
ConvertError::Internal(message) => Self::internal(message),
}
}
}
pub(super) fn text_generate(
request: proto::TextGenerateRequest,
preferred: Option<&PreferredSamplingParams>,
) -> ConvertResult<FrontendRequest> {
let proto::TextGenerateRequest {
text,
sampling_params,
stream,
return_logprob,
top_logprobs_num,
logprob_start_len,
return_text_in_logprobs,
rid,
lora_path,
routing_key,
routed_dp_rank,
trace_headers,
session_id,
disaggregated_params,
priority,
require_reasoning,
max_thinking_tokens,
kv_hints,
} = request;
reject_unsupported([
("kv_hints", kv_hints.is_some()),
("lora_path", lora_path.is_some()),
("routing_key", routing_key.is_some()),
("trace_headers", !trace_headers.is_empty()),
("session_id", session_id.is_some()),
("priority", priority.is_some()),
("require_reasoning", require_reasoning.is_some()),
("max_thinking_tokens", max_thinking_tokens.is_some()),
])?;
let (bootstrap_host, bootstrap_port, bootstrap_room) =
disaggregated_fields(disaggregated_params);
Ok(FrontendRequest {
rid: request_id(rid),
text: Some(text),
input_ids: None,
skip_special_tokens: false,
sampling_params: convert_sampling_params(sampling_params, preferred)?,
stream: stream.unwrap_or(false),
return_logprob: return_logprob.unwrap_or(false),
logprob_start_len: logprob_start_len.map(i64::from).unwrap_or(-1),
top_logprobs_num: top_logprobs_num.map(i64::from).unwrap_or(0),
token_ids_logprob: None,
return_sampling_mask: false,
return_hidden_states: false,
return_text_in_logprobs,
bootstrap_host,
bootstrap_port,
bootstrap_room,
bootstrap_pair_key: None,
decode_tp_size: None,
routed_dp_rank: routed_dp_rank.map(i64::from),
disagg_prefill_dp_rank: None,
mm: None,
mm_buffers: Vec::new(),
})
}
pub(super) fn generate(
request: proto::GenerateRequest,
preferred: Option<&PreferredSamplingParams>,
) -> ConvertResult<FrontendRequest> {
let proto::GenerateRequest {
input_ids,
sampling_params,
stream,
return_logprob,
top_logprobs_num,
logprob_start_len,
rid,
lora_path,
routing_key,
routed_dp_rank,
trace_headers,
session_id,
disaggregated_params,
priority,
require_reasoning,
max_thinking_tokens,
kv_hints,
} = request;
reject_unsupported([
("kv_hints", kv_hints.is_some()),
("lora_path", lora_path.is_some()),
("routing_key", routing_key.is_some()),
("trace_headers", !trace_headers.is_empty()),
("session_id", session_id.is_some()),
("priority", priority.is_some()),
("require_reasoning", require_reasoning.is_some()),
("max_thinking_tokens", max_thinking_tokens.is_some()),
])?;
if input_ids.is_empty() {
return Err(ConvertError::invalid_argument("input_ids cannot be empty"));
}
let (bootstrap_host, bootstrap_port, bootstrap_room) =
disaggregated_fields(disaggregated_params);
Ok(FrontendRequest {
rid: request_id(rid),
text: None,
input_ids: Some(input_ids.into_iter().map(i64::from).collect()),
skip_special_tokens: false,
sampling_params: convert_sampling_params(sampling_params, preferred)?,
stream: stream.unwrap_or(false),
return_logprob: return_logprob.unwrap_or(false),
logprob_start_len: logprob_start_len.map(i64::from).unwrap_or(-1),
top_logprobs_num: top_logprobs_num.map(i64::from).unwrap_or(0),
token_ids_logprob: None,
return_sampling_mask: false,
return_hidden_states: false,
return_text_in_logprobs: None,
bootstrap_host,
bootstrap_port,
bootstrap_room,
bootstrap_pair_key: None,
decode_tp_size: None,
routed_dp_rank: routed_dp_rank.map(i64::from),
disagg_prefill_dp_rank: None,
mm: None,
mm_buffers: Vec::new(),
})
}
fn request_id(rid: Option<String>) -> Rid {
rid.map_or_else(Rid::new, |rid| Rid::from_client(&rid))
}
fn reject_unsupported<const N: usize>(fields: [(&'static str, bool); N]) -> ConvertResult<()> {
let unsupported = fields
.into_iter()
.filter_map(|(name, present)| present.then_some(name))
.collect::<Vec<_>>();
if unsupported.is_empty() {
return Ok(());
}
Err(ConvertError::Unsupported(format!(
"the Rust frontend does not yet support runtime.v1 field(s): {}",
unsupported.join(", ")
)))
}
fn disaggregated_fields(
params: Option<proto::DisaggregatedParams>,
) -> (Option<String>, Option<i64>, Option<i64>) {
params.map_or((None, None, None), |params| {
(
Some(params.bootstrap_host),
Some(i64::from(params.bootstrap_port)),
Some(params.bootstrap_room),
)
})
}
/// Merge launch-time preferred values first, then overwrite only fields that
/// are explicitly present in protobuf. This is the same precedence used by
/// the Python-backed gRPC path; proto3 repeated fields cannot distinguish an
/// omitted list from an explicit empty list, so empty stop lists remain absent.
#[allow(deprecated)]
fn convert_sampling_params(
params: Option<proto::SamplingParams>,
preferred: Option<&PreferredSamplingParams>,
) -> ConvertResult<SamplingParams> {
let mut output = SamplingParams::default();
let Some(params) = params else {
if let Some(preferred) = preferred {
output.apply_preferred(&preferred.0).map_err(|error| {
ConvertError::internal(format!("invalid preferred_sampling_params: {error}"))
})?;
}
return Ok(output);
};
if params.guided_decoding.is_some() && (params.json_schema.is_some() || params.regex.is_some())
{
return Err(ConvertError::invalid_argument(
"legacy json_schema/regex cannot be combined with guided_decoding",
));
}
if let Some(value) = params.temperature {
output.temperature = f64::from(value);
output.mark_explicit("temperature");
}
if let Some(value) = params.top_p {
output.top_p = f64::from(value);
output.mark_explicit("top_p");
}
if let Some(value) = params.top_k {
output.top_k = i64::from(value);
output.mark_explicit("top_k");
}
if let Some(value) = params.min_p {
output.min_p = f64::from(value);
output.mark_explicit("min_p");
}
if let Some(value) = params.frequency_penalty {
output.frequency_penalty = f64::from(value);
output.mark_explicit("frequency_penalty");
}
if let Some(value) = params.presence_penalty {
output.presence_penalty = f64::from(value);
output.mark_explicit("presence_penalty");
}
if let Some(value) = params.repetition_penalty {
output.repetition_penalty = f64::from(value);
output.mark_explicit("repetition_penalty");
}
if let Some(value) = params.max_new_tokens {
output.max_new_tokens = Some(i64::from(value));
output.mark_explicit("max_new_tokens");
}
if let Some(value) = params.min_new_tokens {
output.min_new_tokens = i64::from(value);
output.mark_explicit("min_new_tokens");
}
if !params.stop.is_empty() {
output.stop = Some(crate::message::types::OneOrMany::Many(params.stop));
output.mark_explicit("stop");
}
if !params.stop_token_ids.is_empty() {
output.stop_token_ids = Some(params.stop_token_ids.into_iter().map(i64::from).collect());
output.mark_explicit("stop_token_ids");
}
if let Some(value) = params.ignore_eos {
output.ignore_eos = value;
output.mark_explicit("ignore_eos");
}
if let Some(value) = params.n {
output.n = i64::from(value);
output.mark_explicit("n");
}
if let Some(value) = params.seed {
output.sampling_seed = Some(value);
output.mark_explicit("sampling_seed");
}
if let Some(guided) = params.guided_decoding {
apply_guided_decoding(&mut output, guided)?;
} else {
if let Some(value) = params.json_schema {
if value.is_empty() {
return Err(ConvertError::invalid_argument(
"legacy json_schema must not be empty",
));
}
output.json_schema = Some(value);
output.mark_explicit("json_schema");
}
if let Some(value) = params.regex {
if value.is_empty() {
return Err(ConvertError::invalid_argument(
"legacy regex must not be empty",
));
}
output.regex = Some(value);
output.mark_explicit("regex");
}
}
if let Some(preferred) = preferred {
output.apply_preferred(&preferred.0).map_err(|error| {
ConvertError::internal(format!("invalid preferred_sampling_params: {error}"))
})?;
}
Ok(output)
}
fn apply_guided_decoding(
output: &mut SamplingParams,
guided: proto::GuidedDecoding,
) -> ConvertResult<()> {
use proto::guided_decoding::Constraint;
match guided.constraint {
Some(Constraint::JsonSchema(value)) if !value.is_empty() => {
output.json_schema = Some(value);
output.mark_explicit("json_schema");
}
Some(Constraint::Regex(value)) if !value.is_empty() => {
output.regex = Some(value);
output.mark_explicit("regex");
}
Some(Constraint::Ebnf(value)) if !value.is_empty() => {
output.ebnf = Some(value);
output.mark_explicit("ebnf");
}
Some(Constraint::StructuralTag(value)) if !value.is_empty() => {
output.structural_tag = Some(value);
output.mark_explicit("structural_tag");
}
Some(Constraint::Choice(choice))
if !choice.values.is_empty() && choice.values.iter().all(|value| !value.is_empty()) =>
{
let alternatives = choice
.values
.iter()
.map(|value| regex_escape_literal(value))
.collect::<Vec<_>>()
.join("|");
output.regex = Some(format!("(?:{alternatives})"));
output.mark_explicit("regex");
}
Some(Constraint::Choice(_)) => {
return Err(ConvertError::invalid_argument(
"guided choice must contain only non-empty values",
));
}
Some(_) => {
return Err(ConvertError::invalid_argument(
"guided decoding constraint must not be empty",
));
}
None => {
return Err(ConvertError::invalid_argument(
"guided decoding constraint must be specified",
));
}
}
Ok(())
}
fn regex_escape_literal(value: &str) -> String {
let mut escaped = String::with_capacity(value.len());
for character in value.chars() {
if matches!(
character,
'.' | '+' | '*' | '?' | '^' | '$' | '(' | ')' | '[' | ']' | '{' | '}' | '|' | '\\'
) {
escaped.push('\\');
}
escaped.push(character);
}
escaped
}
#[cfg(test)]
mod tests {
use super::*;
use tonic::Code;
#[test]
fn text_request_maps_supported_fields_and_preferred_sampling() {
let preferred = PreferredSamplingParams(serde_json::json!({
"temperature": 0.2,
"top_p": 0.7,
"max_new_tokens": 64,
}));
let request = proto::TextGenerateRequest {
text: "hello".into(),
sampling_params: Some(proto::SamplingParams {
temperature: Some(0.8),
seed: Some(7),
..Default::default()
}),
stream: Some(true),
return_logprob: Some(true),
top_logprobs_num: Some(3),
logprob_start_len: Some(2),
return_text_in_logprobs: Some(true),
rid: Some("client-id".into()),
routed_dp_rank: Some(4),
disaggregated_params: Some(proto::DisaggregatedParams {
bootstrap_host: "10.0.0.1".into(),
bootstrap_port: 8998,
bootstrap_room: 12,
}),
..Default::default()
};
let request = text_generate(request, Some(&preferred)).unwrap();
assert_eq!(request.rid.client_facing(), "client-id");
assert_eq!(request.text.as_deref(), Some("hello"));
assert!(request.input_ids.is_none());
assert!(request.stream);
assert!(request.return_logprob);
assert_eq!(request.top_logprobs_num, 3);
assert_eq!(request.logprob_start_len, 2);
assert_eq!(request.return_text_in_logprobs, Some(true));
assert_eq!(request.routed_dp_rank, Some(4));
assert_eq!(request.bootstrap_host.as_deref(), Some("10.0.0.1"));
assert_eq!(request.bootstrap_port, Some(8998));
assert_eq!(request.bootstrap_room, Some(12));
assert_eq!(request.sampling_params.temperature, f64::from(0.8_f32));
assert_eq!(request.sampling_params.top_p, 0.7);
assert_eq!(request.sampling_params.max_new_tokens, Some(64));
assert_eq!(request.sampling_params.sampling_seed, Some(7));
}
#[test]
fn token_request_validates_input_and_rejects_unported_fields() {
let empty = Status::from(generate(proto::GenerateRequest::default(), None).unwrap_err());
assert_eq!(empty.code(), Code::InvalidArgument);
let unsupported = Status::from(
generate(
proto::GenerateRequest {
input_ids: vec![1],
routing_key: Some("worker-a".into()),
priority: Some(1),
..Default::default()
},
None,
)
.unwrap_err(),
);
assert_eq!(unsupported.code(), Code::Unimplemented);
assert!(unsupported.message().contains("routing_key, priority"));
}
#[test]
fn absent_sampling_params_still_uses_preferred_values() {
let preferred = PreferredSamplingParams(serde_json::json!({
"temperature": 0.25,
"max_new_tokens": 32,
}));
let request = generate(
proto::GenerateRequest {
input_ids: vec![1],
..Default::default()
},
Some(&preferred),
)
.unwrap();
assert_eq!(request.sampling_params.temperature, 0.25);
assert_eq!(request.sampling_params.max_new_tokens, Some(32));
}
#[test]
fn guided_choice_preserves_existing_runtime_v1_mapping() {
let request = generate(
proto::GenerateRequest {
input_ids: vec![1],
sampling_params: Some(proto::SamplingParams {
guided_decoding: Some(proto::GuidedDecoding {
constraint: Some(proto::guided_decoding::Constraint::Choice(
proto::ChoiceConstraint {
values: vec!["a+b".into(), "x.y".into()],
},
)),
}),
..Default::default()
}),
..Default::default()
},
None,
)
.unwrap();
assert_eq!(
request.sampling_params.regex.as_deref(),
Some("(?:a\\+b|x\\.y)")
);
}
#[test]
#[allow(deprecated)]
fn guided_and_legacy_constraints_cannot_be_combined() {
let error = Status::from(
generate(
proto::GenerateRequest {
input_ids: vec![1],
sampling_params: Some(proto::SamplingParams {
regex: Some("a".into()),
guided_decoding: Some(proto::GuidedDecoding {
constraint: Some(proto::guided_decoding::Constraint::Regex("b".into())),
}),
..Default::default()
}),
..Default::default()
},
None,
)
.unwrap_err(),
);
assert_eq!(error.code(), Code::InvalidArgument);
}
}
-303
View File
@@ -1,303 +0,0 @@
//! Tonic adapter for the canonical `sglang.runtime.v1` API.
//!
//! This module translates protobuf requests and responses and mounts that thin
//! adapter on Tonic. The shared [`FrontendHandle`] owns preprocessing,
//! admission, runtime communication, and request cancellation.
//!
//! The thin Tonic-service structure, streamed response approach, and test
//! strategy build on Rain Jiang's multi-protocol prototype in
//! `sgl-project/sglang#36923`. This stack keeps those ideas while using the
//! canonical `runtime.v1` contract and the narrower [`FrontendHandle`] seam.
use std::pin::Pin;
use std::time::Duration;
use futures::Stream;
use sglang_grpc_types::sglang::runtime::v1 as proto;
use sglang_grpc_types::sglang::runtime::v1::sglang_service_server::SglangService;
use tonic::{Request, Response, Status};
use crate::frontend::FrontendHandle;
use crate::message::config::{PreferredSamplingParams, ServerArgs};
mod convert;
mod response;
mod server;
pub(crate) use server::serve;
#[cfg(test)]
mod tests;
const DEFAULT_RESPONSE_TIMEOUT: Duration = Duration::from_secs(300);
type ResponseStream<T> = Pin<Box<dyn Stream<Item = Result<T, Status>> + Send + 'static>>;
/// Narrow snapshot of launch policy required to preserve the existing
/// `runtime.v1` generation behavior. It intentionally excludes listener,
/// authentication, TLS, and lifecycle configuration.
#[derive(Clone)]
struct AdapterConfig {
preferred_sampling_params: Option<PreferredSamplingParams>,
incremental_streaming_output: bool,
response_timeout: Duration,
}
/// Tonic-facing implementation backed by the transport-neutral Rust frontend.
pub(crate) struct GrpcService {
frontend: FrontendHandle,
config: AdapterConfig,
}
impl GrpcService {
pub(crate) fn new(frontend: FrontendHandle, server_args: &ServerArgs) -> Self {
Self {
frontend,
config: AdapterConfig {
preferred_sampling_params: server_args.preferred_sampling_params.clone(),
incremental_streaming_output: server_args.incremental_streaming_output,
response_timeout: DEFAULT_RESPONSE_TIMEOUT,
},
}
}
#[cfg(test)]
fn for_test(
frontend: FrontendHandle,
preferred_sampling_params: Option<PreferredSamplingParams>,
incremental_streaming_output: bool,
response_timeout: Duration,
) -> Self {
Self {
frontend,
config: AdapterConfig {
preferred_sampling_params,
incremental_streaming_output,
response_timeout,
},
}
}
}
fn unimplemented_rpc(name: &'static str) -> Status {
Status::unimplemented(format!(
"{name} is not implemented by the Rust frontend gRPC adapter"
))
}
#[tonic::async_trait]
impl SglangService for GrpcService {
async fn text_generate(
&self,
request: Request<proto::TextGenerateRequest>,
) -> Result<Response<ResponseStream<proto::TextGenerateResponse>>, Status> {
let request = convert::text_generate(
request.into_inner(),
self.config.preferred_sampling_params.as_ref(),
)
.map_err(Status::from)?;
let call = self
.frontend
.generate(request)
.await
.map_err(response::status)?;
Ok(Response::new(response::text_generate_stream(
call,
self.config.incremental_streaming_output,
self.config.response_timeout,
)))
}
async fn generate(
&self,
request: Request<proto::GenerateRequest>,
) -> Result<Response<ResponseStream<proto::GenerateResponse>>, Status> {
let request = convert::generate(
request.into_inner(),
self.config.preferred_sampling_params.as_ref(),
)
.map_err(Status::from)?;
let call = self
.frontend
.generate(request)
.await
.map_err(response::status)?;
Ok(Response::new(response::generate_stream(
call,
self.config.incremental_streaming_output,
self.config.response_timeout,
)))
}
// runtime.v1 is shared with the broader Python-backed service, so its
// generated trait contains more operations than this initial Rust adapter.
// Keep that support boundary explicit: adding an RPC is a deliberate future
// change, never an accidental empty success.
async fn text_embed(
&self,
_request: Request<proto::TextEmbedRequest>,
) -> Result<Response<proto::TextEmbedResponse>, Status> {
Err(unimplemented_rpc("text_embed"))
}
async fn embed(
&self,
_request: Request<proto::EmbedRequest>,
) -> Result<Response<proto::EmbedResponse>, Status> {
Err(unimplemented_rpc("embed"))
}
async fn classify(
&self,
_request: Request<proto::ClassifyRequest>,
) -> Result<Response<proto::ClassifyResponse>, Status> {
Err(unimplemented_rpc("classify"))
}
async fn tokenize(
&self,
_request: Request<proto::TokenizeRequest>,
) -> Result<Response<proto::TokenizeResponse>, Status> {
Err(unimplemented_rpc("tokenize"))
}
async fn detokenize(
&self,
_request: Request<proto::DetokenizeRequest>,
) -> Result<Response<proto::DetokenizeResponse>, Status> {
Err(unimplemented_rpc("detokenize"))
}
async fn health_check(
&self,
_request: Request<proto::HealthCheckRequest>,
) -> Result<Response<proto::HealthCheckResponse>, Status> {
Err(unimplemented_rpc("health_check"))
}
async fn get_model_info(
&self,
_request: Request<proto::GetModelInfoRequest>,
) -> Result<Response<proto::GetModelInfoResponse>, Status> {
Err(unimplemented_rpc("get_model_info"))
}
async fn get_server_info(
&self,
_request: Request<proto::GetServerInfoRequest>,
) -> Result<Response<proto::GetServerInfoResponse>, Status> {
Err(unimplemented_rpc("get_server_info"))
}
async fn list_models(
&self,
_request: Request<proto::ListModelsRequest>,
) -> Result<Response<proto::ListModelsResponse>, Status> {
Err(unimplemented_rpc("list_models"))
}
async fn get_load(
&self,
_request: Request<proto::GetLoadRequest>,
) -> Result<Response<proto::GetLoadResponse>, Status> {
Err(unimplemented_rpc("get_load"))
}
async fn abort(
&self,
_request: Request<proto::AbortRequest>,
) -> Result<Response<proto::AbortResponse>, Status> {
Err(unimplemented_rpc("abort"))
}
async fn flush_cache(
&self,
_request: Request<proto::FlushCacheRequest>,
) -> Result<Response<proto::FlushCacheResponse>, Status> {
Err(unimplemented_rpc("flush_cache"))
}
async fn pause_generation(
&self,
_request: Request<proto::PauseGenerationRequest>,
) -> Result<Response<proto::PauseGenerationResponse>, Status> {
Err(unimplemented_rpc("pause_generation"))
}
async fn watch_engine_state(
&self,
_request: Request<proto::WatchEngineStateRequest>,
) -> Result<Response<ResponseStream<proto::EngineStateSnapshot>>, Status> {
Err(unimplemented_rpc("watch_engine_state"))
}
async fn continue_generation(
&self,
_request: Request<proto::ContinueGenerationRequest>,
) -> Result<Response<proto::ContinueGenerationResponse>, Status> {
Err(unimplemented_rpc("continue_generation"))
}
async fn chat_complete(
&self,
_request: Request<proto::OpenAiRequest>,
) -> Result<Response<ResponseStream<proto::OpenAiStreamChunk>>, Status> {
Err(unimplemented_rpc("chat_complete"))
}
async fn complete(
&self,
_request: Request<proto::OpenAiRequest>,
) -> Result<Response<ResponseStream<proto::OpenAiStreamChunk>>, Status> {
Err(unimplemented_rpc("complete"))
}
async fn open_ai_embed(
&self,
_request: Request<proto::OpenAiRequest>,
) -> Result<Response<proto::OpenAiResponse>, Status> {
Err(unimplemented_rpc("open_ai_embed"))
}
async fn open_ai_classify(
&self,
_request: Request<proto::OpenAiRequest>,
) -> Result<Response<proto::OpenAiResponse>, Status> {
Err(unimplemented_rpc("open_ai_classify"))
}
async fn score(
&self,
_request: Request<proto::OpenAiRequest>,
) -> Result<Response<proto::OpenAiResponse>, Status> {
Err(unimplemented_rpc("score"))
}
async fn rerank(
&self,
_request: Request<proto::OpenAiRequest>,
) -> Result<Response<proto::OpenAiResponse>, Status> {
Err(unimplemented_rpc("rerank"))
}
async fn start_profile(
&self,
_request: Request<proto::StartProfileRequest>,
) -> Result<Response<proto::StartProfileResponse>, Status> {
Err(unimplemented_rpc("start_profile"))
}
async fn stop_profile(
&self,
_request: Request<proto::StopProfileRequest>,
) -> Result<Response<proto::StopProfileResponse>, Status> {
Err(unimplemented_rpc("stop_profile"))
}
async fn update_weights_from_disk(
&self,
_request: Request<proto::UpdateWeightsRequest>,
) -> Result<Response<proto::UpdateWeightsResponse>, Status> {
Err(unimplemented_rpc("update_weights_from_disk"))
}
}
-200
View File
@@ -1,200 +0,0 @@
//! Semantic frontend events to `runtime.v1` stream items and statuses.
use std::collections::HashMap;
use std::time::Duration;
use sglang_grpc_types::sglang::runtime::v1 as proto;
use tonic::{Code, Status};
use super::ResponseStream;
use crate::frontend::{
FrontendCall, FrontendError, FrontendErrorKind, FrontendEvent, FrontendOutput,
};
use crate::native_generation::meta_info_value;
pub(super) fn text_generate_stream(
call: FrontendCall,
incremental: bool,
response_timeout: Duration,
) -> ResponseStream<proto::TextGenerateResponse> {
generation_stream(
call,
incremental,
response_timeout,
|output, meta_info, finished| {
Ok(proto::TextGenerateResponse {
text: output.text.clone(),
meta_info,
finished,
})
},
)
}
pub(super) fn generate_stream(
call: FrontendCall,
incremental: bool,
response_timeout: Duration,
) -> ResponseStream<proto::GenerateResponse> {
generation_stream(
call,
incremental,
response_timeout,
|output, meta_info, finished| {
Ok(proto::GenerateResponse {
output_ids: output
.token_ids
.iter()
.copied()
.map(i32::try_from)
.collect::<Result<Vec<_>, _>>()
.map_err(|_| {
Status::internal("output token ID exceeds runtime.v1 int32 range")
})?,
meta_info,
finished,
})
},
)
}
fn generation_stream<T, F>(
mut call: FrontendCall,
incremental: bool,
response_timeout: Duration,
build: F,
) -> ResponseStream<T>
where
T: Send + 'static,
F: Fn(&FrontendOutput, HashMap<String, String>, bool) -> Result<T, Status> + Send + 'static,
{
let public_id = call.public_id().to_owned();
let stream = async_stream::stream! {
let mut cumulative = FrontendOutput::default();
let mut cumulative_completion_tokens = 0;
loop {
let event = match tokio::time::timeout(response_timeout, call.recv()).await {
Ok(Some(event)) => event,
// The loop stops on terminal events, so this is defensive only.
Ok(None) => break,
Err(_) => {
yield Err(Status::deadline_exceeded("Stream chunk timed out"));
break;
}
};
let (mut delta, finished) = match event {
FrontendEvent::Delta(output) => (output, false),
FrontendEvent::Finished(output) => (output, true),
FrontendEvent::Failed(error) => {
yield Err(status(error));
break;
}
};
let output = if incremental {
// Python's incremental native stream still reports the cumulative
// completion-token count in every frame. Do not retain the full
// generated output just to calculate that one cumulative field.
cumulative_completion_tokens += delta.completion_tokens;
delta.completion_tokens = cumulative_completion_tokens;
&delta
} else {
cumulative.append_delta(&delta);
&cumulative
};
let meta_info = grpc_meta_info(output, &public_id);
match build(output, meta_info, finished) {
Ok(response) => yield Ok(response),
Err(error) => {
yield Err(error);
break;
}
}
if finished {
break;
}
}
// `call` remains owned by this stream. Dropping a non-terminal stream or
// timing out drops the armed FrontendCall and uses the core abort lane;
// observing a terminal event has already disarmed it.
};
Box::pin(stream)
}
/// runtime.v1 represents heterogeneous metadata as strings. Match the existing
/// Python-backed server by JSON-encoding each value independently—including
/// string values such as the request ID.
fn grpc_meta_info(output: &FrontendOutput, public_id: &str) -> HashMap<String, String> {
let serde_json::Value::Object(meta_info) = meta_info_value(output, public_id) else {
unreachable!("native meta_info is always a JSON object");
};
meta_info
.into_iter()
.map(|(key, value)| (key, value.to_string()))
.collect()
}
pub(super) fn status(error: FrontendError) -> Status {
let code = match error.kind() {
FrontendErrorKind::InvalidArgument => Code::InvalidArgument,
FrontendErrorKind::NotFound => Code::NotFound,
FrontendErrorKind::FailedPrecondition => Code::FailedPrecondition,
FrontendErrorKind::ResourceExhausted => Code::ResourceExhausted,
FrontendErrorKind::Cancelled => Code::Cancelled,
FrontendErrorKind::DeadlineExceeded => Code::DeadlineExceeded,
FrontendErrorKind::Unavailable => Code::Unavailable,
FrontendErrorKind::Internal => Code::Internal,
};
Status::new(code, error.to_string())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn semantic_errors_map_to_canonical_grpc_codes() {
let cases = [
(
FrontendError::InvalidArgument("bad".into()),
Code::InvalidArgument,
),
(
FrontendError::RuntimeRejected {
kind: FrontendErrorKind::NotFound,
message: "missing".into(),
legacy_http_status: 404,
},
Code::NotFound,
),
(
FrontendError::RuntimeRejected {
kind: FrontendErrorKind::FailedPrecondition,
message: "not ready".into(),
legacy_http_status: 412,
},
Code::FailedPrecondition,
),
(
FrontendError::Overloaded("full".into()),
Code::ResourceExhausted,
),
(FrontendError::Cancelled("gone".into()), Code::Cancelled),
(
FrontendError::RuntimeRejected {
kind: FrontendErrorKind::DeadlineExceeded,
message: "late".into(),
legacy_http_status: 504,
},
Code::DeadlineExceeded,
),
(FrontendError::Unavailable, Code::Unavailable),
(FrontendError::Internal("bug".into()), Code::Internal),
];
for (error, expected) in cases {
assert_eq!(status(error).code(), expected);
}
}
}
-361
View File
@@ -1,361 +0,0 @@
use std::time::Duration;
use futures::StreamExt;
use sglang_grpc_types::sglang::runtime::v1 as proto;
use sglang_grpc_types::sglang::runtime::v1::sglang_service_server::SglangService;
use tonic::{Code, Request};
use super::GrpcService;
use crate::frontend::{FrontendConfig, FrontendHandle, FrontendMetadata};
use crate::message::finish_reason::FinishReason;
use crate::message::ids::Rid;
use crate::message::request::{GenerateRequest, Request as RuntimeRequest, RequestKind};
use crate::message::response::{ChunkEvent, ResponseItem, ResponseSink};
use crate::tokenizer_manager::wiring::{AbortSource, RequestAdmission, TmEvent};
struct Harness {
service: GrpcService,
intake_rx: flume::Receiver<TmEvent>,
abort_rx: flume::Receiver<AbortSource>,
}
struct GenerationIntake {
request: Box<GenerateRequest>,
rid: Rid,
sink: ResponseSink,
admission: RequestAdmission,
}
impl Harness {
fn new(response_capacity: usize, incremental: bool, response_timeout: Duration) -> Self {
let (intake_tx, intake_rx) = flume::unbounded();
let (abort_tx, abort_rx) = flume::unbounded();
let frontend = FrontendHandle::new(
intake_tx,
abort_tx,
FrontendConfig {
response_capacity,
response_activity: Default::default(),
startup_ready: true,
is_disaggregation: false,
mm_limits: Default::default(),
metadata: FrontendMetadata::default(),
},
);
Self {
service: GrpcService::for_test(frontend, None, incremental, response_timeout),
intake_rx,
abort_rx,
}
}
async fn next_generation(&self) -> GenerationIntake {
let TmEvent::Intake { request, admission } = self
.intake_rx
.recv_async()
.await
.expect("request submitted")
else {
panic!("expected generation intake");
};
let RuntimeRequest {
rid,
sink,
kind,
state: _,
} = request;
let RequestKind::Generate(request) = kind else {
panic!("expected generation request");
};
GenerationIntake {
request,
rid,
sink,
admission,
}
}
}
fn chunk(rid: &Rid, text: &str, token_id: i64, finished: bool) -> ChunkEvent {
ChunkEvent {
rid: rid.clone(),
token_ids: vec![token_id],
finish_reason: finished.then(stop_reason),
prompt_tokens: 3,
text: text.into(),
completion_tokens: 1,
extras: None,
}
}
fn stop_reason() -> FinishReason {
serde_json::from_value(serde_json::json!({"type": "stop"})).unwrap()
}
#[tokio::test]
async fn text_generate_maps_request_and_streams_cumulative_responses() {
let harness = Harness::new(2, false, Duration::from_secs(1));
let response = harness
.service
.text_generate(Request::new(proto::TextGenerateRequest {
text: "prompt".into(),
stream: Some(true),
rid: Some("request-1".into()),
..Default::default()
}))
.await
.unwrap();
let mut stream = response.into_inner();
let intake = harness.next_generation().await;
assert!(intake.admission.try_accept());
assert_eq!(intake.request.text.as_deref(), Some("prompt"));
assert!(intake.request.input_ids.is_none());
assert!(intake.request.stream);
assert_eq!(intake.rid.client_facing(), "request-1");
intake
.sink
.try_send(ResponseItem::Frame(chunk(&intake.rid, "Hel", 1, false)))
.unwrap();
intake
.sink
.try_send(ResponseItem::Done(chunk(&intake.rid, "lo", 2, true)))
.unwrap();
let first = stream.next().await.unwrap().unwrap();
assert_eq!(first.text, "Hel");
assert!(!first.finished);
assert_eq!(first.meta_info["id"], r#""request-1""#);
assert_eq!(first.meta_info["prompt_tokens"], "3");
assert_eq!(first.meta_info["completion_tokens"], "1");
assert_eq!(first.meta_info["finish_reason"], "null");
let finished = stream.next().await.unwrap().unwrap();
assert_eq!(finished.text, "Hello");
assert!(finished.finished);
assert_eq!(finished.meta_info["completion_tokens"], "2");
assert!(finished.meta_info["finish_reason"].contains(r#""type":"stop""#));
assert!(stream.next().await.is_none());
assert!(harness.abort_rx.try_recv().is_err());
}
#[tokio::test]
async fn generate_streams_incremental_token_ids_with_cumulative_count() {
let harness = Harness::new(2, true, Duration::from_secs(1));
let response = harness
.service
.generate(Request::new(proto::GenerateRequest {
input_ids: vec![4, 5],
stream: Some(true),
rid: Some("tokens".into()),
..Default::default()
}))
.await
.unwrap();
let mut stream = response.into_inner();
let intake = harness.next_generation().await;
assert!(intake.admission.try_accept());
assert_eq!(intake.request.input_ids.as_deref(), Some(&[4, 5][..]));
assert!(intake.request.text.is_none());
intake
.sink
.try_send(ResponseItem::Frame(chunk(&intake.rid, "A", 10, false)))
.unwrap();
intake
.sink
.try_send(ResponseItem::Done(chunk(&intake.rid, "B", 11, true)))
.unwrap();
let first = stream.next().await.unwrap().unwrap();
let finished = stream.next().await.unwrap().unwrap();
assert_eq!(first.output_ids, vec![10]);
assert_eq!(finished.output_ids, vec![11]);
assert_eq!(finished.meta_info["completion_tokens"], "2");
assert!(finished.finished);
}
#[tokio::test]
async fn stream_drop_aborts_admitted_request() {
let harness = Harness::new(1, false, Duration::from_secs(1));
let mut stream = harness
.service
.generate(Request::new(proto::GenerateRequest {
input_ids: vec![1],
rid: Some("cancel-me".into()),
..Default::default()
}))
.await
.unwrap()
.into_inner();
let intake = harness.next_generation().await;
assert!(intake.admission.try_accept());
intake
.sink
.try_send(ResponseItem::Frame(chunk(&intake.rid, "partial", 1, false)))
.unwrap();
assert!(stream.next().await.unwrap().is_ok());
drop(stream);
let abort = harness.abort_rx.recv_async().await.unwrap();
assert_eq!(abort.rid().client_facing(), "cancel-me");
assert!(harness.abort_rx.try_recv().is_err());
}
#[tokio::test]
async fn runtime_failure_is_an_in_stream_status_and_disarms_abort() {
let harness = Harness::new(1, false, Duration::from_secs(1));
let mut stream = harness
.service
.generate(Request::new(proto::GenerateRequest {
input_ids: vec![1],
..Default::default()
}))
.await
.unwrap()
.into_inner();
let intake = harness.next_generation().await;
assert!(intake.admission.try_accept());
intake
.sink
.try_send(ResponseItem::Error(crate::utils::error::Error::Validation(
"bad sampling".into(),
)))
.unwrap();
let error = stream.next().await.unwrap().unwrap_err();
assert_eq!(error.code(), Code::InvalidArgument);
drop(stream);
assert!(harness.abort_rx.try_recv().is_err());
}
#[tokio::test]
async fn per_chunk_timeout_after_progress_returns_deadline_exceeded_and_aborts() {
let harness = Harness::new(1, false, Duration::from_millis(10));
let mut stream = harness
.service
.generate(Request::new(proto::GenerateRequest {
input_ids: vec![1],
rid: Some("too-slow".into()),
..Default::default()
}))
.await
.unwrap()
.into_inner();
let intake = harness.next_generation().await;
assert!(intake.admission.try_accept());
intake
.sink
.try_send(ResponseItem::Frame(chunk(&intake.rid, "partial", 1, false)))
.unwrap();
assert!(stream.next().await.unwrap().is_ok());
let error = stream.next().await.unwrap().unwrap_err();
assert_eq!(error.code(), Code::DeadlineExceeded);
drop(stream);
let abort = harness.abort_rx.recv_async().await.unwrap();
assert_eq!(abort.rid().client_facing(), "too-slow");
}
#[tokio::test]
async fn closed_intake_is_a_top_level_unavailable_status() {
let (intake_tx, intake_rx) = flume::unbounded();
drop(intake_rx);
let (abort_tx, _abort_rx) = flume::unbounded();
let frontend = FrontendHandle::new(
intake_tx,
abort_tx,
FrontendConfig {
response_capacity: 1,
response_activity: Default::default(),
startup_ready: true,
is_disaggregation: false,
mm_limits: Default::default(),
metadata: FrontendMetadata::default(),
},
);
let service = GrpcService::for_test(frontend, None, false, Duration::from_secs(1));
let result = service
.generate(Request::new(proto::GenerateRequest {
input_ids: vec![1],
..Default::default()
}))
.await;
let error = match result {
Ok(_) => panic!("closed intake must reject the RPC"),
Err(error) => error,
};
assert_eq!(error.code(), Code::Unavailable);
}
#[tokio::test]
async fn new_proto_fields_and_engine_watch_are_explicitly_unsupported() {
let harness = Harness::new(1, false, Duration::from_secs(1));
let text_error = harness
.service
.text_generate(Request::new(proto::TextGenerateRequest {
text: "hello".into(),
kv_hints: Some(Default::default()),
..Default::default()
}))
.await
.err()
.expect("KV hints must not be silently ignored");
assert_eq!(text_error.code(), Code::Unimplemented);
let token_error = harness
.service
.generate(Request::new(proto::GenerateRequest {
input_ids: vec![1],
kv_hints: Some(Default::default()),
..Default::default()
}))
.await
.err()
.expect("KV hints must not be silently ignored");
assert_eq!(token_error.code(), Code::Unimplemented);
let watch_error = harness
.service
.watch_engine_state(Request::new(proto::WatchEngineStateRequest {}))
.await
.err()
.expect("engine watch is outside the generation adapter");
assert_eq!(watch_error.code(), Code::Unimplemented);
assert!(harness.intake_rx.try_recv().is_err());
}
#[tokio::test]
async fn output_token_overflow_returns_error_instead_of_truncating() {
let harness = Harness::new(1, true, Duration::from_secs(1));
let mut stream = harness
.service
.generate(Request::new(proto::GenerateRequest {
input_ids: vec![i32::MAX],
..Default::default()
}))
.await
.unwrap()
.into_inner();
let intake = harness.next_generation().await;
assert!(intake.admission.try_accept());
assert_eq!(
intake.request.input_ids.as_deref(),
Some(&[i64::from(i32::MAX)][..])
);
intake
.sink
.try_send(ResponseItem::Frame(chunk(
&intake.rid,
"",
i64::from(i32::MAX) + 1,
false,
)))
.unwrap();
assert_eq!(
stream.next().await.unwrap().unwrap_err().code(),
Code::Internal
);
assert!(stream.next().await.is_none());
assert!(
matches!(harness.abort_rx.recv_async().await.unwrap(), AbortSource::Guard(rid) if rid == intake.rid)
);
}
+1 -4
View File
@@ -12,11 +12,8 @@
//! a `PyObject`.
mod api_server;
mod frontend;
mod grpc;
mod message;
mod multi_modality;
mod native_generation;
mod tokenizer_manager;
mod utils;
@@ -25,7 +22,7 @@ pub use message::config::{
RustServerServerArgs, ServerArgs,
};
pub use message::multimodal::MmItem;
pub use message::request::{MmData, ProcessorExtensions};
pub use message::request::MmData;
pub use message::types::TokenIds;
pub use multi_modality::encoded::{
MRope, MmEncodedEntry, MmEncodedItem, MmMetaValue, MmModality, MmTokenIds,
+2
View File
@@ -7,9 +7,11 @@ pub mod config;
pub mod detok;
pub mod finish_reason;
pub mod ids;
pub mod info;
pub mod io_struct;
pub mod multimodal;
pub mod request;
pub mod response;
pub mod sampling;
pub mod types;
pub mod wire;
+6 -2
View File
@@ -572,8 +572,12 @@ impl ServerArgs {
}
}
if let Some(preferred) = &self.preferred_sampling_params {
super::sampling::SamplingParamsInput::from_preferred(&preferred.0)
.map_err(|e| format!("invalid preferred_sampling_params: {e}"))?;
// The wire schema is the contract: a preferred value the decoder
// rejects would 400 every request, so reject it at boot instead.
serde_json::from_value::<sglang_api_types::api::v1::SamplingParams>(
preferred.0.clone(),
)
.map_err(|e| format!("invalid preferred_sampling_params: {e}"))?;
}
Ok(())
}
@@ -98,7 +98,7 @@ impl FinishReason {
}
/// `Some((status, message))` when this is an abort carrying the scheduler's
/// legacy HTTP status — a request failure the frontend boundary converts
/// legacy HTTP status — a request failure the core boundary converts
/// into a semantic error event. A plain abort (no code) reads as `None`.
pub fn abort_status(&self) -> Option<(u16, &str)> {
match self {
@@ -128,7 +128,7 @@ mod tests {
Some((code, message.to_string()))
}
/// The classifier the frontend boundary uses: a validation abort yields its
/// The classifier the core boundary uses: a validation abort yields its
/// `(code, message)` so it can become a semantic failure event; anything
/// else yields `None`.
#[test]
+89
View File
@@ -0,0 +1,89 @@
//! Public model and server metadata: the typed results of `/get_model_info`
//! and `/server_info` on every transport.
use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
use super::config::{DisaggregationMode, PreferredSamplingParams};
/// Static model metadata shared by every public transport.
#[derive(Clone, Debug, Serialize)]
pub(crate) struct ModelInfo {
pub(crate) model_path: String,
pub(crate) served_model_name: String,
pub(crate) tokenizer_path: String,
pub(crate) is_generation: bool,
pub(crate) preferred_sampling_params: Option<PreferredSamplingParams>,
pub(crate) weight_version: Option<String>,
pub(crate) load_format: Option<String>,
pub(crate) reasoning_parser: Option<String>,
pub(crate) tool_call_parser: Option<String>,
pub(crate) disaggregation_mode: DisaggregationMode,
}
/// Public server metadata plus scheduler-owned runtime metrics.
#[derive(Clone, Debug, Serialize)]
pub(crate) struct ServerInfo {
pub(crate) model_path: String,
pub(crate) served_model_name: String,
pub(crate) tokenizer_path: String,
pub(crate) max_context_length: u64,
pub(crate) max_total_num_tokens: u64,
pub(crate) version: String,
pub(crate) frontend: &'static str,
pub(crate) internal_states: Vec<InternalState>,
}
/// Scheduler-owned runtime metrics used by the public server-info operation.
///
/// Deserializing into this allowlisted shape is intentional: the scheduler's
/// raw state also contains its full launch arguments, including credentials.
/// Unknown fields are discarded before any transport receives the result.
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
#[serde(default)]
pub(crate) struct InternalState {
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) last_gen_throughput: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) memory_usage: Option<MemoryUsage>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) effective_max_running_requests_per_dp: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) avg_spec_accept_length: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) step_time_dict: Option<BTreeMap<u64, Vec<f64>>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) rust_mm_transport: Option<BTreeMap<String, u64>>,
}
/// Public subset of scheduler memory metrics.
///
/// This nested allowlist is intentional: adding a scheduler field does not
/// silently add different public data to HTTP before the gRPC contract can add
/// the same field. New metrics should be promoted here deliberately.
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
#[serde(default)]
pub(crate) struct MemoryUsage {
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) weight: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) kvcache: Option<MemoryMeasurement>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) startup_available: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) token_capacity: Option<u64>,
pub(crate) token_capacity_swa: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) graph: Option<BTreeMap<String, f64>>,
}
/// The scheduler's KV-cache measurement can be a native float or a NumPy
/// scalar stringified by the MessagePack bridge. Preserve that representation
/// so extracting the typed contract does not change public responses.
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[serde(untagged)]
pub(crate) enum MemoryMeasurement {
Number(f64),
String(String),
}
+20 -209
View File
@@ -1,12 +1,6 @@
//! Typed multimodal inputs of the `/generate` body — the Rust form of Python
//! `MultimodalDataInputFormat` (`io_struct.py`) — and their per-request fan-out.
use std::fmt;
use serde::de::value::{MapAccessDeserializer, SeqAccessDeserializer};
use serde::de::{MapAccess, SeqAccess, Visitor};
use serde::{Deserialize, Deserializer};
use super::request::{HeapBytes, check_broadcast_budget};
use crate::utils::error::Error;
@@ -22,19 +16,13 @@ pub enum MmItem {
/// are read by model families this pipeline does not run, and Python's
/// `load_image` itself reduces the item to `.url`.
Ref { url: String },
/// A preprocessed item (`{"format": "processor_output" | "precomputed_embedding", …}`).
/// Parsed only far enough to be rejected by name at the MM stage; Python
/// ignores it the same way on a text-only model.
Preprocessed { format: String },
}
impl MmItem {
/// The raw source string for the modality pipeline, `None` for a
/// preprocessed item.
/// The raw source string for the modality pipeline.
pub fn source(&self) -> Option<&str> {
match self {
MmItem::Source(source) | MmItem::Ref { url: source } => Some(source),
MmItem::Preprocessed { .. } => None,
}
}
}
@@ -42,68 +30,11 @@ impl MmItem {
impl HeapBytes for MmItem {
fn heap_bytes(&self) -> usize {
match self {
MmItem::Source(s) | MmItem::Ref { url: s } | MmItem::Preprocessed { format: s } => {
s.len()
}
MmItem::Source(s) | MmItem::Ref { url: s } => s.len(),
}
}
}
/// The object form of an item, as Python's `Dict[str, Any]`: `format` marks a
/// preprocessed item (checked first, as `glm4v` does), `url` an `ImageData`.
#[derive(Deserialize)]
struct ItemObject {
#[serde(default)]
url: Option<String>,
#[serde(default)]
format: Option<String>,
}
impl TryFrom<ItemObject> for MmItem {
type Error = &'static str;
fn try_from(object: ItemObject) -> Result<Self, Self::Error> {
match (object.format, object.url) {
(Some(format), _) => Ok(MmItem::Preprocessed { format }),
(None, Some(url)) => Ok(MmItem::Ref { url }),
(None, None) => Err("a multimodal item object needs a `url` or a `format` key"),
}
}
}
/// Hand-written rather than `#[serde(untagged)]` so a bad item is reported as
/// what it is ("expected a source string or an item object"), not as "did not
/// match any variant".
impl<'de> Deserialize<'de> for MmItem {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
struct ItemVisitor;
impl<'de> Visitor<'de> for ItemVisitor {
type Value = MmItem;
fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str("a media source string or a multimodal item object")
}
fn visit_str<E: serde::de::Error>(self, value: &str) -> Result<Self::Value, E> {
Ok(MmItem::Source(value.to_owned()))
}
fn visit_string<E: serde::de::Error>(self, value: String) -> Result<Self::Value, E> {
Ok(MmItem::Source(value))
}
fn visit_map<A: MapAccess<'de>>(self, map: A) -> Result<Self::Value, A::Error> {
ItemObject::deserialize(MapAccessDeserializer::new(map))?
.try_into()
.map_err(serde::de::Error::custom)
}
}
deserializer.deserialize_any(ItemVisitor)
}
}
/// One `image_data` / `video_data` / `audio_data` field as sent: Python
/// `MultimodalDataInputFormat`, whose three shapes read differently for a single
/// request and a batch (see [`fan_out`]).
@@ -117,119 +48,6 @@ pub enum MmDataInput {
Nested(Vec<Option<Vec<Option<MmItem>>>>),
}
/// One element of a list-form field, before the list is known to be flat or nested.
enum ListElement {
Null,
Item(MmItem),
List(Vec<Option<MmItem>>),
}
impl<'de> Deserialize<'de> for ListElement {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
struct ElementVisitor;
impl<'de> Visitor<'de> for ElementVisitor {
type Value = ListElement;
fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str("null, a media source string, an item object, or a list of items")
}
fn visit_unit<E: serde::de::Error>(self) -> Result<Self::Value, E> {
Ok(ListElement::Null)
}
fn visit_none<E: serde::de::Error>(self) -> Result<Self::Value, E> {
Ok(ListElement::Null)
}
fn visit_str<E: serde::de::Error>(self, value: &str) -> Result<Self::Value, E> {
Ok(ListElement::Item(MmItem::Source(value.to_owned())))
}
fn visit_string<E: serde::de::Error>(self, value: String) -> Result<Self::Value, E> {
Ok(ListElement::Item(MmItem::Source(value)))
}
fn visit_map<A: MapAccess<'de>>(self, map: A) -> Result<Self::Value, A::Error> {
ItemObject::deserialize(MapAccessDeserializer::new(map))?
.try_into()
.map(ListElement::Item)
.map_err(serde::de::Error::custom)
}
fn visit_seq<A: SeqAccess<'de>>(self, seq: A) -> Result<Self::Value, A::Error> {
Vec::<Option<MmItem>>::deserialize(SeqAccessDeserializer::new(seq))
.map(ListElement::List)
}
}
deserializer.deserialize_any(ElementVisitor)
}
}
impl<'de> Deserialize<'de> for MmDataInput {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
struct InputVisitor;
impl<'de> Visitor<'de> for InputVisitor {
type Value = MmDataInput;
fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str("a media item, a list of items, or a list of item lists")
}
fn visit_str<E: serde::de::Error>(self, value: &str) -> Result<Self::Value, E> {
Ok(MmDataInput::One(MmItem::Source(value.to_owned())))
}
fn visit_string<E: serde::de::Error>(self, value: String) -> Result<Self::Value, E> {
Ok(MmDataInput::One(MmItem::Source(value)))
}
fn visit_map<A: MapAccess<'de>>(self, map: A) -> Result<Self::Value, A::Error> {
ItemObject::deserialize(MapAccessDeserializer::new(map))?
.try_into()
.map(MmDataInput::One)
.map_err(serde::de::Error::custom)
}
fn visit_seq<A: SeqAccess<'de>>(self, seq: A) -> Result<Self::Value, A::Error> {
let elements = Vec::<ListElement>::deserialize(SeqAccessDeserializer::new(seq))?;
let nested = elements
.iter()
.any(|element| matches!(element, ListElement::List(_)));
if !nested {
return Ok(MmDataInput::Many(
elements
.into_iter()
.map(|element| match element {
ListElement::Item(item) => Some(item),
ListElement::Null => None,
ListElement::List(_) => unreachable!("checked above"),
})
.collect(),
));
}
elements
.into_iter()
.map(|element| match element {
ListElement::List(items) => Ok(Some(items)),
ListElement::Null => Ok(None),
ListElement::Item(_) => Err(serde::de::Error::custom(
"a nested list cannot mix bare items with item lists",
)),
})
.collect::<Result<_, _>>()
.map(MmDataInput::Nested)
}
}
deserializer.deserialize_any(InputVisitor)
}
}
/// The items of one modality for one request, `null` entries dropped.
fn present(items: Vec<Option<MmItem>>) -> Vec<MmItem> {
items.into_iter().flatten().collect()
}
@@ -300,8 +118,12 @@ fn check_len(len: usize, n: usize, name: &str) -> Result<(), Error> {
mod tests {
use super::*;
fn parse(json: &str) -> Result<MmDataInput, serde_json::Error> {
serde_json::from_str(json)
/// Parsed as the wire type (the schema's `MediaInput`), then converted;
/// either step's rejection is an `Err`.
fn parse(json: &str) -> Result<MmDataInput, String> {
let wire: sglang_api_types::api::v1::MediaInput =
serde_json::from_str(json).map_err(|e| e.to_string())?;
crate::message::wire::media_input(wire).map_err(|e| e.to_string())
}
fn src(s: &str) -> MmItem {
@@ -324,39 +146,28 @@ mod tests {
);
}
/// Object items: `format` wins over `url` (a preprocessed item may carry
/// both), and an object with neither is named in the error.
/// Object items are typed `MediaRef`s: only `url` reaches this pipeline, a
/// hint the schema does not name is an error, and a preprocessed-input
/// dict is not a wire shape at all (its values are tensors; Engine-only).
#[test]
fn parses_item_objects() {
assert_eq!(
parse(r#"{"url": "u", "detail": "high"}"#).unwrap(),
MmDataInput::One(MmItem::Ref { url: "u".into() })
);
assert_eq!(
parse(r#"[{"format": "processor_output", "url": "u", "pixel_values": [1]}]"#).unwrap(),
MmDataInput::Many(vec![Some(MmItem::Preprocessed {
format: "processor_output".into()
})])
);
let err = parse(r#"{"detail": "high"}"#).unwrap_err().to_string();
assert!(err.contains("`url` or a `format`"), "{err}");
let err = parse(r#"{"url": "u", "detai": "high"}"#)
.unwrap_err()
.to_string(); // codespell:ignore detai
assert!(err.contains("unknown field"), "{err}");
assert!(parse(r#"[{"format": "processor_output", "url": "u"}]"#).is_err());
assert!(parse(r#"{"detail": "high"}"#).is_err(), "a ref needs a url");
}
/// Anything Python's item union does not cover is rejected up front, with
/// the expected shape in the message.
/// Anything Python's item union does not cover is rejected up front.
#[test]
fn rejects_non_items() {
for (json, expect) in [
("5", "expected a media item, a list of items"),
(r#"["a", 5]"#, "expected null, a media source string"),
(r#"["a", ["b"]]"#, "cannot mix"),
(
r#"[[["a"]]]"#,
"expected a media source string or a multimodal item object",
),
] {
let err = parse(json).unwrap_err().to_string();
assert!(err.contains(expect), "{json}: {err}");
for json in ["5", r#"["a", 5]"#, r#"["a", ["b"]]"#, r#"[[["a"]]]"#] {
assert!(parse(json).is_err(), "{json}");
}
}
File diff suppressed because it is too large Load Diff
+8 -8
View File
@@ -1,4 +1,4 @@
//! The runtime response direction: per-request channels into the frontend
//! The runtime response direction: per-request channels into the core
//! boundary ([`ResponseSink`] / [`ResponseItem`]), response frame encodings,
//! and columnar batch decoding into per-request [`ChunkEvent`]s.
@@ -12,7 +12,7 @@ use crate::message::types::TokenIds;
use crate::utils::error::Error;
/// Per-request back-channel the detok shard writes runtime outputs to and the
/// frontend boundary drains; bounded, and receiver-drop cancels the producer.
/// core boundary drains; bounded, and receiver-drop cancels the producer.
#[derive(Clone, Debug)]
pub enum ResponseSink {
Local(mpsc::Sender<ResponseItem>),
@@ -38,12 +38,12 @@ impl ResponseSink {
}
}
#[allow(dead_code)] // the receiver half is owned inside the frontend boundary.
#[allow(dead_code)] // the receiver half is owned inside the core boundary.
pub type ResponseSource = mpsc::Receiver<ResponseItem>;
/// Runtime-facing output delivered to the frontend boundary: a detok-decoded
/// Runtime-facing output delivered to the core boundary: a detok-decoded
/// [`ChunkEvent`], a serialized internal-operation result, or a stage error.
/// [`crate::frontend::FrontendHandle`] translates these variants before a
/// [`crate::api_server::core::CoreHandle`] translates these variants before a
/// transport adapter sees them.
#[derive(Debug)]
pub enum ResponseItem {
@@ -54,10 +54,10 @@ pub enum ResponseItem {
/// A control-request result encoded by the Python scheduler.
Control(Bytes),
/// Reply to an internal service request (`RequestKind::Detokenize`): raw
/// bytes for the frontend to decode (e.g. detokenized text), not a
/// bytes for the core to decode (e.g. detokenized text), not a
/// client-facing wire payload and not a generation frame.
Data(Bytes),
/// Terminal runtime failure; the frontend maps it to a semantic error.
/// Terminal runtime failure; the core maps it to a semantic error.
Error(Error),
}
@@ -558,7 +558,7 @@ pub fn frame_error(rid: &str, message: &str) -> Bytes {
#[derive(Debug, Clone, Default)]
pub struct ChunkEvent {
/// Runtime correlation ID. Moved out of the frame header (which owns it and
/// drops it), so carrying it costs no allocation. The frontend strips this
/// drops it), so carrying it costs no allocation. The core strips this
/// field before exposing semantic output; the shard routes it via `Rid::shard`.
pub rid: Rid,
/// New token ids for this step, widened from the scheduler's int32 wire
+2 -148
View File
@@ -3,12 +3,9 @@
//! `__post_init__` → `normalize` → `verify` pipeline (run in that order, as
//! `TokenizerManager._create_tokenized_object` does).
use std::collections::{BTreeMap, BTreeSet};
use std::fmt;
use std::collections::BTreeMap;
use serde::de::value::{MapAccessDeserializer, SeqAccessDeserializer};
use serde::de::{MapAccess, SeqAccess, Visitor};
use serde::{Deserialize, Deserializer, Serialize};
use serde::{Deserialize, Serialize};
use super::types::OneOrMany;
use crate::utils::{error::Error, regex::RegexPattern};
@@ -241,90 +238,6 @@ pub struct SamplingParams {
/// strip that wrapper from its own grammar, so it is a pipeline output only.
#[serde(skip_deserializing)]
pub ebnf_full_assistant: bool,
/// API fields present in the request object. Serde defaults erase this
/// distinction, but preferred sampling parameters must not overwrite an
/// explicit request value, including an explicit default or null.
#[serde(skip)]
pub(crate) explicit_fields: BTreeSet<String>,
}
/// The `/generate` body's `sampling_params`: one object (broadcast to every
/// prompt) or a list of them (one per prompt), fanned out by `GenerateBody::into_requests`.
///
/// Hand-written `Deserialize` rather than `#[serde(untagged)]`: untagged buffers
/// the input and, on failure, reports only "data did not match any variant" —
/// losing the field-level message ("unknown field `temperature`, expected one of
/// …") that makes a typo actionable. Object-vs-list is unambiguous here, so a
/// single `deserialize_any` dispatch keeps the inner error verbatim.
#[derive(Debug, Clone, PartialEq)]
pub enum SamplingParamsInput {
/// Boxed: `SamplingParams` is ~440 bytes, so an inline variant would make
/// every `GenerateBody` that big regardless of which form arrived.
One(Box<SamplingParams>),
Many(Vec<SamplingParams>),
}
impl<'de> Deserialize<'de> for SamplingParamsInput {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
struct InputVisitor;
impl<'de> Visitor<'de> for InputVisitor {
type Value = SamplingParamsInput;
fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str("a sampling_params object, or a list of them (one per prompt)")
}
fn visit_map<A: MapAccess<'de>>(self, map: A) -> Result<Self::Value, A::Error> {
let value = serde_json::Value::deserialize(MapAccessDeserializer::new(map))?;
sampling_params_from_value(value)
.map(|p| SamplingParamsInput::One(Box::new(p)))
.map_err(serde::de::Error::custom)
}
fn visit_seq<A: SeqAccess<'de>>(self, seq: A) -> Result<Self::Value, A::Error> {
let values =
Vec::<serde_json::Value>::deserialize(SeqAccessDeserializer::new(seq))?;
values
.into_iter()
.map(sampling_params_from_value)
.collect::<Result<Vec<_>, _>>()
.map(SamplingParamsInput::Many)
.map_err(serde::de::Error::custom)
}
}
deserializer.deserialize_any(InputVisitor)
}
}
fn sampling_params_from_value(value: serde_json::Value) -> Result<SamplingParams, String> {
let explicit_fields = value
.as_object()
.ok_or_else(|| "sampling_params must be an object".to_string())?
.keys()
.cloned()
.collect();
let mut params: SamplingParams = serde_json::from_value(value).map_err(|e| e.to_string())?;
params.explicit_fields = explicit_fields;
Ok(params)
}
impl SamplingParamsInput {
/// Merge launch-time preferred params beneath request params. A request key
/// wins even when it explicitly carries the type's default or null.
pub fn apply_preferred(&mut self, preferred: &serde_json::Value) -> Result<(), String> {
match self {
Self::One(params) => params.apply_preferred(preferred),
Self::Many(params) => params
.iter_mut()
.try_for_each(|params| params.apply_preferred(preferred)),
}
}
pub fn from_preferred(preferred: &serde_json::Value) -> Result<Self, String> {
sampling_params_from_value(preferred.clone()).map(|params| Self::One(Box::new(params)))
}
}
impl Default for SamplingParams {
@@ -366,39 +279,11 @@ impl Default for SamplingParams {
stop_regex_max_len: 0,
is_normalized: false,
ebnf_full_assistant: false,
explicit_fields: BTreeSet::new(),
}
}
}
impl SamplingParams {
/// Record a field supplied by a non-Serde adapter. Preferred launch values
/// are merged underneath these fields just as they are underneath keys
/// present in an HTTP sampling object.
pub(crate) fn mark_explicit(&mut self, field: &'static str) {
self.explicit_fields.insert(field.to_owned());
}
/// Merge operator-provided sampling defaults beneath explicitly supplied
/// request fields. HTTP and gRPC adapters use the same precedence policy.
pub(crate) fn apply_preferred(&mut self, preferred: &serde_json::Value) -> Result<(), String> {
let mut merged = preferred
.as_object()
.ok_or_else(|| "preferred_sampling_params must be a JSON object".to_string())?
.clone();
let request_value = serde_json::to_value(&*self).map_err(|e| e.to_string())?;
let request = request_value
.as_object()
.ok_or_else(|| "SamplingParams did not serialize as an object".to_string())?;
for field in &self.explicit_fields {
if let Some(value) = request.get(field) {
merged.insert(field.clone(), value.clone());
}
}
*self = sampling_params_from_value(serde_json::Value::Object(merged))?;
Ok(())
}
/// `__post_init__` → `normalize` → `verify`, the order
/// `TokenizerManager._create_tokenized_object` runs them in. `Err` is a
/// request-local 400. `skip_tokenizer_init` stands in for Python's
@@ -1220,35 +1105,4 @@ mod tests {
let err = norm_err(&json).to_string();
assert!(err.contains("at most"), "{err}");
}
#[test]
fn preferred_params_fill_only_omitted_request_fields() {
let preferred = serde_json::json!({
"temperature": 0.25,
"top_p": 0.75,
"max_new_tokens": 4096
});
let mut input: SamplingParamsInput =
serde_json::from_str(r#"{"temperature": 1.0, "top_p": null}"#).unwrap();
input.apply_preferred(&preferred).unwrap();
let SamplingParamsInput::One(params) = input else {
panic!("expected scalar params")
};
assert_eq!(params.temperature, 1.0, "explicit default wins");
assert_eq!(params.top_p, 1.0, "explicit null keeps the type default");
assert_eq!(params.max_new_tokens, Some(4096), "omitted uses preferred");
}
#[test]
fn preferred_params_apply_to_every_batched_object() {
let preferred = serde_json::json!({"temperature": 0.25, "top_p": 0.75});
let mut input: SamplingParamsInput =
serde_json::from_str(r#"[{"temperature": 0.5}, {"top_p": 0.9}]"#).unwrap();
input.apply_preferred(&preferred).unwrap();
let SamplingParamsInput::Many(params) = input else {
panic!("expected batched params")
};
assert_eq!((params[0].temperature, params[0].top_p), (0.5, 0.75));
assert_eq!((params[1].temperature, params[1].top_p), (0.25, 0.9));
}
}
+410
View File
@@ -0,0 +1,410 @@
//! The `/generate` wire contract is the generated `sglang_api_types` request
//! (the proto schema is the single source of truth for both servers). This
//! module is the one place its carriers become the internal request types:
//! scalar-or-list columns into [`OneOrMany`], media containers into
//! [`MmDataInput`], and the wire `SamplingParams` into the internal one that
//! carries normalization state. Nothing here validates batch shape; that is
//! [`into_requests`](super::request::into_requests).
use std::collections::BTreeMap;
use sglang_api_types::api::v1 as api;
use super::multimodal::{MmDataInput, MmItem};
use super::sampling::{CustomParamValue, SamplingParams};
use super::types::{OneOrMany, TokenIds};
use crate::utils::error::Error;
pub fn string_or_list(v: api::StringOrList) -> Option<OneOrMany<String>> {
use api::string_or_list::Value;
v.value.map(|v| match v {
Value::One(s) => OneOrMany::One(s),
Value::Many(l) => OneOrMany::Many(l.items),
})
}
pub fn int64_or_list(v: api::Int64OrList) -> Option<OneOrMany<i64>> {
use api::int64_or_list::Value;
v.value.map(|v| match v {
Value::One(x) => OneOrMany::One(x),
Value::Many(l) => OneOrMany::Many(l.items),
})
}
pub fn bool_or_list(v: api::BoolOrList) -> Option<OneOrMany<bool>> {
use api::bool_or_list::Value;
v.value.map(|v| match v {
Value::One(x) => OneOrMany::One(x),
Value::Many(l) => OneOrMany::Many(l.items),
})
}
fn token_ids(v: api::TokenIds) -> TokenIds {
v.ids
}
pub fn token_ids_or_list(v: api::TokenIdsOrList) -> Option<OneOrMany<TokenIds>> {
use api::token_ids_or_list::Value;
v.value.map(|v| match v {
Value::One(ids) => OneOrMany::One(token_ids(ids)),
Value::Many(l) => OneOrMany::Many(l.items.into_iter().map(token_ids).collect()),
})
}
pub fn optional_string_or_list(v: api::OptionalStringOrList) -> Option<OneOrMany<Option<String>>> {
use api::optional_string_or_list::Value;
v.value.map(|v| match v {
Value::One(s) => OneOrMany::One(s.value),
Value::Many(l) => OneOrMany::Many(l.items.into_iter().map(|s| s.value).collect()),
})
}
pub fn optional_int64_or_list(v: api::OptionalInt64OrList) -> Option<OneOrMany<Option<i64>>> {
use api::optional_int64_or_list::Value;
v.value.map(|v| match v {
Value::One(x) => OneOrMany::One(x.value),
Value::Many(l) => OneOrMany::Many(l.items.into_iter().map(|x| x.value).collect()),
})
}
pub fn string_list_or_list(v: api::StringListOrList) -> Option<OneOrMany<Vec<String>>> {
use api::string_list_or_list::Value;
v.value.map(|v| match v {
Value::One(l) => OneOrMany::One(l.items),
Value::Many(ll) => OneOrMany::Many(ll.items.into_iter().map(|l| l.items).collect()),
})
}
/// A media item: a bare source, or a ref whose `url` must be present. proto3
/// strings carry no presence, so an object without `url` decodes as an empty
/// one and is rejected here (Python's `load_image` fails on it the same way).
pub fn media_item(item: api::MediaItem) -> Result<MmItem, Error> {
use api::media_item::Value;
match item.value {
// Only `url` reaches the Rust MM pipeline: the hints are read by model
// families it does not run (Python's `load_image` reduces an item to
// `.url` the same way).
Some(Value::Ref(r)) if !r.url.is_empty() => Ok(MmItem::Ref { url: r.url }),
Some(Value::Ref(_)) => Err(Error::Validation(
"a multimodal item object needs a non-empty `url`".into(),
)),
Some(Value::Source(s)) => Ok(MmItem::Source(s)),
// Unreachable from JSON (the untagged union always sets an arm).
None => Err(Error::Validation("empty multimodal item".into())),
}
}
pub fn media_input(input: api::MediaInput) -> Result<MmDataInput, Error> {
use api::media_input::Value;
let item = |i: api::OptionalMediaItem| i.item.map(media_item).transpose();
Ok(match input.value {
Some(Value::Item(i)) => MmDataInput::One(media_item(i)?),
Some(Value::Many(l)) => {
MmDataInput::Many(l.items.into_iter().map(item).collect::<Result<_, _>>()?)
}
Some(Value::Nested(ll)) => MmDataInput::Nested(
ll.items
.into_iter()
.map(|entry| {
entry
.items
.map(|l| l.items.into_iter().map(item).collect::<Result<_, _>>())
.transpose()
})
.collect::<Result<_, _>>()?,
),
None => MmDataInput::Many(Vec::new()),
})
}
/// The wire params, with the schema defaults already applied by the generated
/// decoder (an absent key reads its default), become the internal params. Only
/// `custom_params` needs parsing: it crosses the wire as JSON text.
impl TryFrom<api::SamplingParams> for SamplingParams {
type Error = String;
fn try_from(p: api::SamplingParams) -> Result<Self, String> {
let defaults = SamplingParams::default();
let custom_params: Option<BTreeMap<String, CustomParamValue>> = match p.custom_params {
None => None,
Some(text) => {
Some(serde_json::from_str(&text).map_err(|e| format!("custom_params: {e}"))?)
}
};
Ok(SamplingParams {
max_new_tokens: p.max_new_tokens,
stop: p.stop.and_then(string_or_list),
stop_token_ids: p.stop_token_ids.map(|l| l.items),
stop_regex: p.stop_regex.and_then(string_or_list),
temperature: p.temperature.unwrap_or(defaults.temperature),
top_p: p.top_p.unwrap_or(defaults.top_p),
top_k: p.top_k.unwrap_or(defaults.top_k),
min_p: p.min_p.unwrap_or(defaults.min_p),
frequency_penalty: p.frequency_penalty.unwrap_or(defaults.frequency_penalty),
presence_penalty: p.presence_penalty.unwrap_or(defaults.presence_penalty),
repetition_penalty: p.repetition_penalty.unwrap_or(defaults.repetition_penalty),
min_new_tokens: p.min_new_tokens.unwrap_or(defaults.min_new_tokens),
n: p.n.unwrap_or(defaults.n),
beam_width: p.beam_width,
json_schema: p.json_schema,
regex: p.regex,
ebnf: p.ebnf,
structural_tag: p.structural_tag,
ignore_eos: p.ignore_eos.unwrap_or(defaults.ignore_eos),
skip_special_tokens: p
.skip_special_tokens
.unwrap_or(defaults.skip_special_tokens),
spaces_between_special_tokens: p
.spaces_between_special_tokens
.unwrap_or(defaults.spaces_between_special_tokens),
no_stop_trim: p.no_stop_trim.unwrap_or(defaults.no_stop_trim),
stream_interval: p.stream_interval,
logit_bias: (!p.logit_bias.is_empty()).then(|| p.logit_bias.into_iter().collect()),
sampling_seed: p.sampling_seed,
custom_params,
..defaults
})
}
}
/// `sampling_params` as sent: one object (a broadcast in a batch) or one per
/// prompt. Not a [`OneOrMany`]: that carrier is sealed to wire-shaped items,
/// and the internal params are not one.
pub enum SamplingInput {
/// Boxed: the params are ~450 bytes, so an inline variant would make
/// every input that big regardless of which form arrived.
One(Box<SamplingParams>),
Many(Vec<SamplingParams>),
}
pub fn sampling_params_or_list(
v: api::SamplingParamsOrList,
) -> Result<Option<SamplingInput>, String> {
use api::sampling_params_or_list::Value;
Ok(match v.value {
None => None,
Some(Value::One(p)) => Some(SamplingInput::One(Box::new(p.try_into()?))),
Some(Value::Many(l)) => Some(SamplingInput::Many(
l.items
.into_iter()
.map(SamplingParams::try_from)
.collect::<Result<_, _>>()?,
)),
})
}
/// Launch-time preferred sampling params fill the keys a request did not
/// send, at the JSON level, before the body is decoded: `{**preferred,
/// **request}` per object, which is Python TokenizerManager's precedence. A
/// request key wins even when it carries the type's default or null, which
/// a decoded struct can no longer tell apart from an absent key.
pub fn merge_preferred_sampling(
body: &mut serde_json::Value,
preferred: &serde_json::Value,
) -> Result<(), String> {
let preferred = preferred
.as_object()
.ok_or_else(|| "preferred_sampling_params must be a JSON object".to_string())?;
let Some(body) = body.as_object_mut() else {
return Ok(()); // not an object: the decoder reports it
};
let merge_into = |request: &mut serde_json::Value| {
if let Some(obj) = request.as_object_mut() {
for (k, v) in preferred {
obj.entry(k.clone()).or_insert_with(|| v.clone());
}
}
};
match body.get_mut("sampling_params") {
None | Some(serde_json::Value::Null) => {
body.insert(
"sampling_params".into(),
serde_json::Value::Object(preferred.clone()),
);
}
Some(serde_json::Value::Array(items)) => items.iter_mut().for_each(merge_into),
Some(one) => merge_into(one),
}
Ok(())
}
/// The same fill for a request that arrived already decoded (gRPC): the
/// carrier round-trips through its JSON form so protobuf and HTTP share one
/// precedence rule. Protobuf has no null, so an unset field is an absent key
/// (the schema's null-emitting fields are stripped before the merge), never
/// the explicit null a JSON client can send.
pub fn fill_preferred_sampling(
params: Option<api::SamplingParamsOrList>,
preferred: &serde_json::Value,
) -> Result<Option<api::SamplingParamsOrList>, String> {
let mut body = serde_json::json!({ "sampling_params": params });
let strip_nulls = |object: &mut serde_json::Value| {
if let Some(map) = object.as_object_mut() {
map.retain(|_, value| !value.is_null());
}
};
match &mut body["sampling_params"] {
serde_json::Value::Array(items) => items.iter_mut().for_each(strip_nulls),
one => strip_nulls(one),
}
merge_preferred_sampling(&mut body, preferred)?;
serde_json::from_value(body["sampling_params"].take()).map_err(|e| e.to_string())
}
#[cfg(test)]
mod tests {
use super::*;
fn decode(body: &str, preferred: serde_json::Value) -> Vec<SamplingParams> {
let mut value: serde_json::Value = serde_json::from_str(body).unwrap();
merge_preferred_sampling(&mut value, &preferred).unwrap();
let req: api::GenerateRequest = serde_json::from_value(value).unwrap();
match sampling_params_or_list(req.sampling_params.unwrap())
.unwrap()
.unwrap()
{
SamplingInput::One(p) => vec![*p],
SamplingInput::Many(v) => v,
}
}
/// A request key wins over the preferred value, even as an explicit
/// default or null; only omitted keys take the preferred value.
#[test]
fn preferred_params_fill_only_omitted_request_fields() {
let preferred =
serde_json::json!({"temperature": 0.25, "top_p": 0.75, "max_new_tokens": 4096});
let p = &decode(
r#"{"sampling_params": {"temperature": 1.0, "top_p": null}}"#,
preferred.clone(),
)[0];
assert_eq!(p.temperature, 1.0, "explicit default wins");
assert_eq!(p.top_p, 1.0, "explicit null keeps the type default");
assert_eq!(p.max_new_tokens, Some(4096), "omitted uses preferred");
let p = &decode(r#"{"text": "hi"}"#, preferred)[0];
assert_eq!(
(p.temperature, p.max_new_tokens),
(0.25, Some(4096)),
"absent object takes all"
);
}
#[test]
fn preferred_params_apply_to_every_batched_object() {
let preferred = serde_json::json!({"temperature": 0.25, "top_p": 0.75});
let ps = decode(
r#"{"sampling_params": [{"temperature": 0.5}, {"top_p": 0.9}]}"#,
preferred,
);
assert_eq!((ps[0].temperature, ps[0].top_p), (0.5, 0.75));
assert_eq!((ps[1].temperature, ps[1].top_p), (0.25, 0.9));
}
/// The generated `TokenIdsOrList` and the crate's own `OneOrMany<TokenIds>`
/// accept and reject the same shapes (the first-element rule), so the
/// swap to the wire type changed no input_ids contract.
#[test]
fn token_ids_decoding_matches_one_or_many() {
for value in [
"null",
"[]",
"[0]",
"[-2147483648,2147483647]",
"[[]]",
"[[],[1,-2]]",
"[[1,2],[3]]",
"1",
"true",
"{}",
r#""1""#,
"[null]",
"[true]",
r#"["1"]"#,
"[1.0]",
"[-0]",
"[2147483648]",
"[0,2147483648]",
"[1,[2]]",
"[[1],2]",
"[[[1]]]",
"[[1.0]]",
] {
let expected = serde_json::from_str::<Option<OneOrMany<TokenIds>>>(value);
let actual = serde_json::from_str::<Option<api::TokenIdsOrList>>(value)
.map(|v| v.and_then(token_ids_or_list));
match (expected, actual) {
(Ok(e), Ok(a)) => assert_eq!(a, e, "{value}"),
(Err(_), Err(_)) => {}
(e, a) => panic!("{value}: expected {e:?}, got {a:?}"),
}
}
}
#[test]
fn media_input_converts_every_container_form() {
let conv = |json: &str| -> MmDataInput {
media_input(serde_json::from_str::<api::MediaInput>(json).unwrap()).unwrap()
};
let src = |s: &str| MmItem::Source(s.to_owned());
assert_eq!(conv(r#""u""#), MmDataInput::One(src("u")));
assert_eq!(
conv(r#"{"url": "u", "detail": "high"}"#),
MmDataInput::One(MmItem::Ref { url: "u".into() })
);
assert_eq!(
conv(r#"["a", null, "b"]"#),
MmDataInput::Many(vec![Some(src("a")), None, Some(src("b"))])
);
assert_eq!(conv("[]"), MmDataInput::Many(vec![]));
assert_eq!(
conv(r#"[["a", null], null, []]"#),
MmDataInput::Nested(vec![Some(vec![Some(src("a")), None]), None, Some(vec![])])
);
}
#[test]
fn custom_params_cross_as_json_text() {
let p: api::SamplingParams =
serde_json::from_str(r#"{"custom_params": {"k": [1, "x"]}, "logit_bias": {"5": 1.5}}"#)
.unwrap();
let p = SamplingParams::try_from(p).unwrap();
assert!(
p.custom_params
.as_ref()
.is_some_and(|m| m.contains_key("k"))
);
assert_eq!(
p.logit_bias.as_ref().and_then(|m| m.get("5")).copied(),
Some(1.5)
);
assert_eq!(p.temperature, 1.0, "schema default applied by the decoder");
}
/// The protobuf entry fills the same keys the JSON entry does: a set field
/// wins, an unset one takes the preferred value, and an absent carrier
/// takes them all.
#[test]
fn preferred_params_fill_unset_protobuf_fields() {
use api::sampling_params_or_list::Value;
let preferred = serde_json::json!({"temperature": 0.25, "max_new_tokens": 4096});
let sent = api::SamplingParamsOrList {
value: Some(Value::One(api::SamplingParams {
temperature: Some(1.0),
..Default::default()
})),
};
let filled = fill_preferred_sampling(Some(sent), &preferred)
.unwrap()
.unwrap();
let Some(Value::One(p)) = filled.value else {
panic!("one object in, one object out");
};
assert_eq!((p.temperature, p.max_new_tokens), (Some(1.0), Some(4096)));
let filled = fill_preferred_sampling(None, &preferred).unwrap().unwrap();
let Some(Value::One(p)) = filled.value else {
panic!("an absent carrier becomes the preferred object");
};
assert_eq!((p.temperature, p.max_new_tokens), (Some(0.25), Some(4096)));
}
}
@@ -10,7 +10,7 @@ use sglang_mm::common::fetch::{ByteBudget, fetch_bytes_budgeted};
use sglang_mm::driver::{ImageSource, MmInput};
use crate::message::multimodal::MmItem;
use crate::message::request::{MmData, ProcessorExtensions};
use crate::message::request::MmData;
use crate::message::types::TokenIds;
/// Fully resolved media for a multimodal processor. I/O sources were
@@ -23,9 +23,6 @@ pub struct ResolvedMediaWork {
pub images: Vec<Bytes>,
pub videos: Vec<Bytes>,
pub audios: Vec<Bytes>,
/// Request fields owned by the selected processor rather than this shared
/// payload layer.
pub processor_extensions: ProcessorExtensions,
}
/// Resolve all modality fields in the fixed image/video/audio prefetch order.
@@ -42,7 +39,6 @@ fn resolve_media_work_with_budget(
image_data,
video_data,
audio_data,
processor_extensions,
prefetched,
mm_hashes: _,
} = mm;
@@ -59,7 +55,6 @@ fn resolve_media_work_with_budget(
images,
videos,
audios,
processor_extensions,
})
}
@@ -83,16 +78,13 @@ fn collect_media(
fetch_bytes_budgeted(&source, budget).map(Bytes::from)
}
}
MmItem::Preprocessed { format } => Err(format!(
"unsupported {field} item: preprocessed `{format}` input"
)),
})
.collect()
}
/// True for sources the API layer must resolve before MM dispatch: I/O — network
/// *or* disk, since a network mount can hang past any HTTP timeout — never runs
/// on the fixed MM worker pool (see `frontend::prefetch`). `data:` and bare
/// on the fixed MM worker pool (see `core::prefetch`). `data:` and bare
/// base64 are pure CPU and stay on the worker. Lives next to [`image_source`]
/// so the prefetch walk and the parse walk cannot drift.
pub fn is_io_source(src: &str) -> bool {
@@ -120,16 +112,12 @@ pub fn to_mm_input(input_ids: TokenIds, mm: MmData) -> Result<MmInput, String> {
image_data,
video_data,
audio_data,
processor_extensions,
prefetched,
mm_hashes: _,
} = mm;
if !video_data.is_empty() || !audio_data.is_empty() {
return Err("unsupported modality: video/audio input".into());
}
if !processor_extensions.is_empty() {
return Err("unsupported generate extensions for this processor".into());
}
let mut prefetched = prefetched.iter();
let images = image_data
.into_iter()
@@ -155,9 +143,6 @@ fn image_source(
.map(|bytes| ImageSource::Bytes(bytes.to_vec()))
.ok_or_else(|| "I/O-backed image source was not prefetched".to_string())
}
MmItem::Preprocessed { format } => Err(format!(
"unsupported image_data item: preprocessed `{format}` input"
)),
}
}
@@ -207,31 +192,6 @@ mod tests {
.unwrap()
.contains("video/audio")
);
let err = to_mm_input(
IDS.to_vec(),
image_work(vec![MmItem::Preprocessed {
format: "processor_output".into(),
}]),
)
.err()
.unwrap();
assert!(err.contains("preprocessed `processor_output`"), "{err}");
let extension = MmData {
processor_extensions: std::iter::once((
"multimodal_custom".to_owned(),
rmpv::Value::Boolean(true),
))
.collect(),
..Default::default()
};
assert!(
to_mm_input(IDS.to_vec(), extension)
.err()
.unwrap()
.contains("unsupported generate extensions")
);
}
/// I/O-backed sources (URLs, file paths) take their prefetched bytes in walk
-160
View File
@@ -1,160 +0,0 @@
//! Shared wire semantics for SGLang-native generation responses.
//!
//! Native HTTP places these values in a JSON `meta_info` object, while
//! `runtime.v1` gRPC JSON-encodes each value into `map<string, string>`. Keeping
//! the logical shape here prevents the sibling adapters from drifting without
//! making either adapter depend on the other.
use crate::frontend::FrontendOutput;
/// The text slot of a `[logprob, token_id, text]` tuple: the decoded token when
/// `return_text_in_logprobs` supplied a text buffer, else `null`.
pub(crate) fn text_slot(texts: Option<&[String]>, j: usize) -> serde_json::Value {
texts
.and_then(|texts| texts.get(j))
.map(|text| serde_json::json!(text))
.unwrap_or(serde_json::Value::Null)
}
/// A decoded-text column becomes the tuples' text source only when populated.
pub(crate) fn opt_texts(texts: &[String]) -> Option<&[String]> {
(!texts.is_empty()).then_some(texts)
}
/// The logprob slot of a tuple: a finite value, or `null` for the `NaN` sentinel.
pub(crate) fn lp_value(value: f32) -> serde_json::Value {
if value.is_nan() {
serde_json::Value::Null
} else {
serde_json::json!(value)
}
}
/// SGLang logprob shape: a list of `[logprob, token_id, text]` tuples.
pub(crate) fn logprob_tuples(
values: &[f32],
token_ids: &[i32],
texts: Option<&[String]>,
) -> serde_json::Value {
let tuples = values
.iter()
.zip(token_ids.iter())
.enumerate()
.map(|(index, (&value, &token_id))| {
serde_json::json!([lp_value(value), token_id, text_slot(texts, index)])
})
.collect();
serde_json::Value::Array(tuples)
}
/// Ragged top-k / requested-token shape: one entry per position, or `null`
/// where that position's length is zero.
pub(crate) fn ragged_logprob_tuples(
values: &[f32],
token_ids: &[i32],
lengths: &[u32],
texts: Option<&[String]>,
) -> serde_json::Value {
let mut positions = Vec::with_capacity(lengths.len());
let mut offset = 0usize;
for &length in lengths {
let length = length as usize;
if length == 0 {
positions.push(serde_json::Value::Null);
} else {
// Bounds checks turn malformed parallel columns into a shortened
// row instead of panicking a transport worker.
let tuples = (offset..offset + length)
.filter_map(|index| {
Some(serde_json::json!([
lp_value(*values.get(index)?),
*token_ids.get(index)?,
text_slot(texts, index)
]))
})
.collect();
positions.push(serde_json::Value::Array(tuples));
}
offset += length;
}
serde_json::Value::Array(positions)
}
/// Reshape flat hidden-state values and per-row lengths into nested rows.
pub(crate) fn hidden_states_rows(values: &[f32], lengths: &[u32]) -> serde_json::Value {
let mut rows = Vec::with_capacity(lengths.len());
let mut offset = 0usize;
for &length in lengths {
let length = length as usize;
rows.push(serde_json::json!(
values.get(offset..offset + length).unwrap_or(&[])
));
offset += length;
}
serde_json::Value::Array(rows)
}
/// Build the heterogeneous native `meta_info` object before either adapter
/// chooses its wire encoding.
pub(crate) fn meta_info_value(out: &FrontendOutput, rid: &str) -> serde_json::Value {
let mut meta = serde_json::json!({
"id": rid,
"prompt_tokens": out.prompt_tokens,
"completion_tokens": out.completion_tokens,
"finish_reason": out.finish_reason,
});
let Some(extras) = out.extras.as_deref() else {
return meta;
};
// Python always sets input and output token logprobs together, including
// the empty input list emitted by a disaggregated decode node.
if !extras.out_lp_val.is_empty() || !extras.in_lp_val.is_empty() {
meta["output_token_logprobs"] = logprob_tuples(
&extras.out_lp_val,
&extras.out_lp_idx,
opt_texts(&extras.out_lp_txt),
);
meta["input_token_logprobs"] = logprob_tuples(
&extras.in_lp_val,
&extras.in_lp_idx,
opt_texts(&extras.in_lp_txt),
);
}
if !extras.out_top_lens.is_empty() {
meta["output_top_logprobs"] = ragged_logprob_tuples(
&extras.out_top_val,
&extras.out_top_idx,
&extras.out_top_lens,
opt_texts(&extras.out_top_txt),
);
}
if !extras.in_top_lens.is_empty() {
meta["input_top_logprobs"] = ragged_logprob_tuples(
&extras.in_top_val,
&extras.in_top_idx,
&extras.in_top_lens,
opt_texts(&extras.in_top_txt),
);
}
if !extras.out_tid_lens.is_empty() {
meta["output_token_ids_logprobs"] = ragged_logprob_tuples(
&extras.out_tid_val,
&extras.out_tid_idx,
&extras.out_tid_lens,
opt_texts(&extras.out_tid_txt),
);
}
if !extras.in_tid_lens.is_empty() {
meta["input_token_ids_logprobs"] = ragged_logprob_tuples(
&extras.in_tid_val,
&extras.in_tid_idx,
&extras.in_tid_lens,
opt_texts(&extras.in_tid_txt),
);
}
if !extras.hidden_lens.is_empty() {
meta["hidden_states"] = hidden_states_rows(&extras.hidden_val, &extras.hidden_lens);
}
meta
}
@@ -184,7 +184,7 @@ impl Runnable for DetokenizerWorker {
// Plain `recv`: exits when the `DetokMsg` channel closes (every `Senders`
// clone gone). On shutdown that happens once to-scheduler/from-scheduler
// exit — frontend calls own only their abort sender, not detok channels.
// exit — core calls own only their abort sender, not detok channels.
while let Ok(msg) = self.rx.recv() {
match msg {
DetokMsg::Register {
@@ -395,7 +395,7 @@ fn handle_chunk(
}
let _ = st.fsm.apply(Event::Disconnect);
// Abort ONLY when the sink is full. `Closed` means the handler future is
// already gone, so its `FrontendCall` has queued the abort; do not emit
// already gone, so its `CoreCall` has queued the abort; do not emit
// the same lifecycle event a second time. A full sink still has a live
// handler, so the detokenizer must initiate cleanup itself.
if matches!(e, SinkError::Full) {
@@ -102,7 +102,7 @@ fn ids_len(req: &SchedulerRequest) -> usize {
/// further chunk can be delivered, and tell the scheduler to stop generating.
///
/// Neither releases anything, and nothing needs them to. Release ordering used
/// to be the delicate part here — `FrontendCall::drop` releasing a rid right
/// to be the delicate part here — `CoreCall::drop` releasing a rid right
/// after enqueuing the abort ordered the SEND, not the EFFECT, so a retry of
/// the same rid could `Register` ahead of the stale abort and be torn down by
/// it. `Rid::from_client` removes the premise: a retry carries a different
@@ -45,12 +45,12 @@ const ADMISSION_PENDING: u8 = 0;
const ADMISSION_ACCEPTED: u8 = 1;
const ADMISSION_CANCELLED: u8 = 2;
/// Cancellation-safe ownership handoff between a frontend call and Intake.
/// Cancellation-safe ownership handoff between a core call and Intake.
///
/// A bounded `send_async` can enqueue a request and wake its sender without the
/// sender being polled again. If that task is then cancelled, this token decides
/// exactly one outcome: either Intake observes the prior cancellation and drops
/// the request, or the frontend observes prior admission and emits an abort.
/// the request, or the core observes prior admission and emits an abort.
#[derive(Clone)]
pub struct RequestAdmission {
state: Arc<AtomicU8>,
@@ -91,7 +91,7 @@ impl RequestAdmission {
/// cannot be tangled up with an abort still in flight for the original.
#[derive(Clone, Debug)]
pub enum AbortSource {
/// From an in-flight frontend call drop. Owns the release.
/// From an in-flight core call drop. Owns the release.
Guard(Rid),
/// From a detokenizer terminal path. Aborts the scheduler work.
Detok(Rid),
+55 -38
View File
@@ -301,16 +301,16 @@ pub fn start(cfg: RuntimeConfig) -> Result<Runtime, String> {
// One transport-neutral entrance to the runtime. Each configured listener
// receives a clone; no listener owns or reconstructs scheduler wiring.
let frontend = crate::frontend::FrontendHandle::new(
let core = crate::api_server::core::CoreHandle::new(
senders.tok_manager_tx.clone(),
senders.abort_tx.clone(),
crate::frontend::FrontendConfig {
crate::api_server::core::CoreConfig {
response_capacity: cfg.rust_server_args.stage_channel_cap,
response_activity: response_activity.clone(),
startup_ready: cfg.server_args.skip_server_warmup,
is_disaggregation: cfg.server_args.is_disaggregation(),
mm_limits: cfg.server_args.limit_mm_data_per_request.clone(),
metadata: crate::frontend::FrontendMetadata::from(cfg.server_args.as_ref()),
metadata: crate::api_server::core::CoreMetadata::from(cfg.server_args.as_ref()),
},
);
@@ -320,7 +320,8 @@ pub fn start(cfg: RuntimeConfig) -> Result<Runtime, String> {
let api_cores = plan.as_ref().map(|p| p.api.clone());
let shutdown_rx = shutdown_rx.clone();
let grpc_server = grpc_listener.map(|listener| {
let service = crate::grpc::GrpcService::new(frontend.clone(), &cfg.server_args);
let service =
api_server::grpc::service::GrpcService::new(core.clone(), &cfg.server_args);
(listener, service)
});
let handle = std::thread::Builder::new()
@@ -341,14 +342,17 @@ pub fn start(cfg: RuntimeConfig) -> Result<Runtime, String> {
}
let rt = builder.build().expect("build api runtime");
rt.block_on(async move {
let http = api_server::app::serve(
let http = api_server::http::app::serve(
http_listener,
frontend,
core,
cfg.server_args.clone(),
shutdown_rx.clone(),
);
if let Some((listener, service)) = grpc_server {
tokio::join!(http, crate::grpc::serve(listener, service, shutdown_rx));
tokio::join!(
http,
api_server::grpc::app::serve(listener, service, shutdown_rx)
);
} else {
http.await;
}
@@ -375,8 +379,8 @@ mod tests {
use super::*;
use crate::message::config::{RuntimeConfig, RustServerServerArgs, ServerArgs};
use crate::message::response::{BatchHeader, frame_decode_batch_cols};
use sglang_grpc_types::sglang::runtime::v1 as proto;
use sglang_grpc_types::sglang::runtime::v1::sglang_service_client::SglangServiceClient;
use sglang_api_types::api::v1 as proto;
use sglang_api_types::api::v1::sglang_service_client::SglangServiceClient;
fn free_loopback_addrs() -> (std::net::SocketAddr, std::net::SocketAddr) {
// Hold both probes at once so the OS cannot return the same ephemeral
@@ -412,6 +416,21 @@ mod tests {
SglangServiceClient::new(endpoint.connect().await.expect("connect gRPC client"))
}
fn grpc_ids_request(ids: &[i64], rid: &str) -> proto::GenerateRequest {
proto::GenerateRequest {
input_ids: Some(proto::TokenIdsOrList {
value: Some(proto::token_ids_or_list::Value::One(proto::TokenIds {
ids: ids.to_vec(),
})),
}),
rid: Some(proto::StringOrList {
value: Some(proto::string_or_list::Value::One(rid.into())),
}),
stream: Some(true),
..Default::default()
}
}
fn scheduler_rid(header: &[u8]) -> String {
let value = rmpv::decode::read_value(&mut std::io::Cursor::new(header)).unwrap();
value
@@ -643,9 +662,9 @@ mod tests {
drop(rebound);
}
/// End to end over the configured socket: the generated runtime.v1 client
/// End to end over the configured socket: the generated api.v1 client
/// reaches the existing scheduler ring, and an existing scheduler frame is
/// translated back into a protobuf response. Shutdown then closes both
/// translated back into a protobuf frame. Shutdown then closes both
/// transport listeners.
#[test]
fn configured_grpc_serves_generation_and_closes_with_http() {
@@ -663,12 +682,7 @@ mod tests {
let mut client = connect_grpc(grpc_addr).await;
tokio::time::timeout(
Duration::from_secs(5),
client.generate(proto::GenerateRequest {
input_ids: vec![1, 2, 3],
stream: Some(true),
rid: Some("network-generation".into()),
..Default::default()
}),
client.generate(grpc_ids_request(&[1, 2, 3], "network-generation")),
)
.await
.expect("timed out starting Generate RPC")
@@ -694,25 +708,29 @@ mod tests {
.push(terminal_token_frame(rid, &[9, 10]))
);
let response = client_runtime
let item = client_runtime
.block_on(async {
tokio::time::timeout(Duration::from_secs(2), stream.message()).await
})
.expect("timed out waiting for gRPC response")
.expect("gRPC stream error")
.expect("gRPC stream ended without a response");
assert_eq!(response.output_ids, vec![9, 10]);
assert!(response.finished);
assert_eq!(response.meta_info["id"], r#""network-generation""#);
let Some(proto::generate_stream_item::Item::Frame(response)) = item.item else {
panic!("expected a frame, got {item:?}");
};
assert_eq!(response.output_ids.map(|ids| ids.ids), Some(vec![9, 10]));
let meta_info = response.meta_info.expect("frame meta_info");
assert_eq!(meta_info.id, "network-generation");
assert!(meta_info.finish_reason.is_some());
rt.request_shutdown();
assert!(std::net::TcpStream::connect(http_addr).is_err());
assert!(std::net::TcpStream::connect(grpc_addr).is_err());
}
/// The existing runtime.v1 listener accepts bodies above Tonic's 4 MiB
/// default. Keep that transport policy when mounting the same protocol on
/// the Rust frontend; reaching adapter validation proves the body decoded.
/// The Python-backed gRPC listener accepts bodies above Tonic's 4 MiB
/// default. Keep that transport policy on the Rust frontend; reaching
/// adapter validation proves the body decoded.
#[test]
fn grpc_listener_preserves_existing_large_message_limit() {
use tonic::Code;
@@ -729,19 +747,23 @@ mod tests {
tokio::time::timeout(
std::time::Duration::from_secs(10),
client.generate(proto::GenerateRequest {
input_ids: vec![1],
// This field is intentionally unsupported. An
// UNIMPLEMENTED status proves the >4 MiB protobuf reached
// the adapter instead of Tonic rejecting it on size.
routing_key: Some("x".repeat(5 * 1024 * 1024)),
..Default::default()
// `text` and `input_ids` together is a normalization
// error: an INVALID_ARGUMENT status proves the >4 MiB
// protobuf reached the adapter instead of Tonic rejecting
// it on size.
text: Some(proto::StringOrList {
value: Some(proto::string_or_list::Value::One(
"x".repeat(5 * 1024 * 1024),
)),
}),
..grpc_ids_request(&[1], "large")
}),
)
.await
.expect("timed out sending large Generate RPC")
.expect_err("unsupported routing_key must fail")
.expect_err("text and input_ids together must fail")
});
assert_eq!(error.code(), Code::Unimplemented);
assert_eq!(error.code(), Code::InvalidArgument);
rt.request_shutdown();
assert!(std::net::TcpStream::connect(grpc_addr).is_err());
@@ -765,12 +787,7 @@ mod tests {
let mut client = connect_grpc(grpc_addr).await;
let stream = tokio::time::timeout(
Duration::from_secs(5),
client.generate(proto::GenerateRequest {
input_ids: vec![1, 2, 3],
stream: Some(true),
rid: Some("in-flight".into()),
..Default::default()
}),
client.generate(grpc_ids_request(&[1, 2, 3], "in-flight")),
)
.await
.expect("timed out starting Generate RPC")
+1 -1
View File
@@ -11,7 +11,7 @@ from sglang.test.test_utils import CustomTestCase
BUILD_AND_RUN_TIMEOUT_S = 900
RUST_WORKSPACE = Path(__file__).resolve().parents[3] / "rust"
GRPC_TYPES_PACKAGE = "sglang-grpc-types"
GRPC_TYPES_PACKAGE = "sglang-api-types"
register_cpu_ci(est_time=170, suite="base-a-test-cpu")
+5 -3
View File
@@ -590,21 +590,23 @@ crate-type = ["cdylib"]
self.assertNotIn(module_name, sys.modules)
def test_checked_in_crates_are_discovered_from_wheel_metadata(self):
grpc_proto = (rust_extension._RUST_WORKSPACE.parent / "proto").resolve()
# No crate reads `proto/` at build time: the generated bindings are
# checked in under rust/sglang-api-types (`cargo gen-api`), so the
# workspace is every extension's only source input.
for python_module, package, library, features, source_inputs in (
(
"sglang.srt.rust_extensions._server",
"sglang-server",
"sglang_server",
(),
(grpc_proto,),
(),
),
(
"sglang.srt.rust_extensions._grpc",
"sglang-grpc",
"sglang_grpc_core",
(),
(grpc_proto,),
(),
),
(
"sglang.srt.rust_extensions._multimodal",
@@ -0,0 +1,325 @@
"""JSON-contract tests for the generated Python API types (sglang.api.v1).
Mirrors rust/sglang-api-types/tests/json_contract.rs case for case (minus the
protobuf-arrival accessors, which have no Python counterpart), so the Rust and
Python emitters of sglang-api-codegen are held to one contract.
"""
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
import json
import unittest
from sglang.api.v1.api_types import (
FinishAbort,
FinishLength,
FinishStop,
GenerateMetaInfo,
GenerateRequest,
GenerateResponse,
JsonContractError,
LogprobEntry,
MediaRef,
SamplingParams,
decode_FinishReason,
decode_MediaInput,
decode_OptionalInt64OrList,
decode_OptionalStringOrList,
decode_SamplingParamsOrList,
decode_StringListOrList,
decode_StringOrList,
decode_TokenIdsOrList,
encode_FinishReason,
encode_MediaInput,
encode_OptionalStringOrList,
encode_StringOrList,
)
from sglang.test.test_utils import CustomTestCase
def compact(value) -> str:
"""serde_json::to_string parity: no whitespace."""
return json.dumps(value, separators=(",", ":"))
def sp(text: str) -> SamplingParams:
return SamplingParams.from_json_value(json.loads(text))
class TestSamplingParams(CustomTestCase):
def test_defaults_match_hand_written(self):
"""Absent keys yield the schema defaults."""
p = sp("{}")
self.assertEqual(p.max_new_tokens, 128)
self.assertEqual(p.temperature, 1.0)
self.assertEqual(p.top_p, 1.0)
self.assertEqual(p.top_k, 1 << 30)
self.assertEqual(p.min_p, 0.0)
self.assertEqual(p.repetition_penalty, 1.0)
self.assertEqual(p.min_new_tokens, 0)
self.assertEqual(p.n, 1)
self.assertIs(p.ignore_eos, False)
self.assertIs(p.skip_special_tokens, True)
self.assertIs(p.spaces_between_special_tokens, True)
self.assertIs(p.no_stop_trim, False)
def test_constructor_defaults_match_absent_keys(self):
"""A dataclass built by hand carries the same schema defaults."""
self.assertEqual(SamplingParams(), sp("{}"))
def test_null_resets_default(self):
"""null_resets_default: an explicit null is the default, not an error."""
p = sp('{"temperature": null, "top_k": null, "ignore_eos": null}')
self.assertEqual(p.temperature, 1.0)
self.assertEqual(p.top_k, 1 << 30)
self.assertIs(p.ignore_eos, False)
def test_max_new_tokens_absent_vs_null(self):
"""null_is_none: absent = 128, null = unbounded."""
self.assertEqual(sp("{}").max_new_tokens, 128)
self.assertIsNone(sp('{"max_new_tokens": null}').max_new_tokens)
self.assertEqual(sp('{"max_new_tokens": 7}').max_new_tokens, 7)
def test_denies_unknown_keys(self):
"""DENY policy with serde's own error text."""
with self.assertRaises(JsonContractError) as ctx:
sp('{"temperatur": 0.5}') # codespell:ignore temperatur
self.assertTrue(
str(ctx.exception).startswith(
"unknown field `temperatur`, expected one of" # codespell:ignore temperatur
),
str(ctx.exception),
)
def test_type_mismatch_uses_serde_text(self):
with self.assertRaises(JsonContractError) as ctx:
sp('{"temperature": "hot"}')
self.assertEqual(str(ctx.exception), 'invalid type: string "hot", expected f64')
def test_or_list_keeps_field_errors(self):
"""one_or_many over SamplingParams keeps field-level error texts."""
with self.assertRaises(JsonContractError) as ctx:
decode_SamplingParamsOrList(
{"temperatur": 1} # codespell:ignore temperatur
)
self.assertTrue(
str(ctx.exception).startswith(
"unknown field `temperatur`" # codespell:ignore temperatur
),
str(ctx.exception),
)
many = decode_SamplingParamsOrList([{"n": 2}, {}])
self.assertEqual([p.n for p in many], [2, 1])
def test_raw_json_passthrough(self):
"""raw_json: custom_params crosses untouched, integers included."""
p = sp('{"custom_params": {"k": 3}}')
self.assertEqual(p.custom_params, {"k": 3})
self.assertEqual(p.to_json_value()["custom_params"], {"k": 3})
def test_int_key_map(self):
p = sp('{"logit_bias": {"42": -1}}')
self.assertEqual(p.logit_bias, {"42": -1.0})
class TestGenerateRequest(CustomTestCase):
def test_ignores_unknown_keys(self):
"""IGNORE policy: unported Python fields must not 400."""
req = GenerateRequest.from_json_value(
{
"text": "hi",
"priority": 3,
"session_id": "s",
"custom_logit_processor": "x",
}
)
self.assertEqual(req.text, "hi")
def test_media_input_shapes_dispatch(self):
"""Typed media: the value's shape picks the container form, nulls are
kept per element, and a ref object decodes to a MediaRef."""
req = GenerateRequest.from_json_value(
{"image_data": [["data:image/png;base64,xx"], None, []]}
)
self.assertEqual(req.image_data, [["data:image/png;base64,xx"], None, []])
flat = decode_MediaInput(["a", None, {"url": "b", "detail": "high"}])
self.assertEqual(flat[:2], ["a", None])
self.assertIsInstance(flat[2], MediaRef)
self.assertEqual((flat[2].url, flat[2].detail), ("b", "high"))
self.assertEqual(decode_MediaInput("u"), "u")
self.assertEqual(decode_MediaInput([]), [])
for value in (
["a", None, {"url": "b", "detail": "high"}],
[["a", None], None, []],
"u",
):
self.assertEqual(encode_MediaInput(decode_MediaInput(value)), value)
def test_media_ref_is_typed_and_strict(self):
with self.assertRaises(JsonContractError) as ctx:
decode_MediaInput(
{"url": "u", "max_dynamic_patc": 6}
) # codespell:ignore patc
self.assertIn("unknown field", str(ctx.exception))
with self.assertRaises(JsonContractError):
decode_MediaInput([{"format": "processor_output"}])
with self.assertRaises(JsonContractError):
decode_MediaInput(["a", ["b"]])
def test_string_list_or_list_shape_dispatch(self):
self.assertEqual(decode_StringListOrList(["a", "b"]), ["a", "b"])
self.assertEqual(
decode_StringListOrList([["a"], ["b", "c"]]), [["a"], ["b", "c"]]
)
self.assertEqual(decode_StringListOrList([]), [])
def test_stream_null_resets_default(self):
self.assertIs(GenerateRequest.from_json_value({"stream": None}).stream, False)
self.assertIs(GenerateRequest.from_json_value({"stream": True}).stream, True)
def test_round_trip_batch(self):
body = {
"text": ["a", "b"],
"sampling_params": [{"n": 2}, {"temperature": 0.5}],
"return_logprob": True,
"bootstrap_host": [None, "h"],
}
req = GenerateRequest.from_json_value(body)
back = req.to_json_value()
self.assertEqual(back["text"], ["a", "b"])
self.assertEqual(back["return_logprob"], True)
self.assertEqual(back["bootstrap_host"], [None, "h"])
self.assertEqual(back["sampling_params"][1]["temperature"], 0.5)
class TestCarriers(CustomTestCase):
def test_one_or_many_flattens(self):
self.assertEqual(decode_StringOrList("a"), "a")
self.assertEqual(compact(encode_StringOrList("a")), '"a"')
self.assertEqual(decode_StringOrList(["a", "b"]), ["a", "b"])
self.assertEqual(compact(encode_StringOrList(["a", "b"])), '["a","b"]')
with self.assertRaises(JsonContractError) as ctx:
decode_StringOrList({"x": 1})
self.assertEqual(
str(ctx.exception),
"invalid type: map, expected a value or an array of values for StringOrList",
)
def test_token_ids_or_list_shape_dispatch(self):
"""[1,2] is one id list, [[1],[2,3]] is a batch, [] is one empty list."""
self.assertEqual(decode_TokenIdsOrList([1, 2]), [1, 2])
self.assertEqual(decode_TokenIdsOrList([[1], [2, 3]]), [[1], [2, 3]])
self.assertEqual(decode_TokenIdsOrList([]), [])
def test_optional_carriers_keep_element_nulls(self):
"""PD bootstrap columns: [null, "h"] keeps per-item presence."""
v = decode_OptionalStringOrList([None, "h"])
self.assertEqual(v, [None, "h"])
self.assertEqual(compact(encode_OptionalStringOrList(v)), '[null,"h"]')
self.assertEqual(decode_OptionalInt64OrList(17000), 17000)
class TestFinishReason(CustomTestCase):
def test_round_trips(self):
"""Tagged by "type"; matched keeps its wire shape (str / int / int list)."""
for text in [
'{"type":"stop","matched":"</s>"}',
'{"type":"stop","matched":7}',
'{"type":"stop","matched":[1,2]}',
'{"type":"length","length":2}',
]:
want = json.loads(text)
parsed = decode_FinishReason(want)
self.assertEqual(encode_FinishReason(parsed), want, text)
def test_variant_classes(self):
self.assertIsInstance(decode_FinishReason({"type": "stop"}), FinishStop)
self.assertIsInstance(
decode_FinishReason({"type": "length", "length": 2}), FinishLength
)
self.assertIsInstance(decode_FinishReason({"type": "abort"}), FinishAbort)
def test_abort_emits_null_keys(self):
"""An abort carries all three keys, nulls included."""
parsed = decode_FinishReason({"type": "abort", "message": "m"})
back = encode_FinishReason(parsed)
self.assertEqual(
back,
{"type": "abort", "message": "m", "status_code": None, "err_type": None},
)
def test_unknown_type_passes_through(self):
"""The forward-compat escape hatch: unknown tags round-trip unchanged."""
raw = {"type": "paused", "step": 3}
parsed = decode_FinishReason(raw)
self.assertIsInstance(parsed, dict)
self.assertEqual(encode_FinishReason(parsed), raw)
def test_malformed_known_type_passes_through(self):
parsed = decode_FinishReason({"type": "stop", "matched": {"bad": "shape"}})
self.assertIsInstance(parsed, dict)
class TestFrames(CustomTestCase):
def test_logprob_tuple_shape(self):
"""[logprob, token_id, text|null]; the logprob slot is nullable."""
e = LogprobEntry(logprob=-0.25, token_id=7, text=None)
self.assertEqual(compact(e.to_json_value()), "[-0.25,7,null]")
parsed = LogprobEntry.from_json_value([-0.5, 9, "x"])
self.assertEqual(parsed.token_id, 9)
self.assertEqual(parsed.text, "x")
first_prefill = LogprobEntry.from_json_value([None, 10, None])
self.assertIsNone(first_prefill.logprob)
self.assertEqual(compact(first_prefill.to_json_value()), "[null,10,null]")
with self.assertRaises(JsonContractError) as ctx:
LogprobEntry.from_json_value([1.0, 2])
self.assertEqual(
str(ctx.exception), "invalid length 2, expected a 3-element tuple"
)
def test_generate_response_matches_frame_value_shape(self):
"""Key order (text, meta_info, output_ids), conditional keys, and
finish_reason always present (null pre-terminal)."""
frame = GenerateResponse(
text="ok",
output_ids=[7, 8],
meta_info=GenerateMetaInfo(
id="client-rid",
prompt_tokens=5,
completion_tokens=2,
finish_reason=None,
),
index=None,
)
self.assertEqual(
compact(frame.to_json_value()),
'{"text":"ok","meta_info":{"id":"client-rid","prompt_tokens":5,"completion_tokens":2,"finish_reason":null},"output_ids":[7,8]}',
)
def test_frame_round_trip_with_logprobs(self):
wire = {
"text": "ok",
"meta_info": {
"id": "r",
"prompt_tokens": 1,
"completion_tokens": 1,
"finish_reason": {"type": "stop", "matched": 2},
"output_token_logprobs": [[-0.1, 5, "x"]],
"output_top_logprobs": [None, [[-0.2, 6, None]]],
"hidden_states": [[0.5, 0.25]],
"e2e_latency": 0.01,
},
"output_ids": [5],
"index": 0,
}
frame = GenerateResponse.from_json_value(wire)
self.assertIsInstance(frame.meta_info.finish_reason, FinishStop)
self.assertEqual(frame.meta_info.output_top_logprobs[0], None)
self.assertEqual(frame.meta_info.output_top_logprobs[1][0].token_id, 6)
self.assertEqual(frame.to_json_value(), wire)
if __name__ == "__main__":
unittest.main()
+250
View File
@@ -0,0 +1,250 @@
"""Parity of the three `/generate` vocabularies: the hand-written Python
request structs (`io_struct.GenerateReqInput`, `SamplingParams`), the
generated Python types (`sglang.api.v1.api_types`), and the generated Rust
types (`rust/sglang-api-types`).
The cross-language half runs through `rust/sglang-api-types/testdata/
json_parity.json`: both generators must turn each fixture input into the same
canonical bytes (`tests/json_parity.rs` checks the Rust side). The Python half
pins the field sets against the hand-written structs and proves a schema
request still normalizes through the dataclass the Python server runs.
"""
import dataclasses
import json
import unittest
from pathlib import Path
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel
maybe_stub_sgl_kernel() # must precede any import that pulls in sgl_kernel
from sglang.api.v1 import api_types
from sglang.api.v1.api_types import (
GenerateMetaInfo,
GenerateRequest,
GenerateResponse,
SamplingParams,
)
from sglang.srt.managers.io_struct import GenerateReqInput
from sglang.srt.sampling.sampling_params import SamplingParams as PySamplingParams
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
REPO_ROOT = Path(__file__).resolve().parents[5]
FIXTURES = REPO_ROOT / "rust" / "sglang-api-types" / "testdata" / "json_parity.json"
# `GenerateReqInput` fields with no schema counterpart. Each is either
# Engine-only (tensors, callbacks), set by the server after admission, or a
# feature the HTTP contract has not adopted. A new dataclass field must be
# added to the proto or listed here, so a client-facing field cannot slip
# into the Python server without the Rust server and the contract seeing it.
PYTHON_ONLY_REQUEST_FIELDS = frozenset(
{
"background",
"cache_salt",
"conversation_id",
"custom_labels",
"custom_logit_processor",
"data_parallel_rank",
"encoder_urls",
"external_trace_header",
"extra_key",
"http_worker_ipc",
"image_max_dynamic_patch",
"images_config",
"input_embeds",
"kv_hints",
"log_metrics",
"lora_id",
"lora_path",
"max_dynamic_patch",
"max_thinking_tokens",
"min_dynamic_patch",
"mm_content_hashes",
"modalities",
"multi_item_delimiter_indices",
"need_wait_for_mm_inputs",
"no_logs",
"num_items_assigned",
"positional_embed_overrides",
"priority",
"received_time",
"require_reasoning",
"return_bytes",
"return_entropy",
"return_flat_raw_top_logprobs",
"return_flat_raw_top_logprobs_b64",
"return_indexer_topk",
"return_prompt_token_ids",
"return_routed_experts",
"return_sampling_mask",
"routed_experts_start_len",
"routing_key",
"sampling_logprobs_mode",
"session_id",
"session_params",
"token_indices_to_pool",
"use_audio_in_video",
"video_config",
"video_max_dynamic_patch",
}
)
# `SamplingParams` members that `normalize()` derives; never on the wire.
DERIVED_SAMPLING_FIELDS = frozenset(
{
"stop_strs",
"stop_regex_strs",
"stop_str_max_len",
"stop_regex_max_len",
"is_normalized",
"ebnf_full_assistant",
}
)
def compact(value) -> str:
"""serde_json::to_string parity: no whitespace."""
return json.dumps(value, separators=(",", ":"))
def load_fixtures():
with open(FIXTURES) as f:
return json.load(f)["cases"]
class TestCrossLanguageFixtures(CustomTestCase):
def test_fixtures_decode_to_their_canonical_bytes(self):
"""The Python emitter agrees with the Rust emitter on every fixture:
same keys, same order, same presence and null rules, and the
canonical form is a fixed point."""
cases = load_fixtures()
self.assertTrue(cases)
for case in cases:
with self.subTest(case=f"{case['type']}: {case['name']}"):
cls = getattr(api_types, case["type"])
decoded = cls.from_json_value(case["input"])
self.assertEqual(compact(decoded.to_json_value()), case["canonical"])
again = cls.from_json_value(json.loads(case["canonical"]))
self.assertEqual(compact(again.to_json_value()), case["canonical"])
def test_every_fixture_type_is_a_generated_struct(self):
"""A fixture naming a type the Python module lacks would be skipped
silently by a lookup-by-name; make it a failure."""
for case in load_fixtures():
self.assertTrue(hasattr(api_types, case["type"]), case["type"])
class TestRequestFieldParity(CustomTestCase):
def test_schema_request_fields_are_a_subset_of_the_dataclass(self):
"""Every schema field is a `GenerateReqInput` field, so a schema body
constructs the dataclass as-is, and the dataclass-only fields are
exactly the pinned list."""
schema = set(GenerateRequest.__struct_fields__)
dataclass_fields = {f.name for f in dataclasses.fields(GenerateReqInput)}
self.assertEqual(schema - dataclass_fields, set())
self.assertEqual(dataclass_fields - schema, PYTHON_ONLY_REQUEST_FIELDS)
def test_schema_sampling_fields_match_the_python_api(self):
"""The schema's SamplingParams is the Python struct minus the fields
`normalize()` derives, in the same order."""
python_api = [
name
for name in PySamplingParams.__struct_fields__
if name not in DERIVED_SAMPLING_FIELDS
]
self.assertEqual(set(python_api), set(SamplingParams.__struct_fields__))
self.assertEqual(
set(PySamplingParams.__struct_fields__) - set(python_api),
DERIVED_SAMPLING_FIELDS,
)
def test_request_fixtures_normalize_through_the_dataclass(self):
"""A schema-decoded body, re-encoded, is a valid `GenerateReqInput`
and survives `normalize_batch_and_arguments` with its prompt, batch
shape, stream flag, and sampling values intact."""
for case in load_fixtures():
if case["type"] != "GenerateRequest":
continue
with self.subTest(case=case["name"]):
req = GenerateRequest.from_json_value(case["input"])
wire = req.to_json_value()
dataclass_req = GenerateReqInput(**wire)
dataclass_req.normalize_batch_and_arguments()
self.assertEqual(dataclass_req.stream, wire["stream"])
if "text" in wire:
self.assertEqual(dataclass_req.text, wire["text"])
batch = isinstance(wire["text"], list)
else:
self.assertEqual(dataclass_req.input_ids, wire["input_ids"])
batch = isinstance(wire["input_ids"][0], list)
self.assertEqual(dataclass_req.is_single, not batch)
if "sampling_params" in wire:
sent = wire["sampling_params"]
got = dataclass_req.sampling_params
if isinstance(sent, list):
self.assertEqual(len(got), len(sent))
for sent_item, got_item in zip(sent, got):
self.assertEqual({**got_item, **sent_item}, got_item)
else:
self.assertEqual({**got, **sent}, got)
if "rid" in wire:
self.assertEqual(dataclass_req.rid, wire["rid"])
class TestResponseParity(CustomTestCase):
def test_python_server_meta_info_decodes_to_the_schema_subset(self):
"""The Python server's `meta_info` carries keys the schema does not
model (weight_version, num_retractions, spec stats, ...). The
generated types ignore them and re-emit the schema subset, so a
Rust-server client and a Python-server client see one shape."""
python_frame = {
"text": "ok",
"meta_info": {
"id": "rid",
"finish_reason": {"type": "stop", "matched": 2},
"prompt_tokens": 3,
"weight_version": "default",
"num_retractions": 0,
"completion_tokens": 2,
"cached_tokens": 1,
"spec_accept_rate": 0.5,
"e2e_latency": 0.25,
},
}
frame = GenerateResponse.from_json_value(python_frame)
back = frame.to_json_value()
self.assertEqual(
list(back["meta_info"]),
[
"id",
"prompt_tokens",
"completion_tokens",
"finish_reason",
"e2e_latency",
],
)
self.assertEqual(
back["meta_info"]["finish_reason"], {"type": "stop", "matched": 2}
)
self.assertNotIn("output_ids", back)
def test_response_key_order_is_the_proto_field_order(self):
"""The Rust HTTP server's memoized cumulative frame writes these keys
by hand in this order (`Accumulated::frame_json`); the generated
serializers must keep matching it."""
self.assertEqual(
list(GenerateResponse.__struct_fields__),
["text", "meta_info", "output_ids", "index"],
)
self.assertEqual(
list(GenerateMetaInfo.__struct_fields__)[:4],
["id", "prompt_tokens", "completion_tokens", "finish_reason"],
)
self.assertEqual(list(GenerateMetaInfo.__struct_fields__)[-1], "e2e_latency")
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,73 @@
"""The `/generate` schema gate (`entrypoints/api_contract.py`): the typed media
contract that replaced the raw_json passthrough, checked without a server."""
import unittest
from sglang.srt.entrypoints.api_contract import generate_contract_error
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=3, suite="base-a-test-cpu")
class TestGenerateContract(CustomTestCase):
def test_every_media_container_form_is_accepted(self):
for image_data in (
"u",
{"url": "u", "detail": "high"},
["u", None, {"url": "v"}],
[],
[["a", "b"], None, []],
):
with self.subTest(image_data=image_data):
self.assertIsNone(
generate_contract_error({"text": "hi", "image_data": image_data})
)
self.assertIsNone(
generate_contract_error({"text": "hi", "mm_hashes": ["a1b2", "0xff"]})
)
self.assertIsNone(
generate_contract_error(
{
"text": "hi",
"video_data": {"url": "v", "fps": 2.0, "use_audio": True},
}
)
)
def test_unknown_media_hint_is_rejected(self):
"""A hint the schema does not name used to be silently dropped."""
error = generate_contract_error(
{
"text": "hi",
"image_data": {"url": "u", "max_dynamic_patc": 4},
} # codespell:ignore patc
)
self.assertIsNotNone(error)
self.assertIn("unknown field", error)
def test_preprocessed_inputs_are_not_an_http_shape(self):
"""processor_output / precomputed_embedding carry tensors: Engine-only."""
error = generate_contract_error(
{"input_ids": [1, 2], "image_data": [{"format": "processor_output"}]}
)
self.assertIsNotNone(error)
def test_mixed_list_shapes_are_rejected(self):
error = generate_contract_error({"text": "hi", "image_data": ["a", ["b"]]})
self.assertIsNotNone(error)
def test_sampling_params_follow_the_schema(self):
self.assertIsNone(
generate_contract_error(
{"text": "hi", "sampling_params": {"beam_width": 2, "n": 2}}
)
)
bad = {"temperatur": 1} # codespell:ignore temperatur
error = generate_contract_error({"text": "hi", "sampling_params": bad})
self.assertIsNotNone(error)
self.assertIn("unknown field", error)
if __name__ == "__main__":
unittest.main()
@@ -484,7 +484,8 @@ class TestSamplingParamsMsgspecStruct(CustomTestCase):
)
source = rust_path.read_text()
start = source.index("pub struct SamplingParams {")
end = source.index("\n}\n\n/// The `/generate`", start)
# The struct's own closing brace: the first one back at column 0.
end = source.index("\n}\n", start)
rust_fields = tuple(
re.findall(
r"^\s*pub ([a-z][a-z0-9_]*):",