The sgalng api specs (#41585)
This commit is contained in:
+1
-1
@@ -1,3 +1,3 @@
|
||||
[codespell]
|
||||
ignore-words-list = ans,als,hel,boostrap,childs,te,vas,hsa,ment,cann,thi,makro,wil,rouge,PRIS,ather,MIS,medias,allready,inout,nd,fo,visibles,nothink,renderD,ond,tbe,CopyIn,notin,subtile,subtiles,dout,IST,kInf,datas,abl,valu
|
||||
ignore-words-list = ans,als,hel,boostrap,childs,te,vas,hsa,ment,cann,thi,makro,wil,rouge,PRIS,ather,MIS,medias,allready,inout,nd,fo,visibles,nothink,renderD,ond,tbe,CopyIn,notin,subtile,subtiles,dout,IST,kInf,datas,abl,valu,ser
|
||||
skip = *.json, *.jsonl, *.patch, *.txt, *.lock
|
||||
|
||||
@@ -29,7 +29,7 @@ repos:
|
||||
rev: 7.0.0
|
||||
hooks:
|
||||
- id: isort
|
||||
exclude: '^python/sglang/srt/grpc/.*_pb2\.py$|^python/sglang/srt/grpc/.*_pb2_grpc\.py$|^python/sglang/srt/grpc/.*_pb2\.pyi$|^python/sglang/srt/grpc/.*_pb2_grpc\.pyi$'
|
||||
exclude: '^python/sglang/srt/grpc/.*_pb2\.py$|^python/sglang/srt/grpc/.*_pb2_grpc\.py$|^python/sglang/srt/grpc/.*_pb2\.pyi$|^python/sglang/srt/grpc/.*_pb2_grpc\.pyi$|^python/sglang/(api|json)/v1/.*\.py$'
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.15.1
|
||||
hooks:
|
||||
@@ -47,9 +47,10 @@ repos:
|
||||
python/sglang/srt/grpc/.*_pb2_grpc\.py$|
|
||||
python/sglang/srt/grpc/.*_pb2\.pyi$|
|
||||
python/sglang/srt/grpc/.*_pb2_grpc\.pyi$|
|
||||
python/sglang/(api|json)/v1/.*\.py$|
|
||||
)$
|
||||
- id: ruff-format
|
||||
exclude: '^python/sglang/srt/grpc/.*_pb2\.py$|^python/sglang/srt/grpc/.*_pb2_grpc\.py$|^python/sglang/srt/grpc/.*_pb2\.pyi$|^python/sglang/srt/grpc/.*_pb2_grpc\.pyi$'
|
||||
exclude: '^python/sglang/srt/grpc/.*_pb2\.py$|^python/sglang/srt/grpc/.*_pb2_grpc\.py$|^python/sglang/srt/grpc/.*_pb2\.pyi$|^python/sglang/srt/grpc/.*_pb2_grpc\.pyi$|^python/sglang/(api|json)/v1/.*\.py$'
|
||||
- repo: https://github.com/codespell-project/codespell
|
||||
rev: v2.4.1
|
||||
hooks:
|
||||
|
||||
+3
-3
@@ -202,13 +202,13 @@ RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
# --no-deps prevents uv from pulling torch from default PyPI
|
||||
&& uv pip install sglang-kernel==${SGL_KERNEL_VERSION} --reinstall --no-deps
|
||||
|
||||
# Copy the dependency spec, Rust extension sources, and shared protocol crate.
|
||||
# Split from the dependency install so changes to these build inputs invalidate the
|
||||
# Copy the dependency spec, Rust extension sources, and shared API types.
|
||||
# Split from the pip install so changes to these build inputs invalidate the
|
||||
# dependency layer, while unrelated Python source changes do not.
|
||||
COPY python/pyproject.toml /tmp/sglang_deps/python/pyproject.toml
|
||||
COPY rust/sglang-grpc /tmp/sglang_deps/rust/sglang-grpc
|
||||
COPY rust/sglang-mm /tmp/sglang_deps/rust/sglang-mm
|
||||
COPY proto /tmp/sglang_deps/proto
|
||||
COPY rust/sglang-api-types /tmp/sglang_deps/rust/sglang-api-types
|
||||
|
||||
# Install sglang dependencies (torch, transformers, etc.).
|
||||
# Generate constraints.txt to prevent reinstalling these deps in later stages.
|
||||
|
||||
@@ -422,13 +422,13 @@ RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
; \
|
||||
fi
|
||||
|
||||
# Copy the dependency spec, Rust extension sources, and shared protocol crate.
|
||||
# Copy the dependency spec, Rust extension sources, and shared API types.
|
||||
# Split from the pip install so changes to these build inputs invalidate the
|
||||
# dependency layer, while unrelated Python source changes do not.
|
||||
COPY python/pyproject.toml /tmp/sglang_deps/python/pyproject.toml
|
||||
COPY rust/sglang-grpc /tmp/sglang_deps/rust/sglang-grpc
|
||||
COPY rust/sglang-mm /tmp/sglang_deps/rust/sglang-mm
|
||||
COPY proto /tmp/sglang_deps/proto
|
||||
COPY rust/sglang-api-types /tmp/sglang_deps/rust/sglang-api-types
|
||||
|
||||
# Install sglang dependencies (torch, transformers, etc.).
|
||||
# Generate constraints.txt to prevent reinstalling these deps in later stages.
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
[package]
|
||||
name = "sglang-grpc-types"
|
||||
description = "Generated Rust bindings for the SGLang runtime.v1 API"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
rust-version = "1.92"
|
||||
license.workspace = true
|
||||
repository = "https://github.com/sgl-project/sglang"
|
||||
readme = "README.md"
|
||||
workspace = "../rust"
|
||||
include = [
|
||||
"build.rs",
|
||||
"README.md",
|
||||
"src/**",
|
||||
"sglang/**/*.proto",
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
prost = { workspace = true }
|
||||
prost-types = { workspace = true }
|
||||
tonic = { workspace = true }
|
||||
|
||||
[build-dependencies]
|
||||
prost-build = { workspace = true }
|
||||
protoc-bin-vendored = { workspace = true }
|
||||
tonic-build = { workspace = true }
|
||||
@@ -1,15 +0,0 @@
|
||||
# SGLang gRPC types
|
||||
|
||||
This crate provides the generated Prost messages and Tonic client/server
|
||||
interfaces for SGLang's canonical `sglang.runtime.v1` API.
|
||||
|
||||
The language-independent contract remains in
|
||||
`sglang/runtime/v1/sglang.proto`. Rust bindings are generated from that schema
|
||||
at build time with a vendored `protoc`, so consumers do not need a system
|
||||
protobuf compiler.
|
||||
|
||||
The generated API currently uses Tonic 0.12 to remain compatible with
|
||||
SGLang's existing gRPC server.
|
||||
|
||||
This crate contains protocol types only. SGLang's request handling and server
|
||||
implementation remain in their respective frontend crates.
|
||||
@@ -0,0 +1,134 @@
|
||||
// Shared carriers for the native SGLang API. The ground truth for the JSON
|
||||
// shapes is the sglang.json.v1 options on each field; the protobuf shapes are
|
||||
// the explicit oneofs/messages themselves.
|
||||
|
||||
syntax = "proto3";
|
||||
|
||||
package sglang.api.v1;
|
||||
|
||||
import "sglang/json/v1/options.proto";
|
||||
|
||||
// A token-id sequence. int64 matches the servers' token id width (Rust
|
||||
// `TokenIds = Vec<i64>`, Python `array("q")`).
|
||||
message TokenIds {
|
||||
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST;
|
||||
repeated int64 ids = 1;
|
||||
}
|
||||
|
||||
// Scalar-or-list carriers: `/generate` accepts every per-request field as one
|
||||
// value (single request) or a list (batch). JSON flattens via the one_or_many
|
||||
// option; gRPC clients pick the variant explicitly.
|
||||
message StringOrList {
|
||||
oneof value {
|
||||
option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_ONE_OR_MANY;
|
||||
string one = 1;
|
||||
StringList many = 2;
|
||||
}
|
||||
}
|
||||
|
||||
message StringList {
|
||||
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST;
|
||||
repeated string items = 1;
|
||||
}
|
||||
|
||||
// A flat string list for one request, or one list per batch entry. JSON:
|
||||
// `["a", "b"]` is `one`; `[["a"], ["b", "c"]]` is `many` (first element
|
||||
// decides, `[]` is an empty `one`).
|
||||
message StringListOrList {
|
||||
oneof value {
|
||||
option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_ONE_OR_MANY;
|
||||
StringList one = 1;
|
||||
StringListList many = 2;
|
||||
}
|
||||
}
|
||||
|
||||
message StringListList {
|
||||
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST;
|
||||
repeated StringList items = 1;
|
||||
}
|
||||
|
||||
message BoolOrList {
|
||||
oneof value {
|
||||
option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_ONE_OR_MANY;
|
||||
bool one = 1;
|
||||
BoolList many = 2;
|
||||
}
|
||||
}
|
||||
|
||||
message BoolList {
|
||||
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST;
|
||||
repeated bool items = 1;
|
||||
}
|
||||
|
||||
message Int64OrList {
|
||||
oneof value {
|
||||
option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_ONE_OR_MANY;
|
||||
int64 one = 1;
|
||||
Int64List many = 2;
|
||||
}
|
||||
}
|
||||
|
||||
message Int64List {
|
||||
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST;
|
||||
repeated int64 items = 1;
|
||||
}
|
||||
|
||||
message TokenIdsOrList {
|
||||
oneof value {
|
||||
option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_ONE_OR_MANY;
|
||||
TokenIds one = 1;
|
||||
TokenIdsList many = 2;
|
||||
}
|
||||
}
|
||||
|
||||
message TokenIdsList {
|
||||
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST;
|
||||
repeated TokenIds items = 1;
|
||||
}
|
||||
|
||||
// List variants whose ELEMENTS may be JSON null (PD bootstrap columns: a
|
||||
// batch mixes bootstrapped and plain requests). Proto carries element
|
||||
// presence as a per-element oneof-free wrapper.
|
||||
message OptionalString {
|
||||
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_TRANSPARENT;
|
||||
optional string value = 1;
|
||||
}
|
||||
|
||||
message OptionalStringOrList {
|
||||
oneof value {
|
||||
option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_ONE_OR_MANY;
|
||||
OptionalString one = 1;
|
||||
OptionalStringList many = 2;
|
||||
}
|
||||
}
|
||||
|
||||
message OptionalStringList {
|
||||
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST;
|
||||
repeated OptionalString items = 1;
|
||||
}
|
||||
|
||||
message OptionalInt64 {
|
||||
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_TRANSPARENT;
|
||||
optional int64 value = 1;
|
||||
}
|
||||
|
||||
message OptionalInt64OrList {
|
||||
oneof value {
|
||||
option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_ONE_OR_MANY;
|
||||
OptionalInt64 one = 1;
|
||||
OptionalInt64List many = 2;
|
||||
}
|
||||
}
|
||||
|
||||
message OptionalInt64List {
|
||||
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST;
|
||||
repeated OptionalInt64 items = 1;
|
||||
}
|
||||
|
||||
// The native error body: `{"error": {"message", "code"}}` (Python
|
||||
// `http_server.generate_request` parity). Transports frame it themselves
|
||||
// (unary status + body, SSE error frame, gRPC status detail).
|
||||
message ErrorBody {
|
||||
string message = 1;
|
||||
uint32 code = 2;
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
// The terminal finish reason — Python's `FinishReasonDict`
|
||||
// (`BaseFinishReason.to_json()` in schedule_batch.py), echoed verbatim as
|
||||
// `meta_info.finish_reason`. JSON is internally tagged by "type" with the
|
||||
// variant payload inlined; a type this build doesn't know round-trips
|
||||
// unchanged (unknown_variant_passthrough) so a Python-side addition never
|
||||
// fails a frame decode.
|
||||
|
||||
syntax = "proto3";
|
||||
|
||||
package sglang.api.v1;
|
||||
|
||||
import "sglang/json/v1/options.proto";
|
||||
|
||||
// Python's `matched` key, `Union[str, int, List[int]]`: the three wire shapes
|
||||
// are disjoint, so shape alone picks the arm (untagged).
|
||||
message Matched {
|
||||
oneof value {
|
||||
option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_UNTAGGED;
|
||||
int64 token = 1;
|
||||
string str = 2;
|
||||
MatchedTokens tokens = 3;
|
||||
}
|
||||
}
|
||||
|
||||
message MatchedTokens {
|
||||
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST;
|
||||
repeated int64 ids = 1;
|
||||
}
|
||||
|
||||
// `FINISH_MATCHED_TOKEN` / `FINISH_MATCHED_STR` / `FINISHED_MATCHED_REGEX` —
|
||||
// all three report `type: "stop"`, matching the OpenAI API's value.
|
||||
message FinishStop {
|
||||
optional Matched matched = 1;
|
||||
}
|
||||
|
||||
// `FINISH_LENGTH` — hit `max_new_tokens` (or the context limit).
|
||||
message FinishLength {
|
||||
optional uint64 length = 1;
|
||||
}
|
||||
|
||||
// The `FINISH_ABORT` payload. Python emits all three keys, nulls included.
|
||||
message FinishAbort {
|
||||
optional string message = 1 [(sglang.json.v1.field_json).emit_null_when_absent = true];
|
||||
optional uint32 status_code = 2 [(sglang.json.v1.field_json).emit_null_when_absent = true];
|
||||
optional string err_type = 3 [(sglang.json.v1.field_json).emit_null_when_absent = true];
|
||||
}
|
||||
|
||||
message FinishReason {
|
||||
oneof kind {
|
||||
option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_TAGGED;
|
||||
option (sglang.json.v1.oneof_json).tag = "type";
|
||||
option (sglang.json.v1.oneof_json).unknown_variant_passthrough = true;
|
||||
FinishStop stop = 1;
|
||||
FinishLength length = 2;
|
||||
FinishAbort abort = 3;
|
||||
// The passthrough arm: an unrecognized `type`, kept as its raw map
|
||||
// (JSON text — Struct would coerce integer values to doubles).
|
||||
string unknown = 4 [(sglang.json.v1.field_json).raw_json = true];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
// The native `/generate` surface — the ground truth for
|
||||
// rust/sglang-server/src/message/request.rs's GenerateBody (request) and
|
||||
// api_core/frame.rs's frame shape (response). Unknown JSON keys on the
|
||||
// request are IGNORED: `deny_unknown_fields` broke real clients that send
|
||||
// Python-only `GenerateReqInput` fields (priority, extra_key, session_id,
|
||||
// ...), pinned by request.rs's unported-fields test.
|
||||
|
||||
syntax = "proto3";
|
||||
|
||||
package sglang.api.v1;
|
||||
|
||||
import "sglang/api/v1/common.proto";
|
||||
import "sglang/api/v1/finish.proto";
|
||||
import "sglang/api/v1/sampling.proto";
|
||||
import "sglang/json/v1/options.proto";
|
||||
|
||||
message GenerateRequest {
|
||||
// (message_json).unknown_fields defaults to IGNORE — stated here as schema
|
||||
// intent, load-bearing for client compat.
|
||||
|
||||
StringOrList rid = 1;
|
||||
StringOrList text = 2;
|
||||
TokenIdsOrList input_ids = 3;
|
||||
optional bool stream = 4 [
|
||||
(sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT,
|
||||
(sglang.json.v1.field_json).default_bool = false
|
||||
];
|
||||
// One set of params, or one per batch item. JSON errors keep serde's
|
||||
// field-level texts (the hand-written SamplingParamsInput contract).
|
||||
SamplingParamsOrList sampling_params = 5;
|
||||
BoolOrList return_logprob = 6;
|
||||
Int64OrList logprob_start_len = 7;
|
||||
Int64OrList top_logprobs_num = 8;
|
||||
TokenIdsOrList token_ids_logprob = 9;
|
||||
BoolOrList return_hidden_states = 10;
|
||||
optional bool return_text_in_logprobs = 11;
|
||||
// PD bootstrap columns: a batch mixes bootstrapped and plain requests, so
|
||||
// elements are individually nullable.
|
||||
OptionalStringOrList bootstrap_host = 12;
|
||||
OptionalInt64OrList bootstrap_port = 13;
|
||||
OptionalInt64OrList bootstrap_room = 14;
|
||||
OptionalStringOrList bootstrap_pair_key = 15;
|
||||
OptionalInt64OrList decode_tp_size = 16;
|
||||
optional int64 routed_dp_rank = 17;
|
||||
optional int64 disagg_prefill_dp_rank = 18;
|
||||
// Multimodal payloads pass through untyped (data URIs, URLs, file paths, or
|
||||
// nested batch lists) to the MM pipeline, which owns their validation.
|
||||
// Media inputs, one container per modality (see MediaInput). The item
|
||||
// hints a modality does not support are rejected at the processor.
|
||||
optional MediaInput image_data = 19;
|
||||
// Per-item content hashes overriding the computed ones, in item order; a
|
||||
// flat list for one request (the batch form is rejected downstream).
|
||||
optional StringListOrList mm_hashes = 20;
|
||||
optional MediaInput video_data = 21;
|
||||
optional MediaInput audio_data = 22;
|
||||
}
|
||||
|
||||
message SamplingParamsOrList {
|
||||
oneof value {
|
||||
option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_ONE_OR_MANY;
|
||||
SamplingParams one = 1;
|
||||
SamplingParamsList many = 2;
|
||||
}
|
||||
}
|
||||
|
||||
message SamplingParamsList {
|
||||
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST;
|
||||
repeated SamplingParams items = 1;
|
||||
}
|
||||
|
||||
// One `[logprob, token_id, text|null]` entry (text present when
|
||||
// `return_text_in_logprobs`, decoded on the detok shard). The logprob slot is
|
||||
// optional: Python emits null for a position with no logprob (the first
|
||||
// prefill token; Rust maps the scheduler's NaN to the same null).
|
||||
message LogprobEntry {
|
||||
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_TUPLE;
|
||||
optional double logprob = 1;
|
||||
int64 token_id = 2;
|
||||
optional string text = 3;
|
||||
}
|
||||
|
||||
message LogprobEntries {
|
||||
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST;
|
||||
repeated LogprobEntry entries = 1;
|
||||
}
|
||||
|
||||
// One decode position's top-k alternatives.
|
||||
message TopLogprobRow {
|
||||
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST;
|
||||
repeated LogprobEntry entries = 1;
|
||||
}
|
||||
|
||||
// A position's row, or null: a position that recorded no alternatives is
|
||||
// null on the wire (Python parity), never [].
|
||||
message NullableTopLogprobs {
|
||||
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_TRANSPARENT;
|
||||
TopLogprobRow row = 1;
|
||||
}
|
||||
|
||||
message HiddenStateRow {
|
||||
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST;
|
||||
repeated double values = 1;
|
||||
}
|
||||
|
||||
message GenerateMetaInfo {
|
||||
// The client's rid, byte-for-byte (`Rid::client_facing`).
|
||||
string id = 1;
|
||||
uint32 prompt_tokens = 2;
|
||||
uint64 completion_tokens = 3;
|
||||
// Always present in JSON, null until a terminal frame.
|
||||
FinishReason finish_reason = 4 [(sglang.json.v1.field_json).emit_null_when_absent = true];
|
||||
// input+output token logprobs are set together, empty lists included: a PD
|
||||
// decode node never receives input logprobs, but the PD router keys its
|
||||
// merge of prefill's on the key's presence.
|
||||
LogprobEntries output_token_logprobs = 5;
|
||||
LogprobEntries input_token_logprobs = 6;
|
||||
repeated NullableTopLogprobs output_top_logprobs = 7;
|
||||
repeated NullableTopLogprobs input_top_logprobs = 8;
|
||||
repeated NullableTopLogprobs output_token_ids_logprobs = 9;
|
||||
repeated NullableTopLogprobs input_token_ids_logprobs = 10;
|
||||
repeated HiddenStateRow hidden_states = 11;
|
||||
// Seconds from normalized-request acceptance to the terminal output; only
|
||||
// on the terminal frame.
|
||||
optional double e2e_latency = 12;
|
||||
}
|
||||
|
||||
message GenerateResponse {
|
||||
// JSON key order is FIELD-NUMBER order: text, meta_info, then the
|
||||
// conditionally-present output_ids and index — matching frame_value's
|
||||
// insertion order byte-for-byte.
|
||||
string text = 1;
|
||||
GenerateMetaInfo meta_info = 2;
|
||||
// Omitted while empty (a text-only frame).
|
||||
TokenIds output_ids = 3;
|
||||
// Batch position; only on batch responses/frames.
|
||||
optional uint32 index = 4;
|
||||
}
|
||||
|
||||
// One request's failure inside a Generate stream; carries its batch position
|
||||
// (the HTTP SSE shape puts `index` beside the error object the same way).
|
||||
message GenerateStreamError {
|
||||
ErrorBody error = 1;
|
||||
optional uint32 index = 2;
|
||||
}
|
||||
|
||||
// One item of the gRPC Generate stream: a frame, or one request's failure
|
||||
// (the stream continues — other batch items may still be running). On HTTP
|
||||
// the same failure is an in-stream SSE error frame.
|
||||
message GenerateStreamItem {
|
||||
oneof item {
|
||||
GenerateResponse frame = 1;
|
||||
GenerateStreamError error = 2;
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Media inputs ----------------------------------------------------------
|
||||
//
|
||||
// Every form Python's `/generate` accepts, typed. Preprocessed inputs
|
||||
// (`format: "processor_output"` / `"precomputed_embedding"`) are NOT part of
|
||||
// the HTTP contract: their values are tensors, which JSON cannot carry; they
|
||||
// exist for the in-process Engine only.
|
||||
|
||||
// One media item's source and per-item hints. Unknown keys are an error: a
|
||||
// hint this schema does not name would otherwise be silently dropped.
|
||||
message MediaRef {
|
||||
option (sglang.json.v1.message_json).unknown_fields = UNKNOWN_FIELDS_DENY;
|
||||
// URL, `file://` / absolute path, `data:` URI, or bare base64.
|
||||
string url = 1;
|
||||
// Image: "auto" | "low" | "high" (the OpenAI detail hint).
|
||||
optional string detail = 2;
|
||||
// Image/video: dynamic-patch budget (InternVL-style processors).
|
||||
optional int64 max_dynamic_patch = 3;
|
||||
// Image: a caller-supplied content hash that overrides the computed one.
|
||||
optional string content_hash = 4;
|
||||
// Video/image budgets read by the GLM and MiMo processors.
|
||||
optional int64 min_pixels = 5;
|
||||
optional int64 max_pixels = 6;
|
||||
optional int64 total_max_pixels = 7;
|
||||
optional double fps = 8;
|
||||
optional int64 num_frames = 9;
|
||||
optional int64 min_frames = 10;
|
||||
optional int64 max_frames = 11;
|
||||
optional int64 max_tokens_per_frame = 12;
|
||||
optional int64 max_image_tokens = 13;
|
||||
// Video: also decode the audio track.
|
||||
optional bool use_audio = 14;
|
||||
}
|
||||
|
||||
// One media item. JSON: a bare string is the source; an object is a MediaRef.
|
||||
message MediaItem {
|
||||
oneof value {
|
||||
option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_UNTAGGED;
|
||||
string source = 1;
|
||||
MediaRef ref = 2;
|
||||
}
|
||||
}
|
||||
|
||||
// A list element that may be JSON null (Python's `[img, None]`).
|
||||
message OptionalMediaItem {
|
||||
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_TRANSPARENT;
|
||||
optional MediaItem item = 1;
|
||||
}
|
||||
|
||||
// One request's items (`[a, null, b]`), or one batch entry's items.
|
||||
message MediaItems {
|
||||
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST;
|
||||
repeated OptionalMediaItem items = 1;
|
||||
}
|
||||
|
||||
// A batch entry whose item list may be JSON null.
|
||||
message OptionalMediaItems {
|
||||
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_TRANSPARENT;
|
||||
optional MediaItems items = 1;
|
||||
}
|
||||
|
||||
// One item list per batch entry (`[[a, b], null, []]`).
|
||||
message MediaItemsList {
|
||||
option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST;
|
||||
repeated OptionalMediaItems items = 1;
|
||||
}
|
||||
|
||||
// The three container forms Python's `_normalize_image_data` distinguishes.
|
||||
// JSON: a string or object is `item` (one item; a broadcast in a batch); an
|
||||
// array whose first element is not an array is `many` (a flat list; `[]` is
|
||||
// an empty `many`); an array whose first element is an array is `nested`
|
||||
// (one list per batch entry). Which forms a request may use depends on
|
||||
// whether it is a batch, and is checked downstream, not here.
|
||||
message MediaInput {
|
||||
oneof value {
|
||||
option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_UNTAGGED;
|
||||
MediaItem item = 1;
|
||||
MediaItems many = 2;
|
||||
MediaItemsList nested = 3;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
// The API face of SamplingParams — the ground truth for
|
||||
// rust/sglang-server/src/message/sampling.rs's 25 API fields (its 5
|
||||
// normalize-internal fields and the msgpack scheduler encoding are IPC, not
|
||||
// API, and stay hand-written there). Unknown JSON keys are DENIED: parity with
|
||||
// Python's `SamplingParams(**kwargs)` TypeError.
|
||||
|
||||
syntax = "proto3";
|
||||
|
||||
package sglang.api.v1;
|
||||
|
||||
import "sglang/api/v1/common.proto";
|
||||
import "sglang/json/v1/options.proto";
|
||||
|
||||
message SamplingParams {
|
||||
option (sglang.json.v1.message_json).unknown_fields = UNKNOWN_FIELDS_DENY;
|
||||
|
||||
// Defaulted fields are `optional`: proto3 gives plain scalars no presence,
|
||||
// so an unset double would arrive as 0 instead of its schema default (the
|
||||
// top_p=0 bug the first live gRPC call found). JSON deserialization bakes
|
||||
// the default on absent/null; protobuf arrival reads it through the
|
||||
// generated *_or_default accessors.
|
||||
|
||||
// JSON: absent = 128, explicit null = no limit (Python `Optional[int] =
|
||||
// 128`). Protobuf has no null: unset = 128 (apply_absent_defaults at the
|
||||
// gRPC entry); no limit needs an explicit large value.
|
||||
optional int64 max_new_tokens = 1 [
|
||||
(sglang.json.v1.field_json).null_policy = NULL_POLICY_IS_NONE,
|
||||
(sglang.json.v1.field_json).default_i64 = 128
|
||||
];
|
||||
StringOrList stop = 2;
|
||||
Int64List stop_token_ids = 3;
|
||||
StringOrList stop_regex = 4;
|
||||
optional double temperature = 5 [
|
||||
(sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT,
|
||||
(sglang.json.v1.field_json).default_f64 = 1.0
|
||||
];
|
||||
optional double top_p = 6 [
|
||||
(sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT,
|
||||
(sglang.json.v1.field_json).default_f64 = 1.0
|
||||
];
|
||||
// Default 1 << 30 (TOP_K_ALL): "consider the whole vocabulary".
|
||||
optional int64 top_k = 7 [
|
||||
(sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT,
|
||||
(sglang.json.v1.field_json).default_i64 = 1073741824
|
||||
];
|
||||
optional double min_p = 8 [
|
||||
(sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT,
|
||||
(sglang.json.v1.field_json).default_f64 = 0.0
|
||||
];
|
||||
optional double frequency_penalty = 9 [
|
||||
(sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT,
|
||||
(sglang.json.v1.field_json).default_f64 = 0.0
|
||||
];
|
||||
optional double presence_penalty = 10 [
|
||||
(sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT,
|
||||
(sglang.json.v1.field_json).default_f64 = 0.0
|
||||
];
|
||||
optional double repetition_penalty = 11 [
|
||||
(sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT,
|
||||
(sglang.json.v1.field_json).default_f64 = 1.0
|
||||
];
|
||||
optional int64 min_new_tokens = 12 [
|
||||
(sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT,
|
||||
(sglang.json.v1.field_json).default_i64 = 0
|
||||
];
|
||||
optional int64 n = 13 [
|
||||
(sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT,
|
||||
(sglang.json.v1.field_json).default_i64 = 1
|
||||
];
|
||||
optional string json_schema = 14;
|
||||
optional string regex = 15;
|
||||
optional string ebnf = 16;
|
||||
optional string structural_tag = 17;
|
||||
optional bool ignore_eos = 18 [
|
||||
(sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT,
|
||||
(sglang.json.v1.field_json).default_bool = false
|
||||
];
|
||||
optional bool skip_special_tokens = 19 [
|
||||
(sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT,
|
||||
(sglang.json.v1.field_json).default_bool = true
|
||||
];
|
||||
optional bool spaces_between_special_tokens = 20 [
|
||||
(sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT,
|
||||
(sglang.json.v1.field_json).default_bool = true
|
||||
];
|
||||
optional bool no_stop_trim = 21 [
|
||||
(sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT,
|
||||
(sglang.json.v1.field_json).default_bool = false
|
||||
];
|
||||
optional int64 stream_interval = 22;
|
||||
// Token id (numeric-string key, Python parity) -> bias.
|
||||
map<string, double> logit_bias = 23 [(sglang.json.v1.field_json).int_key_map = true];
|
||||
optional int64 sampling_seed = 24;
|
||||
// Opaque JSON forwarded to a custom logit processor; never inspected here.
|
||||
optional string custom_params = 25 [(sglang.json.v1.field_json).raw_json = true];
|
||||
|
||||
// beam_width > 1 turns the request into a beam search request; `n` is then
|
||||
// the number of returned sequences (n <= beam_width).
|
||||
optional int64 beam_width = 26;
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
// The native SGLang gRPC service served by rust/sglang-server's tonic
|
||||
// transport, over the same api_core the HTTP endpoints use. Covers exactly the
|
||||
// surface the native server implements (the legacy sglang.runtime.v1 bridge's
|
||||
// embed/classify/tokenize RPCs have no native backend here and are not
|
||||
// carried over).
|
||||
|
||||
syntax = "proto3";
|
||||
|
||||
package sglang.api.v1;
|
||||
|
||||
import "sglang/api/v1/generate.proto";
|
||||
import "sglang/json/v1/options.proto";
|
||||
|
||||
service SglangService {
|
||||
// `/generate`: always server-streaming (a unary request is a one-frame
|
||||
// stream); per-batch-item failures ride the stream as GenerateStreamItem
|
||||
// errors, pre-submit failures end the RPC with a gRPC status.
|
||||
rpc Generate(GenerateRequest) returns (stream GenerateStreamItem);
|
||||
|
||||
// `/health_generate`: fires a 1-token probe and watches the response
|
||||
// heartbeat. UNAVAILABLE on a stalled pipeline.
|
||||
rpc HealthCheck(HealthCheckRequest) returns (HealthCheckResponse);
|
||||
|
||||
// `/get_model_info`: static model metadata, no scheduler round-trip.
|
||||
rpc GetModelInfo(GetModelInfoRequest) returns (GetModelInfoResponse);
|
||||
|
||||
// `/server_info`: one control round-trip, shaped through the runtime-metric
|
||||
// allowlist (never the raw server-args dump, which embeds keys).
|
||||
rpc GetServerInfo(GetServerInfoRequest) returns (GetServerInfoResponse);
|
||||
}
|
||||
|
||||
message HealthCheckRequest {}
|
||||
|
||||
message HealthCheckResponse {
|
||||
bool healthy = 1;
|
||||
}
|
||||
|
||||
message GetModelInfoRequest {}
|
||||
|
||||
message GetModelInfoResponse {
|
||||
string model_path = 1;
|
||||
string served_model_name = 2;
|
||||
string tokenizer_path = 3;
|
||||
bool is_generation = 4;
|
||||
// The launch-time `--preferred-sampling-params` object, or null when unset.
|
||||
// JSON carries the object itself (the HTTP `/get_model_info` shape); the
|
||||
// protobuf field holds it as JSON text.
|
||||
optional string preferred_sampling_params = 5 [
|
||||
(sglang.json.v1.field_json).raw_json = true,
|
||||
(sglang.json.v1.field_json).emit_null_when_absent = true
|
||||
];
|
||||
optional string weight_version = 6 [(sglang.json.v1.field_json).emit_null_when_absent = true];
|
||||
optional string load_format = 7 [(sglang.json.v1.field_json).emit_null_when_absent = true];
|
||||
optional string reasoning_parser = 8 [(sglang.json.v1.field_json).emit_null_when_absent = true];
|
||||
optional string tool_call_parser = 9 [(sglang.json.v1.field_json).emit_null_when_absent = true];
|
||||
}
|
||||
|
||||
message GetServerInfoRequest {}
|
||||
|
||||
message GetServerInfoResponse {
|
||||
string model_path = 1;
|
||||
string served_model_name = 2;
|
||||
string tokenizer_path = 3;
|
||||
uint32 max_context_length = 4;
|
||||
optional uint64 max_total_num_tokens = 5;
|
||||
string version = 6;
|
||||
// The allowlisted runtime metrics (last_gen_throughput, memory_usage,
|
||||
// effective_max_running_requests_per_dp, avg_spec_accept_length,
|
||||
// step_time_dict), untyped because the scheduler owns their shapes; JSON
|
||||
// passes them through verbatim.
|
||||
repeated string internal_states = 7 [(sglang.json.v1.field_json).raw_json = true];
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
// The JSON-fidelity option vocabulary: how a message, field, or oneof behaves
|
||||
// on the HTTP JSON wire, declared in the schema itself. The sglang-api-codegen
|
||||
// generator reads these options from the FileDescriptorSet and emits the serde
|
||||
// implementations; proto3's canonical JSON mapping is NOT used. Each option is
|
||||
// one generator template with its own unit tests.
|
||||
//
|
||||
// gRPC never sees these options: the protobuf wire format is untouched.
|
||||
|
||||
syntax = "proto3";
|
||||
|
||||
package sglang.json.v1;
|
||||
|
||||
import "google/protobuf/descriptor.proto";
|
||||
|
||||
// How a message treats unknown JSON keys on deserialization.
|
||||
enum UnknownFields {
|
||||
// Silently dropped (the default): parity with endpoints that accept and
|
||||
// ignore unported fields, where rejecting would break existing clients.
|
||||
UNKNOWN_FIELDS_IGNORE = 0;
|
||||
// A 400 with serde's field-level message: parity with Python constructors
|
||||
// that TypeError on unexpected kwargs.
|
||||
UNKNOWN_FIELDS_DENY = 1;
|
||||
}
|
||||
|
||||
// How a message is shaped on the JSON wire. One shape per message: an enum,
|
||||
// not independent flags, so two cannot be set at once.
|
||||
enum MessageShape {
|
||||
// A plain JSON object keyed by field name (the default).
|
||||
MESSAGE_SHAPE_OBJECT = 0;
|
||||
|
||||
// A single-`repeated`-field carrier (TokenIds, StringList, ...): its JSON
|
||||
// is the bare array, not an {items: [...]} object, wherever it appears.
|
||||
MESSAGE_SHAPE_BARE_LIST = 1;
|
||||
|
||||
// A positional tuple: the fields, in declaration order, form a bare array
|
||||
// (e.g. a logprob entry's [logprob, token_id, text|null]). Optional members
|
||||
// serialize as null.
|
||||
MESSAGE_SHAPE_TUPLE = 2;
|
||||
|
||||
// A single-field message that is invisible in JSON: it serializes as its
|
||||
// sole field's value (null when unset). Exists to give proto a presence
|
||||
// wrapper for nullable LIST ELEMENTS (OptionalString, OptionalInt64).
|
||||
MESSAGE_SHAPE_TRANSPARENT = 3;
|
||||
}
|
||||
|
||||
// Attached to a message: `option (sglang.json.v1.message_json).<name> = ...;`
|
||||
// in its body. Decides the JSON form of the message as a whole, for both
|
||||
// directions of the HTTP wire: `shape` says what the serializer writes and
|
||||
// what the deserializer expects (object, bare array, tuple, or the sole
|
||||
// field's value); `unknown_fields` says what the deserializer does with keys
|
||||
// the message does not declare. Serialization never emits unknown keys.
|
||||
message MessageJson {
|
||||
UnknownFields unknown_fields = 1;
|
||||
MessageShape shape = 2;
|
||||
}
|
||||
|
||||
// How an absent key or an explicit JSON null reads into a field. One policy
|
||||
// per field: an enum, not independent flags, so two cannot be set at once.
|
||||
// Deserialization only; `emit_null_when_absent` is the serialization side.
|
||||
enum NullPolicy {
|
||||
// Absent is unset; an explicit null is a type error (the default).
|
||||
NULL_POLICY_DEFAULT = 0;
|
||||
|
||||
// Explicit null (and an absent key) yields the stated default instead of
|
||||
// an error: parity with Python dataclasses whose __post_init__ maps None
|
||||
// back to the default.
|
||||
NULL_POLICY_RESETS_DEFAULT = 1;
|
||||
|
||||
// An ABSENT key yields the stated default, but an explicit null clears the
|
||||
// field to no-value (e.g. max_new_tokens: absent = 128, null = no limit).
|
||||
NULL_POLICY_IS_NONE = 2;
|
||||
}
|
||||
|
||||
// Attached to one field: `type name = N [(sglang.json.v1.field_json).<name>
|
||||
// = ...];`. Decides how that key behaves, with the read and write sides kept
|
||||
// apart. Deserialization: `null_policy` (with a `default_*`) maps an absent
|
||||
// key or an explicit null onto a value. Serialization: `emit_null_when_absent`
|
||||
// writes the key as null instead of omitting it. Both directions: `raw_json`
|
||||
// carries the value through as JSON text, and `int_key_map` spells numeric
|
||||
// map keys as strings.
|
||||
message FieldJson {
|
||||
NullPolicy null_policy = 1;
|
||||
// The default a non-DEFAULT policy yields; exactly one must accompany it.
|
||||
optional double default_f64 = 2;
|
||||
optional int64 default_i64 = 3;
|
||||
optional bool default_bool = 4;
|
||||
optional string default_string = 5;
|
||||
|
||||
// The key is always serialized, an unset value as an explicit null (e.g.
|
||||
// reasoning_content in chat deltas: Python always emits the key).
|
||||
bool emit_null_when_absent = 6;
|
||||
|
||||
// On a string (or repeated string) field: the string holds JSON text, and
|
||||
// HTTP (de)serialization passes the VALUE through — integers, nesting, and
|
||||
// element shapes intact (google.protobuf.Struct cannot do this: it coerces
|
||||
// integers to doubles). gRPC carries the JSON text.
|
||||
bool raw_json = 7;
|
||||
|
||||
// On a map<string, ...> field whose keys are numeric strings (Python
|
||||
// parity, e.g. logit_bias token ids).
|
||||
bool int_key_map = 8;
|
||||
}
|
||||
|
||||
// How a oneof (a union) is spelled on the JSON wire. One shape per union.
|
||||
enum OneofShape {
|
||||
// Unset: the message owning the oneof is a plain object.
|
||||
ONEOF_SHAPE_UNSPECIFIED = 0;
|
||||
|
||||
// Untagged: the value's shape picks the variant, and variant mismatch
|
||||
// errors keep serde's field-level texts (never "data did not match any
|
||||
// variant").
|
||||
ONEOF_SHAPE_UNTAGGED = 1;
|
||||
|
||||
// On a oneof of exactly { T one; TList many; }: a JSON scalar deserializes
|
||||
// as `one`, a JSON array as `many`; serialization flattens the same way.
|
||||
ONEOF_SHAPE_ONE_OR_MANY = 2;
|
||||
|
||||
// Internally tagged by the key in `tag` (e.g. finish_reason's
|
||||
// {"type": "stop", ...}); variant payload fields inline into the same
|
||||
// object, with each variant's tag value being its field name.
|
||||
ONEOF_SHAPE_TAGGED = 3;
|
||||
}
|
||||
|
||||
// Attached to a oneof: `option (sglang.json.v1.oneof_json).<name> = ...;` in
|
||||
// its body. A oneof is a tagged union on the protobuf wire (the field number
|
||||
// is the tag); JSON has no such tag, so `shape` picks how the union is
|
||||
// spelled there and how the deserializer recovers the variant: by value
|
||||
// shape (untagged), scalar versus array (one_or_many), or a discriminator key
|
||||
// named by `tag` (tagged). `unknown_variant_passthrough` lets a tagged union
|
||||
// round-trip a variant this schema does not know.
|
||||
message OneofJson {
|
||||
OneofShape shape = 1;
|
||||
|
||||
// The discriminator key; required with ONEOF_SHAPE_TAGGED, meaningless
|
||||
// otherwise.
|
||||
string tag = 2;
|
||||
|
||||
// With ONEOF_SHAPE_TAGGED: JSON input matching no variant is captured
|
||||
// verbatim by the designated catch-all variant (a raw_json Struct member)
|
||||
// and round-trips unchanged -- the forward-compatibility escape hatch for
|
||||
// values another component may extend (e.g. finish_reason).
|
||||
bool unknown_variant_passthrough = 3;
|
||||
}
|
||||
|
||||
extend google.protobuf.MessageOptions {
|
||||
// Field numbers from the internal-use extension range (50000-99999).
|
||||
MessageJson message_json = 50100;
|
||||
}
|
||||
|
||||
extend google.protobuf.FieldOptions {
|
||||
FieldJson field_json = 50100;
|
||||
}
|
||||
|
||||
extend google.protobuf.OneofOptions {
|
||||
OneofJson oneof_json = 50100;
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
//! Rust bindings for SGLang's canonical `sglang.runtime.v1` protobuf API.
|
||||
//!
|
||||
//! The generated messages and Tonic client/server interfaces intentionally
|
||||
//! contain no frontend implementation or transport policy.
|
||||
|
||||
/// Protobuf packages exposed with the same namespace as the wire contract.
|
||||
pub mod sglang {
|
||||
pub mod runtime {
|
||||
pub mod v1 {
|
||||
tonic::include_proto!("sglang.runtime.v1");
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,18 @@
|
||||
"""The HTTP JSON contract gate: `proto/sglang/api/v1` is the schema for both
|
||||
the Rust and the Python server, so a `/generate` body the Python dataclass
|
||||
parse admitted but the schema rejects is refused here with the schema's own
|
||||
error text. The generated decoder is the single source of that judgment.
|
||||
"""
|
||||
|
||||
from typing import Any, Optional
|
||||
|
||||
from sglang.api.v1.api_types import GenerateRequest, JsonContractError
|
||||
|
||||
|
||||
def generate_contract_error(body: Any) -> Optional[str]:
|
||||
"""The contract violation in a parsed `/generate` JSON body, or None."""
|
||||
try:
|
||||
GenerateRequest.from_json_value(body)
|
||||
except JsonContractError as e:
|
||||
return str(e)
|
||||
return None
|
||||
@@ -73,6 +73,7 @@ from sglang.srt.entrypoints.anthropic.protocol import (
|
||||
AnthropicMessagesRequest,
|
||||
)
|
||||
from sglang.srt.entrypoints.anthropic.serving import AnthropicServing
|
||||
from sglang.srt.entrypoints.api_contract import generate_contract_error
|
||||
from sglang.srt.entrypoints.engine import (
|
||||
Engine,
|
||||
init_tokenizer_manager,
|
||||
@@ -908,6 +909,11 @@ if os.environ.get("DUMPER_SERVER_PORT") == "reuse":
|
||||
)
|
||||
async def generate_request(obj: GenerateReqInput, request: Request):
|
||||
"""Handle a generate request."""
|
||||
# Starlette caches the parsed body, so this is the schema check, not a
|
||||
# second decode.
|
||||
contract_error = generate_contract_error(await request.json())
|
||||
if contract_error is not None:
|
||||
return ORJSONResponse(status_code=400, content={"error": contract_error})
|
||||
if envs.SGLANG_ENABLE_REQUEST_HEADER_OVERRIDES.get():
|
||||
apply_header_overrides(obj, request.headers)
|
||||
if obj.stream:
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
# Workspace-local cargo aliases.
|
||||
[alias]
|
||||
# Regenerate the API types (Rust + Python) from proto/sglang; see
|
||||
# sglang-api-codegen/src/main.rs.
|
||||
gen-api = "run -p sglang-api-codegen"
|
||||
Generated
+137
-133
@@ -292,40 +292,13 @@ dependencies = [
|
||||
"arrayvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "axum"
|
||||
version = "0.7.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum-core 0.4.5",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"itoa",
|
||||
"matchit 0.7.3",
|
||||
"memchr",
|
||||
"mime",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"rustversion",
|
||||
"serde",
|
||||
"sync_wrapper",
|
||||
"tower 0.5.3",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "axum"
|
||||
version = "0.8.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90"
|
||||
dependencies = [
|
||||
"axum-core 0.5.6",
|
||||
"axum-core",
|
||||
"bytes",
|
||||
"form_urlencoded",
|
||||
"futures-util",
|
||||
@@ -335,7 +308,7 @@ dependencies = [
|
||||
"hyper",
|
||||
"hyper-util",
|
||||
"itoa",
|
||||
"matchit 0.8.4",
|
||||
"matchit",
|
||||
"memchr",
|
||||
"mime",
|
||||
"percent-encoding",
|
||||
@@ -346,32 +319,12 @@ dependencies = [
|
||||
"serde_urlencoded",
|
||||
"sync_wrapper",
|
||||
"tokio",
|
||||
"tower 0.5.3",
|
||||
"tower",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "axum-core"
|
||||
version = "0.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"mime",
|
||||
"pin-project-lite",
|
||||
"rustversion",
|
||||
"sync_wrapper",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "axum-core"
|
||||
version = "0.5.6"
|
||||
@@ -1265,6 +1218,12 @@ version = "1.0.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
|
||||
|
||||
[[package]]
|
||||
name = "foldhash"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
|
||||
|
||||
[[package]]
|
||||
name = "form_urlencoded"
|
||||
version = "1.2.2"
|
||||
@@ -1492,6 +1451,15 @@ version = "0.12.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.15.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
|
||||
dependencies = [
|
||||
"foldhash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.16.1"
|
||||
@@ -1658,7 +1626,7 @@ dependencies = [
|
||||
"libc",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"socket2 0.5.10",
|
||||
"socket2",
|
||||
"system-configuration",
|
||||
"tokio",
|
||||
"tower-service",
|
||||
@@ -2088,12 +2056,6 @@ dependencies = [
|
||||
"regex-automata",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "matchit"
|
||||
version = "0.7.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
|
||||
|
||||
[[package]]
|
||||
name = "matchit"
|
||||
version = "0.8.4"
|
||||
@@ -2558,11 +2520,12 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
|
||||
|
||||
[[package]]
|
||||
name = "petgraph"
|
||||
version = "0.7.1"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772"
|
||||
checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455"
|
||||
dependencies = [
|
||||
"fixedbitset",
|
||||
"hashbrown 0.15.5",
|
||||
"indexmap 2.14.0",
|
||||
]
|
||||
|
||||
@@ -2739,9 +2702,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "prost"
|
||||
version = "0.13.5"
|
||||
version = "0.14.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5"
|
||||
checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"prost-derive",
|
||||
@@ -2749,19 +2712,20 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "prost-build"
|
||||
version = "0.13.5"
|
||||
version = "0.14.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
|
||||
checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"itertools",
|
||||
"log",
|
||||
"multimap",
|
||||
"once_cell",
|
||||
"petgraph",
|
||||
"prettyplease",
|
||||
"prost",
|
||||
"prost-types",
|
||||
"pulldown-cmark",
|
||||
"pulldown-cmark-to-cmark",
|
||||
"regex",
|
||||
"syn 2.0.119",
|
||||
"tempfile",
|
||||
@@ -2769,9 +2733,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "prost-derive"
|
||||
version = "0.13.5"
|
||||
version = "0.14.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
|
||||
checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools",
|
||||
@@ -2781,10 +2745,20 @@ dependencies = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prost-types"
|
||||
version = "0.13.5"
|
||||
name = "prost-reflect"
|
||||
version = "0.16.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16"
|
||||
checksum = "01b80ea363c31af2de2b92e3c07ed1156628f7838c4afb4df75ee78a37fedbd1"
|
||||
dependencies = [
|
||||
"prost",
|
||||
"prost-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prost-types"
|
||||
version = "0.14.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a"
|
||||
dependencies = [
|
||||
"prost",
|
||||
]
|
||||
@@ -2853,6 +2827,26 @@ version = "3.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "95067976aca6421a523e491fce939a3e65249bac4b977adee0ee9771568e8aa3"
|
||||
|
||||
[[package]]
|
||||
name = "pulldown-cmark"
|
||||
version = "0.13.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"memchr",
|
||||
"unicase",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pulldown-cmark-to-cmark"
|
||||
version = "22.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "84bbb29c624230c4bd1047bbdb2aa47e41c860e9665ce62ba9504eebe91bf867"
|
||||
dependencies = [
|
||||
"pulldown-cmark",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pulp"
|
||||
version = "0.22.3"
|
||||
@@ -2967,7 +2961,7 @@ dependencies = [
|
||||
"quinn-udp",
|
||||
"rustc-hash 2.1.3",
|
||||
"rustls",
|
||||
"socket2 0.5.10",
|
||||
"socket2",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tracing",
|
||||
@@ -3005,7 +2999,7 @@ dependencies = [
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
"once_cell",
|
||||
"socket2 0.5.10",
|
||||
"socket2",
|
||||
"tracing",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
@@ -3344,7 +3338,7 @@ dependencies = [
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
"tokio-util",
|
||||
"tower 0.5.3",
|
||||
"tower",
|
||||
"tower-http",
|
||||
"tower-service",
|
||||
"url",
|
||||
@@ -3688,6 +3682,30 @@ dependencies = [
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sglang-api-codegen"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"prost",
|
||||
"prost-build",
|
||||
"prost-reflect",
|
||||
"protoc-bin-vendored",
|
||||
"tonic-prost-build",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sglang-api-types"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"prost",
|
||||
"prost-types",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tonic",
|
||||
"tonic-prost",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sglang-grpc"
|
||||
version = "0.1.0"
|
||||
@@ -3696,28 +3714,17 @@ dependencies = [
|
||||
"prost-types",
|
||||
"pyo3",
|
||||
"serde_json",
|
||||
"sglang-grpc-types",
|
||||
"sglang-api-types",
|
||||
"tokenizers",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tonic",
|
||||
"tonic-prost",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sglang-grpc-types"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"prost",
|
||||
"prost-build",
|
||||
"prost-types",
|
||||
"protoc-bin-vendored",
|
||||
"tonic",
|
||||
"tonic-build",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sglang-mm"
|
||||
version = "0.1.0"
|
||||
@@ -3757,7 +3764,7 @@ name = "sglang-renderer"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"async-stream",
|
||||
"axum 0.8.9",
|
||||
"axum",
|
||||
"clap",
|
||||
"dynamo-parsers",
|
||||
"dynamo-protocols",
|
||||
@@ -3776,7 +3783,7 @@ dependencies = [
|
||||
"sglang-processor",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tower 0.5.3",
|
||||
"tower",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"uuid",
|
||||
@@ -3788,7 +3795,7 @@ version = "0.1.0"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-stream",
|
||||
"axum 0.8.9",
|
||||
"axum",
|
||||
"bytemuck",
|
||||
"bytes",
|
||||
"core_affinity",
|
||||
@@ -3799,6 +3806,7 @@ dependencies = [
|
||||
"flume",
|
||||
"futures",
|
||||
"hf-hub",
|
||||
"http",
|
||||
"itertools",
|
||||
"minijinja",
|
||||
"numpy",
|
||||
@@ -3809,14 +3817,14 @@ dependencies = [
|
||||
"rustix",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sglang-grpc-types",
|
||||
"sglang-api-types",
|
||||
"sglang-mm",
|
||||
"socket2 0.6.5",
|
||||
"socket2",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tonic",
|
||||
"tower 0.5.3",
|
||||
"tower",
|
||||
"tracing",
|
||||
"tracing-appender",
|
||||
"tracing-subscriber",
|
||||
@@ -3903,16 +3911,6 @@ version = "1.15.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
|
||||
|
||||
[[package]]
|
||||
name = "socket2"
|
||||
version = "0.5.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "socket2"
|
||||
version = "0.6.5"
|
||||
@@ -4253,7 +4251,7 @@ dependencies = [
|
||||
"parking_lot",
|
||||
"pin-project-lite",
|
||||
"signal-hook-registry",
|
||||
"socket2 0.6.5",
|
||||
"socket2",
|
||||
"tokio-macros",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
@@ -4306,16 +4304,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tonic"
|
||||
version = "0.12.3"
|
||||
version = "0.14.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52"
|
||||
checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef"
|
||||
dependencies = [
|
||||
"async-stream",
|
||||
"async-trait",
|
||||
"axum 0.7.9",
|
||||
"axum",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"flate2",
|
||||
"h2",
|
||||
"http",
|
||||
"http-body",
|
||||
@@ -4325,11 +4321,11 @@ dependencies = [
|
||||
"hyper-util",
|
||||
"percent-encoding",
|
||||
"pin-project",
|
||||
"prost",
|
||||
"socket2 0.5.10",
|
||||
"socket2",
|
||||
"sync_wrapper",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tower 0.4.13",
|
||||
"tower",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
@@ -4337,9 +4333,32 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tonic-build"
|
||||
version = "0.12.3"
|
||||
version = "0.14.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9557ce109ea773b399c9b9e5dca39294110b74f1f342cb347a80d1fce8c26a11"
|
||||
checksum = "c68f61875ac5293cf72e6c8cf0158086428c82c37229e98c840878f1706b0322"
|
||||
dependencies = [
|
||||
"prettyplease",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tonic-prost"
|
||||
version = "0.14.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"prost",
|
||||
"tonic",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tonic-prost-build"
|
||||
version = "0.14.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "654e5643eff75d7f8c99197ce1440ed19a3474eada74c12bbac488b2cafdae27"
|
||||
dependencies = [
|
||||
"prettyplease",
|
||||
"proc-macro2",
|
||||
@@ -4347,26 +4366,8 @@ dependencies = [
|
||||
"prost-types",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tower"
|
||||
version = "0.4.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"indexmap 1.9.3",
|
||||
"pin-project",
|
||||
"pin-project-lite",
|
||||
"rand 0.8.7",
|
||||
"slab",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
"tempfile",
|
||||
"tonic-build",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4377,9 +4378,12 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"indexmap 2.14.0",
|
||||
"pin-project-lite",
|
||||
"slab",
|
||||
"sync_wrapper",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
@@ -4397,7 +4401,7 @@ dependencies = [
|
||||
"http",
|
||||
"http-body",
|
||||
"pin-project-lite",
|
||||
"tower 0.5.3",
|
||||
"tower",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
"url",
|
||||
|
||||
+9
-6
@@ -1,7 +1,8 @@
|
||||
[workspace]
|
||||
resolver = "3"
|
||||
members = [
|
||||
"../proto",
|
||||
"sglang-api-codegen",
|
||||
"sglang-api-types",
|
||||
"sglang-grpc",
|
||||
"sglang-mm",
|
||||
"sglang-processor",
|
||||
@@ -30,9 +31,9 @@ futures = "0.3"
|
||||
pyo3 = { version = "0.29.0", features = ["extension-module"] }
|
||||
# These versions define the generated public gRPC API. Update them together
|
||||
# and deliberately rather than changing bindings during an unrelated build.
|
||||
prost = "=0.13.5"
|
||||
prost-types = "=0.13.5"
|
||||
prost-build = "=0.13.5"
|
||||
prost = "0.14"
|
||||
prost-types = "0.14"
|
||||
prost-build = "0.14"
|
||||
protoc-bin-vendored = "=3.2.0"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
@@ -40,11 +41,13 @@ socket2 = "0.6"
|
||||
thiserror = "2"
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
tokio-stream = { version = "0.1", features = ["net"] }
|
||||
tonic = "0.14"
|
||||
tonic-build = "0.14"
|
||||
tonic-prost = "0.14"
|
||||
tower = { version = "0.5", features = ["util"] }
|
||||
tracing = "0.1"
|
||||
tracing-appender = "0.2"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
tonic = { version = "=0.12.3", features = ["gzip", "transport"] }
|
||||
tonic-build = "=0.12.3"
|
||||
uuid = { version = "1", features = ["v4"] }
|
||||
|
||||
# Profiles only take effect at the workspace root; per-member profile sections
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
[package]
|
||||
name = "sglang-api-codegen"
|
||||
description = "Generator: proto/sglang -> checked-in bindings (prost structs, tonic services, schema-driven serde, Python types)"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
# No [package.metadata.sglang]: not a Python extension; wheel builds skip it.
|
||||
|
||||
[[bin]]
|
||||
name = "sglang-api-codegen"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
prost = { workspace = true }
|
||||
prost-build = "0.14"
|
||||
prost-reflect = "0.16"
|
||||
protoc-bin-vendored = "3"
|
||||
tonic-prost-build = "0.14"
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,179 @@
|
||||
//! The generator: every checked-in binding of `proto/sglang/`, from one protoc.
|
||||
//!
|
||||
//! Targets (see [`TARGETS`]):
|
||||
//! Both targets land in `rust/sglang-api-types/src/generated/`:
|
||||
//! - `sglang.api.v1`: prost structs, tonic services, schema-driven serde, plus
|
||||
//! `python/sglang/api/v1/types.py`.
|
||||
//! - `sglang.runtime.v1`: prost structs and tonic services only.
|
||||
//!
|
||||
//! Pass 1 (prost-build + tonic-prost-build) emits the prost structs and tonic
|
||||
//! service code, plus a `FileDescriptorSet`. For targets with a JSON contract
|
||||
//! pass 2 reads the `sglang.json.v1` options from that set through
|
||||
//! prost-reflect and emits, from one model (`model`), the Rust serde
|
||||
//! implementations (`emit`) and the Python interface definition (`emit_py`);
|
||||
//! proto3's canonical JSON mapping is NOT used, the options are the contract.
|
||||
//!
|
||||
//! Regeneration is manual. From anywhere in the `rust/` workspace:
|
||||
//!
|
||||
//! ```text
|
||||
//! cargo gen-api
|
||||
//! ```
|
||||
//!
|
||||
//! Each target has a single root .proto: every API must be defined (or
|
||||
//! imported) there, and protoc's import closure supplies the rest of the
|
||||
//! package; a file the root does not reach gets no generated code. All output
|
||||
//! is checked in, so commit it together with the .proto change. The vendored
|
||||
//! protoc (pinned by Cargo.lock) is used unless `PROTOC` is set, so the output
|
||||
//! is byte-stable across machines.
|
||||
|
||||
mod emit;
|
||||
mod emit_py;
|
||||
mod model;
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
/// One generated binding set. Paths are relative to the repository root.
|
||||
struct Target {
|
||||
/// The root .proto under `proto/`; its import closure is the API surface.
|
||||
root: &'static str,
|
||||
package: &'static str,
|
||||
rust_out: &'static str,
|
||||
/// Set when the schema carries `sglang.json.v1` options: emits the serde
|
||||
/// implementations next to the structs and the Python module here.
|
||||
python_out: Option<&'static str>,
|
||||
/// prost `boxed` paths (message fields stored as `Box<T>`).
|
||||
boxed: &'static [&'static str],
|
||||
/// Emit every service method with an UNIMPLEMENTED default body, so a
|
||||
/// server may implement a subset (the runtime.v1 follower metadata
|
||||
/// service). Off for a contract every server implements in full.
|
||||
default_stubs: bool,
|
||||
}
|
||||
|
||||
const TARGETS: &[Target] = &[
|
||||
Target {
|
||||
root: "sglang/api/v1/service.proto",
|
||||
package: "sglang.api.v1",
|
||||
rust_out: "rust/sglang-api-types/src/generated",
|
||||
python_out: Some("python/sglang/api/v1/api_types.py"),
|
||||
// Boxed abort payload won't set the size of every ChunkEvent.
|
||||
boxed: &[".sglang.api.v1.FinishReason.kind.abort"],
|
||||
default_stubs: false,
|
||||
},
|
||||
Target {
|
||||
root: "sglang/runtime/v1/sglang.proto",
|
||||
package: "sglang.runtime.v1",
|
||||
rust_out: "rust/sglang-api-types/src/generated",
|
||||
python_out: None,
|
||||
boxed: &[],
|
||||
default_stubs: true,
|
||||
},
|
||||
];
|
||||
|
||||
fn main() {
|
||||
let repo_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../..");
|
||||
let proto_root = repo_root.join("proto");
|
||||
|
||||
// Vendored protoc first: the checked-in output must not depend on
|
||||
// whichever protoc a machine has. `PROTOC` still overrides for platforms
|
||||
// the vendored crate lacks.
|
||||
if std::env::var_os("PROTOC").is_none()
|
||||
&& let Ok(vendored) = protoc_bin_vendored::protoc_bin_path()
|
||||
{
|
||||
// SAFETY: nothing else is running yet; the generator is single-threaded.
|
||||
unsafe { std::env::set_var("PROTOC", vendored) };
|
||||
}
|
||||
|
||||
// The descriptor set is only pass 2's input, not part of any crate; it is
|
||||
// a protoc-version-dependent binary, so keep it out of the tree.
|
||||
let scratch_dir = std::env::temp_dir().join("sglang-api-codegen");
|
||||
std::fs::create_dir_all(&scratch_dir).expect("create scratch dir");
|
||||
|
||||
for target in TARGETS {
|
||||
generate(target, &repo_root, &proto_root, &scratch_dir);
|
||||
}
|
||||
}
|
||||
|
||||
fn generate(target: &Target, repo_root: &Path, proto_root: &Path, scratch_dir: &Path) {
|
||||
let package = target.package;
|
||||
let out_dir = repo_root.join(target.rust_out);
|
||||
std::fs::create_dir_all(&out_dir).expect("create generated dir");
|
||||
let descriptor_path = scratch_dir.join(format!("{package}.descriptor.bin"));
|
||||
|
||||
// Pass 1: structs + tonic services (+ the descriptor set pass 2 reads).
|
||||
// Everything the root imports comes in transitively via the include path,
|
||||
// in topological order, so the output is stable.
|
||||
let mut config = prost_build::Config::new();
|
||||
config
|
||||
.out_dir(&out_dir)
|
||||
.file_descriptor_set_path(&descriptor_path)
|
||||
.service_generator(
|
||||
tonic_prost_build::configure()
|
||||
.generate_default_stubs(target.default_stubs)
|
||||
.service_generator(),
|
||||
)
|
||||
.protoc_arg("--experimental_allow_proto3_optional");
|
||||
for path in target.boxed {
|
||||
config.boxed(path);
|
||||
}
|
||||
config
|
||||
.compile_protos(&[proto_root.join(target.root)], &[proto_root])
|
||||
.unwrap_or_else(|e| panic!("pass 1 ({package}): {e}"));
|
||||
|
||||
// prost writes structs + services into one file; split the tonic modules
|
||||
// out so the include! layout stays stable.
|
||||
split_tonic(&out_dir, package);
|
||||
|
||||
if let Some(python_out) = target.python_out {
|
||||
let bytes = std::fs::read(&descriptor_path).expect("descriptor set");
|
||||
|
||||
// Pass 2: schema-driven serde from the options in the descriptor set.
|
||||
let serde_src = emit::emit_serde(&bytes, package);
|
||||
std::fs::write(out_dir.join(format!("{package}.serde.rs")), serde_src)
|
||||
.expect("write serde");
|
||||
|
||||
// Pass 2 (Python): the same model rendered as Structs + codecs.
|
||||
let py_out = repo_root.join(python_out);
|
||||
std::fs::create_dir_all(py_out.parent().expect("py out dir")).expect("create py dir");
|
||||
std::fs::write(&py_out, emit_py::emit_python(&bytes, package)).expect("write python types");
|
||||
println!("generated {}", py_out.display());
|
||||
|
||||
// The options package itself needs no Rust types (it exists for the
|
||||
// generator); drop pass-1 output for it if prost emitted one.
|
||||
let _ = std::fs::remove_file(out_dir.join("sglang.json.v1.rs"));
|
||||
}
|
||||
|
||||
rustfmt(&out_dir);
|
||||
println!("generated into {}", out_dir.display());
|
||||
}
|
||||
|
||||
/// Move the tonic `pub mod *_client/_server` blocks from the prost output
|
||||
/// `<package>.rs` into `<package>.tonic.rs`.
|
||||
fn split_tonic(out_dir: &Path, package: &str) {
|
||||
let prost_file = out_dir.join(format!("{package}.rs"));
|
||||
let src = std::fs::read_to_string(&prost_file).expect("read pass-1 output");
|
||||
let marker = "/// Generated client implementations.";
|
||||
let (structs, tonic) = match src.find(marker) {
|
||||
Some(pos) => src.split_at(pos),
|
||||
None => (src.as_str(), ""),
|
||||
};
|
||||
std::fs::write(&prost_file, structs.trim_end().to_string() + "\n").expect("write structs");
|
||||
std::fs::write(
|
||||
out_dir.join(format!("{package}.tonic.rs")),
|
||||
tonic.trim_start(),
|
||||
)
|
||||
.expect("write tonic");
|
||||
}
|
||||
|
||||
fn rustfmt(out_dir: &Path) {
|
||||
for entry in std::fs::read_dir(out_dir).expect("read generated dir") {
|
||||
let path = entry.expect("dir entry").path();
|
||||
if path.extension().is_some_and(|e| e == "rs") {
|
||||
// Best effort: the output is already well-formed; rustfmt only
|
||||
// normalizes.
|
||||
let _ = std::process::Command::new("rustfmt")
|
||||
.arg("--edition=2024")
|
||||
.arg(&path)
|
||||
.status();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,550 @@
|
||||
//! Pass 2's input model: each message classified by the `sglang.json.v1`
|
||||
//! options into the JSON shape the emitter renders. The options are read from
|
||||
//! the FileDescriptorSet through prost-reflect, so the schema file is the
|
||||
//! single source of truth.
|
||||
|
||||
use prost_reflect::{
|
||||
DescriptorPool, ExtensionDescriptor, FieldDescriptor, Kind, MessageDescriptor,
|
||||
};
|
||||
|
||||
pub struct Extensions {
|
||||
message_json: ExtensionDescriptor,
|
||||
field_json: ExtensionDescriptor,
|
||||
oneof_json: ExtensionDescriptor,
|
||||
}
|
||||
|
||||
impl Extensions {
|
||||
pub fn resolve(pool: &DescriptorPool) -> Extensions {
|
||||
let get = |name: &str| {
|
||||
pool.get_extension_by_name(name)
|
||||
.unwrap_or_else(|| panic!("options.proto extension missing: {name}"))
|
||||
};
|
||||
Extensions {
|
||||
message_json: get("sglang.json.v1.message_json"),
|
||||
field_json: get("sglang.json.v1.field_json"),
|
||||
oneof_json: get("sglang.json.v1.oneof_json"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn opt_bool(options: &prost_reflect::DynamicMessage, name: &str) -> bool {
|
||||
options
|
||||
.get_field_by_name(name)
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// An enum-typed option as its number; 0 (the unset/default variant) when absent.
|
||||
fn opt_enum(options: &prost_reflect::DynamicMessage, name: &str) -> i32 {
|
||||
options
|
||||
.get_field_by_name(name)
|
||||
.and_then(|v| v.as_enum_number())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
fn opt_str(options: &prost_reflect::DynamicMessage, name: &str) -> Option<String> {
|
||||
options
|
||||
.get_field_by_name(name)
|
||||
.and_then(|v| v.as_str().map(str::to_owned))
|
||||
.filter(|s| !s.is_empty())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum ScalarTy {
|
||||
F64,
|
||||
I32,
|
||||
I64,
|
||||
U32,
|
||||
U64,
|
||||
Bool,
|
||||
Str,
|
||||
}
|
||||
|
||||
impl ScalarTy {
|
||||
pub fn rust(&self) -> &'static str {
|
||||
match self {
|
||||
ScalarTy::F64 => "f64",
|
||||
ScalarTy::I32 => "i32",
|
||||
ScalarTy::I64 => "i64",
|
||||
ScalarTy::U32 => "u32",
|
||||
ScalarTy::U64 => "u64",
|
||||
ScalarTy::Bool => "bool",
|
||||
ScalarTy::Str => "::prost::alloc::string::String",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum DefaultLit {
|
||||
F64(f64),
|
||||
I64(i64),
|
||||
Bool(bool),
|
||||
Str(String),
|
||||
}
|
||||
|
||||
impl DefaultLit {
|
||||
pub fn rust(&self) -> String {
|
||||
match self {
|
||||
DefaultLit::F64(v) => format!("{v:?}f64"),
|
||||
DefaultLit::I64(v) => format!("{v}i64"),
|
||||
DefaultLit::Bool(v) => format!("{v}"),
|
||||
DefaultLit::Str(v) => format!("{v:?}.to_string()"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum FieldKind {
|
||||
/// A plain proto3 scalar (no presence). `null_default` set =
|
||||
/// null_resets_default: absent AND explicit null yield the default.
|
||||
Scalar {
|
||||
ty: ScalarTy,
|
||||
null_default: Option<DefaultLit>,
|
||||
},
|
||||
/// proto3 `optional` scalar. `null_is_none_default` set: absent yields the
|
||||
/// default, explicit null clears to None. `null_default` set
|
||||
/// (null_resets_default): absent AND null yield Some(default), and an
|
||||
/// `*_or_default` accessor covers protobuf arrival (unset = None).
|
||||
OptScalar {
|
||||
ty: ScalarTy,
|
||||
emit_null: bool,
|
||||
null_is_none_default: Option<DefaultLit>,
|
||||
null_default: Option<DefaultLit>,
|
||||
},
|
||||
/// A message-typed field: `Option<T>` presence; the value's JSON shape is
|
||||
/// the message's own impl.
|
||||
Message {
|
||||
rust_path: String,
|
||||
emit_null: bool,
|
||||
},
|
||||
RepeatedScalar {
|
||||
ty: ScalarTy,
|
||||
},
|
||||
RepeatedMessage {
|
||||
rust_path: String,
|
||||
},
|
||||
/// repeated string with raw_json: Vec<String> of JSON texts passing
|
||||
/// through as an array of raw values.
|
||||
RepeatedRawJson,
|
||||
/// map<string, scalar>; int_key means numeric-string keys (no transform
|
||||
/// needed on the JSON side, documented parity).
|
||||
Map {
|
||||
value: ScalarTy,
|
||||
},
|
||||
/// A string field holding JSON text (raw_json): the value passes through
|
||||
/// verbatim on the HTTP side.
|
||||
RawJson {
|
||||
emit_null: bool,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FieldModel {
|
||||
/// The JSON key (= the proto field name).
|
||||
pub json_name: String,
|
||||
/// The prost struct field ident (keyword-escaped).
|
||||
pub rust_ident: String,
|
||||
pub kind: FieldKind,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct VariantModel {
|
||||
/// The JSON discriminator value / shape source (= the proto field name).
|
||||
pub json_name: String,
|
||||
/// The prost oneof enum variant (PascalCase).
|
||||
pub rust_variant: String,
|
||||
pub kind: FieldKind,
|
||||
/// For tagged unions: the payload message's fields, inlined into the
|
||||
/// tagged object.
|
||||
pub payload_fields: Vec<FieldModel>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum MessageKind {
|
||||
/// Single-repeated-field carrier: JSON is the bare array.
|
||||
BareList { item: FieldModel },
|
||||
/// Positional tuple: fields in declaration order form a bare array.
|
||||
Tuple { fields: Vec<FieldModel> },
|
||||
/// Single-field wrapper serializing as its sole field's value.
|
||||
Transparent { field: FieldModel },
|
||||
/// Single oneof { one, many }: scalar-or-list flattening.
|
||||
OneOrMany {
|
||||
oneof_mod: String,
|
||||
one: VariantModel,
|
||||
many: VariantModel,
|
||||
},
|
||||
/// Single oneof, internally tagged by `tag`; `passthrough` catches
|
||||
/// unknown tags as a raw map (and any known-tag payload that fails to
|
||||
/// parse, matching the hand-written FinishReason).
|
||||
TaggedUnion {
|
||||
oneof_mod: String,
|
||||
oneof_ident: String,
|
||||
tag: String,
|
||||
variants: Vec<VariantModel>,
|
||||
passthrough: Option<VariantModel>,
|
||||
},
|
||||
/// Single oneof, untagged: the value's shape picks the variant.
|
||||
Untagged {
|
||||
oneof_mod: String,
|
||||
oneof_ident: String,
|
||||
variants: Vec<VariantModel>,
|
||||
},
|
||||
/// A plain object.
|
||||
Struct {
|
||||
deny_unknown: bool,
|
||||
fields: Vec<FieldModel>,
|
||||
},
|
||||
/// gRPC-only (a plain oneof container like GenerateStreamItem): no HTTP
|
||||
/// JSON contract, no serde emitted.
|
||||
Skip,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct MessageModel {
|
||||
/// Rust type name (= proto message name).
|
||||
pub rust_name: String,
|
||||
pub kind: MessageKind,
|
||||
}
|
||||
|
||||
pub fn build_models(pool: &DescriptorPool, package: &str) -> Vec<MessageModel> {
|
||||
let ext = Extensions::resolve(pool);
|
||||
pool.all_messages()
|
||||
.filter(|m| m.full_name().starts_with(package))
|
||||
// Map-entry synthetic messages have no JSON identity of their own.
|
||||
.filter(|m| !m.is_map_entry())
|
||||
.map(|m| message_model(&m, &ext))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn message_model(msg: &MessageDescriptor, ext: &Extensions) -> MessageModel {
|
||||
let rust_name = msg.name().to_string();
|
||||
let options = msg.options();
|
||||
let mopts = options.get_extension(&ext.message_json);
|
||||
let mopts = mopts.as_message();
|
||||
|
||||
// `MessageShape`: one shape per message, by construction of the enum.
|
||||
let shape = mopts.map(|m| opt_enum(m, "shape")).unwrap_or(0);
|
||||
let (bare_list, tuple, transparent) = (shape == 1, shape == 2, shape == 3);
|
||||
let deny_unknown = mopts
|
||||
.and_then(|m| m.get_field_by_name("unknown_fields"))
|
||||
.and_then(|v| v.as_enum_number())
|
||||
.unwrap_or(0)
|
||||
== 1;
|
||||
|
||||
// proto3 `optional` is implemented as a synthetic oneof: those fields are
|
||||
// plain fields with presence, not union members.
|
||||
let oneofs: Vec<_> = msg.oneofs().filter(|o| !o.is_synthetic()).collect();
|
||||
let plain_fields: Vec<FieldDescriptor> = msg
|
||||
.fields()
|
||||
.filter(|f| f.containing_oneof().is_none_or(|o| o.is_synthetic()))
|
||||
.collect();
|
||||
|
||||
if bare_list {
|
||||
assert_eq!(
|
||||
plain_fields.len(),
|
||||
1,
|
||||
"{}: bare_list needs exactly one repeated field",
|
||||
msg.full_name()
|
||||
);
|
||||
return MessageModel {
|
||||
rust_name,
|
||||
kind: MessageKind::BareList {
|
||||
item: field_model(&plain_fields[0], ext),
|
||||
},
|
||||
};
|
||||
}
|
||||
if transparent {
|
||||
assert_eq!(
|
||||
plain_fields.len(),
|
||||
1,
|
||||
"{}: transparent needs exactly one field",
|
||||
msg.full_name()
|
||||
);
|
||||
return MessageModel {
|
||||
rust_name,
|
||||
kind: MessageKind::Transparent {
|
||||
field: field_model(&plain_fields[0], ext),
|
||||
},
|
||||
};
|
||||
}
|
||||
if tuple {
|
||||
return MessageModel {
|
||||
rust_name,
|
||||
kind: MessageKind::Tuple {
|
||||
fields: plain_fields.iter().map(|f| field_model(f, ext)).collect(),
|
||||
},
|
||||
};
|
||||
}
|
||||
if oneofs.len() == 1 && plain_fields.is_empty() {
|
||||
let oneof = &oneofs[0];
|
||||
let oopts = oneof.options();
|
||||
let oopts = oopts.get_extension(&ext.oneof_json);
|
||||
let oopts = oopts.as_message();
|
||||
// `OneofShape`: one spelling per union, by construction of the enum.
|
||||
let shape = oopts.map(|m| opt_enum(m, "shape")).unwrap_or(0);
|
||||
let (untagged, one_or_many, tagged) = (shape == 1, shape == 2, shape == 3);
|
||||
let tag = oopts.and_then(|m| opt_str(m, "tag"));
|
||||
assert_eq!(
|
||||
tagged,
|
||||
tag.is_some(),
|
||||
"{}: ONEOF_SHAPE_TAGGED and `tag` go together",
|
||||
msg.full_name()
|
||||
);
|
||||
let passthrough_on = oopts
|
||||
.map(|m| opt_bool(m, "unknown_variant_passthrough"))
|
||||
.unwrap_or(false);
|
||||
let oneof_mod = snake(&rust_name);
|
||||
let oneof_ident = pascal(oneof.name());
|
||||
|
||||
let mut variants: Vec<VariantModel> = oneof
|
||||
.fields()
|
||||
.map(|f| VariantModel {
|
||||
json_name: f.name().to_string(),
|
||||
rust_variant: pascal(f.name()),
|
||||
kind: field_model(&f, ext).kind,
|
||||
payload_fields: match f.kind() {
|
||||
// The raw_json passthrough arm carries a well-known type;
|
||||
// its innards are never inlined.
|
||||
Kind::Message(payload)
|
||||
if tag.is_some() && !payload.full_name().starts_with("google.protobuf") =>
|
||||
{
|
||||
payload
|
||||
.fields()
|
||||
.map(|pf| field_model(&pf, ext))
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
_ => vec![],
|
||||
},
|
||||
})
|
||||
.collect();
|
||||
|
||||
if one_or_many {
|
||||
assert_eq!(variants.len(), 2, "{}: one_or_many", msg.full_name());
|
||||
let many = variants.pop().expect("many variant");
|
||||
let one = variants.pop().expect("one variant");
|
||||
return MessageModel {
|
||||
rust_name,
|
||||
kind: MessageKind::OneOrMany {
|
||||
oneof_mod,
|
||||
one,
|
||||
many,
|
||||
},
|
||||
};
|
||||
}
|
||||
if let Some(tag) = tag {
|
||||
let passthrough = passthrough_on.then(|| {
|
||||
let idx = variants
|
||||
.iter()
|
||||
.position(|v| matches!(v.kind, FieldKind::RawJson { .. }))
|
||||
.expect("passthrough needs a raw_json string variant");
|
||||
variants.remove(idx)
|
||||
});
|
||||
return MessageModel {
|
||||
rust_name,
|
||||
kind: MessageKind::TaggedUnion {
|
||||
oneof_mod,
|
||||
oneof_ident,
|
||||
tag,
|
||||
variants,
|
||||
passthrough,
|
||||
},
|
||||
};
|
||||
}
|
||||
if untagged {
|
||||
return MessageModel {
|
||||
rust_name,
|
||||
kind: MessageKind::Untagged {
|
||||
oneof_mod,
|
||||
oneof_ident,
|
||||
variants,
|
||||
},
|
||||
};
|
||||
}
|
||||
// A plain oneof container (e.g. GenerateStreamItem) has no HTTP JSON
|
||||
// contract of its own — it is gRPC-only. Emit nothing.
|
||||
return MessageModel {
|
||||
rust_name,
|
||||
kind: MessageKind::Skip,
|
||||
};
|
||||
}
|
||||
|
||||
MessageModel {
|
||||
rust_name,
|
||||
kind: MessageKind::Struct {
|
||||
deny_unknown,
|
||||
fields: plain_fields.iter().map(|f| field_model(f, ext)).collect(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn field_model(f: &FieldDescriptor, ext: &Extensions) -> FieldModel {
|
||||
let options = f.options();
|
||||
let fopts = options.get_extension(&ext.field_json);
|
||||
let fopts = fopts.as_message();
|
||||
let get_bool = |name: &str| fopts.map(|m| opt_bool(m, name)).unwrap_or(false);
|
||||
let default_lit = |m: &prost_reflect::DynamicMessage| -> Option<DefaultLit> {
|
||||
// has_field_by_name: the default_* members are proto3 optional, so
|
||||
// presence is meaningful.
|
||||
if m.has_field_by_name("default_f64") {
|
||||
return m
|
||||
.get_field_by_name("default_f64")
|
||||
.and_then(|v| v.as_f64())
|
||||
.map(DefaultLit::F64);
|
||||
}
|
||||
if m.has_field_by_name("default_i64") {
|
||||
return m
|
||||
.get_field_by_name("default_i64")
|
||||
.and_then(|v| v.as_i64())
|
||||
.map(DefaultLit::I64);
|
||||
}
|
||||
if m.has_field_by_name("default_bool") {
|
||||
return m
|
||||
.get_field_by_name("default_bool")
|
||||
.and_then(|v| v.as_bool())
|
||||
.map(DefaultLit::Bool);
|
||||
}
|
||||
if m.has_field_by_name("default_string") {
|
||||
return m
|
||||
.get_field_by_name("default_string")
|
||||
.and_then(|v| v.as_str().map(str::to_owned))
|
||||
.map(DefaultLit::Str);
|
||||
}
|
||||
None
|
||||
};
|
||||
let raw_json = get_bool("raw_json");
|
||||
let emit_null = get_bool("emit_null_when_absent");
|
||||
// `NullPolicy`: one policy per field, by construction of the enum.
|
||||
let null_policy = fopts.map(|m| opt_enum(m, "null_policy")).unwrap_or(0);
|
||||
let (null_resets, null_is_none) = (null_policy == 1, null_policy == 2);
|
||||
let default = fopts.and_then(default_lit);
|
||||
|
||||
let kind = if raw_json {
|
||||
if f.is_list() {
|
||||
FieldKind::RepeatedRawJson
|
||||
} else {
|
||||
assert!(
|
||||
matches!(f.kind(), Kind::String),
|
||||
"{}: raw_json requires a string field",
|
||||
f.full_name()
|
||||
);
|
||||
FieldKind::RawJson { emit_null }
|
||||
}
|
||||
} else if f.is_map() {
|
||||
let value_ty = match f.kind() {
|
||||
Kind::Message(entry) => scalar_ty(
|
||||
&entry
|
||||
.get_field_by_name("value")
|
||||
.expect("map value field")
|
||||
.kind(),
|
||||
),
|
||||
_ => unreachable!("map fields are message-kinded"),
|
||||
};
|
||||
FieldKind::Map { value: value_ty }
|
||||
} else if f.is_list() {
|
||||
match f.kind() {
|
||||
Kind::Message(m) => FieldKind::RepeatedMessage {
|
||||
rust_path: m.name().to_string(),
|
||||
},
|
||||
other => FieldKind::RepeatedScalar {
|
||||
ty: scalar_ty(&other),
|
||||
},
|
||||
}
|
||||
} else {
|
||||
match f.kind() {
|
||||
Kind::Message(m) => FieldKind::Message {
|
||||
rust_path: m.name().to_string(),
|
||||
emit_null,
|
||||
},
|
||||
other => {
|
||||
let ty = scalar_ty(&other);
|
||||
if f.supports_presence() {
|
||||
FieldKind::OptScalar {
|
||||
ty,
|
||||
emit_null,
|
||||
null_is_none_default: null_is_none.then(|| {
|
||||
default
|
||||
.clone()
|
||||
.expect("NULL_POLICY_IS_NONE requires a default_*")
|
||||
}),
|
||||
null_default: null_resets.then(|| {
|
||||
default
|
||||
.clone()
|
||||
.expect("NULL_POLICY_RESETS_DEFAULT requires a default_*")
|
||||
}),
|
||||
}
|
||||
} else {
|
||||
FieldKind::Scalar {
|
||||
ty,
|
||||
null_default: null_resets.then(|| {
|
||||
default
|
||||
.clone()
|
||||
.expect("NULL_POLICY_RESETS_DEFAULT requires a default_*")
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
FieldModel {
|
||||
json_name: f.name().to_string(),
|
||||
rust_ident: escape_ident(f.name()),
|
||||
kind,
|
||||
}
|
||||
}
|
||||
|
||||
fn scalar_ty(kind: &Kind) -> ScalarTy {
|
||||
match kind {
|
||||
Kind::Double | Kind::Float => ScalarTy::F64,
|
||||
Kind::Int64 | Kind::Sint64 | Kind::Sfixed64 => ScalarTy::I64,
|
||||
Kind::Int32 | Kind::Sint32 | Kind::Sfixed32 => ScalarTy::I32,
|
||||
Kind::Uint32 | Kind::Fixed32 => ScalarTy::U32,
|
||||
Kind::Uint64 | Kind::Fixed64 => ScalarTy::U64,
|
||||
Kind::Bool => ScalarTy::Bool,
|
||||
Kind::String => ScalarTy::Str,
|
||||
other => panic!("unsupported scalar kind {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// prost's keyword escaping for field idents.
|
||||
fn escape_ident(name: &str) -> String {
|
||||
const KEYWORDS: &[&str] = &[
|
||||
"as", "break", "const", "continue", "crate", "dyn", "else", "enum", "extern", "false",
|
||||
"fn", "for", "if", "impl", "in", "let", "loop", "match", "mod", "move", "mut", "pub",
|
||||
"ref", "return", "self", "static", "struct", "super", "trait", "true", "type", "unsafe",
|
||||
"use", "where", "while",
|
||||
];
|
||||
if KEYWORDS.contains(&name) {
|
||||
format!("r#{name}")
|
||||
} else {
|
||||
name.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn pascal(name: &str) -> String {
|
||||
name.split('_')
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(|s| {
|
||||
let mut chars = s.chars();
|
||||
match chars.next() {
|
||||
Some(first) => first.to_ascii_uppercase().to_string() + chars.as_str(),
|
||||
None => String::new(),
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn snake(name: &str) -> String {
|
||||
let mut out = String::new();
|
||||
for (i, ch) in name.chars().enumerate() {
|
||||
if ch.is_ascii_uppercase() {
|
||||
if i > 0 {
|
||||
out.push('_');
|
||||
}
|
||||
out.push(ch.to_ascii_lowercase());
|
||||
} else {
|
||||
out.push(ch);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
[package]
|
||||
name = "sglang-api-types"
|
||||
description = "Generated SGLang API types: prost structs, tonic services, schema-driven serde"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
# No [package.metadata.sglang]: not a Python extension; wheel builds skip it.
|
||||
|
||||
[lib]
|
||||
name = "sglang_api_types"
|
||||
crate-type = ["rlib"]
|
||||
|
||||
[dependencies]
|
||||
bytes = { workspace = true }
|
||||
prost = { workspace = true }
|
||||
prost-types = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
# preserve_order: raw_json passthrough (custom_params, internal_states, an
|
||||
# unknown finish reason) re-serializes through serde_json::Value, and the
|
||||
# contract is "verbatim": key order must not depend on what else links the
|
||||
# crate.
|
||||
serde_json = { workspace = true, features = ["preserve_order"] }
|
||||
tonic = { workspace = true }
|
||||
tonic-prost = { workspace = true }
|
||||
@@ -0,0 +1,63 @@
|
||||
//! Hand-written extensions on the generated types: the accessors and
|
||||
//! constructors server logic needs beyond the wire shapes. Grows with each
|
||||
//! migration stage.
|
||||
|
||||
use std::borrow::Cow;
|
||||
|
||||
use crate::api::v1::{FinishReason, FinishStop, Matched, MatchedTokens, finish_reason, matched};
|
||||
|
||||
impl FinishReason {
|
||||
/// A stop reason (`type: "stop"`), optionally carrying what matched.
|
||||
pub fn stop(matched: Option<matched::Value>) -> Self {
|
||||
FinishReason {
|
||||
kind: Some(finish_reason::Kind::Stop(FinishStop {
|
||||
matched: matched.map(|value| Matched { value: Some(value) }),
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the wire `type` without reserializing the finish reason (an
|
||||
/// unknown reason's is read out of its raw JSON, hence the `Cow`).
|
||||
pub fn kind_name(&self) -> Option<Cow<'_, str>> {
|
||||
match self.kind.as_ref()? {
|
||||
finish_reason::Kind::Stop(_) => Some(Cow::Borrowed("stop")),
|
||||
finish_reason::Kind::Length(_) => Some(Cow::Borrowed("length")),
|
||||
finish_reason::Kind::Abort(_) => Some(Cow::Borrowed("abort")),
|
||||
finish_reason::Kind::Unknown(raw) => {
|
||||
let parsed: serde_json::Value = serde_json::from_str(raw).ok()?;
|
||||
parsed
|
||||
.get("type")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.map(|s| Cow::Owned(s.to_owned()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The stop this request matched, if it stopped on one. `None` for
|
||||
/// length/abort and for an unknown type.
|
||||
pub fn matched(&self) -> Option<&matched::Value> {
|
||||
match self.kind.as_ref()? {
|
||||
finish_reason::Kind::Stop(stop) => stop.matched.as_ref()?.value.as_ref(),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// `Some((status, message))` when this is an abort carrying a `status_code` —
|
||||
/// a scheduler-side request error the API surfaces as that HTTP status instead
|
||||
/// of as a normal completion. A plain abort (no code) reads as `None`.
|
||||
pub fn abort_status(&self) -> Option<(u16, &str)> {
|
||||
match self.kind.as_ref()? {
|
||||
finish_reason::Kind::Abort(abort) => Some((
|
||||
u16::try_from(abort.status_code?).ok()?,
|
||||
abort.message.as_deref().unwrap_or("request aborted"),
|
||||
)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Vec<i64>> for MatchedTokens {
|
||||
fn from(ids: Vec<i64>) -> Self {
|
||||
MatchedTokens { ids }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,657 @@
|
||||
// This file is @generated by prost-build.
|
||||
/// A token-id sequence. int64 matches the servers' token id width (Rust
|
||||
/// `TokenIds = Vec<i64>`, Python `array("q")`).
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct TokenIds {
|
||||
#[prost(int64, repeated, tag = "1")]
|
||||
pub ids: ::prost::alloc::vec::Vec<i64>,
|
||||
}
|
||||
/// Scalar-or-list carriers: `/generate` accepts every per-request field as one
|
||||
/// value (single request) or a list (batch). JSON flattens via the one_or_many
|
||||
/// option; gRPC clients pick the variant explicitly.
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct StringOrList {
|
||||
#[prost(oneof = "string_or_list::Value", tags = "1, 2")]
|
||||
pub value: ::core::option::Option<string_or_list::Value>,
|
||||
}
|
||||
/// Nested message and enum types in `StringOrList`.
|
||||
pub mod string_or_list {
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)]
|
||||
pub enum Value {
|
||||
#[prost(string, tag = "1")]
|
||||
One(::prost::alloc::string::String),
|
||||
#[prost(message, tag = "2")]
|
||||
Many(super::StringList),
|
||||
}
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct StringList {
|
||||
#[prost(string, repeated, tag = "1")]
|
||||
pub items: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
|
||||
}
|
||||
/// A flat string list for one request, or one list per batch entry. JSON:
|
||||
/// `\["a", "b"\]` is `one`; `\[["a"\], \["b", "c"]\]` is `many` (first element
|
||||
/// decides, `\[\]` is an empty `one`).
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct StringListOrList {
|
||||
#[prost(oneof = "string_list_or_list::Value", tags = "1, 2")]
|
||||
pub value: ::core::option::Option<string_list_or_list::Value>,
|
||||
}
|
||||
/// Nested message and enum types in `StringListOrList`.
|
||||
pub mod string_list_or_list {
|
||||
#[derive(Clone, PartialEq, ::prost::Oneof)]
|
||||
pub enum Value {
|
||||
#[prost(message, tag = "1")]
|
||||
One(super::StringList),
|
||||
#[prost(message, tag = "2")]
|
||||
Many(super::StringListList),
|
||||
}
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct StringListList {
|
||||
#[prost(message, repeated, tag = "1")]
|
||||
pub items: ::prost::alloc::vec::Vec<StringList>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct BoolOrList {
|
||||
#[prost(oneof = "bool_or_list::Value", tags = "1, 2")]
|
||||
pub value: ::core::option::Option<bool_or_list::Value>,
|
||||
}
|
||||
/// Nested message and enum types in `BoolOrList`.
|
||||
pub mod bool_or_list {
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)]
|
||||
pub enum Value {
|
||||
#[prost(bool, tag = "1")]
|
||||
One(bool),
|
||||
#[prost(message, tag = "2")]
|
||||
Many(super::BoolList),
|
||||
}
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct BoolList {
|
||||
#[prost(bool, repeated, tag = "1")]
|
||||
pub items: ::prost::alloc::vec::Vec<bool>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct Int64OrList {
|
||||
#[prost(oneof = "int64_or_list::Value", tags = "1, 2")]
|
||||
pub value: ::core::option::Option<int64_or_list::Value>,
|
||||
}
|
||||
/// Nested message and enum types in `Int64OrList`.
|
||||
pub mod int64_or_list {
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)]
|
||||
pub enum Value {
|
||||
#[prost(int64, tag = "1")]
|
||||
One(i64),
|
||||
#[prost(message, tag = "2")]
|
||||
Many(super::Int64List),
|
||||
}
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct Int64List {
|
||||
#[prost(int64, repeated, tag = "1")]
|
||||
pub items: ::prost::alloc::vec::Vec<i64>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct TokenIdsOrList {
|
||||
#[prost(oneof = "token_ids_or_list::Value", tags = "1, 2")]
|
||||
pub value: ::core::option::Option<token_ids_or_list::Value>,
|
||||
}
|
||||
/// Nested message and enum types in `TokenIdsOrList`.
|
||||
pub mod token_ids_or_list {
|
||||
#[derive(Clone, PartialEq, ::prost::Oneof)]
|
||||
pub enum Value {
|
||||
#[prost(message, tag = "1")]
|
||||
One(super::TokenIds),
|
||||
#[prost(message, tag = "2")]
|
||||
Many(super::TokenIdsList),
|
||||
}
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct TokenIdsList {
|
||||
#[prost(message, repeated, tag = "1")]
|
||||
pub items: ::prost::alloc::vec::Vec<TokenIds>,
|
||||
}
|
||||
/// List variants whose ELEMENTS may be JSON null (PD bootstrap columns: a
|
||||
/// batch mixes bootstrapped and plain requests). Proto carries element
|
||||
/// presence as a per-element oneof-free wrapper.
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct OptionalString {
|
||||
#[prost(string, optional, tag = "1")]
|
||||
pub value: ::core::option::Option<::prost::alloc::string::String>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct OptionalStringOrList {
|
||||
#[prost(oneof = "optional_string_or_list::Value", tags = "1, 2")]
|
||||
pub value: ::core::option::Option<optional_string_or_list::Value>,
|
||||
}
|
||||
/// Nested message and enum types in `OptionalStringOrList`.
|
||||
pub mod optional_string_or_list {
|
||||
#[derive(Clone, PartialEq, ::prost::Oneof)]
|
||||
pub enum Value {
|
||||
#[prost(message, tag = "1")]
|
||||
One(super::OptionalString),
|
||||
#[prost(message, tag = "2")]
|
||||
Many(super::OptionalStringList),
|
||||
}
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct OptionalStringList {
|
||||
#[prost(message, repeated, tag = "1")]
|
||||
pub items: ::prost::alloc::vec::Vec<OptionalString>,
|
||||
}
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct OptionalInt64 {
|
||||
#[prost(int64, optional, tag = "1")]
|
||||
pub value: ::core::option::Option<i64>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct OptionalInt64OrList {
|
||||
#[prost(oneof = "optional_int64_or_list::Value", tags = "1, 2")]
|
||||
pub value: ::core::option::Option<optional_int64_or_list::Value>,
|
||||
}
|
||||
/// Nested message and enum types in `OptionalInt64OrList`.
|
||||
pub mod optional_int64_or_list {
|
||||
#[derive(Clone, PartialEq, ::prost::Oneof)]
|
||||
pub enum Value {
|
||||
#[prost(message, tag = "1")]
|
||||
One(super::OptionalInt64),
|
||||
#[prost(message, tag = "2")]
|
||||
Many(super::OptionalInt64List),
|
||||
}
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct OptionalInt64List {
|
||||
#[prost(message, repeated, tag = "1")]
|
||||
pub items: ::prost::alloc::vec::Vec<OptionalInt64>,
|
||||
}
|
||||
/// The native error body: `{"error": {"message", "code"}}` (Python
|
||||
/// `http_server.generate_request` parity). Transports frame it themselves
|
||||
/// (unary status + body, SSE error frame, gRPC status detail).
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct ErrorBody {
|
||||
#[prost(string, tag = "1")]
|
||||
pub message: ::prost::alloc::string::String,
|
||||
#[prost(uint32, tag = "2")]
|
||||
pub code: u32,
|
||||
}
|
||||
/// Python's `matched` key, `Union\[str, int, List[int]\]`: the three wire shapes
|
||||
/// are disjoint, so shape alone picks the arm (untagged).
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct Matched {
|
||||
#[prost(oneof = "matched::Value", tags = "1, 2, 3")]
|
||||
pub value: ::core::option::Option<matched::Value>,
|
||||
}
|
||||
/// Nested message and enum types in `Matched`.
|
||||
pub mod matched {
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)]
|
||||
pub enum Value {
|
||||
#[prost(int64, tag = "1")]
|
||||
Token(i64),
|
||||
#[prost(string, tag = "2")]
|
||||
Str(::prost::alloc::string::String),
|
||||
#[prost(message, tag = "3")]
|
||||
Tokens(super::MatchedTokens),
|
||||
}
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct MatchedTokens {
|
||||
#[prost(int64, repeated, tag = "1")]
|
||||
pub ids: ::prost::alloc::vec::Vec<i64>,
|
||||
}
|
||||
/// `FINISH_MATCHED_TOKEN` / `FINISH_MATCHED_STR` / `FINISHED_MATCHED_REGEX` —
|
||||
/// all three report `type: "stop"`, matching the OpenAI API's value.
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct FinishStop {
|
||||
#[prost(message, optional, tag = "1")]
|
||||
pub matched: ::core::option::Option<Matched>,
|
||||
}
|
||||
/// `FINISH_LENGTH` — hit `max_new_tokens` (or the context limit).
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct FinishLength {
|
||||
#[prost(uint64, optional, tag = "1")]
|
||||
pub length: ::core::option::Option<u64>,
|
||||
}
|
||||
/// The `FINISH_ABORT` payload. Python emits all three keys, nulls included.
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct FinishAbort {
|
||||
#[prost(string, optional, tag = "1")]
|
||||
pub message: ::core::option::Option<::prost::alloc::string::String>,
|
||||
#[prost(uint32, optional, tag = "2")]
|
||||
pub status_code: ::core::option::Option<u32>,
|
||||
#[prost(string, optional, tag = "3")]
|
||||
pub err_type: ::core::option::Option<::prost::alloc::string::String>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct FinishReason {
|
||||
#[prost(oneof = "finish_reason::Kind", tags = "1, 2, 3, 4")]
|
||||
pub kind: ::core::option::Option<finish_reason::Kind>,
|
||||
}
|
||||
/// Nested message and enum types in `FinishReason`.
|
||||
pub mod finish_reason {
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)]
|
||||
pub enum Kind {
|
||||
#[prost(message, tag = "1")]
|
||||
Stop(super::FinishStop),
|
||||
#[prost(message, tag = "2")]
|
||||
Length(super::FinishLength),
|
||||
#[prost(message, tag = "3")]
|
||||
Abort(::prost::alloc::boxed::Box<super::FinishAbort>),
|
||||
/// The passthrough arm: an unrecognized `type`, kept as its raw map
|
||||
/// (JSON text — Struct would coerce integer values to doubles).
|
||||
#[prost(string, tag = "4")]
|
||||
Unknown(::prost::alloc::string::String),
|
||||
}
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct SamplingParams {
|
||||
/// JSON: absent = 128, explicit null = no limit (Python `Optional\[int\] = 128`). Protobuf has no null: unset = 128 (apply_absent_defaults at the
|
||||
/// gRPC entry); no limit needs an explicit large value.
|
||||
#[prost(int64, optional, tag = "1")]
|
||||
pub max_new_tokens: ::core::option::Option<i64>,
|
||||
#[prost(message, optional, tag = "2")]
|
||||
pub stop: ::core::option::Option<StringOrList>,
|
||||
#[prost(message, optional, tag = "3")]
|
||||
pub stop_token_ids: ::core::option::Option<Int64List>,
|
||||
#[prost(message, optional, tag = "4")]
|
||||
pub stop_regex: ::core::option::Option<StringOrList>,
|
||||
#[prost(double, optional, tag = "5")]
|
||||
pub temperature: ::core::option::Option<f64>,
|
||||
#[prost(double, optional, tag = "6")]
|
||||
pub top_p: ::core::option::Option<f64>,
|
||||
/// Default 1 \<\< 30 (TOP_K_ALL): "consider the whole vocabulary".
|
||||
#[prost(int64, optional, tag = "7")]
|
||||
pub top_k: ::core::option::Option<i64>,
|
||||
#[prost(double, optional, tag = "8")]
|
||||
pub min_p: ::core::option::Option<f64>,
|
||||
#[prost(double, optional, tag = "9")]
|
||||
pub frequency_penalty: ::core::option::Option<f64>,
|
||||
#[prost(double, optional, tag = "10")]
|
||||
pub presence_penalty: ::core::option::Option<f64>,
|
||||
#[prost(double, optional, tag = "11")]
|
||||
pub repetition_penalty: ::core::option::Option<f64>,
|
||||
#[prost(int64, optional, tag = "12")]
|
||||
pub min_new_tokens: ::core::option::Option<i64>,
|
||||
#[prost(int64, optional, tag = "13")]
|
||||
pub n: ::core::option::Option<i64>,
|
||||
#[prost(string, optional, tag = "14")]
|
||||
pub json_schema: ::core::option::Option<::prost::alloc::string::String>,
|
||||
#[prost(string, optional, tag = "15")]
|
||||
pub regex: ::core::option::Option<::prost::alloc::string::String>,
|
||||
#[prost(string, optional, tag = "16")]
|
||||
pub ebnf: ::core::option::Option<::prost::alloc::string::String>,
|
||||
#[prost(string, optional, tag = "17")]
|
||||
pub structural_tag: ::core::option::Option<::prost::alloc::string::String>,
|
||||
#[prost(bool, optional, tag = "18")]
|
||||
pub ignore_eos: ::core::option::Option<bool>,
|
||||
#[prost(bool, optional, tag = "19")]
|
||||
pub skip_special_tokens: ::core::option::Option<bool>,
|
||||
#[prost(bool, optional, tag = "20")]
|
||||
pub spaces_between_special_tokens: ::core::option::Option<bool>,
|
||||
#[prost(bool, optional, tag = "21")]
|
||||
pub no_stop_trim: ::core::option::Option<bool>,
|
||||
#[prost(int64, optional, tag = "22")]
|
||||
pub stream_interval: ::core::option::Option<i64>,
|
||||
/// Token id (numeric-string key, Python parity) -> bias.
|
||||
#[prost(map = "string, double", tag = "23")]
|
||||
pub logit_bias: ::std::collections::HashMap<::prost::alloc::string::String, f64>,
|
||||
#[prost(int64, optional, tag = "24")]
|
||||
pub sampling_seed: ::core::option::Option<i64>,
|
||||
/// Opaque JSON forwarded to a custom logit processor; never inspected here.
|
||||
#[prost(string, optional, tag = "25")]
|
||||
pub custom_params: ::core::option::Option<::prost::alloc::string::String>,
|
||||
/// beam_width > 1 turns the request into a beam search request; `n` is then
|
||||
/// the number of returned sequences (n \<= beam_width).
|
||||
#[prost(int64, optional, tag = "26")]
|
||||
pub beam_width: ::core::option::Option<i64>,
|
||||
}
|
||||
/// (message_json).unknown_fields defaults to IGNORE — stated here as schema
|
||||
/// intent, load-bearing for client compat.
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct GenerateRequest {
|
||||
#[prost(message, optional, tag = "1")]
|
||||
pub rid: ::core::option::Option<StringOrList>,
|
||||
#[prost(message, optional, tag = "2")]
|
||||
pub text: ::core::option::Option<StringOrList>,
|
||||
#[prost(message, optional, tag = "3")]
|
||||
pub input_ids: ::core::option::Option<TokenIdsOrList>,
|
||||
#[prost(bool, optional, tag = "4")]
|
||||
pub stream: ::core::option::Option<bool>,
|
||||
/// One set of params, or one per batch item. JSON errors keep serde's
|
||||
/// field-level texts (the hand-written SamplingParamsInput contract).
|
||||
#[prost(message, optional, tag = "5")]
|
||||
pub sampling_params: ::core::option::Option<SamplingParamsOrList>,
|
||||
#[prost(message, optional, tag = "6")]
|
||||
pub return_logprob: ::core::option::Option<BoolOrList>,
|
||||
#[prost(message, optional, tag = "7")]
|
||||
pub logprob_start_len: ::core::option::Option<Int64OrList>,
|
||||
#[prost(message, optional, tag = "8")]
|
||||
pub top_logprobs_num: ::core::option::Option<Int64OrList>,
|
||||
#[prost(message, optional, tag = "9")]
|
||||
pub token_ids_logprob: ::core::option::Option<TokenIdsOrList>,
|
||||
#[prost(message, optional, tag = "10")]
|
||||
pub return_hidden_states: ::core::option::Option<BoolOrList>,
|
||||
#[prost(bool, optional, tag = "11")]
|
||||
pub return_text_in_logprobs: ::core::option::Option<bool>,
|
||||
/// PD bootstrap columns: a batch mixes bootstrapped and plain requests, so
|
||||
/// elements are individually nullable.
|
||||
#[prost(message, optional, tag = "12")]
|
||||
pub bootstrap_host: ::core::option::Option<OptionalStringOrList>,
|
||||
#[prost(message, optional, tag = "13")]
|
||||
pub bootstrap_port: ::core::option::Option<OptionalInt64OrList>,
|
||||
#[prost(message, optional, tag = "14")]
|
||||
pub bootstrap_room: ::core::option::Option<OptionalInt64OrList>,
|
||||
#[prost(message, optional, tag = "15")]
|
||||
pub bootstrap_pair_key: ::core::option::Option<OptionalStringOrList>,
|
||||
#[prost(message, optional, tag = "16")]
|
||||
pub decode_tp_size: ::core::option::Option<OptionalInt64OrList>,
|
||||
#[prost(int64, optional, tag = "17")]
|
||||
pub routed_dp_rank: ::core::option::Option<i64>,
|
||||
#[prost(int64, optional, tag = "18")]
|
||||
pub disagg_prefill_dp_rank: ::core::option::Option<i64>,
|
||||
/// Multimodal payloads pass through untyped (data URIs, URLs, file paths, or
|
||||
/// nested batch lists) to the MM pipeline, which owns their validation.
|
||||
/// Media inputs, one container per modality (see MediaInput). The item
|
||||
/// hints a modality does not support are rejected at the processor.
|
||||
#[prost(message, optional, tag = "19")]
|
||||
pub image_data: ::core::option::Option<MediaInput>,
|
||||
/// Per-item content hashes overriding the computed ones, in item order; a
|
||||
/// flat list for one request (the batch form is rejected downstream).
|
||||
#[prost(message, optional, tag = "20")]
|
||||
pub mm_hashes: ::core::option::Option<StringListOrList>,
|
||||
#[prost(message, optional, tag = "21")]
|
||||
pub video_data: ::core::option::Option<MediaInput>,
|
||||
#[prost(message, optional, tag = "22")]
|
||||
pub audio_data: ::core::option::Option<MediaInput>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct SamplingParamsOrList {
|
||||
#[prost(oneof = "sampling_params_or_list::Value", tags = "1, 2")]
|
||||
pub value: ::core::option::Option<sampling_params_or_list::Value>,
|
||||
}
|
||||
/// Nested message and enum types in `SamplingParamsOrList`.
|
||||
pub mod sampling_params_or_list {
|
||||
#[derive(Clone, PartialEq, ::prost::Oneof)]
|
||||
pub enum Value {
|
||||
#[prost(message, tag = "1")]
|
||||
One(super::SamplingParams),
|
||||
#[prost(message, tag = "2")]
|
||||
Many(super::SamplingParamsList),
|
||||
}
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct SamplingParamsList {
|
||||
#[prost(message, repeated, tag = "1")]
|
||||
pub items: ::prost::alloc::vec::Vec<SamplingParams>,
|
||||
}
|
||||
/// One `\[logprob, token_id, text|null\]` entry (text present when
|
||||
/// `return_text_in_logprobs`, decoded on the detok shard). The logprob slot is
|
||||
/// optional: Python emits null for a position with no logprob (the first
|
||||
/// prefill token; Rust maps the scheduler's NaN to the same null).
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct LogprobEntry {
|
||||
#[prost(double, optional, tag = "1")]
|
||||
pub logprob: ::core::option::Option<f64>,
|
||||
#[prost(int64, tag = "2")]
|
||||
pub token_id: i64,
|
||||
#[prost(string, optional, tag = "3")]
|
||||
pub text: ::core::option::Option<::prost::alloc::string::String>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct LogprobEntries {
|
||||
#[prost(message, repeated, tag = "1")]
|
||||
pub entries: ::prost::alloc::vec::Vec<LogprobEntry>,
|
||||
}
|
||||
/// One decode position's top-k alternatives.
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct TopLogprobRow {
|
||||
#[prost(message, repeated, tag = "1")]
|
||||
pub entries: ::prost::alloc::vec::Vec<LogprobEntry>,
|
||||
}
|
||||
/// A position's row, or null: a position that recorded no alternatives is
|
||||
/// null on the wire (Python parity), never \[\].
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct NullableTopLogprobs {
|
||||
#[prost(message, optional, tag = "1")]
|
||||
pub row: ::core::option::Option<TopLogprobRow>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct HiddenStateRow {
|
||||
#[prost(double, repeated, tag = "1")]
|
||||
pub values: ::prost::alloc::vec::Vec<f64>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct GenerateMetaInfo {
|
||||
/// The client's rid, byte-for-byte (`Rid::client_facing`).
|
||||
#[prost(string, tag = "1")]
|
||||
pub id: ::prost::alloc::string::String,
|
||||
#[prost(uint32, tag = "2")]
|
||||
pub prompt_tokens: u32,
|
||||
#[prost(uint64, tag = "3")]
|
||||
pub completion_tokens: u64,
|
||||
/// Always present in JSON, null until a terminal frame.
|
||||
#[prost(message, optional, tag = "4")]
|
||||
pub finish_reason: ::core::option::Option<FinishReason>,
|
||||
/// input+output token logprobs are set together, empty lists included: a PD
|
||||
/// decode node never receives input logprobs, but the PD router keys its
|
||||
/// merge of prefill's on the key's presence.
|
||||
#[prost(message, optional, tag = "5")]
|
||||
pub output_token_logprobs: ::core::option::Option<LogprobEntries>,
|
||||
#[prost(message, optional, tag = "6")]
|
||||
pub input_token_logprobs: ::core::option::Option<LogprobEntries>,
|
||||
#[prost(message, repeated, tag = "7")]
|
||||
pub output_top_logprobs: ::prost::alloc::vec::Vec<NullableTopLogprobs>,
|
||||
#[prost(message, repeated, tag = "8")]
|
||||
pub input_top_logprobs: ::prost::alloc::vec::Vec<NullableTopLogprobs>,
|
||||
#[prost(message, repeated, tag = "9")]
|
||||
pub output_token_ids_logprobs: ::prost::alloc::vec::Vec<NullableTopLogprobs>,
|
||||
#[prost(message, repeated, tag = "10")]
|
||||
pub input_token_ids_logprobs: ::prost::alloc::vec::Vec<NullableTopLogprobs>,
|
||||
#[prost(message, repeated, tag = "11")]
|
||||
pub hidden_states: ::prost::alloc::vec::Vec<HiddenStateRow>,
|
||||
/// Seconds from normalized-request acceptance to the terminal output; only
|
||||
/// on the terminal frame.
|
||||
#[prost(double, optional, tag = "12")]
|
||||
pub e2e_latency: ::core::option::Option<f64>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct GenerateResponse {
|
||||
/// JSON key order is FIELD-NUMBER order: text, meta_info, then the
|
||||
/// conditionally-present output_ids and index — matching frame_value's
|
||||
/// insertion order byte-for-byte.
|
||||
#[prost(string, tag = "1")]
|
||||
pub text: ::prost::alloc::string::String,
|
||||
#[prost(message, optional, tag = "2")]
|
||||
pub meta_info: ::core::option::Option<GenerateMetaInfo>,
|
||||
/// Omitted while empty (a text-only frame).
|
||||
#[prost(message, optional, tag = "3")]
|
||||
pub output_ids: ::core::option::Option<TokenIds>,
|
||||
/// Batch position; only on batch responses/frames.
|
||||
#[prost(uint32, optional, tag = "4")]
|
||||
pub index: ::core::option::Option<u32>,
|
||||
}
|
||||
/// One request's failure inside a Generate stream; carries its batch position
|
||||
/// (the HTTP SSE shape puts `index` beside the error object the same way).
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct GenerateStreamError {
|
||||
#[prost(message, optional, tag = "1")]
|
||||
pub error: ::core::option::Option<ErrorBody>,
|
||||
#[prost(uint32, optional, tag = "2")]
|
||||
pub index: ::core::option::Option<u32>,
|
||||
}
|
||||
/// One item of the gRPC Generate stream: a frame, or one request's failure
|
||||
/// (the stream continues — other batch items may still be running). On HTTP
|
||||
/// the same failure is an in-stream SSE error frame.
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct GenerateStreamItem {
|
||||
#[prost(oneof = "generate_stream_item::Item", tags = "1, 2")]
|
||||
pub item: ::core::option::Option<generate_stream_item::Item>,
|
||||
}
|
||||
/// Nested message and enum types in `GenerateStreamItem`.
|
||||
pub mod generate_stream_item {
|
||||
#[derive(Clone, PartialEq, ::prost::Oneof)]
|
||||
pub enum Item {
|
||||
#[prost(message, tag = "1")]
|
||||
Frame(super::GenerateResponse),
|
||||
#[prost(message, tag = "2")]
|
||||
Error(super::GenerateStreamError),
|
||||
}
|
||||
}
|
||||
/// One media item's source and per-item hints. Unknown keys are an error: a
|
||||
/// hint this schema does not name would otherwise be silently dropped.
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct MediaRef {
|
||||
/// URL, `file://` / absolute path, `data:` URI, or bare base64.
|
||||
#[prost(string, tag = "1")]
|
||||
pub url: ::prost::alloc::string::String,
|
||||
/// Image: "auto" | "low" | "high" (the OpenAI detail hint).
|
||||
#[prost(string, optional, tag = "2")]
|
||||
pub detail: ::core::option::Option<::prost::alloc::string::String>,
|
||||
/// Image/video: dynamic-patch budget (InternVL-style processors).
|
||||
#[prost(int64, optional, tag = "3")]
|
||||
pub max_dynamic_patch: ::core::option::Option<i64>,
|
||||
/// Image: a caller-supplied content hash that overrides the computed one.
|
||||
#[prost(string, optional, tag = "4")]
|
||||
pub content_hash: ::core::option::Option<::prost::alloc::string::String>,
|
||||
/// Video/image budgets read by the GLM and MiMo processors.
|
||||
#[prost(int64, optional, tag = "5")]
|
||||
pub min_pixels: ::core::option::Option<i64>,
|
||||
#[prost(int64, optional, tag = "6")]
|
||||
pub max_pixels: ::core::option::Option<i64>,
|
||||
#[prost(int64, optional, tag = "7")]
|
||||
pub total_max_pixels: ::core::option::Option<i64>,
|
||||
#[prost(double, optional, tag = "8")]
|
||||
pub fps: ::core::option::Option<f64>,
|
||||
#[prost(int64, optional, tag = "9")]
|
||||
pub num_frames: ::core::option::Option<i64>,
|
||||
#[prost(int64, optional, tag = "10")]
|
||||
pub min_frames: ::core::option::Option<i64>,
|
||||
#[prost(int64, optional, tag = "11")]
|
||||
pub max_frames: ::core::option::Option<i64>,
|
||||
#[prost(int64, optional, tag = "12")]
|
||||
pub max_tokens_per_frame: ::core::option::Option<i64>,
|
||||
#[prost(int64, optional, tag = "13")]
|
||||
pub max_image_tokens: ::core::option::Option<i64>,
|
||||
/// Video: also decode the audio track.
|
||||
#[prost(bool, optional, tag = "14")]
|
||||
pub use_audio: ::core::option::Option<bool>,
|
||||
}
|
||||
/// One media item. JSON: a bare string is the source; an object is a MediaRef.
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct MediaItem {
|
||||
#[prost(oneof = "media_item::Value", tags = "1, 2")]
|
||||
pub value: ::core::option::Option<media_item::Value>,
|
||||
}
|
||||
/// Nested message and enum types in `MediaItem`.
|
||||
pub mod media_item {
|
||||
#[derive(Clone, PartialEq, ::prost::Oneof)]
|
||||
pub enum Value {
|
||||
#[prost(string, tag = "1")]
|
||||
Source(::prost::alloc::string::String),
|
||||
#[prost(message, tag = "2")]
|
||||
Ref(super::MediaRef),
|
||||
}
|
||||
}
|
||||
/// A list element that may be JSON null (Python's `\[img, None\]`).
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct OptionalMediaItem {
|
||||
#[prost(message, optional, tag = "1")]
|
||||
pub item: ::core::option::Option<MediaItem>,
|
||||
}
|
||||
/// One request's items (`\[a, null, b\]`), or one batch entry's items.
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct MediaItems {
|
||||
#[prost(message, repeated, tag = "1")]
|
||||
pub items: ::prost::alloc::vec::Vec<OptionalMediaItem>,
|
||||
}
|
||||
/// A batch entry whose item list may be JSON null.
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct OptionalMediaItems {
|
||||
#[prost(message, optional, tag = "1")]
|
||||
pub items: ::core::option::Option<MediaItems>,
|
||||
}
|
||||
/// One item list per batch entry (`\[[a, b\], null, \[]\]`).
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct MediaItemsList {
|
||||
#[prost(message, repeated, tag = "1")]
|
||||
pub items: ::prost::alloc::vec::Vec<OptionalMediaItems>,
|
||||
}
|
||||
/// The three container forms Python's `_normalize_image_data` distinguishes.
|
||||
/// JSON: a string or object is `item` (one item; a broadcast in a batch); an
|
||||
/// array whose first element is not an array is `many` (a flat list; `\[\]` is
|
||||
/// an empty `many`); an array whose first element is an array is `nested`
|
||||
/// (one list per batch entry). Which forms a request may use depends on
|
||||
/// whether it is a batch, and is checked downstream, not here.
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct MediaInput {
|
||||
#[prost(oneof = "media_input::Value", tags = "1, 2, 3")]
|
||||
pub value: ::core::option::Option<media_input::Value>,
|
||||
}
|
||||
/// Nested message and enum types in `MediaInput`.
|
||||
pub mod media_input {
|
||||
#[derive(Clone, PartialEq, ::prost::Oneof)]
|
||||
pub enum Value {
|
||||
#[prost(message, tag = "1")]
|
||||
Item(super::MediaItem),
|
||||
#[prost(message, tag = "2")]
|
||||
Many(super::MediaItems),
|
||||
#[prost(message, tag = "3")]
|
||||
Nested(super::MediaItemsList),
|
||||
}
|
||||
}
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct HealthCheckRequest {}
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct HealthCheckResponse {
|
||||
#[prost(bool, tag = "1")]
|
||||
pub healthy: bool,
|
||||
}
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct GetModelInfoRequest {}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct GetModelInfoResponse {
|
||||
#[prost(string, tag = "1")]
|
||||
pub model_path: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "2")]
|
||||
pub served_model_name: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "3")]
|
||||
pub tokenizer_path: ::prost::alloc::string::String,
|
||||
#[prost(bool, tag = "4")]
|
||||
pub is_generation: bool,
|
||||
/// The launch-time `--preferred-sampling-params` object, or null when unset.
|
||||
/// JSON carries the object itself (the HTTP `/get_model_info` shape); the
|
||||
/// protobuf field holds it as JSON text.
|
||||
#[prost(string, optional, tag = "5")]
|
||||
pub preferred_sampling_params: ::core::option::Option<::prost::alloc::string::String>,
|
||||
#[prost(string, optional, tag = "6")]
|
||||
pub weight_version: ::core::option::Option<::prost::alloc::string::String>,
|
||||
#[prost(string, optional, tag = "7")]
|
||||
pub load_format: ::core::option::Option<::prost::alloc::string::String>,
|
||||
#[prost(string, optional, tag = "8")]
|
||||
pub reasoning_parser: ::core::option::Option<::prost::alloc::string::String>,
|
||||
#[prost(string, optional, tag = "9")]
|
||||
pub tool_call_parser: ::core::option::Option<::prost::alloc::string::String>,
|
||||
}
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct GetServerInfoRequest {}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct GetServerInfoResponse {
|
||||
#[prost(string, tag = "1")]
|
||||
pub model_path: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "2")]
|
||||
pub served_model_name: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "3")]
|
||||
pub tokenizer_path: ::prost::alloc::string::String,
|
||||
#[prost(uint32, tag = "4")]
|
||||
pub max_context_length: u32,
|
||||
#[prost(uint64, optional, tag = "5")]
|
||||
pub max_total_num_tokens: ::core::option::Option<u64>,
|
||||
#[prost(string, tag = "6")]
|
||||
pub version: ::prost::alloc::string::String,
|
||||
/// The allowlisted runtime metrics (last_gen_throughput, memory_usage,
|
||||
/// effective_max_running_requests_per_dp, avg_spec_accept_length,
|
||||
/// step_time_dict), untyped because the scheduler owns their shapes; JSON
|
||||
/// passes them through verbatim.
|
||||
#[prost(string, repeated, tag = "7")]
|
||||
pub internal_states: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,489 @@
|
||||
/// Generated client implementations.
|
||||
pub mod sglang_service_client {
|
||||
#![allow(
|
||||
unused_variables,
|
||||
dead_code,
|
||||
missing_docs,
|
||||
clippy::wildcard_imports,
|
||||
clippy::let_unit_value
|
||||
)]
|
||||
use tonic::codegen::http::Uri;
|
||||
use tonic::codegen::*;
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SglangServiceClient<T> {
|
||||
inner: tonic::client::Grpc<T>,
|
||||
}
|
||||
impl SglangServiceClient<tonic::transport::Channel> {
|
||||
/// Attempt to create a new client by connecting to a given endpoint.
|
||||
pub async fn connect<D>(dst: D) -> Result<Self, tonic::transport::Error>
|
||||
where
|
||||
D: TryInto<tonic::transport::Endpoint>,
|
||||
D::Error: Into<StdError>,
|
||||
{
|
||||
let conn = tonic::transport::Endpoint::new(dst)?.connect().await?;
|
||||
Ok(Self::new(conn))
|
||||
}
|
||||
}
|
||||
impl<T> SglangServiceClient<T>
|
||||
where
|
||||
T: tonic::client::GrpcService<tonic::body::Body>,
|
||||
T::Error: Into<StdError>,
|
||||
T::ResponseBody: Body<Data = Bytes> + std::marker::Send + 'static,
|
||||
<T::ResponseBody as Body>::Error: Into<StdError> + std::marker::Send,
|
||||
{
|
||||
pub fn new(inner: T) -> Self {
|
||||
let inner = tonic::client::Grpc::new(inner);
|
||||
Self { inner }
|
||||
}
|
||||
pub fn with_origin(inner: T, origin: Uri) -> Self {
|
||||
let inner = tonic::client::Grpc::with_origin(inner, origin);
|
||||
Self { inner }
|
||||
}
|
||||
pub fn with_interceptor<F>(
|
||||
inner: T,
|
||||
interceptor: F,
|
||||
) -> SglangServiceClient<InterceptedService<T, F>>
|
||||
where
|
||||
F: tonic::service::Interceptor,
|
||||
T::ResponseBody: Default,
|
||||
T: tonic::codegen::Service<
|
||||
http::Request<tonic::body::Body>,
|
||||
Response = http::Response<
|
||||
<T as tonic::client::GrpcService<tonic::body::Body>>::ResponseBody,
|
||||
>,
|
||||
>,
|
||||
<T as tonic::codegen::Service<http::Request<tonic::body::Body>>>::Error:
|
||||
Into<StdError> + std::marker::Send + std::marker::Sync,
|
||||
{
|
||||
SglangServiceClient::new(InterceptedService::new(inner, interceptor))
|
||||
}
|
||||
/// Compress requests with the given encoding.
|
||||
///
|
||||
/// This requires the server to support it otherwise it might respond with an
|
||||
/// error.
|
||||
#[must_use]
|
||||
pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self {
|
||||
self.inner = self.inner.send_compressed(encoding);
|
||||
self
|
||||
}
|
||||
/// Enable decompressing responses.
|
||||
#[must_use]
|
||||
pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self {
|
||||
self.inner = self.inner.accept_compressed(encoding);
|
||||
self
|
||||
}
|
||||
/// Limits the maximum size of a decoded message.
|
||||
///
|
||||
/// Default: `4MB`
|
||||
#[must_use]
|
||||
pub fn max_decoding_message_size(mut self, limit: usize) -> Self {
|
||||
self.inner = self.inner.max_decoding_message_size(limit);
|
||||
self
|
||||
}
|
||||
/// Limits the maximum size of an encoded message.
|
||||
///
|
||||
/// Default: `usize::MAX`
|
||||
#[must_use]
|
||||
pub fn max_encoding_message_size(mut self, limit: usize) -> Self {
|
||||
self.inner = self.inner.max_encoding_message_size(limit);
|
||||
self
|
||||
}
|
||||
/// `/generate`: always server-streaming (a unary request is a one-frame
|
||||
/// stream); per-batch-item failures ride the stream as GenerateStreamItem
|
||||
/// errors, pre-submit failures end the RPC with a gRPC status.
|
||||
pub async fn generate(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::GenerateRequest>,
|
||||
) -> std::result::Result<
|
||||
tonic::Response<tonic::codec::Streaming<super::GenerateStreamItem>>,
|
||||
tonic::Status,
|
||||
> {
|
||||
self.inner.ready().await.map_err(|e| {
|
||||
tonic::Status::unknown(format!("Service was not ready: {}", e.into()))
|
||||
})?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path =
|
||||
http::uri::PathAndQuery::from_static("/sglang.api.v1.SglangService/Generate");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("sglang.api.v1.SglangService", "Generate"));
|
||||
self.inner.server_streaming(req, path, codec).await
|
||||
}
|
||||
/// `/health_generate`: fires a 1-token probe and watches the response
|
||||
/// heartbeat. UNAVAILABLE on a stalled pipeline.
|
||||
pub async fn health_check(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::HealthCheckRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::HealthCheckResponse>, tonic::Status>
|
||||
{
|
||||
self.inner.ready().await.map_err(|e| {
|
||||
tonic::Status::unknown(format!("Service was not ready: {}", e.into()))
|
||||
})?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path =
|
||||
http::uri::PathAndQuery::from_static("/sglang.api.v1.SglangService/HealthCheck");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut().insert(GrpcMethod::new(
|
||||
"sglang.api.v1.SglangService",
|
||||
"HealthCheck",
|
||||
));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
/// `/get_model_info`: static model metadata, no scheduler round-trip.
|
||||
pub async fn get_model_info(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::GetModelInfoRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::GetModelInfoResponse>, tonic::Status>
|
||||
{
|
||||
self.inner.ready().await.map_err(|e| {
|
||||
tonic::Status::unknown(format!("Service was not ready: {}", e.into()))
|
||||
})?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path =
|
||||
http::uri::PathAndQuery::from_static("/sglang.api.v1.SglangService/GetModelInfo");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut().insert(GrpcMethod::new(
|
||||
"sglang.api.v1.SglangService",
|
||||
"GetModelInfo",
|
||||
));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
/// `/server_info`: one control round-trip, shaped through the runtime-metric
|
||||
/// allowlist (never the raw server-args dump, which embeds keys).
|
||||
pub async fn get_server_info(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::GetServerInfoRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::GetServerInfoResponse>, tonic::Status>
|
||||
{
|
||||
self.inner.ready().await.map_err(|e| {
|
||||
tonic::Status::unknown(format!("Service was not ready: {}", e.into()))
|
||||
})?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path =
|
||||
http::uri::PathAndQuery::from_static("/sglang.api.v1.SglangService/GetServerInfo");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut().insert(GrpcMethod::new(
|
||||
"sglang.api.v1.SglangService",
|
||||
"GetServerInfo",
|
||||
));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
}
|
||||
}
|
||||
/// Generated server implementations.
|
||||
pub mod sglang_service_server {
|
||||
#![allow(
|
||||
unused_variables,
|
||||
dead_code,
|
||||
missing_docs,
|
||||
clippy::wildcard_imports,
|
||||
clippy::let_unit_value
|
||||
)]
|
||||
use tonic::codegen::*;
|
||||
/// Generated trait containing gRPC methods that should be implemented for use with SglangServiceServer.
|
||||
#[async_trait]
|
||||
pub trait SglangService: std::marker::Send + std::marker::Sync + 'static {
|
||||
/// Server streaming response type for the Generate method.
|
||||
type GenerateStream: tonic::codegen::tokio_stream::Stream<
|
||||
Item = std::result::Result<super::GenerateStreamItem, tonic::Status>,
|
||||
> + std::marker::Send
|
||||
+ 'static;
|
||||
/// `/generate`: always server-streaming (a unary request is a one-frame
|
||||
/// stream); per-batch-item failures ride the stream as GenerateStreamItem
|
||||
/// errors, pre-submit failures end the RPC with a gRPC status.
|
||||
async fn generate(
|
||||
&self,
|
||||
request: tonic::Request<super::GenerateRequest>,
|
||||
) -> std::result::Result<tonic::Response<Self::GenerateStream>, tonic::Status>;
|
||||
/// `/health_generate`: fires a 1-token probe and watches the response
|
||||
/// heartbeat. UNAVAILABLE on a stalled pipeline.
|
||||
async fn health_check(
|
||||
&self,
|
||||
request: tonic::Request<super::HealthCheckRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::HealthCheckResponse>, tonic::Status>;
|
||||
/// `/get_model_info`: static model metadata, no scheduler round-trip.
|
||||
async fn get_model_info(
|
||||
&self,
|
||||
request: tonic::Request<super::GetModelInfoRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::GetModelInfoResponse>, tonic::Status>;
|
||||
/// `/server_info`: one control round-trip, shaped through the runtime-metric
|
||||
/// allowlist (never the raw server-args dump, which embeds keys).
|
||||
async fn get_server_info(
|
||||
&self,
|
||||
request: tonic::Request<super::GetServerInfoRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::GetServerInfoResponse>, tonic::Status>;
|
||||
}
|
||||
#[derive(Debug)]
|
||||
pub struct SglangServiceServer<T> {
|
||||
inner: Arc<T>,
|
||||
accept_compression_encodings: EnabledCompressionEncodings,
|
||||
send_compression_encodings: EnabledCompressionEncodings,
|
||||
max_decoding_message_size: Option<usize>,
|
||||
max_encoding_message_size: Option<usize>,
|
||||
}
|
||||
impl<T> SglangServiceServer<T> {
|
||||
pub fn new(inner: T) -> Self {
|
||||
Self::from_arc(Arc::new(inner))
|
||||
}
|
||||
pub fn from_arc(inner: Arc<T>) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
accept_compression_encodings: Default::default(),
|
||||
send_compression_encodings: Default::default(),
|
||||
max_decoding_message_size: None,
|
||||
max_encoding_message_size: None,
|
||||
}
|
||||
}
|
||||
pub fn with_interceptor<F>(inner: T, interceptor: F) -> InterceptedService<Self, F>
|
||||
where
|
||||
F: tonic::service::Interceptor,
|
||||
{
|
||||
InterceptedService::new(Self::new(inner), interceptor)
|
||||
}
|
||||
/// Enable decompressing requests with the given encoding.
|
||||
#[must_use]
|
||||
pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self {
|
||||
self.accept_compression_encodings.enable(encoding);
|
||||
self
|
||||
}
|
||||
/// Compress responses with the given encoding, if the client supports it.
|
||||
#[must_use]
|
||||
pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self {
|
||||
self.send_compression_encodings.enable(encoding);
|
||||
self
|
||||
}
|
||||
/// Limits the maximum size of a decoded message.
|
||||
///
|
||||
/// Default: `4MB`
|
||||
#[must_use]
|
||||
pub fn max_decoding_message_size(mut self, limit: usize) -> Self {
|
||||
self.max_decoding_message_size = Some(limit);
|
||||
self
|
||||
}
|
||||
/// Limits the maximum size of an encoded message.
|
||||
///
|
||||
/// Default: `usize::MAX`
|
||||
#[must_use]
|
||||
pub fn max_encoding_message_size(mut self, limit: usize) -> Self {
|
||||
self.max_encoding_message_size = Some(limit);
|
||||
self
|
||||
}
|
||||
}
|
||||
impl<T, B> tonic::codegen::Service<http::Request<B>> for SglangServiceServer<T>
|
||||
where
|
||||
T: SglangService,
|
||||
B: Body + std::marker::Send + 'static,
|
||||
B::Error: Into<StdError> + std::marker::Send + 'static,
|
||||
{
|
||||
type Response = http::Response<tonic::body::Body>;
|
||||
type Error = std::convert::Infallible;
|
||||
type Future = BoxFuture<Self::Response, Self::Error>;
|
||||
fn poll_ready(
|
||||
&mut self,
|
||||
_cx: &mut Context<'_>,
|
||||
) -> Poll<std::result::Result<(), Self::Error>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
fn call(&mut self, req: http::Request<B>) -> Self::Future {
|
||||
match req.uri().path() {
|
||||
"/sglang.api.v1.SglangService/Generate" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct GenerateSvc<T: SglangService>(pub Arc<T>);
|
||||
impl<T: SglangService>
|
||||
tonic::server::ServerStreamingService<super::GenerateRequest>
|
||||
for GenerateSvc<T>
|
||||
{
|
||||
type Response = super::GenerateStreamItem;
|
||||
type ResponseStream = T::GenerateStream;
|
||||
type Future =
|
||||
BoxFuture<tonic::Response<Self::ResponseStream>, tonic::Status>;
|
||||
fn call(
|
||||
&mut self,
|
||||
request: tonic::Request<super::GenerateRequest>,
|
||||
) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut = async move {
|
||||
<T as SglangService>::generate(&inner, request).await
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = GenerateSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(
|
||||
accept_compression_encodings,
|
||||
send_compression_encodings,
|
||||
)
|
||||
.apply_max_message_size_config(
|
||||
max_decoding_message_size,
|
||||
max_encoding_message_size,
|
||||
);
|
||||
let res = grpc.server_streaming(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/sglang.api.v1.SglangService/HealthCheck" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct HealthCheckSvc<T: SglangService>(pub Arc<T>);
|
||||
impl<T: SglangService> tonic::server::UnaryService<super::HealthCheckRequest>
|
||||
for HealthCheckSvc<T>
|
||||
{
|
||||
type Response = super::HealthCheckResponse;
|
||||
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
|
||||
fn call(
|
||||
&mut self,
|
||||
request: tonic::Request<super::HealthCheckRequest>,
|
||||
) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut = async move {
|
||||
<T as SglangService>::health_check(&inner, request).await
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = HealthCheckSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(
|
||||
accept_compression_encodings,
|
||||
send_compression_encodings,
|
||||
)
|
||||
.apply_max_message_size_config(
|
||||
max_decoding_message_size,
|
||||
max_encoding_message_size,
|
||||
);
|
||||
let res = grpc.unary(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/sglang.api.v1.SglangService/GetModelInfo" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct GetModelInfoSvc<T: SglangService>(pub Arc<T>);
|
||||
impl<T: SglangService> tonic::server::UnaryService<super::GetModelInfoRequest>
|
||||
for GetModelInfoSvc<T>
|
||||
{
|
||||
type Response = super::GetModelInfoResponse;
|
||||
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
|
||||
fn call(
|
||||
&mut self,
|
||||
request: tonic::Request<super::GetModelInfoRequest>,
|
||||
) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut = async move {
|
||||
<T as SglangService>::get_model_info(&inner, request).await
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = GetModelInfoSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(
|
||||
accept_compression_encodings,
|
||||
send_compression_encodings,
|
||||
)
|
||||
.apply_max_message_size_config(
|
||||
max_decoding_message_size,
|
||||
max_encoding_message_size,
|
||||
);
|
||||
let res = grpc.unary(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/sglang.api.v1.SglangService/GetServerInfo" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct GetServerInfoSvc<T: SglangService>(pub Arc<T>);
|
||||
impl<T: SglangService> tonic::server::UnaryService<super::GetServerInfoRequest>
|
||||
for GetServerInfoSvc<T>
|
||||
{
|
||||
type Response = super::GetServerInfoResponse;
|
||||
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
|
||||
fn call(
|
||||
&mut self,
|
||||
request: tonic::Request<super::GetServerInfoRequest>,
|
||||
) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut = async move {
|
||||
<T as SglangService>::get_server_info(&inner, request).await
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = GetServerInfoSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(
|
||||
accept_compression_encodings,
|
||||
send_compression_encodings,
|
||||
)
|
||||
.apply_max_message_size_config(
|
||||
max_decoding_message_size,
|
||||
max_encoding_message_size,
|
||||
);
|
||||
let res = grpc.unary(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
_ => Box::pin(async move {
|
||||
let mut response = http::Response::new(tonic::body::Body::default());
|
||||
let headers = response.headers_mut();
|
||||
headers.insert(
|
||||
tonic::Status::GRPC_STATUS,
|
||||
(tonic::Code::Unimplemented as i32).into(),
|
||||
);
|
||||
headers.insert(
|
||||
http::header::CONTENT_TYPE,
|
||||
tonic::metadata::GRPC_CONTENT_TYPE,
|
||||
);
|
||||
Ok(response)
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
impl<T> Clone for SglangServiceServer<T> {
|
||||
fn clone(&self) -> Self {
|
||||
let inner = self.inner.clone();
|
||||
Self {
|
||||
inner,
|
||||
accept_compression_encodings: self.accept_compression_encodings,
|
||||
send_compression_encodings: self.send_compression_encodings,
|
||||
max_decoding_message_size: self.max_decoding_message_size,
|
||||
max_encoding_message_size: self.max_encoding_message_size,
|
||||
}
|
||||
}
|
||||
}
|
||||
/// Generated gRPC service name
|
||||
pub const SERVICE_NAME: &str = "sglang.api.v1.SglangService";
|
||||
impl<T> tonic::server::NamedService for SglangServiceServer<T> {
|
||||
const NAME: &'static str = SERVICE_NAME;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,457 @@
|
||||
// This file is @generated by prost-build.
|
||||
/// Disaggregated prefill/decode rendezvous parameters.
|
||||
///
|
||||
/// Carried on Generate/TextGenerate requests when the worker is part of a
|
||||
/// disaggregated deployment. The prefill leg writes its KV cache to the
|
||||
/// bootstrap server identified by (bootstrap_host, bootstrap_port) under
|
||||
/// the room id; the decode leg fetches it from the same room. Both legs
|
||||
/// receive the same `bootstrap_room` from the router so they rendezvous
|
||||
/// on the KV transfer.
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct DisaggregatedParams {
|
||||
#[prost(string, tag = "1")]
|
||||
pub bootstrap_host: ::prost::alloc::string::String,
|
||||
#[prost(int32, tag = "2")]
|
||||
pub bootstrap_port: i32,
|
||||
#[prost(int64, tag = "3")]
|
||||
pub bootstrap_room: i64,
|
||||
}
|
||||
/// Sampling parameters shared across text and tokenized RPCs.
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct SamplingParams {
|
||||
#[prost(float, optional, tag = "1")]
|
||||
pub temperature: ::core::option::Option<f32>,
|
||||
#[prost(float, optional, tag = "2")]
|
||||
pub top_p: ::core::option::Option<f32>,
|
||||
#[prost(int32, optional, tag = "3")]
|
||||
pub top_k: ::core::option::Option<i32>,
|
||||
#[prost(float, optional, tag = "4")]
|
||||
pub min_p: ::core::option::Option<f32>,
|
||||
#[prost(float, optional, tag = "5")]
|
||||
pub frequency_penalty: ::core::option::Option<f32>,
|
||||
#[prost(float, optional, tag = "6")]
|
||||
pub presence_penalty: ::core::option::Option<f32>,
|
||||
#[prost(float, optional, tag = "7")]
|
||||
pub repetition_penalty: ::core::option::Option<f32>,
|
||||
#[prost(int32, optional, tag = "8")]
|
||||
pub max_new_tokens: ::core::option::Option<i32>,
|
||||
#[prost(int32, optional, tag = "9")]
|
||||
pub min_new_tokens: ::core::option::Option<i32>,
|
||||
#[prost(string, repeated, tag = "10")]
|
||||
pub stop: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
|
||||
#[prost(int32, repeated, tag = "11")]
|
||||
pub stop_token_ids: ::prost::alloc::vec::Vec<i32>,
|
||||
#[prost(bool, optional, tag = "12")]
|
||||
pub ignore_eos: ::core::option::Option<bool>,
|
||||
#[prost(int32, optional, tag = "13")]
|
||||
pub n: ::core::option::Option<i32>,
|
||||
#[deprecated]
|
||||
#[prost(string, optional, tag = "14")]
|
||||
pub json_schema: ::core::option::Option<::prost::alloc::string::String>,
|
||||
#[deprecated]
|
||||
#[prost(string, optional, tag = "15")]
|
||||
pub regex: ::core::option::Option<::prost::alloc::string::String>,
|
||||
#[prost(int64, optional, tag = "16")]
|
||||
pub seed: ::core::option::Option<i64>,
|
||||
#[prost(message, optional, tag = "17")]
|
||||
pub guided_decoding: ::core::option::Option<GuidedDecoding>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct GuidedDecoding {
|
||||
#[prost(oneof = "guided_decoding::Constraint", tags = "1, 2, 3, 4, 5")]
|
||||
pub constraint: ::core::option::Option<guided_decoding::Constraint>,
|
||||
}
|
||||
/// Nested message and enum types in `GuidedDecoding`.
|
||||
pub mod guided_decoding {
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)]
|
||||
pub enum Constraint {
|
||||
#[prost(string, tag = "1")]
|
||||
JsonSchema(::prost::alloc::string::String),
|
||||
#[prost(string, tag = "2")]
|
||||
Regex(::prost::alloc::string::String),
|
||||
#[prost(string, tag = "3")]
|
||||
Ebnf(::prost::alloc::string::String),
|
||||
#[prost(message, tag = "4")]
|
||||
Choice(super::ChoiceConstraint),
|
||||
#[prost(string, tag = "5")]
|
||||
StructuralTag(::prost::alloc::string::String),
|
||||
}
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct ChoiceConstraint {
|
||||
#[prost(string, repeated, tag = "1")]
|
||||
pub values: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct TextGenerateRequest {
|
||||
#[prost(string, tag = "1")]
|
||||
pub text: ::prost::alloc::string::String,
|
||||
#[prost(message, optional, tag = "2")]
|
||||
pub sampling_params: ::core::option::Option<SamplingParams>,
|
||||
#[prost(bool, optional, tag = "3")]
|
||||
pub stream: ::core::option::Option<bool>,
|
||||
#[prost(bool, optional, tag = "4")]
|
||||
pub return_logprob: ::core::option::Option<bool>,
|
||||
#[prost(int32, optional, tag = "5")]
|
||||
pub top_logprobs_num: ::core::option::Option<i32>,
|
||||
#[prost(int32, optional, tag = "6")]
|
||||
pub logprob_start_len: ::core::option::Option<i32>,
|
||||
#[prost(bool, optional, tag = "7")]
|
||||
pub return_text_in_logprobs: ::core::option::Option<bool>,
|
||||
#[prost(string, optional, tag = "8")]
|
||||
pub rid: ::core::option::Option<::prost::alloc::string::String>,
|
||||
#[prost(string, optional, tag = "9")]
|
||||
pub lora_path: ::core::option::Option<::prost::alloc::string::String>,
|
||||
#[prost(string, optional, tag = "10")]
|
||||
pub routing_key: ::core::option::Option<::prost::alloc::string::String>,
|
||||
#[prost(int32, optional, tag = "11")]
|
||||
pub routed_dp_rank: ::core::option::Option<i32>,
|
||||
#[prost(map = "string, string", tag = "12")]
|
||||
pub trace_headers:
|
||||
::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>,
|
||||
#[prost(string, optional, tag = "13")]
|
||||
pub session_id: ::core::option::Option<::prost::alloc::string::String>,
|
||||
#[prost(message, optional, tag = "14")]
|
||||
pub disaggregated_params: ::core::option::Option<DisaggregatedParams>,
|
||||
#[prost(int32, optional, tag = "15")]
|
||||
pub priority: ::core::option::Option<i32>,
|
||||
#[prost(bool, optional, tag = "16")]
|
||||
pub require_reasoning: ::core::option::Option<bool>,
|
||||
#[prost(uint32, optional, tag = "17")]
|
||||
pub max_thinking_tokens: ::core::option::Option<u32>,
|
||||
#[prost(message, optional, tag = "18")]
|
||||
pub kv_hints: ::core::option::Option<KvHintsEnvelope>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct TextGenerateResponse {
|
||||
#[prost(string, tag = "1")]
|
||||
pub text: ::prost::alloc::string::String,
|
||||
#[prost(map = "string, string", tag = "2")]
|
||||
pub meta_info:
|
||||
::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>,
|
||||
#[prost(bool, tag = "3")]
|
||||
pub finished: bool,
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct GenerateRequest {
|
||||
#[prost(int32, repeated, tag = "1")]
|
||||
pub input_ids: ::prost::alloc::vec::Vec<i32>,
|
||||
#[prost(message, optional, tag = "2")]
|
||||
pub sampling_params: ::core::option::Option<SamplingParams>,
|
||||
#[prost(bool, optional, tag = "3")]
|
||||
pub stream: ::core::option::Option<bool>,
|
||||
#[prost(bool, optional, tag = "4")]
|
||||
pub return_logprob: ::core::option::Option<bool>,
|
||||
#[prost(int32, optional, tag = "5")]
|
||||
pub top_logprobs_num: ::core::option::Option<i32>,
|
||||
#[prost(int32, optional, tag = "6")]
|
||||
pub logprob_start_len: ::core::option::Option<i32>,
|
||||
#[prost(string, optional, tag = "7")]
|
||||
pub rid: ::core::option::Option<::prost::alloc::string::String>,
|
||||
#[prost(string, optional, tag = "8")]
|
||||
pub lora_path: ::core::option::Option<::prost::alloc::string::String>,
|
||||
#[prost(string, optional, tag = "9")]
|
||||
pub routing_key: ::core::option::Option<::prost::alloc::string::String>,
|
||||
#[prost(int32, optional, tag = "10")]
|
||||
pub routed_dp_rank: ::core::option::Option<i32>,
|
||||
#[prost(map = "string, string", tag = "11")]
|
||||
pub trace_headers:
|
||||
::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>,
|
||||
#[prost(string, optional, tag = "12")]
|
||||
pub session_id: ::core::option::Option<::prost::alloc::string::String>,
|
||||
#[prost(message, optional, tag = "13")]
|
||||
pub disaggregated_params: ::core::option::Option<DisaggregatedParams>,
|
||||
#[prost(int32, optional, tag = "14")]
|
||||
pub priority: ::core::option::Option<i32>,
|
||||
#[prost(bool, optional, tag = "15")]
|
||||
pub require_reasoning: ::core::option::Option<bool>,
|
||||
#[prost(uint32, optional, tag = "16")]
|
||||
pub max_thinking_tokens: ::core::option::Option<u32>,
|
||||
#[prost(message, optional, tag = "17")]
|
||||
pub kv_hints: ::core::option::Option<KvHintsEnvelope>,
|
||||
}
|
||||
/// Versioned KV-hint envelope forwarded from a trusted orchestrator.
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct KvHintsEnvelope {
|
||||
#[prost(string, tag = "1")]
|
||||
pub protocol_version: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "2")]
|
||||
pub message_id: ::prost::alloc::string::String,
|
||||
#[prost(message, repeated, tag = "3")]
|
||||
pub actions: ::prost::alloc::vec::Vec<KvHintAction>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct KvHintAction {
|
||||
#[prost(string, tag = "1")]
|
||||
pub action_id: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "2")]
|
||||
pub action_type: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "3")]
|
||||
pub action_version: ::prost::alloc::string::String,
|
||||
/// Action-specific; validated by the component implementing action_type.
|
||||
#[prost(message, optional, tag = "4")]
|
||||
pub payload: ::core::option::Option<::prost_types::Struct>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct GenerateResponse {
|
||||
#[prost(int32, repeated, tag = "1")]
|
||||
pub output_ids: ::prost::alloc::vec::Vec<i32>,
|
||||
#[prost(map = "string, string", tag = "2")]
|
||||
pub meta_info:
|
||||
::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>,
|
||||
#[prost(bool, tag = "3")]
|
||||
pub finished: bool,
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct TextEmbedRequest {
|
||||
#[prost(string, tag = "1")]
|
||||
pub text: ::prost::alloc::string::String,
|
||||
#[prost(string, optional, tag = "2")]
|
||||
pub rid: ::core::option::Option<::prost::alloc::string::String>,
|
||||
#[prost(string, optional, tag = "3")]
|
||||
pub routing_key: ::core::option::Option<::prost::alloc::string::String>,
|
||||
#[prost(map = "string, string", tag = "4")]
|
||||
pub trace_headers:
|
||||
::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct TextEmbedResponse {
|
||||
#[prost(float, repeated, tag = "1")]
|
||||
pub embedding: ::prost::alloc::vec::Vec<f32>,
|
||||
#[prost(map = "string, string", tag = "2")]
|
||||
pub meta_info:
|
||||
::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct EmbedRequest {
|
||||
#[prost(int32, repeated, tag = "1")]
|
||||
pub input_ids: ::prost::alloc::vec::Vec<i32>,
|
||||
#[prost(string, optional, tag = "2")]
|
||||
pub rid: ::core::option::Option<::prost::alloc::string::String>,
|
||||
#[prost(string, optional, tag = "3")]
|
||||
pub routing_key: ::core::option::Option<::prost::alloc::string::String>,
|
||||
#[prost(map = "string, string", tag = "4")]
|
||||
pub trace_headers:
|
||||
::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct EmbedResponse {
|
||||
#[prost(float, repeated, tag = "1")]
|
||||
pub embedding: ::prost::alloc::vec::Vec<f32>,
|
||||
#[prost(map = "string, string", tag = "2")]
|
||||
pub meta_info:
|
||||
::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>,
|
||||
}
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct HealthCheckRequest {}
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct HealthCheckResponse {
|
||||
#[prost(bool, tag = "1")]
|
||||
pub healthy: bool,
|
||||
}
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct WatchEngineStateRequest {}
|
||||
/// A complete discovery and lifecycle snapshot. The JSON discovery payloads
|
||||
/// intentionally retain the same wire shape as GetModelInfo/GetServerInfo.
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct EngineStateSnapshot {
|
||||
/// Unix time in nanoseconds, captured once for this engine process.
|
||||
#[prost(uint64, tag = "1")]
|
||||
pub instance_id: u64,
|
||||
/// Starts at one and increases for each snapshot from this instance.
|
||||
#[prost(uint64, tag = "2")]
|
||||
pub revision: u64,
|
||||
#[prost(bool, tag = "3")]
|
||||
pub healthy: bool,
|
||||
#[prost(bool, tag = "4")]
|
||||
pub is_pause: bool,
|
||||
#[prost(message, optional, tag = "5")]
|
||||
pub model_info: ::core::option::Option<GetModelInfoResponse>,
|
||||
#[prost(message, optional, tag = "6")]
|
||||
pub server_info: ::core::option::Option<GetServerInfoResponse>,
|
||||
}
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct GetModelInfoRequest {}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct GetModelInfoResponse {
|
||||
#[prost(string, tag = "1")]
|
||||
pub model_path: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "2")]
|
||||
pub json_info: ::prost::alloc::string::String,
|
||||
}
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct GetServerInfoRequest {}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct GetServerInfoResponse {
|
||||
#[prost(string, tag = "1")]
|
||||
pub json_info: ::prost::alloc::string::String,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct AbortRequest {
|
||||
#[prost(string, tag = "1")]
|
||||
pub rid: ::prost::alloc::string::String,
|
||||
#[prost(bool, tag = "2")]
|
||||
pub abort_all: bool,
|
||||
}
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct AbortResponse {
|
||||
#[prost(bool, tag = "1")]
|
||||
pub success: bool,
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct ClassifyRequest {
|
||||
#[prost(string, tag = "1")]
|
||||
pub text: ::prost::alloc::string::String,
|
||||
#[prost(int32, repeated, tag = "2")]
|
||||
pub input_ids: ::prost::alloc::vec::Vec<i32>,
|
||||
#[prost(string, optional, tag = "3")]
|
||||
pub rid: ::core::option::Option<::prost::alloc::string::String>,
|
||||
#[prost(string, optional, tag = "4")]
|
||||
pub routing_key: ::core::option::Option<::prost::alloc::string::String>,
|
||||
#[prost(map = "string, string", tag = "5")]
|
||||
pub trace_headers:
|
||||
::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct ClassifyResponse {
|
||||
#[prost(float, repeated, tag = "1")]
|
||||
pub embedding: ::prost::alloc::vec::Vec<f32>,
|
||||
#[prost(map = "string, string", tag = "2")]
|
||||
pub meta_info:
|
||||
::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct TokenizeRequest {
|
||||
#[prost(string, tag = "1")]
|
||||
pub text: ::prost::alloc::string::String,
|
||||
#[prost(bool, optional, tag = "2")]
|
||||
pub add_special_tokens: ::core::option::Option<bool>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct TokenizeResponse {
|
||||
#[prost(int32, repeated, tag = "1")]
|
||||
pub tokens: ::prost::alloc::vec::Vec<i32>,
|
||||
#[prost(int32, tag = "2")]
|
||||
pub count: i32,
|
||||
#[prost(int32, tag = "3")]
|
||||
pub max_model_len: i32,
|
||||
#[prost(string, tag = "4")]
|
||||
pub input_text: ::prost::alloc::string::String,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct DetokenizeRequest {
|
||||
#[prost(int32, repeated, tag = "1")]
|
||||
pub tokens: ::prost::alloc::vec::Vec<i32>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct DetokenizeResponse {
|
||||
#[prost(string, tag = "1")]
|
||||
pub text: ::prost::alloc::string::String,
|
||||
}
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct ListModelsRequest {}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct ListModelsResponse {
|
||||
#[prost(message, repeated, tag = "1")]
|
||||
pub models: ::prost::alloc::vec::Vec<ModelCard>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct ModelCard {
|
||||
#[prost(string, tag = "1")]
|
||||
pub id: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "2")]
|
||||
pub root: ::prost::alloc::string::String,
|
||||
#[prost(string, optional, tag = "3")]
|
||||
pub parent: ::core::option::Option<::prost::alloc::string::String>,
|
||||
#[prost(int32, optional, tag = "4")]
|
||||
pub max_model_len: ::core::option::Option<i32>,
|
||||
}
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct GetLoadRequest {
|
||||
#[prost(int32, optional, tag = "1")]
|
||||
pub dp_rank: ::core::option::Option<i32>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct GetLoadResponse {
|
||||
#[prost(string, tag = "1")]
|
||||
pub json_info: ::prost::alloc::string::String,
|
||||
}
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct FlushCacheRequest {}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct FlushCacheResponse {
|
||||
#[prost(bool, tag = "1")]
|
||||
pub success: bool,
|
||||
#[prost(string, tag = "2")]
|
||||
pub message: ::prost::alloc::string::String,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct PauseGenerationRequest {
|
||||
#[prost(string, tag = "1")]
|
||||
pub mode: ::prost::alloc::string::String,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct PauseGenerationResponse {
|
||||
#[prost(string, tag = "1")]
|
||||
pub message: ::prost::alloc::string::String,
|
||||
}
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct ContinueGenerationRequest {}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct ContinueGenerationResponse {
|
||||
#[prost(string, tag = "1")]
|
||||
pub message: ::prost::alloc::string::String,
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct OpenAiRequest {
|
||||
#[prost(bytes = "vec", tag = "1")]
|
||||
pub json_body: ::prost::alloc::vec::Vec<u8>,
|
||||
#[prost(map = "string, string", tag = "2")]
|
||||
pub trace_headers:
|
||||
::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct OpenAiStreamChunk {
|
||||
#[prost(bytes = "vec", tag = "1")]
|
||||
pub json_chunk: ::prost::alloc::vec::Vec<u8>,
|
||||
#[prost(bool, tag = "2")]
|
||||
pub finished: bool,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct OpenAiResponse {
|
||||
#[prost(bytes = "vec", tag = "1")]
|
||||
pub json_body: ::prost::alloc::vec::Vec<u8>,
|
||||
#[prost(int32, tag = "2")]
|
||||
pub status_code: i32,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct StartProfileRequest {
|
||||
#[prost(string, optional, tag = "1")]
|
||||
pub output_dir: ::core::option::Option<::prost::alloc::string::String>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct StartProfileResponse {
|
||||
#[prost(string, tag = "1")]
|
||||
pub message: ::prost::alloc::string::String,
|
||||
}
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct StopProfileRequest {}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct StopProfileResponse {
|
||||
#[prost(string, tag = "1")]
|
||||
pub message: ::prost::alloc::string::String,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct UpdateWeightsRequest {
|
||||
#[prost(string, tag = "1")]
|
||||
pub model_path: ::prost::alloc::string::String,
|
||||
#[prost(string, optional, tag = "2")]
|
||||
pub load_format: ::core::option::Option<::prost::alloc::string::String>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct UpdateWeightsResponse {
|
||||
#[prost(bool, tag = "1")]
|
||||
pub success: bool,
|
||||
#[prost(string, tag = "2")]
|
||||
pub message: ::prost::alloc::string::String,
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,28 @@
|
||||
//! Generated SGLang API types. `src/generated/` is written by
|
||||
//! `cargo gen-api` from `proto/sglang/` -- the schema is the ground truth;
|
||||
//! edit the .proto files and regenerate, never these files, and commit the
|
||||
//! regenerated output with the .proto change.
|
||||
//!
|
||||
//! Module paths mirror the proto packages: `api::v1` is `sglang.api.v1` (the
|
||||
//! HTTP/gRPC contract, with schema-driven JSON serde), `runtime::v1` is
|
||||
//! `sglang.runtime.v1` (the gRPC bridge's protocol types, prost + tonic only).
|
||||
|
||||
mod ext;
|
||||
|
||||
pub mod api {
|
||||
// Generated code optimizes for template uniformity, not lint cleanliness.
|
||||
#[allow(clippy::all)]
|
||||
pub mod v1 {
|
||||
include!("generated/sglang.api.v1.rs");
|
||||
include!("generated/sglang.api.v1.serde.rs");
|
||||
include!("generated/sglang.api.v1.tonic.rs");
|
||||
}
|
||||
}
|
||||
|
||||
pub mod runtime {
|
||||
#[allow(clippy::all)]
|
||||
pub mod v1 {
|
||||
include!("generated/sglang.runtime.v1.rs");
|
||||
include!("generated/sglang.runtime.v1.tonic.rs");
|
||||
}
|
||||
}
|
||||
+373
@@ -0,0 +1,373 @@
|
||||
{
|
||||
"_": "Cross-language JSON parity fixtures for sglang.api.v1. Each case: decoding `input` and re-encoding yields `canonical` byte for byte, and `canonical` re-encodes to itself. Checked by rust/sglang-api-types/tests/json_parity.rs and test/registered/unit/api/v1/test_parity.py.",
|
||||
"cases": [
|
||||
{
|
||||
"type": "GenerateRequest",
|
||||
"name": "single text prompt with a sampling object; stream absent",
|
||||
"input": {
|
||||
"text": "hi",
|
||||
"sampling_params": {
|
||||
"temperature": 0.5,
|
||||
"max_new_tokens": 8
|
||||
}
|
||||
},
|
||||
"canonical": "{\"text\":\"hi\",\"stream\":false,\"sampling_params\":{\"max_new_tokens\":8,\"temperature\":0.5,\"top_p\":1.0,\"top_k\":1073741824,\"min_p\":0.0,\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"repetition_penalty\":1.0,\"min_new_tokens\":0,\"n\":1,\"ignore_eos\":false,\"skip_special_tokens\":true,\"spaces_between_special_tokens\":true,\"no_stop_trim\":false}}"
|
||||
},
|
||||
{
|
||||
"type": "GenerateRequest",
|
||||
"name": "batch form: per-item sampling, broadcast flag, nullable PD columns",
|
||||
"input": {
|
||||
"text": [
|
||||
"a",
|
||||
"b"
|
||||
],
|
||||
"stream": true,
|
||||
"sampling_params": [
|
||||
{
|
||||
"top_p": 0.9
|
||||
},
|
||||
{
|
||||
"temperature": 0.5
|
||||
}
|
||||
],
|
||||
"return_logprob": true,
|
||||
"bootstrap_host": [
|
||||
null,
|
||||
"h"
|
||||
],
|
||||
"bootstrap_port": [
|
||||
8998,
|
||||
null
|
||||
]
|
||||
},
|
||||
"canonical": "{\"text\":[\"a\",\"b\"],\"stream\":true,\"sampling_params\":[{\"max_new_tokens\":128,\"temperature\":1.0,\"top_p\":0.9,\"top_k\":1073741824,\"min_p\":0.0,\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"repetition_penalty\":1.0,\"min_new_tokens\":0,\"n\":1,\"ignore_eos\":false,\"skip_special_tokens\":true,\"spaces_between_special_tokens\":true,\"no_stop_trim\":false},{\"max_new_tokens\":128,\"temperature\":0.5,\"top_p\":1.0,\"top_k\":1073741824,\"min_p\":0.0,\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"repetition_penalty\":1.0,\"min_new_tokens\":0,\"n\":1,\"ignore_eos\":false,\"skip_special_tokens\":true,\"spaces_between_special_tokens\":true,\"no_stop_trim\":false}],\"return_logprob\":true,\"bootstrap_host\":[null,\"h\"],\"bootstrap_port\":[8998,null]}"
|
||||
},
|
||||
{
|
||||
"type": "GenerateRequest",
|
||||
"name": "token prompt with every logprob knob; stream null resets to false",
|
||||
"input": {
|
||||
"input_ids": [
|
||||
1,
|
||||
2,
|
||||
3
|
||||
],
|
||||
"rid": "r-1",
|
||||
"stream": null,
|
||||
"return_logprob": true,
|
||||
"logprob_start_len": 0,
|
||||
"top_logprobs_num": 2,
|
||||
"token_ids_logprob": [
|
||||
5,
|
||||
6
|
||||
],
|
||||
"return_text_in_logprobs": true,
|
||||
"return_hidden_states": true,
|
||||
"routed_dp_rank": 1
|
||||
},
|
||||
"canonical": "{\"rid\":\"r-1\",\"input_ids\":[1,2,3],\"stream\":false,\"return_logprob\":true,\"logprob_start_len\":0,\"top_logprobs_num\":2,\"token_ids_logprob\":[5,6],\"return_hidden_states\":true,\"return_text_in_logprobs\":true,\"routed_dp_rank\":1}"
|
||||
},
|
||||
{
|
||||
"type": "GenerateRequest",
|
||||
"name": "media containers: nested, ref object, bare source; unknown keys ignored",
|
||||
"input": {
|
||||
"text": "see",
|
||||
"image_data": [
|
||||
[
|
||||
"a",
|
||||
null
|
||||
],
|
||||
null,
|
||||
[]
|
||||
],
|
||||
"video_data": {
|
||||
"url": "v",
|
||||
"fps": 2.0,
|
||||
"use_audio": true
|
||||
},
|
||||
"audio_data": "data:audio/wav;base64,AA==",
|
||||
"mm_hashes": [
|
||||
"h1",
|
||||
"h2"
|
||||
],
|
||||
"priority": 3,
|
||||
"session_id": "ignored"
|
||||
},
|
||||
"canonical": "{\"text\":\"see\",\"stream\":false,\"image_data\":[[\"a\",null],null,[]],\"mm_hashes\":[\"h1\",\"h2\"],\"video_data\":{\"url\":\"v\",\"fps\":2.0,\"use_audio\":true},\"audio_data\":\"data:audio/wav;base64,AA==\"}"
|
||||
},
|
||||
{
|
||||
"type": "SamplingParams",
|
||||
"name": "empty object: every schema default is emitted",
|
||||
"input": {},
|
||||
"canonical": "{\"max_new_tokens\":128,\"temperature\":1.0,\"top_p\":1.0,\"top_k\":1073741824,\"min_p\":0.0,\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"repetition_penalty\":1.0,\"min_new_tokens\":0,\"n\":1,\"ignore_eos\":false,\"skip_special_tokens\":true,\"spaces_between_special_tokens\":true,\"no_stop_trim\":false}"
|
||||
},
|
||||
{
|
||||
"type": "SamplingParams",
|
||||
"name": "explicit null resets a default; lists, maps and raw custom_params round-trip",
|
||||
"input": {
|
||||
"temperature": null,
|
||||
"stop": [
|
||||
"a",
|
||||
"b"
|
||||
],
|
||||
"stop_token_ids": [
|
||||
1,
|
||||
2
|
||||
],
|
||||
"logit_bias": {
|
||||
"7": -1.5
|
||||
},
|
||||
"custom_params": {
|
||||
"k": [
|
||||
1,
|
||||
"x",
|
||||
null
|
||||
]
|
||||
},
|
||||
"json_schema": "{}",
|
||||
"beam_width": 2
|
||||
},
|
||||
"canonical": "{\"max_new_tokens\":128,\"stop\":[\"a\",\"b\"],\"stop_token_ids\":[1,2],\"temperature\":1.0,\"top_p\":1.0,\"top_k\":1073741824,\"min_p\":0.0,\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"repetition_penalty\":1.0,\"min_new_tokens\":0,\"n\":1,\"json_schema\":\"{}\",\"ignore_eos\":false,\"skip_special_tokens\":true,\"spaces_between_special_tokens\":true,\"no_stop_trim\":false,\"logit_bias\":{\"7\":-1.5},\"custom_params\":{\"k\":[1,\"x\",null]},\"beam_width\":2}"
|
||||
},
|
||||
{
|
||||
"type": "GenerateResponse",
|
||||
"name": "plain frame: finish_reason null pre-terminal, output_ids present",
|
||||
"input": {
|
||||
"text": "ok",
|
||||
"meta_info": {
|
||||
"id": "client-rid",
|
||||
"prompt_tokens": 5,
|
||||
"completion_tokens": 2,
|
||||
"finish_reason": null
|
||||
},
|
||||
"output_ids": [
|
||||
7,
|
||||
8
|
||||
]
|
||||
},
|
||||
"canonical": "{\"text\":\"ok\",\"meta_info\":{\"id\":\"client-rid\",\"prompt_tokens\":5,\"completion_tokens\":2,\"finish_reason\":null},\"output_ids\":[7,8]}"
|
||||
},
|
||||
{
|
||||
"type": "GenerateResponse",
|
||||
"name": "terminal batch frame with every logprob family, null rows, hidden states, e2e_latency, index",
|
||||
"input": {
|
||||
"text": "ok",
|
||||
"meta_info": {
|
||||
"id": "r",
|
||||
"prompt_tokens": 1,
|
||||
"completion_tokens": 1,
|
||||
"finish_reason": {
|
||||
"type": "stop",
|
||||
"matched": [
|
||||
1,
|
||||
2
|
||||
]
|
||||
},
|
||||
"output_token_logprobs": [
|
||||
[
|
||||
-0.5,
|
||||
5,
|
||||
"x"
|
||||
],
|
||||
[
|
||||
null,
|
||||
6,
|
||||
null
|
||||
]
|
||||
],
|
||||
"input_token_logprobs": [],
|
||||
"output_top_logprobs": [
|
||||
null,
|
||||
[
|
||||
[
|
||||
-0.25,
|
||||
6,
|
||||
null
|
||||
]
|
||||
]
|
||||
],
|
||||
"input_top_logprobs": [
|
||||
[
|
||||
[
|
||||
-0.125,
|
||||
1,
|
||||
"<s>"
|
||||
]
|
||||
]
|
||||
],
|
||||
"output_token_ids_logprobs": [
|
||||
[
|
||||
[
|
||||
-0.5,
|
||||
9,
|
||||
null
|
||||
]
|
||||
]
|
||||
],
|
||||
"input_token_ids_logprobs": [
|
||||
null
|
||||
],
|
||||
"hidden_states": [
|
||||
[
|
||||
0.5,
|
||||
0.25
|
||||
],
|
||||
[]
|
||||
],
|
||||
"e2e_latency": 0.01
|
||||
},
|
||||
"output_ids": [
|
||||
5
|
||||
],
|
||||
"index": 0
|
||||
},
|
||||
"canonical": "{\"text\":\"ok\",\"meta_info\":{\"id\":\"r\",\"prompt_tokens\":1,\"completion_tokens\":1,\"finish_reason\":{\"type\":\"stop\",\"matched\":[1,2]},\"output_token_logprobs\":[[-0.5,5,\"x\"],[null,6,null]],\"input_token_logprobs\":[],\"output_top_logprobs\":[null,[[-0.25,6,null]]],\"input_top_logprobs\":[[[-0.125,1,\"<s>\"]]],\"output_token_ids_logprobs\":[[[-0.5,9,null]]],\"input_token_ids_logprobs\":[null],\"hidden_states\":[[0.5,0.25],[]],\"e2e_latency\":0.01},\"output_ids\":[5],\"index\":0}"
|
||||
},
|
||||
{
|
||||
"type": "GenerateResponse",
|
||||
"name": "finish length",
|
||||
"input": {
|
||||
"text": "",
|
||||
"meta_info": {
|
||||
"id": "r",
|
||||
"prompt_tokens": 1,
|
||||
"completion_tokens": 9,
|
||||
"finish_reason": {
|
||||
"type": "length",
|
||||
"length": 9
|
||||
}
|
||||
}
|
||||
},
|
||||
"canonical": "{\"text\":\"\",\"meta_info\":{\"id\":\"r\",\"prompt_tokens\":1,\"completion_tokens\":9,\"finish_reason\":{\"type\":\"length\",\"length\":9}}}"
|
||||
},
|
||||
{
|
||||
"type": "GenerateResponse",
|
||||
"name": "finish abort emits its null keys",
|
||||
"input": {
|
||||
"text": "",
|
||||
"meta_info": {
|
||||
"id": "r",
|
||||
"prompt_tokens": 1,
|
||||
"completion_tokens": 0,
|
||||
"finish_reason": {
|
||||
"type": "abort",
|
||||
"message": "over the limit",
|
||||
"status_code": 400,
|
||||
"err_type": null
|
||||
}
|
||||
}
|
||||
},
|
||||
"canonical": "{\"text\":\"\",\"meta_info\":{\"id\":\"r\",\"prompt_tokens\":1,\"completion_tokens\":0,\"finish_reason\":{\"type\":\"abort\",\"message\":\"over the limit\",\"status_code\":400,\"err_type\":null}}}"
|
||||
},
|
||||
{
|
||||
"type": "GenerateResponse",
|
||||
"name": "unknown finish type passes through unchanged",
|
||||
"input": {
|
||||
"text": "",
|
||||
"meta_info": {
|
||||
"id": "r",
|
||||
"prompt_tokens": 1,
|
||||
"completion_tokens": 0,
|
||||
"finish_reason": {
|
||||
"detail": 3,
|
||||
"type": "novel"
|
||||
}
|
||||
}
|
||||
},
|
||||
"canonical": "{\"text\":\"\",\"meta_info\":{\"id\":\"r\",\"prompt_tokens\":1,\"completion_tokens\":0,\"finish_reason\":{\"detail\":3,\"type\":\"novel\"}}}"
|
||||
},
|
||||
{
|
||||
"type": "GenerateStreamError",
|
||||
"name": "in-stream error item with batch index",
|
||||
"input": {
|
||||
"error": {
|
||||
"message": "bad sampling",
|
||||
"code": 400
|
||||
},
|
||||
"index": 1
|
||||
},
|
||||
"canonical": "{\"error\":{\"message\":\"bad sampling\",\"code\":400},\"index\":1}"
|
||||
},
|
||||
{
|
||||
"type": "GetModelInfoResponse",
|
||||
"name": "absent optionals are emitted as null",
|
||||
"input": {
|
||||
"model_path": "/m",
|
||||
"served_model_name": "m",
|
||||
"tokenizer_path": "/t",
|
||||
"is_generation": true,
|
||||
"weight_version": "v1"
|
||||
},
|
||||
"canonical": "{\"model_path\":\"/m\",\"served_model_name\":\"m\",\"tokenizer_path\":\"/t\",\"is_generation\":true,\"preferred_sampling_params\":null,\"weight_version\":\"v1\",\"load_format\":null,\"reasoning_parser\":null,\"tool_call_parser\":null}"
|
||||
},
|
||||
{
|
||||
"type": "GetServerInfoResponse",
|
||||
"name": "internal_states is a list of raw JSON documents",
|
||||
"input": {
|
||||
"model_path": "/m",
|
||||
"served_model_name": "m",
|
||||
"tokenizer_path": "/t",
|
||||
"max_context_length": 4096,
|
||||
"max_total_num_tokens": 8192,
|
||||
"version": "0.5.0",
|
||||
"internal_states": [
|
||||
{
|
||||
"last_gen_throughput": 1.5,
|
||||
"memory_usage": {
|
||||
"weight": 2.0
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"canonical": "{\"model_path\":\"/m\",\"served_model_name\":\"m\",\"tokenizer_path\":\"/t\",\"max_context_length\":4096,\"max_total_num_tokens\":8192,\"version\":\"0.5.0\",\"internal_states\":[{\"last_gen_throughput\":1.5,\"memory_usage\":{\"weight\":2.0}}]}"
|
||||
},
|
||||
{
|
||||
"type": "SamplingParams",
|
||||
"name": "null_is_none: an explicit max_new_tokens null (unbounded) survives the round trip as null",
|
||||
"input": {
|
||||
"max_new_tokens": null,
|
||||
"n": 2
|
||||
},
|
||||
"canonical": "{\"max_new_tokens\":null,\"temperature\":1.0,\"top_p\":1.0,\"top_k\":1073741824,\"min_p\":0.0,\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"repetition_penalty\":1.0,\"min_new_tokens\":0,\"n\":2,\"ignore_eos\":false,\"skip_special_tokens\":true,\"spaces_between_special_tokens\":true,\"no_stop_trim\":false}"
|
||||
},
|
||||
{
|
||||
"type": "GetModelInfoResponse",
|
||||
"name": "preferred_sampling_params is the launch object itself (raw JSON)",
|
||||
"input": {
|
||||
"model_path": "/m",
|
||||
"served_model_name": "m",
|
||||
"tokenizer_path": "/t",
|
||||
"is_generation": true,
|
||||
"preferred_sampling_params": {
|
||||
"temperature": 0.5,
|
||||
"max_new_tokens": 64
|
||||
},
|
||||
"weight_version": "v1"
|
||||
},
|
||||
"canonical": "{\"model_path\":\"/m\",\"served_model_name\":\"m\",\"tokenizer_path\":\"/t\",\"is_generation\":true,\"preferred_sampling_params\":{\"temperature\":0.5,\"max_new_tokens\":64},\"weight_version\":\"v1\",\"load_format\":null,\"reasoning_parser\":null,\"tool_call_parser\":null}"
|
||||
},
|
||||
{
|
||||
"type": "GenerateResponse",
|
||||
"name": "unknown finish type passes through with its key order",
|
||||
"input": {
|
||||
"text": "",
|
||||
"meta_info": {
|
||||
"id": "r",
|
||||
"prompt_tokens": 1,
|
||||
"completion_tokens": 0,
|
||||
"finish_reason": {
|
||||
"type": "novel",
|
||||
"detail": 3,
|
||||
"again": [
|
||||
1,
|
||||
{
|
||||
"z": 1,
|
||||
"a": 2
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"canonical": "{\"text\":\"\",\"meta_info\":{\"id\":\"r\",\"prompt_tokens\":1,\"completion_tokens\":0,\"finish_reason\":{\"type\":\"novel\",\"detail\":3,\"again\":[1,{\"z\":1,\"a\":2}]}}}"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,346 @@
|
||||
//! Golden tests for the schema-driven JSON contract: each sglang.json.v1
|
||||
//! option template, pinned against the behavior of the hand-written types it
|
||||
//! replaces (message/sampling.rs, message/request.rs, message/finish_reason.rs,
|
||||
//! api_server/core/frame.rs). These fixtures are the contract's only gate,
|
||||
//! so they keep the generator honest on their own.
|
||||
|
||||
use sglang_api_types::api::v1::*;
|
||||
|
||||
fn sp(json: &str) -> Result<SamplingParams, serde_json::Error> {
|
||||
serde_json::from_str(json)
|
||||
}
|
||||
|
||||
/// Absent keys yield the schema defaults (sampling.rs `defaulted!` values).
|
||||
#[test]
|
||||
fn sampling_defaults_match_hand_written() {
|
||||
let p = sp("{}").unwrap();
|
||||
assert_eq!(p.max_new_tokens, Some(128));
|
||||
assert_eq!(p.temperature, Some(1.0));
|
||||
assert_eq!(p.top_p, Some(1.0));
|
||||
assert_eq!(p.top_k, Some(1 << 30));
|
||||
assert_eq!(p.min_p, Some(0.0));
|
||||
assert_eq!(p.repetition_penalty, Some(1.0));
|
||||
assert_eq!(p.min_new_tokens, Some(0));
|
||||
assert_eq!(p.n, Some(1));
|
||||
assert_eq!(p.ignore_eos, Some(false));
|
||||
assert_eq!(p.skip_special_tokens, Some(true));
|
||||
assert_eq!(p.spaces_between_special_tokens, Some(true));
|
||||
assert_eq!(p.no_stop_trim, Some(false));
|
||||
}
|
||||
|
||||
/// Protobuf arrival: a prost-decoded (all-unset) SamplingParams reads its
|
||||
/// schema defaults through the accessors — the top_p=0 bug the first live
|
||||
/// gRPC call found.
|
||||
#[test]
|
||||
fn proto_unset_reads_schema_defaults() {
|
||||
let p = SamplingParams::default();
|
||||
assert_eq!(p.temperature_or_default(), 1.0);
|
||||
assert_eq!(p.top_p_or_default(), 1.0);
|
||||
assert_eq!(p.top_k_or_default(), 1 << 30);
|
||||
assert_eq!(p.n_or_default(), 1);
|
||||
assert!(p.skip_special_tokens_or_default());
|
||||
let mut p = SamplingParams::default();
|
||||
p.apply_absent_defaults();
|
||||
assert_eq!(p.max_new_tokens, Some(128));
|
||||
assert_eq!(p.top_p, Some(1.0));
|
||||
}
|
||||
|
||||
/// null_resets_default: an explicit null is the default, not an error
|
||||
/// (Python `__post_init__` parity).
|
||||
#[test]
|
||||
fn sampling_null_resets_default() {
|
||||
let p = sp(r#"{"temperature": null, "top_k": null, "ignore_eos": null}"#).unwrap();
|
||||
assert_eq!(p.temperature, Some(1.0));
|
||||
assert_eq!(p.top_k, Some(1 << 30));
|
||||
assert_eq!(p.ignore_eos, Some(false));
|
||||
}
|
||||
|
||||
/// max_new_tokens is the null_is_none field: absent = 128, null = unbounded.
|
||||
#[test]
|
||||
fn max_new_tokens_absent_vs_null() {
|
||||
assert_eq!(sp("{}").unwrap().max_new_tokens, Some(128));
|
||||
assert_eq!(
|
||||
sp(r#"{"max_new_tokens": null}"#).unwrap().max_new_tokens,
|
||||
None
|
||||
);
|
||||
assert_eq!(
|
||||
sp(r#"{"max_new_tokens": 7}"#).unwrap().max_new_tokens,
|
||||
Some(7)
|
||||
);
|
||||
}
|
||||
|
||||
/// SamplingParams DENIES unknown keys with serde's own error text (parity
|
||||
/// with the hand-written `deny_unknown_fields`).
|
||||
#[test]
|
||||
fn sampling_denies_unknown_keys() {
|
||||
let err = sp(r#"{"temperatur": 0.5}"#).unwrap_err().to_string(); // codespell:ignore temperatur
|
||||
assert!(
|
||||
err.starts_with("unknown field `temperatur`, expected one of"), // codespell:ignore temperatur
|
||||
"{err}"
|
||||
);
|
||||
}
|
||||
|
||||
/// GenerateRequest IGNORES unknown keys (unported Python fields must not 400;
|
||||
/// pinned in message/request.rs).
|
||||
#[test]
|
||||
fn generate_request_ignores_unknown_keys() {
|
||||
let req: GenerateRequest = serde_json::from_str(
|
||||
r#"{"text": "hi", "priority": 3, "session_id": "s", "custom_logit_processor": "x"}"#,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
req.text,
|
||||
Some(StringOrList {
|
||||
value: Some(string_or_list::Value::One(ref s))
|
||||
}) if s == "hi"
|
||||
));
|
||||
}
|
||||
|
||||
/// one_or_many: a scalar and a list both parse; serialization flattens back.
|
||||
#[test]
|
||||
fn one_or_many_flattens() {
|
||||
let one: StringOrList = serde_json::from_str(r#""a""#).unwrap();
|
||||
assert_eq!(serde_json::to_string(&one).unwrap(), r#""a""#);
|
||||
let many: StringOrList = serde_json::from_str(r#"["a", "b"]"#).unwrap();
|
||||
assert_eq!(serde_json::to_string(&many).unwrap(), r#"["a","b"]"#);
|
||||
}
|
||||
|
||||
/// The TokenIds ambiguity: [1,2] is one id list, [[1],[2]] is a batch, [] is
|
||||
/// one empty list (the hand-written untagged first-arm rule).
|
||||
#[test]
|
||||
fn token_ids_or_list_shape_dispatch() {
|
||||
let one: TokenIdsOrList = serde_json::from_str("[1, 2]").unwrap();
|
||||
assert!(matches!(
|
||||
one.value,
|
||||
Some(token_ids_or_list::Value::One(ref t)) if t.ids == vec![1, 2]
|
||||
));
|
||||
let many: TokenIdsOrList = serde_json::from_str("[[1], [2, 3]]").unwrap();
|
||||
assert!(matches!(
|
||||
many.value,
|
||||
Some(token_ids_or_list::Value::Many(ref l)) if l.items.len() == 2
|
||||
));
|
||||
let empty: TokenIdsOrList = serde_json::from_str("[]").unwrap();
|
||||
assert!(matches!(
|
||||
empty.value,
|
||||
Some(token_ids_or_list::Value::One(ref t)) if t.ids.is_empty()
|
||||
));
|
||||
}
|
||||
|
||||
/// Nullable list elements (PD bootstrap columns): [null, "h"] keeps per-item
|
||||
/// presence; a bare scalar is the one-arm.
|
||||
#[test]
|
||||
fn optional_carriers_keep_element_nulls() {
|
||||
let v: OptionalStringOrList = serde_json::from_str(r#"[null, "h"]"#).unwrap();
|
||||
assert_eq!(serde_json::to_string(&v).unwrap(), r#"[null,"h"]"#);
|
||||
match v.value {
|
||||
Some(optional_string_or_list::Value::Many(l)) => {
|
||||
assert_eq!(l.items[0].value, None);
|
||||
assert_eq!(l.items[1].value.as_deref(), Some("h"));
|
||||
}
|
||||
other => panic!("expected many, got {other:?}"),
|
||||
}
|
||||
let one: OptionalInt64OrList = serde_json::from_str("17000").unwrap();
|
||||
assert!(matches!(
|
||||
one.value,
|
||||
Some(optional_int64_or_list::Value::One(OptionalInt64 {
|
||||
value: Some(17000)
|
||||
}))
|
||||
));
|
||||
}
|
||||
|
||||
/// SamplingParams one-or-many keeps serde's field-level error texts (the
|
||||
/// hand-written SamplingParamsInput contract: never "data did not match any
|
||||
/// variant").
|
||||
#[test]
|
||||
fn sampling_or_list_keeps_field_errors() {
|
||||
let err = serde_json::from_str::<SamplingParamsOrList>(r#"{"temperatur": 1}"#) // codespell:ignore temperatur
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(
|
||||
err.starts_with("unknown field `temperatur`"), // codespell:ignore temperatur
|
||||
"field-level text lost: {err}"
|
||||
);
|
||||
}
|
||||
|
||||
/// finish_reason round-trips key-for-key, tagged by "type"; matched keeps its
|
||||
/// wire shape (str / int / int list).
|
||||
#[test]
|
||||
fn finish_reason_round_trips() {
|
||||
for json in [
|
||||
r#"{"type":"stop","matched":"</s>"}"#,
|
||||
r#"{"type":"stop","matched":7}"#,
|
||||
r#"{"type":"stop","matched":[1,2]}"#,
|
||||
r#"{"type":"length","length":2}"#,
|
||||
] {
|
||||
let parsed: FinishReason = serde_json::from_str(json).unwrap();
|
||||
let back: serde_json::Value = serde_json::to_value(&parsed).unwrap();
|
||||
let want: serde_json::Value = serde_json::from_str(json).unwrap();
|
||||
assert_eq!(back, want, "round trip of {json}");
|
||||
}
|
||||
}
|
||||
|
||||
/// An abort carries all three keys, nulls included (Python emits them).
|
||||
#[test]
|
||||
fn finish_abort_emits_null_keys() {
|
||||
let parsed: FinishReason = serde_json::from_str(r#"{"type":"abort","message":"m"}"#).unwrap();
|
||||
let back = serde_json::to_string(&parsed).unwrap();
|
||||
let v: serde_json::Value = serde_json::from_str(&back).unwrap();
|
||||
assert_eq!(v["type"], "abort");
|
||||
assert_eq!(v["message"], "m");
|
||||
assert!(v.as_object().unwrap().contains_key("status_code"));
|
||||
assert!(v["status_code"].is_null());
|
||||
assert!(v.as_object().unwrap().contains_key("err_type"));
|
||||
}
|
||||
|
||||
/// An unknown finish type round-trips unchanged (the forward-compat escape
|
||||
/// hatch: a Python-side addition must not fail frame decode).
|
||||
#[test]
|
||||
fn unknown_finish_reason_passes_through() {
|
||||
let json = r#"{"type":"paused","step":3}"#;
|
||||
let parsed: FinishReason = serde_json::from_str(json).unwrap();
|
||||
assert!(matches!(parsed.kind, Some(finish_reason::Kind::Unknown(_))));
|
||||
let back: serde_json::Value = serde_json::to_value(&parsed).unwrap();
|
||||
let want: serde_json::Value = serde_json::from_str(json).unwrap();
|
||||
assert_eq!(back, want);
|
||||
}
|
||||
|
||||
/// A known tag with a malformed payload also falls back to passthrough (the
|
||||
/// hand-written FinishReason's untagged-outer behavior).
|
||||
#[test]
|
||||
fn malformed_known_finish_reason_passes_through() {
|
||||
let json = r#"{"type":"stop","matched":{"bad":"shape"}}"#;
|
||||
let parsed: FinishReason = serde_json::from_str(json).unwrap();
|
||||
assert!(matches!(parsed.kind, Some(finish_reason::Kind::Unknown(_))));
|
||||
}
|
||||
|
||||
/// Logprob entries are [logprob, token_id, text|null] tuples; the logprob
|
||||
/// slot is nullable (Python emits null for the first prefill position).
|
||||
#[test]
|
||||
fn logprob_tuple_shape() {
|
||||
let e = LogprobEntry {
|
||||
logprob: Some(-0.25),
|
||||
token_id: 7,
|
||||
text: None,
|
||||
};
|
||||
assert_eq!(serde_json::to_string(&e).unwrap(), "[-0.25,7,null]");
|
||||
let parsed: LogprobEntry = serde_json::from_str(r#"[-0.5, 9, "x"]"#).unwrap();
|
||||
assert_eq!(parsed.token_id, 9);
|
||||
assert_eq!(parsed.text.as_deref(), Some("x"));
|
||||
let first_prefill: LogprobEntry = serde_json::from_str(r#"[null, 10, null]"#).unwrap();
|
||||
assert_eq!(first_prefill.logprob, None);
|
||||
assert_eq!(
|
||||
serde_json::to_string(&first_prefill).unwrap(),
|
||||
"[null,10,null]"
|
||||
);
|
||||
}
|
||||
|
||||
/// The /generate frame shape: key order (text, meta_info, output_ids),
|
||||
/// conditional keys, finish_reason always present (null pre-terminal) --
|
||||
/// matching api_core/frame.rs's frame_value.
|
||||
#[test]
|
||||
fn generate_response_matches_frame_value_shape() {
|
||||
let frame = GenerateResponse {
|
||||
text: "ok".into(),
|
||||
output_ids: Some(TokenIds { ids: vec![7, 8] }),
|
||||
meta_info: Some(GenerateMetaInfo {
|
||||
id: "client-rid".into(),
|
||||
prompt_tokens: 5,
|
||||
completion_tokens: 2,
|
||||
finish_reason: None,
|
||||
..Default::default()
|
||||
}),
|
||||
index: None,
|
||||
};
|
||||
assert_eq!(
|
||||
serde_json::to_string(&frame).unwrap(),
|
||||
r#"{"text":"ok","meta_info":{"id":"client-rid","prompt_tokens":5,"completion_tokens":2,"finish_reason":null},"output_ids":[7,8]}"#
|
||||
);
|
||||
}
|
||||
|
||||
/// Typed media input: every container form Python's `/generate` accepts, with
|
||||
/// the value's shape picking the arm (string/object = one item, array = flat
|
||||
/// list, array of arrays = per batch entry) and JSON null kept per element.
|
||||
#[test]
|
||||
fn media_input_shapes_dispatch() {
|
||||
use media_input::Value;
|
||||
let parse = |json: &str| -> MediaInput { serde_json::from_str(json).unwrap() };
|
||||
assert!(matches!(parse(r#""u""#).value, Some(Value::Item(_))));
|
||||
assert!(matches!(
|
||||
parse(r#"{"url": "u", "detail": "high"}"#).value,
|
||||
Some(Value::Item(_))
|
||||
));
|
||||
let flat = parse(r#"["a", null, {"url": "b"}]"#);
|
||||
let Some(Value::Many(items)) = flat.value else {
|
||||
panic!("flat list")
|
||||
};
|
||||
assert_eq!(items.items.len(), 3);
|
||||
assert!(items.items[1].item.is_none(), "null element kept");
|
||||
assert!(
|
||||
matches!(parse("[]").value, Some(Value::Many(_))),
|
||||
"[] is an empty flat list"
|
||||
);
|
||||
let nested = parse(r#"[["a", null], null, []]"#);
|
||||
let Some(Value::Nested(lists)) = nested.value else {
|
||||
panic!("nested list")
|
||||
};
|
||||
assert_eq!(lists.items.len(), 3);
|
||||
assert!(lists.items[1].items.is_none(), "null entry kept");
|
||||
assert_eq!(lists.items[2].items.as_ref().unwrap().items.len(), 0);
|
||||
// Round trip keeps the client's spelling.
|
||||
for json in [
|
||||
r#""u""#,
|
||||
r#"["a",null,{"url":"b"}]"#,
|
||||
r#"[["a",null],null,[]]"#,
|
||||
] {
|
||||
assert_eq!(serde_json::to_string(&parse(json)).unwrap(), json);
|
||||
}
|
||||
}
|
||||
|
||||
/// A media object is a MediaRef with named hints; an unknown hint is an error
|
||||
/// (it used to be silently dropped), and a preprocessed-input dict is not an
|
||||
/// HTTP shape at all.
|
||||
#[test]
|
||||
fn media_ref_is_typed_and_strict() {
|
||||
let r: MediaRef =
|
||||
serde_json::from_str(r#"{"url": "u", "max_dynamic_patch": 6, "fps": 2.5}"#).unwrap();
|
||||
assert_eq!(
|
||||
(r.url.as_str(), r.max_dynamic_patch, r.fps),
|
||||
("u", Some(6), Some(2.5))
|
||||
);
|
||||
let err = serde_json::from_str::<MediaInput>(r#"{"url": "u", "max_dynamic_patc": 6}"#)
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(err.contains("unknown field `max_dynamic_patc`"), "{err}");
|
||||
let err = serde_json::from_str::<MediaInput>(r#"[{"format": "processor_output"}]"#)
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(err.contains("unknown field `format`"), "{err}");
|
||||
let err = serde_json::from_str::<MediaInput>(r#"["a", ["b"]]"#)
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(!err.is_empty(), "mixed flat/nested is rejected");
|
||||
}
|
||||
|
||||
/// mm_hashes: a flat string list for one request, nested for a batch; the
|
||||
/// first element decides (the bare-scalar-list rule, generalized past ids).
|
||||
#[test]
|
||||
fn string_list_or_list_shape_dispatch() {
|
||||
use string_list_or_list::Value;
|
||||
let parse = |json: &str| -> StringListOrList { serde_json::from_str(json).unwrap() };
|
||||
assert!(matches!(parse(r#"["a", "b"]"#).value, Some(Value::One(_))));
|
||||
assert!(matches!(
|
||||
parse(r#"[["a"], ["b", "c"]]"#).value,
|
||||
Some(Value::Many(_))
|
||||
));
|
||||
assert!(matches!(parse("[]").value, Some(Value::One(_))));
|
||||
let req: GenerateRequest =
|
||||
serde_json::from_str(r#"{"mm_hashes": ["a1b2", "0xff"], "image_data": "u"}"#).unwrap();
|
||||
assert!(req.mm_hashes.is_some() && req.image_data.is_some());
|
||||
}
|
||||
|
||||
/// custom_params is the one remaining raw_json passthrough on a request.
|
||||
#[test]
|
||||
fn custom_params_raw_json_passthrough() {
|
||||
let p: SamplingParams = serde_json::from_str(r#"{"custom_params": {"k": 3}}"#).unwrap();
|
||||
assert_eq!(p.custom_params.as_deref(), Some(r#"{"k":3}"#));
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
//! Cross-language parity: the fixtures in `testdata/json_parity.json` are
|
||||
//! checked here against the generated Rust types and, case for case, against
|
||||
//! the generated Python types by `test/registered/unit/api/v1/test_parity.py`.
|
||||
//! Both emitters must decode each `input` to the same `canonical` bytes (field
|
||||
//! set, key order, presence and null rules) and keep `canonical` fixed under
|
||||
//! a second round trip.
|
||||
|
||||
use serde::{Serialize, de::DeserializeOwned};
|
||||
use sglang_api_types::api::v1 as api;
|
||||
|
||||
const FIXTURES: &str = include_str!("../testdata/json_parity.json");
|
||||
|
||||
fn check<T: DeserializeOwned + Serialize>(name: &str, input: &serde_json::Value, canonical: &str) {
|
||||
let decoded: T = serde_json::from_value(input.clone())
|
||||
.unwrap_or_else(|error| panic!("{name}: input does not decode: {error}"));
|
||||
assert_eq!(
|
||||
serde_json::to_string(&decoded).unwrap(),
|
||||
canonical,
|
||||
"{name}: input -> canonical"
|
||||
);
|
||||
let again: T = serde_json::from_str(canonical)
|
||||
.unwrap_or_else(|error| panic!("{name}: canonical does not decode: {error}"));
|
||||
assert_eq!(
|
||||
serde_json::to_string(&again).unwrap(),
|
||||
canonical,
|
||||
"{name}: canonical is not a fixed point"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fixtures_decode_to_their_canonical_bytes() {
|
||||
let fixtures: serde_json::Value = serde_json::from_str(FIXTURES).unwrap();
|
||||
let cases = fixtures["cases"].as_array().expect("cases array");
|
||||
assert!(!cases.is_empty());
|
||||
for case in cases {
|
||||
let ty = case["type"].as_str().expect("type");
|
||||
let name = format!("{ty}: {}", case["name"].as_str().expect("name"));
|
||||
let input = &case["input"];
|
||||
let canonical = case["canonical"].as_str().expect("canonical");
|
||||
match ty {
|
||||
"GenerateRequest" => check::<api::GenerateRequest>(&name, input, canonical),
|
||||
"SamplingParams" => check::<api::SamplingParams>(&name, input, canonical),
|
||||
"GenerateResponse" => check::<api::GenerateResponse>(&name, input, canonical),
|
||||
"GenerateStreamError" => check::<api::GenerateStreamError>(&name, input, canonical),
|
||||
"GetModelInfoResponse" => check::<api::GetModelInfoResponse>(&name, input, canonical),
|
||||
"GetServerInfoResponse" => check::<api::GetServerInfoResponse>(&name, input, canonical),
|
||||
other => panic!("fixture type {other} has no Rust mapping; add it here"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -9,9 +9,6 @@ license.workspace = true
|
||||
# of the main sglang wheel at the given import path.
|
||||
[package.metadata.sglang]
|
||||
python-module = "sglang.srt.rust_extensions._grpc"
|
||||
# The shared protocol dependency lives outside rust/; include it in the
|
||||
# source-loader artifact fingerprint.
|
||||
source-inputs = ["../../proto"]
|
||||
# Always build optimized, even for an editable install.
|
||||
debug = false
|
||||
|
||||
@@ -26,7 +23,7 @@ crate-type = ["cdylib"]
|
||||
[dependencies]
|
||||
async-stream = { workspace = true }
|
||||
pyo3 = { workspace = true }
|
||||
sglang-grpc-types = { path = "../../proto" }
|
||||
sglang-api-types = { path = "../sglang-api-types" }
|
||||
serde_json = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
tokio-stream = { workspace = true }
|
||||
@@ -38,5 +35,9 @@ uuid = { workspace = true }
|
||||
prost-types = { workspace = true }
|
||||
tokenizers = { version = "0.21", default-features = false, features = ["onig"] }
|
||||
|
||||
[dev-dependencies]
|
||||
# `ProstCodec` for the hand-rolled unary call in the metadata-service test.
|
||||
tonic-prost = { workspace = true }
|
||||
|
||||
[features]
|
||||
default = ["pyo3/extension-module"]
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
# Standalone dev builds only (`maturin build` / `maturin develop`); the
|
||||
# extension ships to users inside the main sglang wheel via setuptools-rust
|
||||
# (python/pyproject.toml, target sglang.srt.rust_extensions._grpc). Its shared
|
||||
# protocol dependency lives in ../../proto, so standalone builds require a
|
||||
# repository checkout. A system protoc is unnecessary; that crate uses a
|
||||
# vendored binary.
|
||||
# protocol dependency is the sibling sglang-api-types crate, so standalone
|
||||
# builds require a repository checkout. No protoc is needed: the bindings are
|
||||
# generated ahead of time and checked in.
|
||||
[build-system]
|
||||
requires = ["maturin>=1.5,<2"]
|
||||
build-backend = "maturin"
|
||||
|
||||
@@ -17,7 +17,7 @@ use crate::utils::{
|
||||
build_classify_dict, build_embed_dict, build_generate_dict, build_text_embed_dict,
|
||||
build_text_generate_dict, extract_model_path,
|
||||
};
|
||||
use sglang_grpc_types::sglang::runtime::v1 as proto;
|
||||
use sglang_api_types::runtime::v1 as proto;
|
||||
|
||||
pub struct SglangServiceImpl {
|
||||
pub bridge: Arc<PyBridge>,
|
||||
|
||||
@@ -81,8 +81,8 @@ fn resolve_max_message_size_honors_env_var() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn follower_metadata_server_exposes_only_server_info_and_shuts_down() {
|
||||
use sglang_grpc_types::sglang::runtime::v1::{GetServerInfoRequest, GetServerInfoResponse};
|
||||
use tonic::codec::ProstCodec;
|
||||
use sglang_api_types::runtime::v1::{GetServerInfoRequest, GetServerInfoResponse};
|
||||
use tonic_prost::ProstCodec;
|
||||
|
||||
let _guard = PAYLOAD_ENV_LOCK.lock().await;
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use sglang_grpc_types::sglang::runtime::v1 as proto;
|
||||
use sglang_api_types::runtime::v1 as proto;
|
||||
|
||||
fn regex_escape_literal(value: &str) -> String {
|
||||
let mut escaped = String::with_capacity(value.len());
|
||||
|
||||
@@ -10,9 +10,6 @@ license.workspace = true
|
||||
python-module = "sglang.srt.rust_extensions._server"
|
||||
# Always build optimized, even for an editable install.
|
||||
debug = false
|
||||
# The shared runtime.v1 crate lives outside rust/; include it in extension
|
||||
# source fingerprints so protobuf changes cannot reuse a stale _server build.
|
||||
source-inputs = ["../../proto"]
|
||||
|
||||
[lib]
|
||||
name = "sglang_server"
|
||||
@@ -51,6 +48,7 @@ dynamo-renderer = "5.1.2"
|
||||
minijinja = "2.24"
|
||||
flume = "0.12.0"
|
||||
hf-hub = { version = "0.4", default-features = false }
|
||||
http = "1.5.0"
|
||||
itertools = "0.14"
|
||||
# POSIX shm (`shm_open` family) for the MM feature fan-out (`utils::shm`).
|
||||
rustix = { version = "1", features = ["fs", "mm", "shm"] }
|
||||
@@ -63,13 +61,13 @@ numpy = "0.29.0"
|
||||
regex-syntax = "=0.8.11"
|
||||
rmp-serde = "1"
|
||||
rmpv = { version = "1", features = ["with-serde"] }
|
||||
sglang-grpc-types = { path = "../../proto" }
|
||||
# The pure-Rust core of the MM pipeline. `default-features = false` drops the
|
||||
# pyo3 bindings so it links as a plain rlib; renamed so `use sglang_mm::…` reads
|
||||
# naturally while the crate keeps its own artifact name in the shared target/.
|
||||
sglang_mm = { package = "sglang-mm", path = "../sglang-mm", default-features = false }
|
||||
tonic = { workspace = true }
|
||||
tokio-stream = { workspace = true }
|
||||
sglang_api_types = { package = "sglang-api-types", path = "../sglang-api-types" }
|
||||
|
||||
[dev-dependencies]
|
||||
# `Router::oneshot` for handler-level router tests.
|
||||
|
||||
@@ -1,83 +1,10 @@
|
||||
//! API server (axum / tokio). I/O-bound; own pinned multi-thread runtime. Only
|
||||
//! this module knows HTTP. Its handlers use the transport-neutral
|
||||
//! [`crate::frontend::FrontendHandle`] to enter the shared runtime pipeline.
|
||||
//! Generation handlers render semantic frontend events as unary JSON or SSE;
|
||||
//! control handlers serialize typed frontend results such as server metadata.
|
||||
pub mod app;
|
||||
mod common;
|
||||
//! API server (axum / tokio). I/O-bound; own pinned multi-thread runtime. The
|
||||
//! transport-neutral [`core::CoreHandle`] is the shared entry into the runtime
|
||||
//! pipeline; `http` and `grpc` are the wire adapters on top of it. Generation
|
||||
//! handlers render semantic core events as unary JSON, SSE, or gRPC frames;
|
||||
//! control handlers serialize typed core results such as server metadata.
|
||||
pub(crate) mod core;
|
||||
mod disaggregation;
|
||||
mod frame;
|
||||
pub(crate) mod grpc;
|
||||
pub(crate) mod http;
|
||||
mod log;
|
||||
mod native_api;
|
||||
mod openai;
|
||||
|
||||
use axum::http::StatusCode;
|
||||
|
||||
use crate::frontend::{FrontendError, FrontendErrorKind};
|
||||
|
||||
/// Map transport-neutral failures into HTTP status codes. Response bodies and
|
||||
/// stream framing remain the responsibility of each HTTP API surface.
|
||||
pub(crate) fn frontend_error_status(error: &FrontendError) -> StatusCode {
|
||||
if let FrontendError::RuntimeRejected {
|
||||
legacy_http_status, ..
|
||||
} = error
|
||||
{
|
||||
return StatusCode::from_u16(*legacy_http_status)
|
||||
.unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
|
||||
}
|
||||
|
||||
match error.kind() {
|
||||
FrontendErrorKind::InvalidArgument => StatusCode::BAD_REQUEST,
|
||||
FrontendErrorKind::NotFound => StatusCode::NOT_FOUND,
|
||||
FrontendErrorKind::FailedPrecondition => StatusCode::PRECONDITION_FAILED,
|
||||
// Existing intake backpressure has historically surfaced as 503.
|
||||
FrontendErrorKind::ResourceExhausted | FrontendErrorKind::Unavailable => {
|
||||
StatusCode::SERVICE_UNAVAILABLE
|
||||
}
|
||||
FrontendErrorKind::Cancelled => StatusCode::from_u16(499).expect("499 is valid"),
|
||||
FrontendErrorKind::DeadlineExceeded => StatusCode::GATEWAY_TIMEOUT,
|
||||
FrontendErrorKind::Internal => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn frontend_error_categories_map_at_the_http_boundary() {
|
||||
assert_eq!(
|
||||
frontend_error_status(&FrontendError::InvalidArgument("bad".into())),
|
||||
StatusCode::BAD_REQUEST
|
||||
);
|
||||
assert_eq!(
|
||||
frontend_error_status(&FrontendError::Unavailable),
|
||||
StatusCode::SERVICE_UNAVAILABLE
|
||||
);
|
||||
assert_eq!(
|
||||
frontend_error_status(&FrontendError::Overloaded("full".into())),
|
||||
StatusCode::SERVICE_UNAVAILABLE
|
||||
);
|
||||
assert_eq!(
|
||||
frontend_error_status(&FrontendError::Cancelled("gone".into())).as_u16(),
|
||||
499
|
||||
);
|
||||
assert_eq!(
|
||||
frontend_error_status(&FrontendError::RuntimeRejected {
|
||||
kind: FrontendErrorKind::InvalidArgument,
|
||||
message: "runtime rejected request".into(),
|
||||
legacy_http_status: 432,
|
||||
})
|
||||
.as_u16(),
|
||||
432
|
||||
);
|
||||
assert_eq!(
|
||||
frontend_error_status(&FrontendError::InvalidResponse("bad reply".into())),
|
||||
StatusCode::INTERNAL_SERVER_ERROR
|
||||
);
|
||||
assert_eq!(
|
||||
frontend_error_status(&FrontendError::Internal("bug".into())),
|
||||
StatusCode::INTERNAL_SERVER_ERROR
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
//! Transport-neutral core shared by the API adapters: the entry point into the Rust request pipeline.
|
||||
//!
|
||||
//! Wire adapters normalize their protocol into the in-process
|
||||
//! [`GenerateRequest`](crate::message::request::GenerateRequest), submit it
|
||||
//! through [`CoreHandle`], and render semantic [`CoreEvent`]s for their
|
||||
//! own transport. This module owns shared preprocessing, runtime translation,
|
||||
//! capabilities, request lifetime, and the rendering of outputs into the
|
||||
//! generated `api.v1` response types that both adapters put on the wire; it
|
||||
//! deliberately knows nothing about Axum, SSE framing, or Tonic.
|
||||
|
||||
mod error;
|
||||
mod frame;
|
||||
mod handle;
|
||||
mod prefetch;
|
||||
|
||||
pub(crate) use error::{CoreError, CoreErrorKind};
|
||||
pub(crate) use frame::{CoreEvent, CoreOutput, HealthStatus};
|
||||
pub(crate) use handle::{CoreCall, CoreConfig, CoreHandle, CoreMetadata, recv_indexed};
|
||||
@@ -0,0 +1,186 @@
|
||||
//! Transport-neutral failure categories, and how each wire adapter maps them
|
||||
//! into its own status space.
|
||||
|
||||
use http::StatusCode;
|
||||
use sglang_api_types::api::v1 as api;
|
||||
|
||||
/// Protocol-independent error category used by transport adapters.
|
||||
///
|
||||
/// The set intentionally follows operation semantics rather than HTTP or gRPC
|
||||
/// status spaces. An adapter maps this category into its own wire status.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(crate) enum CoreErrorKind {
|
||||
InvalidArgument,
|
||||
NotFound,
|
||||
FailedPrecondition,
|
||||
ResourceExhausted,
|
||||
Cancelled,
|
||||
DeadlineExceeded,
|
||||
Unavailable,
|
||||
Internal,
|
||||
}
|
||||
|
||||
/// A transport-neutral failure category.
|
||||
///
|
||||
/// HTTP and gRPC adapters map these categories into their own status spaces;
|
||||
/// neither adapter needs to understand errors emitted by runtime stages.
|
||||
#[derive(Clone, Debug, thiserror::Error)]
|
||||
pub(crate) enum CoreError {
|
||||
#[error("{0}")]
|
||||
InvalidArgument(String),
|
||||
|
||||
/// The intake loop has shut down. A caller may retry elsewhere.
|
||||
#[error("service unavailable")]
|
||||
Unavailable,
|
||||
|
||||
/// Runtime capacity was exhausted before the request could make progress.
|
||||
#[error("{0}")]
|
||||
Overloaded(String),
|
||||
|
||||
#[error("{0}")]
|
||||
Cancelled(String),
|
||||
|
||||
/// The internal producer disappeared before returning a terminal result.
|
||||
///
|
||||
/// This is a server-side failure, not evidence that the caller cancelled.
|
||||
#[error("response truncated before completion")]
|
||||
ResponseTruncated,
|
||||
|
||||
/// A scheduler-side abort that carried an error status.
|
||||
///
|
||||
/// The current Python scheduler communicates this category as an HTTP
|
||||
/// status code inside its finish reason. The semantic event is still
|
||||
/// `Failed`; retaining the legacy code only lets the HTTP adapter preserve
|
||||
/// existing behavior while another adapter maps it into its own status
|
||||
/// space. New core failures should use the semantic variants above.
|
||||
#[error("{message}")]
|
||||
RuntimeRejected {
|
||||
kind: CoreErrorKind,
|
||||
message: String,
|
||||
legacy_http_status: u16,
|
||||
},
|
||||
|
||||
/// The runtime replied with an unexpected or malformed result.
|
||||
#[error("{0}")]
|
||||
InvalidResponse(String),
|
||||
|
||||
#[error("{0}")]
|
||||
Internal(String),
|
||||
}
|
||||
|
||||
impl CoreError {
|
||||
/// Semantic status for adapters that do not use the scheduler's legacy
|
||||
/// HTTP code space (notably the future gRPC adapter).
|
||||
pub(crate) fn kind(&self) -> CoreErrorKind {
|
||||
match self {
|
||||
Self::InvalidArgument(_) => CoreErrorKind::InvalidArgument,
|
||||
Self::Unavailable => CoreErrorKind::Unavailable,
|
||||
Self::Overloaded(_) => CoreErrorKind::ResourceExhausted,
|
||||
Self::Cancelled(_) => CoreErrorKind::Cancelled,
|
||||
Self::ResponseTruncated => CoreErrorKind::Internal,
|
||||
Self::RuntimeRejected { kind, .. } => *kind,
|
||||
Self::InvalidResponse(_) | Self::Internal(_) => CoreErrorKind::Internal,
|
||||
}
|
||||
}
|
||||
|
||||
/// Translate the scheduler's inherited HTTP-coded failure into a semantic
|
||||
/// category once, at the runtime boundary. HTTP can retain the exact legacy
|
||||
/// status while other adapters consume [`Self::kind`].
|
||||
pub(super) fn from_runtime_rejection(message: String, legacy_http_status: u16) -> Self {
|
||||
let kind = match legacy_http_status {
|
||||
404 => CoreErrorKind::NotFound,
|
||||
408 | 504 => CoreErrorKind::DeadlineExceeded,
|
||||
412 => CoreErrorKind::FailedPrecondition,
|
||||
413 | 429 => CoreErrorKind::ResourceExhausted,
|
||||
499 => CoreErrorKind::Cancelled,
|
||||
502 | 503 => CoreErrorKind::Unavailable,
|
||||
400..=499 => CoreErrorKind::InvalidArgument,
|
||||
_ => CoreErrorKind::Internal,
|
||||
};
|
||||
Self::RuntimeRejected {
|
||||
kind,
|
||||
message,
|
||||
legacy_http_status,
|
||||
}
|
||||
}
|
||||
|
||||
/// The HTTP status for this failure. Response bodies and stream framing
|
||||
/// remain the responsibility of each HTTP API surface.
|
||||
/// This failure as one `/generate` item: the native error body with the
|
||||
/// HTTP status as `code`, plus the batch position where there is one.
|
||||
pub(crate) fn stream_error(&self, index: Option<u32>) -> api::GenerateStreamError {
|
||||
api::GenerateStreamError {
|
||||
error: Some(api::ErrorBody {
|
||||
message: self.to_string(),
|
||||
code: u32::from(self.http_status().as_u16()),
|
||||
}),
|
||||
index,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn http_status(&self) -> StatusCode {
|
||||
if let CoreError::RuntimeRejected {
|
||||
legacy_http_status, ..
|
||||
} = self
|
||||
{
|
||||
return StatusCode::from_u16(*legacy_http_status)
|
||||
.unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);
|
||||
}
|
||||
|
||||
match self.kind() {
|
||||
CoreErrorKind::InvalidArgument => StatusCode::BAD_REQUEST,
|
||||
CoreErrorKind::NotFound => StatusCode::NOT_FOUND,
|
||||
CoreErrorKind::FailedPrecondition => StatusCode::PRECONDITION_FAILED,
|
||||
// Existing intake backpressure has historically surfaced as 503.
|
||||
CoreErrorKind::ResourceExhausted | CoreErrorKind::Unavailable => {
|
||||
StatusCode::SERVICE_UNAVAILABLE
|
||||
}
|
||||
CoreErrorKind::Cancelled => StatusCode::from_u16(499).expect("499 is valid"),
|
||||
CoreErrorKind::DeadlineExceeded => StatusCode::GATEWAY_TIMEOUT,
|
||||
CoreErrorKind::Internal => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn core_error_categories_map_at_the_http_boundary() {
|
||||
assert_eq!(
|
||||
CoreError::InvalidArgument("bad".into()).http_status(),
|
||||
StatusCode::BAD_REQUEST
|
||||
);
|
||||
assert_eq!(
|
||||
CoreError::Unavailable.http_status(),
|
||||
StatusCode::SERVICE_UNAVAILABLE
|
||||
);
|
||||
assert_eq!(
|
||||
CoreError::Overloaded("full".into()).http_status(),
|
||||
StatusCode::SERVICE_UNAVAILABLE
|
||||
);
|
||||
assert_eq!(
|
||||
CoreError::Cancelled("gone".into()).http_status().as_u16(),
|
||||
499
|
||||
);
|
||||
assert_eq!(
|
||||
CoreError::RuntimeRejected {
|
||||
kind: CoreErrorKind::InvalidArgument,
|
||||
message: "runtime rejected request".into(),
|
||||
legacy_http_status: 432,
|
||||
}
|
||||
.http_status()
|
||||
.as_u16(),
|
||||
432
|
||||
);
|
||||
assert_eq!(
|
||||
CoreError::InvalidResponse("bad reply".into()).http_status(),
|
||||
StatusCode::INTERNAL_SERVER_ERROR
|
||||
);
|
||||
assert_eq!(
|
||||
CoreError::Internal("bug".into()).http_status(),
|
||||
StatusCode::INTERNAL_SERVER_ERROR
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,457 @@
|
||||
//! The semantic events a call yields (incremental and terminal outputs, a
|
||||
//! request-scoped failure, the health probe's result) and the rendering of an
|
||||
//! output into the `api.v1` response types. The generated schema is the wire
|
||||
//! shape for both adapters: HTTP serializes it through serde, gRPC encodes it
|
||||
//! as protobuf, so a column added here reaches both the same way. No Axum,
|
||||
//! SSE, Tonic, or scheduler-channel details.
|
||||
|
||||
use sglang_api_types::api::v1 as api;
|
||||
|
||||
use super::CoreError;
|
||||
use crate::message::finish_reason::{FinishKind, FinishReason, Matched};
|
||||
use crate::message::response::{ChunkEvent, ChunkExtras};
|
||||
use crate::message::types::TokenIds;
|
||||
|
||||
/// Canonical transport-neutral output payload for generation events.
|
||||
///
|
||||
/// The runtime correlation ID is deliberately absent: it is an implementation
|
||||
/// detail used for scheduler routing, while adapters obtain the client-visible
|
||||
/// identity from [`crate::api_server::core::CoreCall::public_id`]. Runtime-only
|
||||
/// response variants such as raw control bytes are likewise hidden by
|
||||
/// [`crate::api_server::core::CoreCall`].
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub(crate) struct CoreOutput {
|
||||
pub(crate) token_ids: TokenIds,
|
||||
pub(crate) finish_reason: Option<FinishReason>,
|
||||
pub(crate) prompt_tokens: u32,
|
||||
pub(crate) text: String,
|
||||
pub(crate) completion_tokens: u64,
|
||||
pub(crate) extras: Option<Box<ChunkExtras>>,
|
||||
}
|
||||
|
||||
impl From<ChunkEvent> for CoreOutput {
|
||||
fn from(output: ChunkEvent) -> Self {
|
||||
let ChunkEvent {
|
||||
rid: _,
|
||||
token_ids,
|
||||
finish_reason,
|
||||
prompt_tokens,
|
||||
text,
|
||||
completion_tokens,
|
||||
extras,
|
||||
} = output;
|
||||
Self {
|
||||
token_ids,
|
||||
finish_reason,
|
||||
prompt_tokens,
|
||||
text,
|
||||
completion_tokens,
|
||||
extras,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl CoreOutput {
|
||||
/// Fold one runtime delta into this cumulative native-generation output.
|
||||
///
|
||||
/// Runtime events are always incremental. Native HTTP and `api.v1`
|
||||
/// gRPC independently decide whether to expose those deltas or the
|
||||
/// cumulative result, so the fold itself lives at their shared semantic
|
||||
/// boundary rather than in either wire adapter.
|
||||
pub(crate) fn append_delta(&mut self, delta: &Self) {
|
||||
self.text.push_str(&delta.text);
|
||||
self.token_ids.extend_from_slice(&delta.token_ids);
|
||||
self.completion_tokens += delta.completion_tokens;
|
||||
self.prompt_tokens = delta.prompt_tokens;
|
||||
if delta.finish_reason.is_some() {
|
||||
self.finish_reason = delta.finish_reason.clone();
|
||||
}
|
||||
|
||||
let Some(delta_extras) = delta.extras.as_deref() else {
|
||||
return;
|
||||
};
|
||||
let extras = self
|
||||
.extras
|
||||
.get_or_insert_with(|| Box::new(ChunkExtras::default()));
|
||||
|
||||
extras
|
||||
.out_lp_val
|
||||
.extend_from_slice(&delta_extras.out_lp_val);
|
||||
extras
|
||||
.out_lp_idx
|
||||
.extend_from_slice(&delta_extras.out_lp_idx);
|
||||
extras
|
||||
.out_top_val
|
||||
.extend_from_slice(&delta_extras.out_top_val);
|
||||
extras
|
||||
.out_top_idx
|
||||
.extend_from_slice(&delta_extras.out_top_idx);
|
||||
extras
|
||||
.out_top_lens
|
||||
.extend_from_slice(&delta_extras.out_top_lens);
|
||||
extras
|
||||
.out_tid_val
|
||||
.extend_from_slice(&delta_extras.out_tid_val);
|
||||
extras
|
||||
.out_tid_idx
|
||||
.extend_from_slice(&delta_extras.out_tid_idx);
|
||||
extras
|
||||
.out_tid_lens
|
||||
.extend_from_slice(&delta_extras.out_tid_lens);
|
||||
extras
|
||||
.out_lp_txt
|
||||
.extend_from_slice(&delta_extras.out_lp_txt);
|
||||
extras
|
||||
.out_top_txt
|
||||
.extend_from_slice(&delta_extras.out_top_txt);
|
||||
extras
|
||||
.out_tid_txt
|
||||
.extend_from_slice(&delta_extras.out_tid_txt);
|
||||
|
||||
// Input logprobs arrive once (during prefill). A later non-empty value
|
||||
// replaces the earlier one, matching the existing native HTTP fold.
|
||||
if !delta_extras.in_lp_val.is_empty() {
|
||||
extras.in_lp_val = delta_extras.in_lp_val.clone();
|
||||
extras.in_lp_idx = delta_extras.in_lp_idx.clone();
|
||||
extras.in_lp_txt = delta_extras.in_lp_txt.clone();
|
||||
}
|
||||
if !delta_extras.in_top_lens.is_empty() {
|
||||
extras.in_top_val = delta_extras.in_top_val.clone();
|
||||
extras.in_top_idx = delta_extras.in_top_idx.clone();
|
||||
extras.in_top_lens = delta_extras.in_top_lens.clone();
|
||||
extras.in_top_txt = delta_extras.in_top_txt.clone();
|
||||
}
|
||||
if !delta_extras.in_tid_lens.is_empty() {
|
||||
extras.in_tid_val = delta_extras.in_tid_val.clone();
|
||||
extras.in_tid_idx = delta_extras.in_tid_idx.clone();
|
||||
extras.in_tid_lens = delta_extras.in_tid_lens.clone();
|
||||
extras.in_tid_txt = delta_extras.in_tid_txt.clone();
|
||||
}
|
||||
// Hidden states are non-cumulative: the latest complete set wins.
|
||||
if !delta_extras.hidden_lens.is_empty() {
|
||||
extras.hidden_val = delta_extras.hidden_val.clone();
|
||||
extras.hidden_lens = delta_extras.hidden_lens.clone();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Semantic result of a deep core health check.
|
||||
///
|
||||
/// Expected lifecycle states are values rather than transport errors so each
|
||||
/// adapter can render them according to its own protocol.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(crate) enum HealthStatus {
|
||||
Healthy,
|
||||
NotReady,
|
||||
Stalled,
|
||||
}
|
||||
|
||||
/// A semantic generation event before any transport-specific framing.
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum CoreEvent {
|
||||
/// One incremental model-output update.
|
||||
Delta(CoreOutput),
|
||||
/// The final model-output update for this request.
|
||||
Finished(CoreOutput),
|
||||
/// A request-scoped failure. Other requests in a batch may continue.
|
||||
Failed(CoreError),
|
||||
}
|
||||
|
||||
impl CoreEvent {
|
||||
pub(crate) fn is_terminal(&self) -> bool {
|
||||
matches!(self, Self::Finished(_) | Self::Failed(_))
|
||||
}
|
||||
}
|
||||
|
||||
impl CoreOutput {
|
||||
/// One `/generate` frame. `index` is the batch position (batch form only);
|
||||
/// `e2e_latency` rides the terminal frame only.
|
||||
///
|
||||
/// Consumes the output so `text` and `output_ids` move onto the wire: a
|
||||
/// delta or terminal frame never copies them. A cumulative intermediate
|
||||
/// frame is the one caller that must clone first, since the accumulated
|
||||
/// output has to stay behind for the next frame.
|
||||
pub(crate) fn frame(
|
||||
self,
|
||||
id: &str,
|
||||
index: Option<u32>,
|
||||
e2e_latency: Option<f64>,
|
||||
) -> api::GenerateResponse {
|
||||
let meta_info = Some(self.meta_info(id, e2e_latency));
|
||||
api::GenerateResponse {
|
||||
text: self.text,
|
||||
meta_info,
|
||||
// Omitted while empty (a text-only frame).
|
||||
output_ids: (!self.token_ids.is_empty()).then_some(api::TokenIds {
|
||||
ids: self.token_ids,
|
||||
}),
|
||||
index,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn meta_info(&self, id: &str, e2e_latency: Option<f64>) -> api::GenerateMetaInfo {
|
||||
let mut meta = api::GenerateMetaInfo {
|
||||
id: id.to_owned(),
|
||||
prompt_tokens: self.prompt_tokens,
|
||||
completion_tokens: self.completion_tokens,
|
||||
finish_reason: self.finish_reason.as_ref().map(finish_reason),
|
||||
e2e_latency,
|
||||
..Default::default()
|
||||
};
|
||||
let Some(extras) = self.extras.as_deref() else {
|
||||
return meta;
|
||||
};
|
||||
|
||||
// Python always sets input and output token logprobs together, including
|
||||
// the empty input list emitted by a disaggregated decode node.
|
||||
if !extras.out_lp_val.is_empty() || !extras.in_lp_val.is_empty() {
|
||||
meta.output_token_logprobs = Some(logprob_entries(
|
||||
&extras.out_lp_val,
|
||||
&extras.out_lp_idx,
|
||||
opt_texts(&extras.out_lp_txt),
|
||||
));
|
||||
meta.input_token_logprobs = Some(logprob_entries(
|
||||
&extras.in_lp_val,
|
||||
&extras.in_lp_idx,
|
||||
opt_texts(&extras.in_lp_txt),
|
||||
));
|
||||
}
|
||||
if !extras.out_top_lens.is_empty() {
|
||||
meta.output_top_logprobs = ragged_logprob_rows(
|
||||
&extras.out_top_val,
|
||||
&extras.out_top_idx,
|
||||
&extras.out_top_lens,
|
||||
opt_texts(&extras.out_top_txt),
|
||||
);
|
||||
}
|
||||
if !extras.in_top_lens.is_empty() {
|
||||
meta.input_top_logprobs = ragged_logprob_rows(
|
||||
&extras.in_top_val,
|
||||
&extras.in_top_idx,
|
||||
&extras.in_top_lens,
|
||||
opt_texts(&extras.in_top_txt),
|
||||
);
|
||||
}
|
||||
if !extras.out_tid_lens.is_empty() {
|
||||
meta.output_token_ids_logprobs = ragged_logprob_rows(
|
||||
&extras.out_tid_val,
|
||||
&extras.out_tid_idx,
|
||||
&extras.out_tid_lens,
|
||||
opt_texts(&extras.out_tid_txt),
|
||||
);
|
||||
}
|
||||
if !extras.in_tid_lens.is_empty() {
|
||||
meta.input_token_ids_logprobs = ragged_logprob_rows(
|
||||
&extras.in_tid_val,
|
||||
&extras.in_tid_idx,
|
||||
&extras.in_tid_lens,
|
||||
opt_texts(&extras.in_tid_txt),
|
||||
);
|
||||
}
|
||||
if !extras.hidden_lens.is_empty() {
|
||||
meta.hidden_states = hidden_state_rows(&extras.hidden_val, &extras.hidden_lens);
|
||||
}
|
||||
meta
|
||||
}
|
||||
}
|
||||
|
||||
/// A decoded-text column becomes the entries' text source only when populated.
|
||||
fn opt_texts(texts: &[String]) -> Option<&[String]> {
|
||||
(!texts.is_empty()).then_some(texts)
|
||||
}
|
||||
|
||||
/// One `[logprob, token_id, text]` entry.
|
||||
fn logprob_entry(value: f32, token_id: i32, text: Option<&String>) -> api::LogprobEntry {
|
||||
api::LogprobEntry {
|
||||
// The scheduler's NaN sentinel means "no logprob at this position".
|
||||
logprob: (!value.is_nan()).then(|| f64::from(value)),
|
||||
token_id: i64::from(token_id),
|
||||
text: text.cloned(),
|
||||
}
|
||||
}
|
||||
|
||||
fn logprob_entries(
|
||||
values: &[f32],
|
||||
token_ids: &[i32],
|
||||
texts: Option<&[String]>,
|
||||
) -> api::LogprobEntries {
|
||||
let entries = values
|
||||
.iter()
|
||||
.zip(token_ids)
|
||||
.enumerate()
|
||||
.map(|(j, (&value, &token_id))| {
|
||||
logprob_entry(value, token_id, texts.and_then(|texts| texts.get(j)))
|
||||
})
|
||||
.collect();
|
||||
api::LogprobEntries { entries }
|
||||
}
|
||||
|
||||
/// Ragged top-k / requested-token rows: one per position, null where that
|
||||
/// position's length is zero.
|
||||
fn ragged_logprob_rows(
|
||||
values: &[f32],
|
||||
token_ids: &[i32],
|
||||
lengths: &[u32],
|
||||
texts: Option<&[String]>,
|
||||
) -> Vec<api::NullableTopLogprobs> {
|
||||
let mut rows = Vec::with_capacity(lengths.len());
|
||||
let mut offset = 0usize;
|
||||
for &length in lengths {
|
||||
let length = length as usize;
|
||||
let row = (length > 0).then(|| {
|
||||
// Bounds checks turn malformed parallel columns into a shortened
|
||||
// row instead of panicking a transport worker.
|
||||
let entries = (offset..offset + length)
|
||||
.filter_map(|j| {
|
||||
Some(logprob_entry(
|
||||
*values.get(j)?,
|
||||
*token_ids.get(j)?,
|
||||
texts.and_then(|texts| texts.get(j)),
|
||||
))
|
||||
})
|
||||
.collect();
|
||||
api::TopLogprobRow { entries }
|
||||
});
|
||||
rows.push(api::NullableTopLogprobs { row });
|
||||
offset += length;
|
||||
}
|
||||
rows
|
||||
}
|
||||
|
||||
/// Reshape flat hidden-state values and per-row lengths into rows.
|
||||
fn hidden_state_rows(values: &[f32], lengths: &[u32]) -> Vec<api::HiddenStateRow> {
|
||||
let mut rows = Vec::with_capacity(lengths.len());
|
||||
let mut offset = 0usize;
|
||||
for &length in lengths {
|
||||
let length = length as usize;
|
||||
let values = values
|
||||
.get(offset..offset + length)
|
||||
.unwrap_or(&[])
|
||||
.iter()
|
||||
.copied()
|
||||
.map(f64::from)
|
||||
.collect();
|
||||
rows.push(api::HiddenStateRow { values });
|
||||
offset += length;
|
||||
}
|
||||
rows
|
||||
}
|
||||
|
||||
fn finish_reason(reason: &FinishReason) -> api::FinishReason {
|
||||
use api::finish_reason::Kind;
|
||||
let kind = match reason {
|
||||
FinishReason::Known(FinishKind::Stop { matched }) => Kind::Stop(api::FinishStop {
|
||||
matched: matched.as_ref().map(matched_value),
|
||||
}),
|
||||
FinishReason::Known(FinishKind::Length { length }) => {
|
||||
Kind::Length(api::FinishLength { length: *length })
|
||||
}
|
||||
FinishReason::Known(FinishKind::Abort(abort)) => Kind::Abort(Box::new(api::FinishAbort {
|
||||
message: abort.message.clone(),
|
||||
status_code: abort.status_code.map(u32::from),
|
||||
err_type: abort.err_type.clone(),
|
||||
})),
|
||||
// The passthrough arm keeps the raw map as JSON text, the proto's own
|
||||
// encoding for a `type` this build does not know.
|
||||
FinishReason::Unknown(fields) => {
|
||||
Kind::Unknown(serde_json::to_string(fields.as_ref()).unwrap_or_default())
|
||||
}
|
||||
};
|
||||
api::FinishReason { kind: Some(kind) }
|
||||
}
|
||||
|
||||
fn matched_value(matched: &Matched) -> api::Matched {
|
||||
use api::matched::Value;
|
||||
let value = match matched {
|
||||
Matched::Token(token) => Value::Token(*token),
|
||||
Matched::Str(text) => Value::Str(text.clone()),
|
||||
Matched::Tokens(ids) => Value::Tokens(api::MatchedTokens { ids: ids.clone() }),
|
||||
};
|
||||
api::Matched { value: Some(value) }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::message::response::ChunkExtras;
|
||||
|
||||
fn json<T: serde::Serialize>(value: &T) -> serde_json::Value {
|
||||
serde_json::to_value(value).unwrap()
|
||||
}
|
||||
|
||||
/// Python parity pinned on the typed shape: a NaN logprob is a null slot,
|
||||
/// input logprobs are set together with output ones, and a zero-length
|
||||
/// top-k position is a null row rather than an empty one.
|
||||
#[test]
|
||||
fn meta_info_keeps_the_native_null_conventions() {
|
||||
let extras = ChunkExtras {
|
||||
out_lp_val: vec![f32::NAN, -0.5],
|
||||
out_lp_idx: vec![7, 8],
|
||||
out_top_val: vec![-0.1],
|
||||
out_top_idx: vec![9],
|
||||
out_top_lens: vec![0, 1],
|
||||
..Default::default()
|
||||
};
|
||||
let output = CoreOutput {
|
||||
extras: Some(Box::new(extras)),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let meta = output.meta_info("rid", None);
|
||||
let entries = meta.output_token_logprobs.unwrap().entries;
|
||||
assert_eq!(entries[0].logprob, None);
|
||||
assert_eq!(entries[1].logprob, Some(f64::from(-0.5_f32)));
|
||||
assert_eq!(entries[1].token_id, 8);
|
||||
assert_eq!(meta.input_token_logprobs.unwrap().entries, []);
|
||||
assert!(meta.output_top_logprobs[0].row.is_none());
|
||||
let row = meta.output_top_logprobs[1].row.as_ref().unwrap();
|
||||
assert_eq!(row.entries[0].token_id, 9);
|
||||
assert!(meta.input_top_logprobs.is_empty());
|
||||
}
|
||||
|
||||
/// The JSON the schema renders is SGLang's `[logprob, token_id, text]`
|
||||
/// tuple list, with the text slot null unless `return_text_in_logprobs`
|
||||
/// supplied a text column, and null positions in the ragged families.
|
||||
#[test]
|
||||
fn logprob_json_keeps_the_tuple_shape() {
|
||||
let texts = vec!["a".to_string(), "b".to_string()];
|
||||
assert_eq!(
|
||||
json(&logprob_entries(&[-0.5, -1.5], &[10, 20], None)),
|
||||
serde_json::json!([[-0.5f32, 10, null], [-1.5f32, 20, null]])
|
||||
);
|
||||
assert_eq!(
|
||||
json(&logprob_entries(&[-0.5, -1.5], &[10, 20], Some(&texts))),
|
||||
serde_json::json!([[-0.5f32, 10, "a"], [-1.5f32, 20, "b"]])
|
||||
);
|
||||
assert_eq!(
|
||||
json(&ragged_logprob_rows(&[-0.3], &[9], &[0, 1], None)),
|
||||
serde_json::json!([null, [[-0.3f32, 9, null]]])
|
||||
);
|
||||
// The NaN sentinel (the Python `None` logprob for the first prompt
|
||||
// token) is a null logprob with its token id preserved.
|
||||
assert_eq!(
|
||||
json(&ragged_logprob_rows(&[f32::NAN], &[7], &[1], None)),
|
||||
serde_json::json!([[[null, 7, null]]])
|
||||
);
|
||||
assert!(opt_texts(&[]).is_none());
|
||||
assert_eq!(opt_texts(&texts), Some(texts.as_slice()));
|
||||
}
|
||||
|
||||
/// A finish type this build does not know still reaches the client, as
|
||||
/// the JSON text the proto's passthrough arm carries.
|
||||
#[test]
|
||||
fn unknown_finish_reason_passes_through_as_json_text() {
|
||||
let reason: FinishReason =
|
||||
serde_json::from_value(serde_json::json!({"type": "novel", "detail": 3})).unwrap();
|
||||
let api::FinishReason {
|
||||
kind: Some(api::finish_reason::Kind::Unknown(text)),
|
||||
} = finish_reason(&reason)
|
||||
else {
|
||||
panic!("unknown type takes the passthrough arm");
|
||||
};
|
||||
assert_eq!(
|
||||
serde_json::from_str::<serde_json::Value>(&text).unwrap(),
|
||||
serde_json::json!({"type": "novel", "detail": 3})
|
||||
);
|
||||
}
|
||||
}
|
||||
+110
-108
@@ -1,10 +1,6 @@
|
||||
//! Transport-neutral entry point into the Rust frontend pipeline.
|
||||
//!
|
||||
//! Wire adapters normalize their protocol into [`FrontendRequest`], submit it
|
||||
//! through [`FrontendHandle`], and render semantic [`FrontendEvent`]s for their
|
||||
//! own transport. This module owns shared preprocessing, runtime translation,
|
||||
//! capabilities, and request lifetime; it deliberately knows nothing about
|
||||
//! Axum, HTTP response shapes, Tonic, or protobuf.
|
||||
//! The shared handle: submission with admission and cancellation ownership,
|
||||
//! runtime-to-semantic event translation, and the typed control operations
|
||||
//! (model info, server info, detokenize, health probe).
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::{
|
||||
@@ -18,7 +14,7 @@ use tokio::sync::mpsc;
|
||||
use crate::message::config::{DisaggregationMode, ServerArgs};
|
||||
use crate::message::ids::Rid;
|
||||
use crate::message::io_struct::{ControlRequest, GetInternalStateReq};
|
||||
use crate::message::request::{Request, RequestKind};
|
||||
use crate::message::request::{GenerateRequest, Request, RequestKind};
|
||||
use crate::message::response::{ResponseItem, ResponseSink};
|
||||
use crate::message::sampling::SamplingParams;
|
||||
use crate::message::types::TokenIds;
|
||||
@@ -27,32 +23,27 @@ use crate::tokenizer_manager::wiring::{AbortSource, RequestAdmission, TmEvent};
|
||||
use crate::utils::error::Error;
|
||||
use crate::utils::fsm::RequestState;
|
||||
|
||||
mod contract;
|
||||
mod prefetch;
|
||||
|
||||
pub(crate) use contract::{
|
||||
FrontendError, FrontendErrorKind, FrontendEvent, FrontendOutput, FrontendRequest, HealthStatus,
|
||||
InternalState, ModelInfo, ServerInfo,
|
||||
};
|
||||
use super::{CoreError, CoreEvent, HealthStatus, prefetch};
|
||||
use crate::message::info::{InternalState, ModelInfo, ServerInfo};
|
||||
|
||||
/// Sentinel host that makes the KV connector no-op. Parity with
|
||||
/// `sglang.srt.disaggregation.utils.FAKE_BOOTSTRAP_HOST`.
|
||||
const FAKE_BOOTSTRAP_HOST: &str = "2.2.2.2";
|
||||
|
||||
pub(crate) struct FrontendConfig {
|
||||
pub(crate) struct CoreConfig {
|
||||
pub(crate) response_capacity: usize,
|
||||
pub(crate) response_activity: ActivityCounter,
|
||||
pub(crate) startup_ready: bool,
|
||||
pub(crate) is_disaggregation: bool,
|
||||
pub(crate) mm_limits: BTreeMap<String, usize>,
|
||||
pub(crate) metadata: FrontendMetadata,
|
||||
pub(crate) metadata: CoreMetadata,
|
||||
}
|
||||
|
||||
/// Immutable, transport-independent metadata retained by the frontend.
|
||||
/// Immutable, transport-independent metadata retained by the core.
|
||||
/// Keeping this snapshot narrow avoids making adapters or the shared handle
|
||||
/// depend on the full launch-configuration object.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub(crate) struct FrontendMetadata {
|
||||
pub(crate) struct CoreMetadata {
|
||||
model_path: String,
|
||||
served_model_name: String,
|
||||
tokenizer_path: String,
|
||||
@@ -67,7 +58,7 @@ pub(crate) struct FrontendMetadata {
|
||||
version: String,
|
||||
}
|
||||
|
||||
impl From<&ServerArgs> for FrontendMetadata {
|
||||
impl From<&ServerArgs> for CoreMetadata {
|
||||
fn from(args: &ServerArgs) -> Self {
|
||||
Self {
|
||||
model_path: args.model_path.clone(),
|
||||
@@ -86,16 +77,16 @@ impl From<&ServerArgs> for FrontendMetadata {
|
||||
}
|
||||
}
|
||||
|
||||
/// Cloneable capability for submitting work to the existing frontend runtime.
|
||||
/// Cloneable capability for submitting work to the runtime pipeline.
|
||||
///
|
||||
/// Only the runtime constructs this handle. Protocol adapters receive clones;
|
||||
/// they cannot reach the raw stage wiring directly.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct FrontendHandle {
|
||||
inner: Arc<FrontendInner>,
|
||||
pub(crate) struct CoreHandle {
|
||||
inner: Arc<CoreInner>,
|
||||
}
|
||||
|
||||
struct FrontendInner {
|
||||
struct CoreInner {
|
||||
intake_tx: flume::Sender<TmEvent>,
|
||||
abort_tx: flume::Sender<AbortSource>,
|
||||
response_capacity: usize,
|
||||
@@ -103,17 +94,17 @@ struct FrontendInner {
|
||||
startup_ready: AtomicBool,
|
||||
is_disaggregation: bool,
|
||||
mm_limits: BTreeMap<String, usize>,
|
||||
metadata: FrontendMetadata,
|
||||
metadata: CoreMetadata,
|
||||
}
|
||||
|
||||
impl FrontendHandle {
|
||||
impl CoreHandle {
|
||||
pub(crate) fn new(
|
||||
intake_tx: flume::Sender<TmEvent>,
|
||||
abort_tx: flume::Sender<AbortSource>,
|
||||
config: FrontendConfig,
|
||||
config: CoreConfig,
|
||||
) -> Self {
|
||||
Self {
|
||||
inner: Arc::new(FrontendInner {
|
||||
inner: Arc::new(CoreInner {
|
||||
intake_tx,
|
||||
abort_tx,
|
||||
response_capacity: config.response_capacity,
|
||||
@@ -143,13 +134,13 @@ impl FrontendHandle {
|
||||
|
||||
pub(crate) async fn generate(
|
||||
&self,
|
||||
mut request: FrontendRequest,
|
||||
) -> Result<FrontendCall, FrontendError> {
|
||||
mut request: GenerateRequest,
|
||||
) -> Result<CoreCall, CoreError> {
|
||||
self.prepare_generations(std::slice::from_mut(&mut request))
|
||||
.await?;
|
||||
self.submit(RequestKind::Generate(Box::new(request)))
|
||||
.await
|
||||
.map(FrontendCall::new)
|
||||
.map(CoreCall::new)
|
||||
}
|
||||
|
||||
/// Preprocess and submit a group atomically from the adapter's perspective:
|
||||
@@ -158,8 +149,8 @@ impl FrontendHandle {
|
||||
/// the already-created calls aborts those accepted requests.
|
||||
pub(crate) async fn generate_batch(
|
||||
&self,
|
||||
mut requests: Vec<FrontendRequest>,
|
||||
) -> Result<Vec<FrontendCall>, FrontendError> {
|
||||
mut requests: Vec<GenerateRequest>,
|
||||
) -> Result<Vec<CoreCall>, CoreError> {
|
||||
self.prepare_generations(&mut requests).await?;
|
||||
|
||||
let mut calls = Vec::with_capacity(requests.len());
|
||||
@@ -167,19 +158,16 @@ impl FrontendHandle {
|
||||
calls.push(
|
||||
self.submit(RequestKind::Generate(Box::new(request)))
|
||||
.await
|
||||
.map(FrontendCall::new)?,
|
||||
.map(CoreCall::new)?,
|
||||
);
|
||||
}
|
||||
Ok(calls)
|
||||
}
|
||||
|
||||
async fn prepare_generations(
|
||||
&self,
|
||||
requests: &mut [FrontendRequest],
|
||||
) -> Result<(), FrontendError> {
|
||||
async fn prepare_generations(&self, requests: &mut [GenerateRequest]) -> Result<(), CoreError> {
|
||||
prefetch::prefetch_all(requests, &self.inner.mm_limits)
|
||||
.await
|
||||
.map_err(FrontendError::InvalidArgument)
|
||||
.map_err(CoreError::InvalidArgument)
|
||||
}
|
||||
|
||||
/// Return static model metadata without exposing the full launch config.
|
||||
@@ -201,8 +189,8 @@ impl FrontendHandle {
|
||||
|
||||
/// Return public server metadata with the current scheduler metrics.
|
||||
/// Raw control bytes and the scheduler's full launch-argument dump never
|
||||
/// cross the frontend boundary.
|
||||
pub(crate) async fn server_info(&self) -> Result<ServerInfo, FrontendError> {
|
||||
/// cross the core boundary.
|
||||
pub(crate) async fn server_info(&self) -> Result<ServerInfo, CoreError> {
|
||||
let internal_state = self.internal_state().await?;
|
||||
let metadata = &self.inner.metadata;
|
||||
Ok(ServerInfo {
|
||||
@@ -217,7 +205,7 @@ impl FrontendHandle {
|
||||
})
|
||||
}
|
||||
|
||||
async fn internal_state(&self) -> Result<InternalState, FrontendError> {
|
||||
async fn internal_state(&self) -> Result<InternalState, CoreError> {
|
||||
let bytes = self
|
||||
.control(ControlRequest::GetInternalStateReq(
|
||||
GetInternalStateReq::new(Rid::new().to_string()),
|
||||
@@ -226,58 +214,54 @@ impl FrontendHandle {
|
||||
rmp_serde::from_slice::<InternalStateEnvelope>(&bytes)
|
||||
.map(|response| response.internal_state)
|
||||
.map_err(|error| {
|
||||
FrontendError::InvalidResponse(format!("invalid internal-state response: {error}"))
|
||||
CoreError::InvalidResponse(format!("invalid internal-state response: {error}"))
|
||||
})
|
||||
}
|
||||
|
||||
async fn control(&self, request: ControlRequest) -> Result<bytes::Bytes, FrontendError> {
|
||||
async fn control(&self, request: ControlRequest) -> Result<bytes::Bytes, CoreError> {
|
||||
let mut call = self.submit(RequestKind::Control(Box::new(request))).await?;
|
||||
match call.recv().await {
|
||||
Some(ResponseItem::Control(bytes)) => Ok(bytes),
|
||||
Some(ResponseItem::Error(error)) => Err(translate_runtime_error(error)),
|
||||
Some(_) => Err(FrontendError::InvalidResponse(
|
||||
Some(_) => Err(CoreError::InvalidResponse(
|
||||
"unexpected generation output for control request".into(),
|
||||
)),
|
||||
None => Err(FrontendError::ResponseTruncated),
|
||||
None => Err(CoreError::ResponseTruncated),
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode token IDs into text without exposing the runtime's byte payload.
|
||||
pub(crate) async fn detokenize(&self, token_ids: TokenIds) -> Result<String, FrontendError> {
|
||||
pub(crate) async fn detokenize(&self, token_ids: TokenIds) -> Result<String, CoreError> {
|
||||
let mut call = self.submit(RequestKind::Detokenize { token_ids }).await?;
|
||||
let bytes = match call.recv().await {
|
||||
Some(ResponseItem::Data(bytes)) => bytes,
|
||||
Some(ResponseItem::Error(Error::Validation(message))) => {
|
||||
return Err(FrontendError::InvalidArgument(message));
|
||||
return Err(CoreError::InvalidArgument(message));
|
||||
}
|
||||
Some(ResponseItem::Error(error)) => return Err(translate_runtime_error(error)),
|
||||
Some(_) => {
|
||||
return Err(FrontendError::InvalidResponse(
|
||||
return Err(CoreError::InvalidResponse(
|
||||
"unexpected output for detokenize request".into(),
|
||||
));
|
||||
}
|
||||
None => {
|
||||
return Err(FrontendError::Internal("reply channel closed".into()));
|
||||
return Err(CoreError::Internal("reply channel closed".into()));
|
||||
}
|
||||
};
|
||||
String::from_utf8(bytes.to_vec()).map_err(|_| {
|
||||
FrontendError::InvalidResponse("detokenized prompt is not valid UTF-8".into())
|
||||
})
|
||||
String::from_utf8(bytes.to_vec())
|
||||
.map_err(|_| CoreError::InvalidResponse("detokenized prompt is not valid UTF-8".into()))
|
||||
}
|
||||
|
||||
/// Confirm that scheduler output is moving. The response heartbeat is the
|
||||
/// signal rather than this probe's own result, matching Python's
|
||||
/// `last_receive_tstamp` behavior on a busy server.
|
||||
pub(crate) async fn probe_health(
|
||||
&self,
|
||||
timeout: Duration,
|
||||
) -> Result<HealthStatus, FrontendError> {
|
||||
pub(crate) async fn probe_health(&self, timeout: Duration) -> Result<HealthStatus, CoreError> {
|
||||
if !self.is_ready() {
|
||||
return Ok(HealthStatus::NotReady);
|
||||
}
|
||||
|
||||
let baseline = self.response_activity();
|
||||
let probe = FrontendRequest {
|
||||
let probe = GenerateRequest {
|
||||
rid: Rid::new_health_check(),
|
||||
input_ids: Some(vec![0]),
|
||||
sampling_params: SamplingParams {
|
||||
@@ -315,7 +299,7 @@ impl FrontendHandle {
|
||||
/// The async send preserves TokenizerManager inbox backpressure. Once the
|
||||
/// request is accepted, the returned call owns cancellation until it
|
||||
/// observes a terminal response.
|
||||
async fn submit(&self, kind: RequestKind) -> Result<RuntimeCall, FrontendError> {
|
||||
async fn submit(&self, kind: RequestKind) -> Result<RuntimeCall, CoreError> {
|
||||
let rid = request_rid(&kind);
|
||||
let expected_response = ExpectedResponse::for_request(&kind);
|
||||
let (response_tx, response_rx) = mpsc::channel(self.inner.response_capacity);
|
||||
@@ -349,21 +333,21 @@ impl FrontendHandle {
|
||||
// Disarm before returning the transport-neutral rejection.
|
||||
call.in_flight = false;
|
||||
tracing::error!(%rid, "tm inbox closed; request rejected");
|
||||
return Err(FrontendError::Unavailable);
|
||||
return Err(CoreError::Unavailable);
|
||||
}
|
||||
|
||||
Ok(call)
|
||||
}
|
||||
}
|
||||
|
||||
fn translate_runtime_error(error: Error) -> FrontendError {
|
||||
fn translate_runtime_error(error: Error) -> CoreError {
|
||||
match error {
|
||||
Error::Validation(message) => {
|
||||
FrontendError::InvalidArgument(format!("validation failed: {message}"))
|
||||
CoreError::InvalidArgument(format!("validation failed: {message}"))
|
||||
}
|
||||
Error::QueueFull => FrontendError::Overloaded("to_scheduler channel full".into()),
|
||||
Error::Disconnected => FrontendError::Cancelled("client disconnected".into()),
|
||||
error => FrontendError::Internal(error.to_string()),
|
||||
Error::QueueFull => CoreError::Overloaded("to_scheduler channel full".into()),
|
||||
Error::Disconnected => CoreError::Cancelled("client disconnected".into()),
|
||||
error => CoreError::Internal(error.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -372,12 +356,12 @@ fn translate_runtime_error(error: Error) -> FrontendError {
|
||||
/// Runtime response variants are translated here so adapters cannot depend on
|
||||
/// scheduler/control encodings. Dropping an unfinished call retains the
|
||||
/// cancellation guarantees owned by [`RuntimeCall`].
|
||||
pub(crate) struct FrontendCall {
|
||||
pub(crate) struct CoreCall {
|
||||
runtime: RuntimeCall,
|
||||
events_finished: bool,
|
||||
}
|
||||
|
||||
impl FrontendCall {
|
||||
impl CoreCall {
|
||||
fn new(runtime: RuntimeCall) -> Self {
|
||||
Self {
|
||||
runtime,
|
||||
@@ -393,14 +377,14 @@ impl FrontendCall {
|
||||
/// Receive the next semantic event. A runtime channel that closes before a
|
||||
/// terminal response is converted into one terminal failure event; adapters
|
||||
/// never need to infer whether an empty runtime channel means success.
|
||||
pub(crate) async fn recv(&mut self) -> Option<FrontendEvent> {
|
||||
pub(crate) async fn recv(&mut self) -> Option<CoreEvent> {
|
||||
if self.events_finished {
|
||||
return None;
|
||||
}
|
||||
|
||||
let event = match self.runtime.recv().await {
|
||||
Some(item) => generation_event(item),
|
||||
None => FrontendEvent::Failed(FrontendError::Internal(
|
||||
None => CoreEvent::Failed(CoreError::Internal(
|
||||
"response truncated before completion".into(),
|
||||
)),
|
||||
};
|
||||
@@ -410,7 +394,7 @@ impl FrontendCall {
|
||||
|
||||
/// Append every semantic event that is ready without waiting. Channel
|
||||
/// emptiness and closure remain private runtime details.
|
||||
pub(crate) fn drain_ready(&mut self, events: &mut Vec<FrontendEvent>) {
|
||||
pub(crate) fn drain_ready(&mut self, events: &mut Vec<CoreEvent>) {
|
||||
while !self.events_finished {
|
||||
let event = match self.runtime.try_recv() {
|
||||
Ok(item) => generation_event(item),
|
||||
@@ -446,39 +430,56 @@ impl FrontendCall {
|
||||
}
|
||||
}
|
||||
|
||||
fn generation_event(item: ResponseItem) -> FrontendEvent {
|
||||
/// Await the next event from `call`, then drain whatever queued behind it (so the
|
||||
/// caller can coalesce a backlog, as Python's `state.out_list` does), handing the
|
||||
/// call back for `FuturesUnordered` to re-poll. An empty result means the semantic
|
||||
/// stream was already exhausted. Shared by the HTTP and gRPC batch multiplexers.
|
||||
pub(crate) async fn recv_indexed(
|
||||
index: usize,
|
||||
mut call: CoreCall,
|
||||
) -> (usize, CoreCall, Vec<CoreEvent>) {
|
||||
let mut items = Vec::new();
|
||||
match call.recv().await {
|
||||
Some(item) => items.push(item),
|
||||
None => return (index, call, items), // already exhausted
|
||||
}
|
||||
call.drain_ready(&mut items);
|
||||
(index, call, items)
|
||||
}
|
||||
|
||||
fn generation_event(item: ResponseItem) -> CoreEvent {
|
||||
match item {
|
||||
ResponseItem::Frame(output) => FrontendEvent::Delta(output.into()),
|
||||
ResponseItem::Frame(output) => CoreEvent::Delta(output.into()),
|
||||
ResponseItem::Done(output) => {
|
||||
if let Some((legacy_http_status, message)) = output
|
||||
.finish_reason
|
||||
.as_ref()
|
||||
.and_then(|reason| reason.abort_status())
|
||||
{
|
||||
FrontendEvent::Failed(FrontendError::from_runtime_rejection(
|
||||
CoreEvent::Failed(CoreError::from_runtime_rejection(
|
||||
message.to_owned(),
|
||||
legacy_http_status,
|
||||
))
|
||||
} else {
|
||||
FrontendEvent::Finished(output.into())
|
||||
CoreEvent::Finished(output.into())
|
||||
}
|
||||
}
|
||||
ResponseItem::Error(error) => FrontendEvent::Failed(translate_runtime_error(error)),
|
||||
ResponseItem::Control(_) | ResponseItem::Data(_) => FrontendEvent::Failed(
|
||||
FrontendError::InvalidResponse("unexpected non-generation response".into()),
|
||||
ResponseItem::Error(error) => CoreEvent::Failed(translate_runtime_error(error)),
|
||||
ResponseItem::Control(_) | ResponseItem::Data(_) => CoreEvent::Failed(
|
||||
CoreError::InvalidResponse("unexpected non-generation response".into()),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// Private scheduler response envelope. This MessagePack shape is decoded at
|
||||
/// the runtime boundary and deliberately is not part of the frontend contract.
|
||||
/// the runtime boundary and deliberately is not part of the core contract.
|
||||
#[derive(serde::Deserialize)]
|
||||
struct InternalStateEnvelope {
|
||||
#[serde(default)]
|
||||
internal_state: InternalState,
|
||||
}
|
||||
|
||||
/// Runtime-facing request guard kept private behind [`FrontendCall`].
|
||||
/// Runtime-facing request guard kept private behind [`CoreCall`].
|
||||
///
|
||||
/// This type is the only owner of the raw response channel and admission/abort
|
||||
/// capabilities. It is also used directly by typed unary operations such as
|
||||
@@ -569,13 +570,14 @@ mod tests {
|
||||
use bytes::Bytes;
|
||||
|
||||
use super::*;
|
||||
use crate::api_server::core::{CoreErrorKind, CoreOutput};
|
||||
use crate::message::io_struct::GetInternalStateReq;
|
||||
use crate::message::multimodal::MmItem;
|
||||
use crate::message::request::{GenerateRequest, MmData};
|
||||
use crate::message::request::MmData;
|
||||
use crate::message::response::ChunkEvent;
|
||||
|
||||
struct Harness {
|
||||
handle: FrontendHandle,
|
||||
handle: CoreHandle,
|
||||
intake_rx: flume::Receiver<TmEvent>,
|
||||
abort_rx: flume::Receiver<AbortSource>,
|
||||
}
|
||||
@@ -596,7 +598,7 @@ mod tests {
|
||||
intake_tx: flume::Sender<TmEvent>,
|
||||
abort_tx: flume::Sender<AbortSource>,
|
||||
response_capacity: usize,
|
||||
) -> FrontendHandle {
|
||||
) -> CoreHandle {
|
||||
configured_handle(
|
||||
intake_tx,
|
||||
abort_tx,
|
||||
@@ -616,17 +618,17 @@ mod tests {
|
||||
startup_ready: bool,
|
||||
is_disaggregation: bool,
|
||||
mm_limits: BTreeMap<String, usize>,
|
||||
) -> FrontendHandle {
|
||||
FrontendHandle::new(
|
||||
) -> CoreHandle {
|
||||
CoreHandle::new(
|
||||
intake_tx,
|
||||
abort_tx,
|
||||
FrontendConfig {
|
||||
CoreConfig {
|
||||
response_capacity,
|
||||
response_activity,
|
||||
startup_ready,
|
||||
is_disaggregation,
|
||||
mm_limits,
|
||||
metadata: FrontendMetadata::default(),
|
||||
metadata: CoreMetadata::default(),
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -700,44 +702,44 @@ mod tests {
|
||||
fn runtime_errors_become_transport_neutral_categories() {
|
||||
assert!(matches!(
|
||||
translate_runtime_error(Error::Validation("bad input".into())),
|
||||
FrontendError::InvalidArgument(message)
|
||||
CoreError::InvalidArgument(message)
|
||||
if message == "validation failed: bad input"
|
||||
));
|
||||
assert!(matches!(
|
||||
translate_runtime_error(Error::QueueFull),
|
||||
FrontendError::Overloaded(message) if message == "to_scheduler channel full"
|
||||
CoreError::Overloaded(message) if message == "to_scheduler channel full"
|
||||
));
|
||||
assert!(matches!(
|
||||
translate_runtime_error(Error::Disconnected),
|
||||
FrontendError::Cancelled(message) if message == "client disconnected"
|
||||
CoreError::Cancelled(message) if message == "client disconnected"
|
||||
));
|
||||
|
||||
assert!(matches!(
|
||||
translate_runtime_error(Error::Internal("bug".into())),
|
||||
FrontendError::Internal(message) if message == "internal error: bug"
|
||||
CoreError::Internal(message) if message == "internal error: bug"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scheduler_status_is_classified_once_for_non_http_adapters() {
|
||||
for (status, expected) in [
|
||||
(400, FrontendErrorKind::InvalidArgument),
|
||||
(404, FrontendErrorKind::NotFound),
|
||||
(408, FrontendErrorKind::DeadlineExceeded),
|
||||
(412, FrontendErrorKind::FailedPrecondition),
|
||||
(413, FrontendErrorKind::ResourceExhausted),
|
||||
(429, FrontendErrorKind::ResourceExhausted),
|
||||
(432, FrontendErrorKind::InvalidArgument),
|
||||
(499, FrontendErrorKind::Cancelled),
|
||||
(500, FrontendErrorKind::Internal),
|
||||
(503, FrontendErrorKind::Unavailable),
|
||||
(504, FrontendErrorKind::DeadlineExceeded),
|
||||
(400, CoreErrorKind::InvalidArgument),
|
||||
(404, CoreErrorKind::NotFound),
|
||||
(408, CoreErrorKind::DeadlineExceeded),
|
||||
(412, CoreErrorKind::FailedPrecondition),
|
||||
(413, CoreErrorKind::ResourceExhausted),
|
||||
(429, CoreErrorKind::ResourceExhausted),
|
||||
(432, CoreErrorKind::InvalidArgument),
|
||||
(499, CoreErrorKind::Cancelled),
|
||||
(500, CoreErrorKind::Internal),
|
||||
(503, CoreErrorKind::Unavailable),
|
||||
(504, CoreErrorKind::DeadlineExceeded),
|
||||
] {
|
||||
let error = FrontendError::from_runtime_rejection("rejected".into(), status);
|
||||
let error = CoreError::from_runtime_rejection("rejected".into(), status);
|
||||
assert_eq!(error.kind(), expected, "legacy status {status}");
|
||||
assert!(matches!(
|
||||
error,
|
||||
FrontendError::RuntimeRejected {
|
||||
CoreError::RuntimeRejected {
|
||||
legacy_http_status,
|
||||
..
|
||||
} if legacy_http_status == status
|
||||
@@ -777,11 +779,11 @@ mod tests {
|
||||
|
||||
assert!(matches!(
|
||||
call.recv().await,
|
||||
Some(FrontendEvent::Delta(FrontendOutput { text, .. })) if text == "delta"
|
||||
Some(CoreEvent::Delta(CoreOutput { text, .. })) if text == "delta"
|
||||
));
|
||||
assert!(matches!(
|
||||
call.recv().await,
|
||||
Some(FrontendEvent::Finished(FrontendOutput { text, .. })) if text == "final"
|
||||
Some(CoreEvent::Finished(CoreOutput { text, .. })) if text == "final"
|
||||
));
|
||||
assert!(call.recv().await.is_none());
|
||||
drop(call);
|
||||
@@ -816,7 +818,7 @@ mod tests {
|
||||
|
||||
assert!(matches!(
|
||||
call.recv().await,
|
||||
Some(FrontendEvent::Finished(FrontendOutput {
|
||||
Some(CoreEvent::Finished(CoreOutput {
|
||||
finish_reason: Some(_),
|
||||
..
|
||||
}))
|
||||
@@ -855,7 +857,7 @@ mod tests {
|
||||
drop(intake_rx);
|
||||
assert!(matches!(
|
||||
submitter.await.unwrap(),
|
||||
Err(FrontendError::Unavailable)
|
||||
Err(CoreError::Unavailable)
|
||||
));
|
||||
|
||||
assert!(matches!(
|
||||
@@ -891,7 +893,7 @@ mod tests {
|
||||
|
||||
assert!(matches!(
|
||||
handle.generate(request).await,
|
||||
Err(FrontendError::InvalidArgument(message))
|
||||
Err(CoreError::InvalidArgument(message))
|
||||
if message == "Image count 2 exceeds limit 1 per request."
|
||||
));
|
||||
assert!(intake_rx.try_recv().is_err());
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
//! Resolve I/O-backed media sources on the async frontend runtime, before MM dispatch.
|
||||
//! Resolve I/O-backed media sources on the async API runtime, before MM dispatch.
|
||||
//!
|
||||
//! The MM worker pool is fixed, core-pinned CPU capacity: a slow image host — or
|
||||
//! a file on a hanging network mount — must never occupy it, and a request's
|
||||
@@ -1,697 +0,0 @@
|
||||
//! Frame shaping for the native `/generate` protocol: the cumulative
|
||||
//! [`OutputAccumulator`] plus the functions that render [`FrontendOutput`]s /
|
||||
//! accumulated state into wire JSON (`meta_info`, logprob tuples, error and
|
||||
//! abort frames). No HTTP here — the sibling `native_api` module owns the handlers
|
||||
//! and streams; it calls these per frame.
|
||||
|
||||
use crate::frontend::FrontendOutput;
|
||||
#[cfg(test)]
|
||||
use crate::message::response::ChunkExtras;
|
||||
use crate::native_generation::{
|
||||
hidden_states_rows, lp_value, meta_info_value, opt_texts, text_slot,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use crate::native_generation::{logprob_tuples, ragged_logprob_tuples};
|
||||
|
||||
/// Append a flat family's `[logprob, token_id, text]` tuples to `dst`, comma
|
||||
/// separated and WITHOUT the enclosing brackets, so a cumulative frame can
|
||||
/// concatenate each delta instead of re-rendering every accumulated position.
|
||||
///
|
||||
/// Byte-identical to [`logprob_tuples`]'s serialization: `serde_json` writes an
|
||||
/// array as `[`, elements joined by `,`, `]` with no spaces, and each element here
|
||||
/// is rendered by the same `Value` Display.
|
||||
fn push_logprob_tuples(dst: &mut String, vals: &[f32], idxs: &[i32], texts: Option<&[String]>) {
|
||||
use std::fmt::Write;
|
||||
for (j, (&v, &tid)) in vals.iter().zip(idxs.iter()).enumerate() {
|
||||
if !dst.is_empty() {
|
||||
dst.push(',');
|
||||
}
|
||||
let _ = write!(dst, "[{},{tid},{}]", lp_value(v), text_slot(texts, j));
|
||||
}
|
||||
}
|
||||
|
||||
/// Ragged counterpart of [`push_logprob_tuples`] — one entry per position, `null`
|
||||
/// where `lens[p] == 0`. Mirrors [`ragged_logprob_tuples`] including its
|
||||
/// bounds-checked skip, so a header whose `lens` run past the value buffer yields
|
||||
/// the same (shortened) row rather than panicking.
|
||||
fn push_ragged_tuples(
|
||||
dst: &mut String,
|
||||
vals: &[f32],
|
||||
idxs: &[i32],
|
||||
lens: &[u32],
|
||||
texts: Option<&[String]>,
|
||||
) {
|
||||
use std::fmt::Write;
|
||||
let mut off = 0usize;
|
||||
for &l in lens {
|
||||
let l = l as usize;
|
||||
if !dst.is_empty() {
|
||||
dst.push(',');
|
||||
}
|
||||
if l == 0 {
|
||||
dst.push_str("null");
|
||||
} else {
|
||||
dst.push('[');
|
||||
let mut first = true;
|
||||
for j in off..off + l {
|
||||
let (Some(&v), Some(&tid)) = (vals.get(j), idxs.get(j)) else {
|
||||
continue;
|
||||
};
|
||||
if !first {
|
||||
dst.push(',');
|
||||
}
|
||||
first = false;
|
||||
let _ = write!(dst, "[{},{tid},{}]", lp_value(v), text_slot(texts, j));
|
||||
}
|
||||
dst.push(']');
|
||||
}
|
||||
off += l;
|
||||
}
|
||||
}
|
||||
|
||||
/// Render one set-once family (they ride the prefill or the final chunk) into a
|
||||
/// standalone JSON array, so it is serialized when it arrives rather than on every
|
||||
/// subsequent frame.
|
||||
fn ragged_array_json(vals: &[f32], idxs: &[i32], lens: &[u32], texts: Option<&[String]>) -> String {
|
||||
let mut body = String::new();
|
||||
push_ragged_tuples(&mut body, vals, idxs, lens, texts);
|
||||
format!("[{body}]")
|
||||
}
|
||||
|
||||
/// Format a decoded [`FrontendOutput`] as one SGLang `/generate` frame's JSON. `rid`
|
||||
/// (response `meta_info.id`) is passed as a string; the event's numeric `rid` is
|
||||
/// just the shard routing key.
|
||||
pub(super) fn frame_value(out: &FrontendOutput, rid: &str) -> serde_json::Value {
|
||||
let mut v = serde_json::json!({
|
||||
"text": out.text,
|
||||
"meta_info": meta_info_value(out, rid),
|
||||
});
|
||||
if !out.token_ids.is_empty() {
|
||||
v["output_ids"] = serde_json::json!(out.token_ids);
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
/// Cumulative frame JSON from the accumulator's memoized parts — O(1) in the
|
||||
/// accumulated length, where rebuilding the `Value` is O(T) per frame and so O(T²)
|
||||
/// per request.
|
||||
///
|
||||
/// Encodes the same JSON document as `frame_value(..).to_string()` — same keys,
|
||||
/// same values, same escaping — pinned for both the plain and the logprob shapes
|
||||
/// by `cumulative_frame_json_matches_serde`.
|
||||
pub(super) fn cumulative_frame_json(
|
||||
acc: &OutputAccumulator,
|
||||
rid: &str,
|
||||
index: Option<usize>,
|
||||
) -> Option<String> {
|
||||
use std::fmt::Write;
|
||||
|
||||
if acc.extras_memo_broken {
|
||||
return None;
|
||||
}
|
||||
let o = acc.snapshot();
|
||||
// Through `Value` rather than `to_string` on the struct: it is the same
|
||||
// encoder the slow path runs the finish reason through, so any representation
|
||||
// quirk is reproduced instead of re-derived.
|
||||
let finish = serde_json::to_value(&o.finish_reason).ok()?.to_string();
|
||||
|
||||
// Alphabetical by convention only — a stable order that is easy to extend and
|
||||
// diff.
|
||||
let mut m = String::new();
|
||||
let _ = write!(m, "{{\"completion_tokens\":{}", o.completion_tokens);
|
||||
let _ = write!(m, ",\"finish_reason\":{finish}");
|
||||
if let Some(h) = &acc.hidden_json {
|
||||
let _ = write!(m, ",\"hidden_states\":{h}");
|
||||
}
|
||||
let _ = write!(m, ",\"id\":{}", serde_json::Value::String(rid.to_string()));
|
||||
if let Some(v) = &acc.in_tid_json {
|
||||
let _ = write!(m, ",\"input_token_ids_logprobs\":{v}");
|
||||
}
|
||||
// Input+output token logprobs are emitted as a PAIR whenever either side has
|
||||
// data (empty list included), matching `frame_value` byte for byte — see the
|
||||
// PD-router rationale there.
|
||||
let lp_pair = acc.in_lp_json.is_some() || !acc.out_lp_json.is_empty();
|
||||
if lp_pair {
|
||||
let v = acc.in_lp_json.as_deref().unwrap_or("[]");
|
||||
let _ = write!(m, ",\"input_token_logprobs\":{v}");
|
||||
}
|
||||
if let Some(v) = &acc.in_top_json {
|
||||
let _ = write!(m, ",\"input_top_logprobs\":{v}");
|
||||
}
|
||||
// The `Value` path keys these off the source columns being non-empty; an empty
|
||||
// source renders to an empty body, so the two guards coincide.
|
||||
if !acc.out_tid_json.is_empty() {
|
||||
let _ = write!(m, ",\"output_token_ids_logprobs\":[{}]", acc.out_tid_json);
|
||||
}
|
||||
if lp_pair {
|
||||
let _ = write!(m, ",\"output_token_logprobs\":[{}]", acc.out_lp_json);
|
||||
}
|
||||
if !acc.out_top_json.is_empty() {
|
||||
let _ = write!(m, ",\"output_top_logprobs\":[{}]", acc.out_top_json);
|
||||
}
|
||||
let _ = write!(m, ",\"prompt_tokens\":{}}}", o.prompt_tokens);
|
||||
|
||||
let mut s = String::with_capacity(acc.text_json.len() + acc.ids_json.len() + m.len() + 40);
|
||||
s.push('{');
|
||||
if let Some(i) = index {
|
||||
let _ = write!(s, "\"index\":{i},");
|
||||
}
|
||||
s.push_str("\"meta_info\":");
|
||||
s.push_str(&m);
|
||||
if !acc.ids_json.is_empty() {
|
||||
s.push_str(",\"output_ids\":[");
|
||||
s.push_str(&acc.ids_json);
|
||||
s.push(']');
|
||||
}
|
||||
s.push_str(",\"text\":\"");
|
||||
s.push_str(&acc.text_json);
|
||||
s.push_str("\"}");
|
||||
Some(s)
|
||||
}
|
||||
|
||||
/// Attach the batch `index` (batch streams only) and render to the SSE `data` text.
|
||||
pub(super) fn tag_value(mut v: serde_json::Value, index: Option<usize>) -> String {
|
||||
if let Some(i) = index {
|
||||
v["index"] = serde_json::json!(i);
|
||||
}
|
||||
v.to_string()
|
||||
}
|
||||
|
||||
/// One streaming frame's JSON: cumulative ignores `delta`, incremental ships it.
|
||||
pub(super) fn stream_frame_string(
|
||||
delta: FrontendOutput,
|
||||
acc: &OutputAccumulator,
|
||||
incremental: bool,
|
||||
rid_str: &str,
|
||||
index: Option<usize>,
|
||||
) -> String {
|
||||
if !incremental {
|
||||
return cumulative_frame_string(acc, rid_str, index);
|
||||
}
|
||||
tag_value(stream_frame_value(delta, acc, true, rid_str), index)
|
||||
}
|
||||
|
||||
/// A cumulative frame's JSON, built purely from the accumulator (which is why a
|
||||
/// backlog can coalesce to its last); falls back to the `Value` builder on extras.
|
||||
pub(super) fn cumulative_frame_string(
|
||||
acc: &OutputAccumulator,
|
||||
rid_str: &str,
|
||||
index: Option<usize>,
|
||||
) -> String {
|
||||
cumulative_frame_json(acc, rid_str, index)
|
||||
.unwrap_or_else(|| tag_value(frame_value(acc.snapshot(), rid_str), index))
|
||||
}
|
||||
|
||||
/// Format one streaming frame: the accumulator's cumulative view (default), or this
|
||||
/// step's delta with the cumulative token count in `meta_info` (matching Python).
|
||||
pub(super) fn stream_frame_value(
|
||||
delta: FrontendOutput,
|
||||
acc: &OutputAccumulator,
|
||||
incremental: bool,
|
||||
rid_str: &str,
|
||||
) -> serde_json::Value {
|
||||
if incremental {
|
||||
let mut d = delta;
|
||||
d.completion_tokens = acc.snapshot().completion_tokens;
|
||||
frame_value(&d, rid_str)
|
||||
} else {
|
||||
frame_value(acc.snapshot(), rid_str)
|
||||
}
|
||||
}
|
||||
|
||||
/// Folds per-chunk [`FrontendOutput`] deltas into a cumulative view — used by the drain
|
||||
/// loops needing cumulative output (every unary response + the cumulative SGLang
|
||||
/// stream; OpenAI streaming forwards deltas and skips this). Holds a single
|
||||
/// [`FrontendOutput`] so `snapshot` hands back a **borrow** per frame — no per-frame
|
||||
/// clone of the growing buffers (that added O(T²) atop the wire's inherent O(T²)).
|
||||
/// Shared with the [`openai`] submodule.
|
||||
#[derive(Default)]
|
||||
pub(super) struct OutputAccumulator {
|
||||
out: FrontendOutput,
|
||||
/// Serialized cumulative `output_ids` body (`"1,2,3"`, no brackets), appended per
|
||||
/// delta so a frame memcpy's it instead of rebuilding the array — O(T), not O(T²).
|
||||
ids_json: String,
|
||||
/// JSON-escaped cumulative text, without the surrounding quotes. Escaping is
|
||||
/// per-character, so `escape(a + b) == escape(a) + escape(b)` and deltas append.
|
||||
text_json: String,
|
||||
/// Memoized bodies (no enclosing brackets) of the three CUMULATIVE logprob
|
||||
/// families, appended per delta — the same O(T) trick `ids_json` uses, extended
|
||||
/// to the families that made a cumulative stream with logprobs O(T²). Cumulative
|
||||
/// is SGLang's default, so that path re-rendered every accumulated position on
|
||||
/// every frame: measured 117 ms for one 500-token top-5 request, versus 1.2 ms
|
||||
/// incremental.
|
||||
out_lp_json: String,
|
||||
out_top_json: String,
|
||||
out_tid_json: String,
|
||||
/// Set-once families — they ride the prefill or the final chunk, so they are
|
||||
/// rendered when they arrive rather than on every frame after.
|
||||
in_lp_json: Option<String>,
|
||||
in_top_json: Option<String>,
|
||||
in_tid_json: Option<String>,
|
||||
hidden_json: Option<String>,
|
||||
/// Set once a family's text column falls out of lockstep with its values, at
|
||||
/// which point the memo is abandoned for the `Value` path.
|
||||
///
|
||||
/// Appending per delta assumes `text_slot(accumulated, global_j)` equals
|
||||
/// `text_slot(delta, local_j)`, which holds only while every delta supplies
|
||||
/// either a text per value or none at all. That is what a real request does —
|
||||
/// `return_text_in_logprobs` is per-request, so the detok shard fills `*_txt`
|
||||
/// for all deltas or none — but a mixed sequence would silently diverge from
|
||||
/// `frame_value`, so it is detected rather than assumed.
|
||||
extras_memo_broken: bool,
|
||||
}
|
||||
|
||||
/// Append `s` JSON-escaped (no surrounding quotes) — `serde_json` quotes it, and the
|
||||
/// quotes are the first and last bytes of a string encoding.
|
||||
fn push_escaped(dst: &mut String, s: &str) {
|
||||
if s.is_empty() {
|
||||
return;
|
||||
}
|
||||
let quoted = serde_json::to_string(s).expect("str-to-json should never fail");
|
||||
dst.push_str("ed[1..quoted.len() - 1]);
|
||||
}
|
||||
|
||||
impl OutputAccumulator {
|
||||
/// Fold one delta frame in. Output families concatenate; input families and
|
||||
/// hidden states are set-once / last-writer-wins (they ride the prefill/final
|
||||
/// chunk), matching the Python `meta_info` assignment.
|
||||
pub(super) fn fold(&mut self, d: &FrontendOutput) {
|
||||
use std::fmt::Write;
|
||||
|
||||
// Grow the memoized serializations alongside the raw cumulative buffers.
|
||||
push_escaped(&mut self.text_json, &d.text);
|
||||
for &id in &d.token_ids {
|
||||
if !self.ids_json.is_empty() {
|
||||
self.ids_json.push(',');
|
||||
}
|
||||
let _ = write!(self.ids_json, "{id}");
|
||||
}
|
||||
|
||||
let Some(de) = d.extras.as_deref() else {
|
||||
self.out.append_delta(d);
|
||||
return;
|
||||
};
|
||||
// Append THIS delta's tuples, indexed within the delta — equivalent to
|
||||
// indexing the accumulated arrays only while texts stay in lockstep, which
|
||||
// the guard below verifies.
|
||||
push_logprob_tuples(
|
||||
&mut self.out_lp_json,
|
||||
&de.out_lp_val,
|
||||
&de.out_lp_idx,
|
||||
opt_texts(&de.out_lp_txt),
|
||||
);
|
||||
push_ragged_tuples(
|
||||
&mut self.out_top_json,
|
||||
&de.out_top_val,
|
||||
&de.out_top_idx,
|
||||
&de.out_top_lens,
|
||||
opt_texts(&de.out_top_txt),
|
||||
);
|
||||
push_ragged_tuples(
|
||||
&mut self.out_tid_json,
|
||||
&de.out_tid_val,
|
||||
&de.out_tid_idx,
|
||||
&de.out_tid_lens,
|
||||
opt_texts(&de.out_tid_txt),
|
||||
);
|
||||
if !de.in_lp_val.is_empty() {
|
||||
let mut body = String::new();
|
||||
push_logprob_tuples(
|
||||
&mut body,
|
||||
&de.in_lp_val,
|
||||
&de.in_lp_idx,
|
||||
opt_texts(&de.in_lp_txt),
|
||||
);
|
||||
self.in_lp_json = Some(format!("[{body}]"));
|
||||
}
|
||||
// Input families ride once (prefill); `lens` non-empty marks their arrival.
|
||||
if !de.in_top_lens.is_empty() {
|
||||
self.in_top_json = Some(ragged_array_json(
|
||||
&de.in_top_val,
|
||||
&de.in_top_idx,
|
||||
&de.in_top_lens,
|
||||
opt_texts(&de.in_top_txt),
|
||||
));
|
||||
}
|
||||
if !de.in_tid_lens.is_empty() {
|
||||
self.in_tid_json = Some(ragged_array_json(
|
||||
&de.in_tid_val,
|
||||
&de.in_tid_idx,
|
||||
&de.in_tid_lens,
|
||||
opt_texts(&de.in_tid_txt),
|
||||
));
|
||||
}
|
||||
// Hidden states are non-cumulative: the latest non-empty set wins.
|
||||
if !de.hidden_lens.is_empty() {
|
||||
self.hidden_json =
|
||||
Some(hidden_states_rows(&de.hidden_val, &de.hidden_lens).to_string());
|
||||
}
|
||||
|
||||
self.out.append_delta(d);
|
||||
let oe = self
|
||||
.out
|
||||
.extras
|
||||
.as_deref()
|
||||
.expect("a delta with extras creates cumulative extras");
|
||||
let lockstep = |txt: &Vec<String>, val: &Vec<f32>| txt.is_empty() || txt.len() == val.len();
|
||||
if !lockstep(&oe.out_lp_txt, &oe.out_lp_val)
|
||||
|| !lockstep(&oe.out_top_txt, &oe.out_top_val)
|
||||
|| !lockstep(&oe.out_tid_txt, &oe.out_tid_val)
|
||||
{
|
||||
self.extras_memo_broken = true;
|
||||
}
|
||||
}
|
||||
|
||||
/// Borrow the cumulative output for an intermediate streaming frame.
|
||||
pub(super) fn snapshot(&self) -> &FrontendOutput {
|
||||
&self.out
|
||||
}
|
||||
|
||||
/// Consume into the final cumulative output.
|
||||
pub(super) fn into_output(self) -> FrontendOutput {
|
||||
self.out
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn flat_logprob_tuples_shape() {
|
||||
let v = logprob_tuples(&[-0.5, -1.5], &[10, 20], None);
|
||||
assert_eq!(
|
||||
v,
|
||||
serde_json::json!([
|
||||
[-0.5f32, 10, serde_json::Value::Null],
|
||||
[-1.5f32, 20, serde_json::Value::Null]
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
/// With a text buffer, the tuple's third slot carries the decoded token.
|
||||
#[test]
|
||||
fn flat_logprob_tuples_with_text() {
|
||||
let texts = vec!["a".to_string(), "b".to_string()];
|
||||
let v = logprob_tuples(&[-0.5, -1.5], &[10, 20], Some(&texts));
|
||||
assert_eq!(
|
||||
v,
|
||||
serde_json::json!([[-0.5f32, 10, "a"], [-1.5f32, 20, "b"]])
|
||||
);
|
||||
}
|
||||
|
||||
/// Ragged reshape restores null positions (len 0) — mirrors
|
||||
/// detokenize_top_logprobs_tokens emitting None for empty positions.
|
||||
#[test]
|
||||
fn ragged_logprob_tuples_restores_null_positions() {
|
||||
// 2 positions: first null (len 0), second k=1.
|
||||
let v = ragged_logprob_tuples(&[-0.3], &[9], &[0, 1], None);
|
||||
assert_eq!(
|
||||
v,
|
||||
serde_json::json!([
|
||||
serde_json::Value::Null,
|
||||
[[-0.3f32, 9, serde_json::Value::Null]]
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
/// The `NaN` sentinel (the Python `None` logprob for the first prompt token)
|
||||
/// becomes a JSON `null` logprob, while its token id in the parallel `idx`
|
||||
/// column is preserved. Guards the scheduler-killing prompt-logprob crash.
|
||||
#[test]
|
||||
fn nan_sentinel_becomes_null_logprob() {
|
||||
// Flat (input/output logprobs): first value absent, second present.
|
||||
let flat = logprob_tuples(&[f32::NAN, -0.5], &[10, 20], None);
|
||||
assert_eq!(
|
||||
flat,
|
||||
serde_json::json!([
|
||||
[serde_json::Value::Null, 10, serde_json::Value::Null],
|
||||
[-0.5f32, 20, serde_json::Value::Null],
|
||||
])
|
||||
);
|
||||
// Ragged (top-k / token-ids logprobs): a NaN inside a position → null.
|
||||
let ragged = ragged_logprob_tuples(&[f32::NAN], &[7], &[1], None);
|
||||
assert_eq!(
|
||||
ragged,
|
||||
serde_json::json!([[[serde_json::Value::Null, 7, serde_json::Value::Null]]])
|
||||
);
|
||||
}
|
||||
|
||||
/// End-to-end: a `FrontendOutput` carrying a prompt-logprob request (first input
|
||||
/// logprob is the `NaN` sentinel) formats without panicking and emits
|
||||
/// `input_token_logprobs` with a leading `[null, token_id, text]`.
|
||||
#[test]
|
||||
fn prompt_logprob_frame_emits_null_first() {
|
||||
let out = FrontendOutput {
|
||||
extras: Some(Box::new(ChunkExtras {
|
||||
in_lp_val: vec![f32::NAN, -0.5],
|
||||
in_lp_idx: vec![10, 20],
|
||||
in_lp_txt: vec!["<s>".into(), "hi".into()],
|
||||
..Default::default()
|
||||
})),
|
||||
..Default::default()
|
||||
};
|
||||
let frame = frame_value(&out, "1");
|
||||
assert_eq!(
|
||||
frame["meta_info"]["input_token_logprobs"],
|
||||
serde_json::json!([[serde_json::Value::Null, 10, "<s>"], [-0.5f32, 20, "hi"]])
|
||||
);
|
||||
}
|
||||
|
||||
/// The accumulator folds deltas cumulatively and `snapshot` borrows the
|
||||
/// running state (no per-frame clone); `into_output` moves the same state.
|
||||
#[test]
|
||||
fn accumulator_snapshot_is_cumulative() {
|
||||
let mut acc = OutputAccumulator::default();
|
||||
acc.fold(&FrontendOutput {
|
||||
text: "he".into(),
|
||||
token_ids: vec![1, 2],
|
||||
completion_tokens: 2,
|
||||
..Default::default()
|
||||
});
|
||||
{
|
||||
let s = acc.snapshot();
|
||||
assert_eq!(s.text, "he");
|
||||
assert_eq!(s.token_ids, vec![1, 2]);
|
||||
}
|
||||
acc.fold(&FrontendOutput {
|
||||
text: "llo".into(),
|
||||
token_ids: vec![3],
|
||||
completion_tokens: 1,
|
||||
..Default::default()
|
||||
});
|
||||
{
|
||||
let s = acc.snapshot();
|
||||
assert_eq!(s.text, "hello"); // cumulative
|
||||
assert_eq!(s.token_ids, vec![1, 2, 3]);
|
||||
assert_eq!(s.completion_tokens, 3);
|
||||
}
|
||||
let out = acc.into_output();
|
||||
assert_eq!(out.text, "hello");
|
||||
}
|
||||
|
||||
/// A populated text column (decoded on the detok shard) → `Some`; empty
|
||||
/// (`return_text_in_logprobs` off) → `None` → null text slots.
|
||||
#[test]
|
||||
fn opt_texts_gates_on_population() {
|
||||
assert!(opt_texts(&[]).is_none());
|
||||
let t = vec!["x".to_string()];
|
||||
assert_eq!(opt_texts(&t), Some(t.as_slice()));
|
||||
}
|
||||
|
||||
/// Parse a frame so the fast and slow paths can be compared as documents.
|
||||
fn as_json(frame: &str) -> serde_json::Value {
|
||||
serde_json::from_str(frame).expect("a frame must be valid JSON")
|
||||
}
|
||||
|
||||
/// The memoized cumulative fast path must emit the **same JSON document** as the
|
||||
/// `serde_json::Value` builder it replaces — same keys, same values, same
|
||||
/// escaping. Covers unicode and control chars, an empty-ids first frame, a
|
||||
/// finish_reason, and the batch `index`. Guards the O(T) rewrite of the O(T²)
|
||||
/// `output_ids` serialization.
|
||||
#[test]
|
||||
fn cumulative_frame_json_matches_serde() {
|
||||
let deltas = [
|
||||
FrontendOutput {
|
||||
text: String::new(),
|
||||
token_ids: vec![],
|
||||
completion_tokens: 0,
|
||||
prompt_tokens: 128,
|
||||
..Default::default()
|
||||
},
|
||||
FrontendOutput {
|
||||
text: "He\"llo\n\t".into(),
|
||||
token_ids: vec![1000],
|
||||
completion_tokens: 1,
|
||||
prompt_tokens: 128,
|
||||
..Default::default()
|
||||
},
|
||||
FrontendOutput {
|
||||
text: " 世界 🌍 \\".into(),
|
||||
token_ids: vec![-2, 3],
|
||||
completion_tokens: 2,
|
||||
prompt_tokens: 128,
|
||||
..Default::default()
|
||||
},
|
||||
FrontendOutput {
|
||||
text: "!".into(),
|
||||
token_ids: vec![9],
|
||||
completion_tokens: 1,
|
||||
prompt_tokens: 128,
|
||||
finish_reason: serde_json::from_value(
|
||||
serde_json::json!({"type": "stop", "matched": 9}),
|
||||
)
|
||||
.expect("finish reason must parse"),
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
|
||||
for index in [None, Some(3usize)] {
|
||||
let mut acc = OutputAccumulator::default();
|
||||
for d in &deltas {
|
||||
acc.fold(d);
|
||||
let fast = cumulative_frame_json(&acc, "7", index).expect("no extras → fast path");
|
||||
let slow = tag_value(frame_value(acc.snapshot(), "7"), index);
|
||||
println!("fast={fast:?}");
|
||||
println!("slow={slow:?}");
|
||||
assert_eq!(
|
||||
as_json(&fast),
|
||||
as_json(&slow),
|
||||
"index={index:?} text={:?}",
|
||||
acc.snapshot().text
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The same equivalence, for the shape that made cumulative streaming O(T²):
|
||||
/// every logprob family at once, across several deltas, with and without
|
||||
/// `return_text_in_logprobs` texts and with a null ragged position.
|
||||
#[test]
|
||||
fn cumulative_frame_json_matches_serde_with_logprobs() {
|
||||
for with_texts in [false, true] {
|
||||
let txt = |v: &[&str]| -> Vec<String> {
|
||||
if with_texts {
|
||||
v.iter().map(|s| (*s).to_string()).collect()
|
||||
} else {
|
||||
Vec::new()
|
||||
}
|
||||
};
|
||||
let deltas = [
|
||||
// Prefill: the set-once input families and a null top-k position.
|
||||
FrontendOutput {
|
||||
prompt_tokens: 4,
|
||||
extras: Some(Box::new(ChunkExtras {
|
||||
in_lp_val: vec![f32::NAN, -1.5],
|
||||
in_lp_idx: vec![10, 11],
|
||||
in_lp_txt: txt(&["a", "b"]),
|
||||
in_top_val: vec![-0.25],
|
||||
in_top_idx: vec![12],
|
||||
in_top_lens: vec![0, 1],
|
||||
in_top_txt: txt(&["c"]),
|
||||
in_tid_val: vec![-2.0],
|
||||
in_tid_idx: vec![13],
|
||||
in_tid_lens: vec![1],
|
||||
in_tid_txt: txt(&["d"]),
|
||||
..Default::default()
|
||||
})),
|
||||
..Default::default()
|
||||
},
|
||||
FrontendOutput {
|
||||
text: "He\"llo".into(),
|
||||
token_ids: vec![100],
|
||||
completion_tokens: 1,
|
||||
prompt_tokens: 4,
|
||||
extras: Some(Box::new(ChunkExtras {
|
||||
out_lp_val: vec![-0.5],
|
||||
out_lp_idx: vec![100],
|
||||
out_lp_txt: txt(&["He\"llo"]),
|
||||
out_top_val: vec![-0.5, -3.0],
|
||||
out_top_idx: vec![100, 7],
|
||||
out_top_lens: vec![2],
|
||||
out_top_txt: txt(&["He\"llo", "x"]),
|
||||
out_tid_val: vec![-0.5],
|
||||
out_tid_idx: vec![100],
|
||||
out_tid_lens: vec![1],
|
||||
out_tid_txt: txt(&["He\"llo"]),
|
||||
..Default::default()
|
||||
})),
|
||||
..Default::default()
|
||||
},
|
||||
FrontendOutput {
|
||||
text: " 世界".into(),
|
||||
token_ids: vec![-2, 3],
|
||||
completion_tokens: 2,
|
||||
prompt_tokens: 4,
|
||||
finish_reason: serde_json::from_value(
|
||||
serde_json::json!({"type": "stop", "matched": 3}),
|
||||
)
|
||||
.expect("finish reason must parse"),
|
||||
extras: Some(Box::new(ChunkExtras {
|
||||
out_lp_val: vec![f32::NAN, -0.125],
|
||||
out_lp_idx: vec![-2, 3],
|
||||
out_lp_txt: txt(&[" 世", "界"]),
|
||||
// A zero-length position must render as `null`, not `[]`.
|
||||
out_top_val: vec![-0.125],
|
||||
out_top_idx: vec![3],
|
||||
out_top_lens: vec![0, 1],
|
||||
out_top_txt: txt(&["界"]),
|
||||
out_tid_val: vec![],
|
||||
out_tid_idx: vec![],
|
||||
out_tid_lens: vec![0, 0],
|
||||
out_tid_txt: txt(&[]),
|
||||
hidden_val: vec![0.5, -0.25, 1.0],
|
||||
hidden_lens: vec![2, 1],
|
||||
..Default::default()
|
||||
})),
|
||||
},
|
||||
];
|
||||
|
||||
for index in [None, Some(2usize)] {
|
||||
let mut acc = OutputAccumulator::default();
|
||||
for d in &deltas {
|
||||
acc.fold(d);
|
||||
let fast = cumulative_frame_json(&acc, "9", index)
|
||||
.expect("the extras memo must stay valid for a well-formed request");
|
||||
let slow = tag_value(frame_value(acc.snapshot(), "9"), index);
|
||||
assert_eq!(
|
||||
as_json(&fast),
|
||||
as_json(&slow),
|
||||
"with_texts={with_texts} index={index:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A delta sequence that supplies texts for some values and not others breaks
|
||||
/// the append-equivalence the memo rests on (`text_slot` is indexed globally,
|
||||
/// so a gap shifts every later text). The accumulator must notice and defer to
|
||||
/// the `Value` builder rather than emit a frame that disagrees with it.
|
||||
#[test]
|
||||
fn mismatched_logprob_texts_fall_back_to_the_value_path() {
|
||||
let mut acc = OutputAccumulator::default();
|
||||
acc.fold(&FrontendOutput {
|
||||
extras: Some(Box::new(ChunkExtras {
|
||||
out_lp_val: vec![-0.5],
|
||||
out_lp_idx: vec![5],
|
||||
..Default::default() // no texts
|
||||
})),
|
||||
..Default::default()
|
||||
});
|
||||
assert!(cumulative_frame_json(&acc, "1", None).is_some());
|
||||
acc.fold(&FrontendOutput {
|
||||
extras: Some(Box::new(ChunkExtras {
|
||||
out_lp_val: vec![-0.25],
|
||||
out_lp_idx: vec![6],
|
||||
out_lp_txt: vec!["b".into()], // …now texts: out of lockstep
|
||||
..Default::default()
|
||||
})),
|
||||
..Default::default()
|
||||
});
|
||||
assert!(
|
||||
cumulative_frame_json(&acc, "1", None).is_none(),
|
||||
"a text column out of lockstep must invalidate the memo"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
//! Tonic adapter for the native `sglang.api.v1` service.
|
||||
//!
|
||||
//! The service is the gRPC rendering of the contract the native HTTP endpoints
|
||||
//! serve: `Generate` is `/generate`, `HealthCheck` is `/health_generate`, and
|
||||
//! `GetModelInfo` / `GetServerInfo` are their HTTP namesakes. Requests decode
|
||||
//! through the shared `/generate` fan-out and responses stream through the
|
||||
//! shared [`CoreHandle`], which owns preprocessing, admission, runtime
|
||||
//! communication, and request cancellation.
|
||||
//!
|
||||
//! The thin Tonic-service structure, streamed response approach.
|
||||
|
||||
pub(crate) mod app;
|
||||
mod info;
|
||||
mod native_api;
|
||||
pub(crate) mod service;
|
||||
+7
-8
@@ -1,18 +1,17 @@
|
||||
//! Tonic listener lifecycle for the native Rust frontend.
|
||||
//!
|
||||
//! The pre-bound socket, shared API runtime, and shutdown shape build on Rain
|
||||
//! Jiang's multi-protocol prototype in `sgl-project/sglang#36923`. Unlike that
|
||||
//! prototype, the listener mounts the canonical `runtime.v1` adapter on the
|
||||
//! transport-neutral [`crate::frontend::FrontendHandle`] contract.
|
||||
//! Jiang's multi-protocol prototype in `sgl-project/sglang#36923`. The listener
|
||||
//! mounts the `api.v1` adapter on the transport-neutral
|
||||
//! [`crate::api_server::core::CoreHandle`] contract.
|
||||
|
||||
use sglang_grpc_types::sglang::runtime::v1::sglang_service_server::SglangServiceServer;
|
||||
use sglang_api_types::api::v1::sglang_service_server::SglangServiceServer;
|
||||
use tokio_stream::wrappers::TcpListenerStream;
|
||||
|
||||
use super::GrpcService;
|
||||
use super::service::GrpcService;
|
||||
|
||||
/// Keep the native Rust listener aligned with the existing Python-backed
|
||||
/// runtime.v1 server. Tonic's 4 MiB default is too small for large token or
|
||||
/// multimodal request bodies.
|
||||
/// Tonic's 4 MiB default is too small for large token or multimodal request
|
||||
/// bodies; keep the limit the Python-backed gRPC server established.
|
||||
const DEFAULT_GRPC_MAX_MESSAGE_SIZE: usize = 64 * 1024 * 1024;
|
||||
|
||||
fn max_message_size() -> usize {
|
||||
@@ -0,0 +1,68 @@
|
||||
//! Frontend metadata to the `api.v1` info responses.
|
||||
//!
|
||||
//! Both protos carry the JSON-typed members (`preferred_sampling_params`, the
|
||||
//! allowlisted `internal_states`) as JSON text, so the gRPC reply holds the
|
||||
//! same documents the HTTP reply embeds.
|
||||
|
||||
use sglang_api_types::api::v1 as api;
|
||||
|
||||
use crate::message::info::{ModelInfo, ServerInfo};
|
||||
|
||||
pub(super) fn model_info(info: ModelInfo) -> Result<api::GetModelInfoResponse, String> {
|
||||
let ModelInfo {
|
||||
model_path,
|
||||
served_model_name,
|
||||
tokenizer_path,
|
||||
is_generation,
|
||||
preferred_sampling_params,
|
||||
weight_version,
|
||||
load_format,
|
||||
reasoning_parser,
|
||||
tool_call_parser,
|
||||
disaggregation_mode: _,
|
||||
} = info;
|
||||
let preferred_sampling_params = preferred_sampling_params
|
||||
.map(|params| serde_json::to_string(¶ms.0))
|
||||
.transpose()
|
||||
.map_err(|error| format!("preferred_sampling_params is not JSON: {error}"))?;
|
||||
Ok(api::GetModelInfoResponse {
|
||||
model_path,
|
||||
served_model_name,
|
||||
tokenizer_path,
|
||||
is_generation,
|
||||
preferred_sampling_params,
|
||||
weight_version,
|
||||
load_format,
|
||||
reasoning_parser,
|
||||
tool_call_parser,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn server_info(info: ServerInfo) -> Result<api::GetServerInfoResponse, String> {
|
||||
let ServerInfo {
|
||||
model_path,
|
||||
served_model_name,
|
||||
tokenizer_path,
|
||||
max_context_length,
|
||||
max_total_num_tokens,
|
||||
version,
|
||||
frontend: _,
|
||||
internal_states,
|
||||
} = info;
|
||||
let max_context_length = u32::try_from(max_context_length)
|
||||
.map_err(|_| format!("max_context_length {max_context_length} exceeds uint32"))?;
|
||||
let internal_states = internal_states
|
||||
.iter()
|
||||
.map(serde_json::to_string)
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.map_err(|error| format!("internal_states are not JSON: {error}"))?;
|
||||
Ok(api::GetServerInfoResponse {
|
||||
model_path,
|
||||
served_model_name,
|
||||
tokenizer_path,
|
||||
max_context_length,
|
||||
max_total_num_tokens: Some(max_total_num_tokens),
|
||||
version,
|
||||
internal_states,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
//! Core events to `api.v1` stream items and statuses.
|
||||
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use futures::StreamExt;
|
||||
use futures::stream::FuturesUnordered;
|
||||
use sglang_api_types::api::v1 as api;
|
||||
use tonic::{Code, Status};
|
||||
|
||||
use super::service::ResponseStream;
|
||||
use crate::api_server::core::{
|
||||
CoreCall, CoreError, CoreErrorKind, CoreEvent, CoreOutput, recv_indexed,
|
||||
};
|
||||
|
||||
pub(super) struct StreamOptions {
|
||||
/// `stream` as sent: false folds each item to its one terminal frame.
|
||||
pub stream: bool,
|
||||
/// Streamed frames are deltas (true) or cumulative (false, the default).
|
||||
pub incremental: bool,
|
||||
/// Batch form: every item carries its position.
|
||||
pub with_index: bool,
|
||||
pub response_timeout: Duration,
|
||||
pub created_at: Instant,
|
||||
}
|
||||
|
||||
/// Multiplex the admitted calls into one `GenerateStreamItem` stream, the
|
||||
/// gRPC twin of the native SSE multiplexer. Each call aborts itself if the
|
||||
/// stream is dropped while it is unfinished.
|
||||
pub(super) fn generate_stream(
|
||||
calls: Vec<CoreCall>,
|
||||
options: StreamOptions,
|
||||
) -> ResponseStream<api::GenerateStreamItem> {
|
||||
let stream = async_stream::stream! {
|
||||
let public_ids: Vec<String> = calls
|
||||
.iter()
|
||||
.map(|call| call.public_id().to_owned())
|
||||
.collect();
|
||||
let mut cumulative: Vec<CoreOutput> =
|
||||
calls.iter().map(|_| CoreOutput::default()).collect();
|
||||
let index = |i: usize| options.with_index.then_some(i as u32);
|
||||
|
||||
// Poll all receivers concurrently; re-arm a receiver's future after each
|
||||
// non-terminal event so its stream keeps flowing.
|
||||
let mut pending = FuturesUnordered::new();
|
||||
for (i, call) in calls.into_iter().enumerate() {
|
||||
pending.push(recv_indexed(i, call));
|
||||
}
|
||||
|
||||
loop {
|
||||
let next = tokio::time::timeout(options.response_timeout, pending.next()).await;
|
||||
let (i, call, events) = match next {
|
||||
Ok(Some(next)) => next,
|
||||
// Every call has reached its terminal event.
|
||||
Ok(None) => break,
|
||||
Err(_) => {
|
||||
yield Err(Status::deadline_exceeded("Stream chunk timed out"));
|
||||
break;
|
||||
}
|
||||
};
|
||||
if events.is_empty() {
|
||||
// Defensive fallback: CoreCall normally turns a premature
|
||||
// runtime close into a Failed event.
|
||||
yield Ok(error_item(&CoreError::ResponseTruncated, index(i)));
|
||||
continue;
|
||||
}
|
||||
|
||||
let acc = &mut cumulative[i];
|
||||
// Cumulative frames supersede one another, so a drained backlog
|
||||
// collapses to its last (Python's `out_list[-1]`); deltas can't be
|
||||
// dropped.
|
||||
let mut coalesced = false;
|
||||
let mut terminal = None;
|
||||
let mut failed = None;
|
||||
for event in events {
|
||||
match event {
|
||||
CoreEvent::Delta(mut delta) => {
|
||||
acc.append_delta(&delta);
|
||||
if !options.stream {
|
||||
continue;
|
||||
}
|
||||
if options.incremental {
|
||||
// Python's incremental stream still reports the
|
||||
// cumulative completion-token count in every frame.
|
||||
delta.completion_tokens = acc.completion_tokens;
|
||||
yield Ok(frame_item(delta, &public_ids[i], index(i), None));
|
||||
} else {
|
||||
coalesced = true;
|
||||
}
|
||||
}
|
||||
CoreEvent::Finished(delta) => {
|
||||
acc.append_delta(&delta);
|
||||
terminal = Some(delta);
|
||||
}
|
||||
CoreEvent::Failed(error) => failed = Some(error),
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(error) = failed {
|
||||
yield Ok(error_item(&error, index(i)));
|
||||
} else if let Some(mut delta) = terminal {
|
||||
let e2e_latency = Some(options.created_at.elapsed().as_secs_f64());
|
||||
// A unary reply is the cumulative result whatever the stream
|
||||
// policy. The terminal frame is the accumulated output's last
|
||||
// use, so it moves out instead of cloning.
|
||||
let output = if options.stream && options.incremental {
|
||||
delta.completion_tokens = acc.completion_tokens;
|
||||
delta
|
||||
} else {
|
||||
std::mem::take(acc)
|
||||
};
|
||||
yield Ok(frame_item(output, &public_ids[i], index(i), e2e_latency));
|
||||
} else {
|
||||
if coalesced {
|
||||
// An intermediate cumulative frame has to clone: the
|
||||
// accumulated output stays behind for the next one.
|
||||
yield Ok(frame_item(acc.clone(), &public_ids[i], index(i), None));
|
||||
}
|
||||
pending.push(recv_indexed(i, call));
|
||||
}
|
||||
}
|
||||
// Unfinished calls are still owned by `pending`: dropping the stream
|
||||
// drops them and uses the core abort lane; a terminal event has
|
||||
// already disarmed its call.
|
||||
};
|
||||
Box::pin(stream)
|
||||
}
|
||||
|
||||
fn frame_item(
|
||||
output: CoreOutput,
|
||||
public_id: &str,
|
||||
index: Option<u32>,
|
||||
e2e_latency: Option<f64>,
|
||||
) -> api::GenerateStreamItem {
|
||||
api::GenerateStreamItem {
|
||||
item: Some(api::generate_stream_item::Item::Frame(output.frame(
|
||||
public_id,
|
||||
index,
|
||||
e2e_latency,
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
/// One item's failure inside the stream. `code` is the native error body's
|
||||
/// HTTP status, as on the SSE error frame.
|
||||
fn error_item(error: &CoreError, index: Option<u32>) -> api::GenerateStreamItem {
|
||||
api::GenerateStreamItem {
|
||||
item: Some(api::generate_stream_item::Item::Error(
|
||||
error.stream_error(index),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn status(error: CoreError) -> Status {
|
||||
let code = match error.kind() {
|
||||
CoreErrorKind::InvalidArgument => Code::InvalidArgument,
|
||||
CoreErrorKind::NotFound => Code::NotFound,
|
||||
CoreErrorKind::FailedPrecondition => Code::FailedPrecondition,
|
||||
CoreErrorKind::ResourceExhausted => Code::ResourceExhausted,
|
||||
CoreErrorKind::Cancelled => Code::Cancelled,
|
||||
CoreErrorKind::DeadlineExceeded => Code::DeadlineExceeded,
|
||||
CoreErrorKind::Unavailable => Code::Unavailable,
|
||||
CoreErrorKind::Internal => Code::Internal,
|
||||
};
|
||||
Status::new(code, error.to_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn semantic_errors_map_to_canonical_grpc_codes() {
|
||||
let cases = [
|
||||
(
|
||||
CoreError::InvalidArgument("bad".into()),
|
||||
Code::InvalidArgument,
|
||||
),
|
||||
(
|
||||
CoreError::RuntimeRejected {
|
||||
kind: CoreErrorKind::NotFound,
|
||||
message: "missing".into(),
|
||||
legacy_http_status: 404,
|
||||
},
|
||||
Code::NotFound,
|
||||
),
|
||||
(
|
||||
CoreError::RuntimeRejected {
|
||||
kind: CoreErrorKind::FailedPrecondition,
|
||||
message: "not ready".into(),
|
||||
legacy_http_status: 412,
|
||||
},
|
||||
Code::FailedPrecondition,
|
||||
),
|
||||
(
|
||||
CoreError::Overloaded("full".into()),
|
||||
Code::ResourceExhausted,
|
||||
),
|
||||
(CoreError::Cancelled("gone".into()), Code::Cancelled),
|
||||
(
|
||||
CoreError::RuntimeRejected {
|
||||
kind: CoreErrorKind::DeadlineExceeded,
|
||||
message: "late".into(),
|
||||
legacy_http_status: 504,
|
||||
},
|
||||
Code::DeadlineExceeded,
|
||||
),
|
||||
(CoreError::Unavailable, Code::Unavailable),
|
||||
(CoreError::Internal("bug".into()), Code::Internal),
|
||||
];
|
||||
|
||||
for (error, expected) in cases {
|
||||
assert_eq!(status(error).code(), expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,718 @@
|
||||
//! The `SglangService` implementation: per-RPC translation between the
|
||||
//! `api.v1` messages and the shared core.
|
||||
|
||||
use std::pin::Pin;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use futures::Stream;
|
||||
use sglang_api_types::api::v1 as api;
|
||||
use sglang_api_types::api::v1::sglang_service_server::SglangService;
|
||||
use tonic::{Request, Response, Status};
|
||||
|
||||
use super::{info, native_api};
|
||||
use crate::api_server::core::{CoreHandle, HealthStatus};
|
||||
use crate::message::config::{PreferredSamplingParams, ServerArgs};
|
||||
use crate::message::request::into_requests;
|
||||
use crate::message::wire::fill_preferred_sampling;
|
||||
use crate::utils::environ;
|
||||
|
||||
const DEFAULT_RESPONSE_TIMEOUT: Duration = Duration::from_secs(300);
|
||||
|
||||
pub(super) type ResponseStream<T> = Pin<Box<dyn Stream<Item = Result<T, Status>> + Send + 'static>>;
|
||||
|
||||
/// Narrow snapshot of launch policy the adapter needs per call. It
|
||||
/// intentionally excludes listener, authentication, TLS, and lifecycle
|
||||
/// configuration.
|
||||
#[derive(Clone)]
|
||||
struct AdapterConfig {
|
||||
preferred_sampling_params: Option<PreferredSamplingParams>,
|
||||
incremental_streaming_output: bool,
|
||||
/// Longest wait for the next event of any in-flight call before the RPC
|
||||
/// ends with DEADLINE_EXCEEDED.
|
||||
response_timeout: Duration,
|
||||
/// `/health_generate`'s heartbeat window (`SGLANG_HEALTH_CHECK_TIMEOUT`).
|
||||
health_timeout: Duration,
|
||||
}
|
||||
|
||||
/// Tonic-facing implementation backed by the transport-neutral core.
|
||||
pub(crate) struct GrpcService {
|
||||
core: CoreHandle,
|
||||
config: AdapterConfig,
|
||||
}
|
||||
|
||||
impl GrpcService {
|
||||
pub(crate) fn new(core: CoreHandle, server_args: &ServerArgs) -> Self {
|
||||
Self {
|
||||
core,
|
||||
config: AdapterConfig {
|
||||
preferred_sampling_params: server_args.preferred_sampling_params.clone(),
|
||||
incremental_streaming_output: server_args.incremental_streaming_output,
|
||||
response_timeout: DEFAULT_RESPONSE_TIMEOUT,
|
||||
health_timeout: Duration::from_secs(
|
||||
environ::env_i64("SGLANG_HEALTH_CHECK_TIMEOUT", 20).max(0) as u64,
|
||||
),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn for_test(
|
||||
core: CoreHandle,
|
||||
preferred_sampling_params: Option<PreferredSamplingParams>,
|
||||
incremental_streaming_output: bool,
|
||||
response_timeout: Duration,
|
||||
) -> Self {
|
||||
Self {
|
||||
core,
|
||||
config: AdapterConfig {
|
||||
preferred_sampling_params,
|
||||
incremental_streaming_output,
|
||||
response_timeout,
|
||||
health_timeout: Duration::from_millis(20),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tonic::async_trait]
|
||||
impl SglangService for GrpcService {
|
||||
type GenerateStream = ResponseStream<api::GenerateStreamItem>;
|
||||
|
||||
/// `/generate` over gRPC: one schema, one fan-out, one admission path. A
|
||||
/// body that fails normalization ends the RPC with INVALID_ARGUMENT before
|
||||
/// anything reaches the scheduler, as the HTTP 400 does.
|
||||
async fn generate(
|
||||
&self,
|
||||
request: Request<api::GenerateRequest>,
|
||||
) -> Result<Response<Self::GenerateStream>, Status> {
|
||||
let mut request = request.into_inner();
|
||||
if let Some(preferred) = &self.config.preferred_sampling_params {
|
||||
request.sampling_params =
|
||||
fill_preferred_sampling(request.sampling_params.take(), &preferred.0)
|
||||
.map_err(Status::internal)?;
|
||||
}
|
||||
// Protobuf has no null: a field the client left unset takes the
|
||||
// schema default, as an absent JSON key does.
|
||||
request.apply_absent_defaults();
|
||||
let stream = request.stream.unwrap_or(false);
|
||||
let (payloads, is_batch) =
|
||||
into_requests(request).map_err(|error| Status::invalid_argument(error.to_string()))?;
|
||||
// Python starts its request clock after normalization and before
|
||||
// tokenization / multimodal preprocessing; `e2e_latency` measures from
|
||||
// the same boundary here.
|
||||
let created_at = Instant::now();
|
||||
let calls = self
|
||||
.core
|
||||
.generate_batch(payloads)
|
||||
.await
|
||||
.map_err(native_api::status)?;
|
||||
Ok(Response::new(native_api::generate_stream(
|
||||
calls,
|
||||
native_api::StreamOptions {
|
||||
stream,
|
||||
incremental: self.config.incremental_streaming_output,
|
||||
with_index: is_batch,
|
||||
response_timeout: self.config.response_timeout,
|
||||
created_at,
|
||||
},
|
||||
)))
|
||||
}
|
||||
|
||||
async fn health_check(
|
||||
&self,
|
||||
_request: Request<api::HealthCheckRequest>,
|
||||
) -> Result<Response<api::HealthCheckResponse>, Status> {
|
||||
match self.core.probe_health(self.config.health_timeout).await {
|
||||
Ok(HealthStatus::Healthy) => {
|
||||
Ok(Response::new(api::HealthCheckResponse { healthy: true }))
|
||||
}
|
||||
Ok(HealthStatus::NotReady) => Err(Status::unavailable(
|
||||
"server is still completing its startup warmup",
|
||||
)),
|
||||
Ok(HealthStatus::Stalled) => Err(Status::unavailable(
|
||||
"no scheduler output within the health-check timeout",
|
||||
)),
|
||||
Err(error) => Err(native_api::status(error)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_model_info(
|
||||
&self,
|
||||
_request: Request<api::GetModelInfoRequest>,
|
||||
) -> Result<Response<api::GetModelInfoResponse>, Status> {
|
||||
info::model_info(self.core.model_info())
|
||||
.map(Response::new)
|
||||
.map_err(Status::internal)
|
||||
}
|
||||
|
||||
async fn get_server_info(
|
||||
&self,
|
||||
_request: Request<api::GetServerInfoRequest>,
|
||||
) -> Result<Response<api::GetServerInfoResponse>, Status> {
|
||||
let server_info = self.core.server_info().await.map_err(native_api::status)?;
|
||||
info::server_info(server_info)
|
||||
.map(Response::new)
|
||||
.map_err(Status::internal)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::time::Duration;
|
||||
|
||||
use futures::StreamExt;
|
||||
use sglang_api_types::api::v1 as api;
|
||||
use sglang_api_types::api::v1::generate_stream_item::Item;
|
||||
use sglang_api_types::api::v1::sglang_service_server::SglangService;
|
||||
use tonic::{Code, Request};
|
||||
|
||||
use super::GrpcService;
|
||||
use crate::api_server::core::{CoreConfig, CoreHandle, CoreMetadata};
|
||||
use crate::message::config::PreferredSamplingParams;
|
||||
use crate::message::finish_reason::FinishReason;
|
||||
use crate::message::ids::Rid;
|
||||
use crate::message::request::{GenerateRequest, Request as RuntimeRequest, RequestKind};
|
||||
use crate::message::response::{ChunkEvent, ResponseItem, ResponseSink};
|
||||
use crate::tokenizer_manager::wiring::{AbortSource, RequestAdmission, TmEvent};
|
||||
|
||||
struct Harness {
|
||||
service: GrpcService,
|
||||
intake_rx: flume::Receiver<TmEvent>,
|
||||
abort_rx: flume::Receiver<AbortSource>,
|
||||
}
|
||||
|
||||
struct GenerationIntake {
|
||||
request: Box<GenerateRequest>,
|
||||
rid: Rid,
|
||||
sink: ResponseSink,
|
||||
admission: RequestAdmission,
|
||||
}
|
||||
|
||||
impl Harness {
|
||||
fn new(response_capacity: usize, incremental: bool, response_timeout: Duration) -> Self {
|
||||
Self::build(response_capacity, incremental, response_timeout, true, None)
|
||||
}
|
||||
|
||||
fn build(
|
||||
response_capacity: usize,
|
||||
incremental: bool,
|
||||
response_timeout: Duration,
|
||||
startup_ready: bool,
|
||||
preferred: Option<PreferredSamplingParams>,
|
||||
) -> Self {
|
||||
let (intake_tx, intake_rx) = flume::unbounded();
|
||||
let (abort_tx, abort_rx) = flume::unbounded();
|
||||
let core = CoreHandle::new(
|
||||
intake_tx,
|
||||
abort_tx,
|
||||
CoreConfig {
|
||||
response_capacity,
|
||||
response_activity: Default::default(),
|
||||
startup_ready,
|
||||
is_disaggregation: false,
|
||||
mm_limits: Default::default(),
|
||||
metadata: CoreMetadata::default(),
|
||||
},
|
||||
);
|
||||
Self {
|
||||
service: GrpcService::for_test(core, preferred, incremental, response_timeout),
|
||||
intake_rx,
|
||||
abort_rx,
|
||||
}
|
||||
}
|
||||
|
||||
async fn next_generation(&self) -> GenerationIntake {
|
||||
let TmEvent::Intake { request, admission } = self
|
||||
.intake_rx
|
||||
.recv_async()
|
||||
.await
|
||||
.expect("request submitted")
|
||||
else {
|
||||
panic!("expected generation intake");
|
||||
};
|
||||
let RuntimeRequest {
|
||||
rid,
|
||||
sink,
|
||||
kind,
|
||||
state: _,
|
||||
} = request;
|
||||
let RequestKind::Generate(request) = kind else {
|
||||
panic!("expected generation request");
|
||||
};
|
||||
GenerationIntake {
|
||||
request,
|
||||
rid,
|
||||
sink,
|
||||
admission,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn chunk(rid: &Rid, text: &str, token_id: i64, finished: bool) -> ChunkEvent {
|
||||
ChunkEvent {
|
||||
rid: rid.clone(),
|
||||
token_ids: vec![token_id],
|
||||
finish_reason: finished.then(stop_reason),
|
||||
prompt_tokens: 3,
|
||||
text: text.into(),
|
||||
completion_tokens: 1,
|
||||
extras: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn stop_reason() -> FinishReason {
|
||||
serde_json::from_value(serde_json::json!({"type": "stop"})).unwrap()
|
||||
}
|
||||
|
||||
fn one_string(value: &str) -> api::StringOrList {
|
||||
api::StringOrList {
|
||||
value: Some(api::string_or_list::Value::One(value.into())),
|
||||
}
|
||||
}
|
||||
|
||||
fn many_strings(values: &[&str]) -> api::StringOrList {
|
||||
api::StringOrList {
|
||||
value: Some(api::string_or_list::Value::Many(api::StringList {
|
||||
items: values.iter().map(|value| (*value).to_owned()).collect(),
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
fn one_token_ids(ids: &[i64]) -> api::TokenIdsOrList {
|
||||
api::TokenIdsOrList {
|
||||
value: Some(api::token_ids_or_list::Value::One(api::TokenIds {
|
||||
ids: ids.to_vec(),
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
fn text_request(text: &str, rid: &str) -> api::GenerateRequest {
|
||||
api::GenerateRequest {
|
||||
text: Some(one_string(text)),
|
||||
rid: Some(one_string(rid)),
|
||||
stream: Some(true),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn ids_request(ids: &[i64], rid: Option<&str>) -> api::GenerateRequest {
|
||||
api::GenerateRequest {
|
||||
input_ids: Some(one_token_ids(ids)),
|
||||
rid: rid.map(one_string),
|
||||
stream: Some(true),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn frame(item: api::GenerateStreamItem) -> api::GenerateResponse {
|
||||
match item.item {
|
||||
Some(Item::Frame(frame)) => frame,
|
||||
other => panic!("expected a frame, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn stream_error(item: api::GenerateStreamItem) -> api::GenerateStreamError {
|
||||
match item.item {
|
||||
Some(Item::Error(error)) => error,
|
||||
other => panic!("expected an error item, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn meta(frame: &api::GenerateResponse) -> &api::GenerateMetaInfo {
|
||||
frame
|
||||
.meta_info
|
||||
.as_ref()
|
||||
.expect("every frame carries meta_info")
|
||||
}
|
||||
|
||||
fn output_ids(frame: &api::GenerateResponse) -> &[i64] {
|
||||
frame
|
||||
.output_ids
|
||||
.as_ref()
|
||||
.map_or(&[], |ids| ids.ids.as_slice())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn generate_streams_cumulative_frames_for_a_text_prompt() {
|
||||
let harness = Harness::new(2, false, Duration::from_secs(1));
|
||||
let mut stream = harness
|
||||
.service
|
||||
.generate(Request::new(text_request("prompt", "request-1")))
|
||||
.await
|
||||
.unwrap()
|
||||
.into_inner();
|
||||
|
||||
let intake = harness.next_generation().await;
|
||||
assert!(intake.admission.try_accept());
|
||||
assert_eq!(intake.request.text.as_deref(), Some("prompt"));
|
||||
assert!(intake.request.input_ids.is_none());
|
||||
assert!(intake.request.stream);
|
||||
assert_eq!(intake.rid.client_facing(), "request-1");
|
||||
// One chunk at a time: a queued backlog of cumulative frames collapses to
|
||||
// its last, so sending both up front would yield only the terminal frame.
|
||||
intake
|
||||
.sink
|
||||
.try_send(ResponseItem::Frame(chunk(&intake.rid, "Hel", 1, false)))
|
||||
.unwrap();
|
||||
let first = frame(stream.next().await.unwrap().unwrap());
|
||||
assert_eq!(first.text, "Hel");
|
||||
assert_eq!(output_ids(&first), [1]);
|
||||
assert_eq!(first.index, None);
|
||||
let first_meta = meta(&first);
|
||||
assert_eq!(first_meta.id, "request-1");
|
||||
assert_eq!(first_meta.prompt_tokens, 3);
|
||||
assert_eq!(first_meta.completion_tokens, 1);
|
||||
assert!(first_meta.finish_reason.is_none());
|
||||
assert!(first_meta.e2e_latency.is_none());
|
||||
|
||||
intake
|
||||
.sink
|
||||
.try_send(ResponseItem::Done(chunk(&intake.rid, "lo", 2, true)))
|
||||
.unwrap();
|
||||
let finished = frame(stream.next().await.unwrap().unwrap());
|
||||
assert_eq!(finished.text, "Hello");
|
||||
assert_eq!(output_ids(&finished), [1, 2]);
|
||||
let finished_meta = meta(&finished);
|
||||
assert_eq!(finished_meta.completion_tokens, 2);
|
||||
assert!(matches!(
|
||||
finished_meta.finish_reason,
|
||||
Some(api::FinishReason {
|
||||
kind: Some(api::finish_reason::Kind::Stop(_))
|
||||
})
|
||||
));
|
||||
assert!(finished_meta.e2e_latency.is_some());
|
||||
assert!(stream.next().await.is_none());
|
||||
assert!(harness.abort_rx.try_recv().is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn incremental_frames_carry_deltas_with_cumulative_count() {
|
||||
let harness = Harness::new(2, true, Duration::from_secs(1));
|
||||
let mut stream = harness
|
||||
.service
|
||||
.generate(Request::new(ids_request(&[4, 5], Some("tokens"))))
|
||||
.await
|
||||
.unwrap()
|
||||
.into_inner();
|
||||
|
||||
let intake = harness.next_generation().await;
|
||||
assert!(intake.admission.try_accept());
|
||||
assert_eq!(intake.request.input_ids.as_deref(), Some(&[4, 5][..]));
|
||||
assert!(intake.request.text.is_none());
|
||||
intake
|
||||
.sink
|
||||
.try_send(ResponseItem::Frame(chunk(&intake.rid, "A", 10, false)))
|
||||
.unwrap();
|
||||
intake
|
||||
.sink
|
||||
.try_send(ResponseItem::Done(chunk(&intake.rid, "B", 11, true)))
|
||||
.unwrap();
|
||||
|
||||
let first = frame(stream.next().await.unwrap().unwrap());
|
||||
let finished = frame(stream.next().await.unwrap().unwrap());
|
||||
assert_eq!(output_ids(&first), [10]);
|
||||
assert_eq!(output_ids(&finished), [11]);
|
||||
assert_eq!(finished.text, "B");
|
||||
assert_eq!(meta(&finished).completion_tokens, 2);
|
||||
assert!(stream.next().await.is_none());
|
||||
}
|
||||
|
||||
/// `stream: false` is a one-frame stream: deltas fold into the terminal
|
||||
/// result, and the server's incremental policy does not split a unary reply.
|
||||
#[tokio::test]
|
||||
async fn unary_request_yields_only_the_cumulative_terminal_frame() {
|
||||
let harness = Harness::new(2, true, Duration::from_secs(1));
|
||||
let mut request = text_request("prompt", "unary");
|
||||
request.stream = Some(false);
|
||||
let mut stream = harness
|
||||
.service
|
||||
.generate(Request::new(request))
|
||||
.await
|
||||
.unwrap()
|
||||
.into_inner();
|
||||
|
||||
let intake = harness.next_generation().await;
|
||||
assert!(intake.admission.try_accept());
|
||||
assert!(!intake.request.stream);
|
||||
intake
|
||||
.sink
|
||||
.try_send(ResponseItem::Frame(chunk(&intake.rid, "Hel", 1, false)))
|
||||
.unwrap();
|
||||
intake
|
||||
.sink
|
||||
.try_send(ResponseItem::Done(chunk(&intake.rid, "lo", 2, true)))
|
||||
.unwrap();
|
||||
|
||||
let only = frame(stream.next().await.unwrap().unwrap());
|
||||
assert_eq!(only.text, "Hello");
|
||||
assert_eq!(output_ids(&only), [1, 2]);
|
||||
assert_eq!(meta(&only).completion_tokens, 2);
|
||||
assert!(stream.next().await.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stream_drop_aborts_admitted_request() {
|
||||
let harness = Harness::new(1, false, Duration::from_secs(1));
|
||||
let mut stream = harness
|
||||
.service
|
||||
.generate(Request::new(ids_request(&[1], Some("cancel-me"))))
|
||||
.await
|
||||
.unwrap()
|
||||
.into_inner();
|
||||
let intake = harness.next_generation().await;
|
||||
assert!(intake.admission.try_accept());
|
||||
intake
|
||||
.sink
|
||||
.try_send(ResponseItem::Frame(chunk(&intake.rid, "partial", 1, false)))
|
||||
.unwrap();
|
||||
assert!(stream.next().await.unwrap().is_ok());
|
||||
drop(stream);
|
||||
let abort = harness.abort_rx.recv_async().await.unwrap();
|
||||
assert_eq!(abort.rid().client_facing(), "cancel-me");
|
||||
assert!(harness.abort_rx.try_recv().is_err());
|
||||
}
|
||||
|
||||
/// A runtime failure after admission is this request's in-stream error item
|
||||
/// (the SSE error frame's twin), not an RPC status; the call is disarmed.
|
||||
#[tokio::test]
|
||||
async fn runtime_failure_rides_the_stream_as_an_error_item() {
|
||||
let harness = Harness::new(1, false, Duration::from_secs(1));
|
||||
let mut stream = harness
|
||||
.service
|
||||
.generate(Request::new(ids_request(&[1], None)))
|
||||
.await
|
||||
.unwrap()
|
||||
.into_inner();
|
||||
let intake = harness.next_generation().await;
|
||||
assert!(intake.admission.try_accept());
|
||||
intake
|
||||
.sink
|
||||
.try_send(ResponseItem::Error(crate::utils::error::Error::Validation(
|
||||
"bad sampling".into(),
|
||||
)))
|
||||
.unwrap();
|
||||
|
||||
let error = stream_error(stream.next().await.unwrap().unwrap());
|
||||
assert_eq!(error.index, None);
|
||||
let body = error.error.unwrap();
|
||||
assert_eq!(body.code, 400);
|
||||
assert!(body.message.contains("bad sampling"));
|
||||
assert!(stream.next().await.is_none());
|
||||
drop(stream);
|
||||
assert!(harness.abort_rx.try_recv().is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn per_chunk_timeout_after_progress_returns_deadline_exceeded_and_aborts() {
|
||||
let harness = Harness::new(1, false, Duration::from_millis(10));
|
||||
let mut stream = harness
|
||||
.service
|
||||
.generate(Request::new(ids_request(&[1], Some("too-slow"))))
|
||||
.await
|
||||
.unwrap()
|
||||
.into_inner();
|
||||
let intake = harness.next_generation().await;
|
||||
assert!(intake.admission.try_accept());
|
||||
intake
|
||||
.sink
|
||||
.try_send(ResponseItem::Frame(chunk(&intake.rid, "partial", 1, false)))
|
||||
.unwrap();
|
||||
assert!(stream.next().await.unwrap().is_ok());
|
||||
|
||||
let error = stream.next().await.unwrap().unwrap_err();
|
||||
assert_eq!(error.code(), Code::DeadlineExceeded);
|
||||
drop(stream);
|
||||
let abort = harness.abort_rx.recv_async().await.unwrap();
|
||||
assert_eq!(abort.rid().client_facing(), "too-slow");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn closed_intake_is_a_top_level_unavailable_status() {
|
||||
let (intake_tx, intake_rx) = flume::unbounded();
|
||||
drop(intake_rx);
|
||||
let (abort_tx, _abort_rx) = flume::unbounded();
|
||||
let core = CoreHandle::new(
|
||||
intake_tx,
|
||||
abort_tx,
|
||||
CoreConfig {
|
||||
response_capacity: 1,
|
||||
response_activity: Default::default(),
|
||||
startup_ready: true,
|
||||
is_disaggregation: false,
|
||||
mm_limits: Default::default(),
|
||||
metadata: CoreMetadata::default(),
|
||||
},
|
||||
);
|
||||
let service = GrpcService::for_test(core, None, false, Duration::from_secs(1));
|
||||
|
||||
let result = service
|
||||
.generate(Request::new(ids_request(&[1], None)))
|
||||
.await;
|
||||
let error = match result {
|
||||
Ok(_) => panic!("closed intake must reject the RPC"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert_eq!(error.code(), Code::Unavailable);
|
||||
}
|
||||
|
||||
/// Normalization failures end the RPC before anything is submitted, with the
|
||||
/// status that mirrors the HTTP 400.
|
||||
#[tokio::test]
|
||||
async fn malformed_request_is_rejected_before_submission() {
|
||||
let harness = Harness::new(1, false, Duration::from_secs(1));
|
||||
let mut both = text_request("prompt", "both");
|
||||
both.input_ids = Some(one_token_ids(&[1]));
|
||||
let error = harness
|
||||
.service
|
||||
.generate(Request::new(both))
|
||||
.await
|
||||
.err()
|
||||
.expect("text and input_ids together must be rejected");
|
||||
assert_eq!(error.code(), Code::InvalidArgument);
|
||||
assert!(error.message().contains("text"));
|
||||
|
||||
let error = harness
|
||||
.service
|
||||
.generate(Request::new(ids_request(&[], None)))
|
||||
.await
|
||||
.err()
|
||||
.expect("empty input_ids must be rejected");
|
||||
assert_eq!(error.code(), Code::InvalidArgument);
|
||||
assert!(harness.intake_rx.try_recv().is_err());
|
||||
}
|
||||
|
||||
/// A list-form request fans out like the HTTP batch: every item is tagged
|
||||
/// with its position, and one item's failure neither ends the stream nor
|
||||
/// touches its siblings.
|
||||
#[tokio::test]
|
||||
async fn batch_items_are_indexed_and_fail_independently() {
|
||||
let harness = Harness::new(2, false, Duration::from_secs(1));
|
||||
let mut stream = harness
|
||||
.service
|
||||
.generate(Request::new(api::GenerateRequest {
|
||||
text: Some(many_strings(&["a", "b"])),
|
||||
stream: Some(true),
|
||||
..Default::default()
|
||||
}))
|
||||
.await
|
||||
.unwrap()
|
||||
.into_inner();
|
||||
|
||||
let first = harness.next_generation().await;
|
||||
let second = harness.next_generation().await;
|
||||
assert!(first.admission.try_accept());
|
||||
assert!(second.admission.try_accept());
|
||||
assert_eq!(first.request.text.as_deref(), Some("a"));
|
||||
assert_eq!(second.request.text.as_deref(), Some("b"));
|
||||
first
|
||||
.sink
|
||||
.try_send(ResponseItem::Error(crate::utils::error::Error::Validation(
|
||||
"first failed".into(),
|
||||
)))
|
||||
.unwrap();
|
||||
second
|
||||
.sink
|
||||
.try_send(ResponseItem::Done(chunk(&second.rid, "ok", 5, true)))
|
||||
.unwrap();
|
||||
|
||||
let mut items = Vec::new();
|
||||
while let Some(item) = stream.next().await {
|
||||
items.push(item.unwrap());
|
||||
}
|
||||
assert_eq!(items.len(), 2);
|
||||
let (errors, frames): (Vec<_>, Vec<_>) = items
|
||||
.into_iter()
|
||||
.partition(|item| matches!(item.item, Some(Item::Error(_))));
|
||||
let error = stream_error(errors.into_iter().next().unwrap());
|
||||
assert_eq!(error.index, Some(0));
|
||||
assert!(error.error.unwrap().message.contains("first failed"));
|
||||
let ok = frame(frames.into_iter().next().unwrap());
|
||||
assert_eq!(ok.index, Some(1));
|
||||
assert_eq!(ok.text, "ok");
|
||||
assert_eq!(meta(&ok).id, second.rid.client_facing());
|
||||
assert!(harness.abort_rx.try_recv().is_err());
|
||||
}
|
||||
|
||||
/// Launch-time preferred sampling params fill the fields a protobuf request
|
||||
/// left unset, beneath the ones it set: the HTTP precedence at the gRPC entry.
|
||||
#[tokio::test]
|
||||
async fn preferred_sampling_params_fill_unset_fields() {
|
||||
let preferred = PreferredSamplingParams(serde_json::json!({
|
||||
"temperature": 0.25,
|
||||
"max_new_tokens": 32,
|
||||
}));
|
||||
let harness = Harness::build(1, false, Duration::from_secs(1), true, Some(preferred));
|
||||
let mut request = ids_request(&[1], None);
|
||||
request.sampling_params = Some(api::SamplingParamsOrList {
|
||||
value: Some(api::sampling_params_or_list::Value::One(
|
||||
api::SamplingParams {
|
||||
temperature: Some(0.8),
|
||||
..Default::default()
|
||||
},
|
||||
)),
|
||||
});
|
||||
let _stream = harness
|
||||
.service
|
||||
.generate(Request::new(request))
|
||||
.await
|
||||
.unwrap()
|
||||
.into_inner();
|
||||
|
||||
let intake = harness.next_generation().await;
|
||||
assert_eq!(intake.request.sampling_params.temperature, 0.8);
|
||||
assert_eq!(intake.request.sampling_params.max_new_tokens, Some(32));
|
||||
}
|
||||
|
||||
/// Protobuf has no null: a sampling field the client left unset takes the
|
||||
/// schema default, as an absent JSON key does. `max_new_tokens` is the one
|
||||
/// field whose Rust `None` means "unbounded", so without the absent-defaults
|
||||
/// pass an unset value would lift the limit instead of applying 128.
|
||||
#[tokio::test]
|
||||
async fn unset_protobuf_sampling_fields_take_schema_defaults() {
|
||||
let harness = Harness::new(1, false, Duration::from_secs(1));
|
||||
let mut request = ids_request(&[1], None);
|
||||
request.sampling_params = Some(api::SamplingParamsOrList {
|
||||
value: Some(api::sampling_params_or_list::Value::One(
|
||||
api::SamplingParams {
|
||||
temperature: Some(0.8),
|
||||
..Default::default()
|
||||
},
|
||||
)),
|
||||
});
|
||||
let _stream = harness
|
||||
.service
|
||||
.generate(Request::new(request))
|
||||
.await
|
||||
.unwrap()
|
||||
.into_inner();
|
||||
|
||||
let intake = harness.next_generation().await;
|
||||
assert_eq!(intake.request.sampling_params.temperature, 0.8);
|
||||
assert_eq!(intake.request.sampling_params.max_new_tokens, Some(128));
|
||||
assert_eq!(intake.request.sampling_params.top_p, 1.0);
|
||||
}
|
||||
|
||||
/// `HealthCheck` is `/health_generate`: not ready and stalled are both
|
||||
/// UNAVAILABLE, and the stalled probe is a real generation submission.
|
||||
#[tokio::test]
|
||||
async fn health_check_reports_not_ready_and_stalled_as_unavailable() {
|
||||
let not_ready = Harness::build(1, false, Duration::from_secs(1), false, None);
|
||||
let error = not_ready
|
||||
.service
|
||||
.health_check(Request::new(api::HealthCheckRequest {}))
|
||||
.await
|
||||
.expect_err("warmup must not report healthy");
|
||||
assert_eq!(error.code(), Code::Unavailable);
|
||||
assert!(not_ready.intake_rx.try_recv().is_err());
|
||||
|
||||
let stalled = Harness::new(1, false, Duration::from_secs(1));
|
||||
let error = stalled
|
||||
.service
|
||||
.health_check(Request::new(api::HealthCheckRequest {}))
|
||||
.await
|
||||
.expect_err("no heartbeat within the window must not report healthy");
|
||||
assert_eq!(error.code(), Code::Unavailable);
|
||||
let probe = stalled.next_generation().await;
|
||||
assert_eq!(probe.request.input_ids.as_deref(), Some(&[0][..]));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
pub(crate) mod app;
|
||||
mod info;
|
||||
mod native_api;
|
||||
mod openai;
|
||||
+24
-27
@@ -12,20 +12,21 @@ use axum::{
|
||||
response::Response,
|
||||
};
|
||||
|
||||
use super::disaggregation::bootstrap as pd_bootstrap;
|
||||
use super::{common, log, native_api, openai};
|
||||
use crate::frontend::FrontendHandle;
|
||||
use super::{info, native_api, openai};
|
||||
use crate::api_server::core::CoreHandle;
|
||||
use crate::api_server::disaggregation::bootstrap as pd_bootstrap;
|
||||
use crate::api_server::log;
|
||||
use crate::message::config::ServerArgs;
|
||||
|
||||
/// HTTP adapter state: the shared frontend capability, immutable server
|
||||
/// HTTP adapter state: the shared core capability, immutable server
|
||||
/// configuration needed for HTTP request preparation, and the chat formatter.
|
||||
///
|
||||
/// axum clones the router state into **every** request, so it is mounted as
|
||||
/// `Arc<AppState>` — one refcount bump per request instead of cloning the
|
||||
/// frontend handle and chat formatter. Deliberately not `Clone`, so it
|
||||
/// core handle and chat formatter. Deliberately not `Clone`, so it
|
||||
/// can only be shared through that `Arc`.
|
||||
pub(super) struct AppState {
|
||||
pub(super) frontend: FrontendHandle,
|
||||
pub(super) core: CoreHandle,
|
||||
pub(super) server_args: Arc<ServerArgs>,
|
||||
pub(super) chat_formatter: Option<openai::ChatFormatter>,
|
||||
}
|
||||
@@ -45,23 +46,19 @@ async fn mark_startup_ready(
|
||||
"/generate" | "/encode" | "/v1/chat/completions"
|
||||
);
|
||||
let response = next.run(req).await;
|
||||
record_startup_warmup_status(&state.frontend, is_startup_warmup, response.status());
|
||||
record_startup_warmup_status(&state.core, is_startup_warmup, response.status());
|
||||
response
|
||||
}
|
||||
|
||||
fn record_startup_warmup_status(
|
||||
frontend: &FrontendHandle,
|
||||
is_startup_warmup: bool,
|
||||
status: StatusCode,
|
||||
) {
|
||||
fn record_startup_warmup_status(core: &CoreHandle, is_startup_warmup: bool, status: StatusCode) {
|
||||
if is_startup_warmup && status.is_success() {
|
||||
frontend.mark_ready();
|
||||
core.mark_ready();
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn serve(
|
||||
listener: std::net::TcpListener,
|
||||
frontend: FrontendHandle,
|
||||
core: CoreHandle,
|
||||
server_args: Arc<ServerArgs>,
|
||||
// The runtime's shutdown signal, shared with every worker stage: it fires
|
||||
// (disconnects) when `Runtime::request_shutdown` drops the sender, at
|
||||
@@ -71,13 +68,13 @@ pub async fn serve(
|
||||
) {
|
||||
let chat_formatter = openai::load_chat_support(&server_args);
|
||||
let state = Arc::new(AppState {
|
||||
frontend,
|
||||
core,
|
||||
server_args: server_args.clone(),
|
||||
chat_formatter,
|
||||
});
|
||||
// Each endpoint module registers its own routes and merges here.
|
||||
let router = Router::new()
|
||||
.merge(common::routes())
|
||||
.merge(info::routes())
|
||||
.merge(native_api::routes())
|
||||
.merge(openai::routes());
|
||||
|
||||
@@ -138,32 +135,32 @@ pub async fn serve(
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn frontend() -> FrontendHandle {
|
||||
FrontendHandle::new(
|
||||
fn core_handle() -> CoreHandle {
|
||||
CoreHandle::new(
|
||||
flume::unbounded().0,
|
||||
flume::unbounded().0,
|
||||
crate::frontend::FrontendConfig {
|
||||
crate::api_server::core::CoreConfig {
|
||||
response_capacity: 8,
|
||||
response_activity: Default::default(),
|
||||
startup_ready: false,
|
||||
is_disaggregation: false,
|
||||
mm_limits: Default::default(),
|
||||
metadata: crate::frontend::FrontendMetadata::default(),
|
||||
metadata: crate::api_server::core::CoreMetadata::default(),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_a_successful_recognized_warmup_marks_frontend_ready() {
|
||||
let frontend = frontend();
|
||||
let core = core_handle();
|
||||
|
||||
record_startup_warmup_status(&frontend, false, StatusCode::OK);
|
||||
assert!(!frontend.is_ready());
|
||||
record_startup_warmup_status(&core, false, StatusCode::OK);
|
||||
assert!(!core.is_ready());
|
||||
|
||||
record_startup_warmup_status(&frontend, true, StatusCode::INTERNAL_SERVER_ERROR);
|
||||
assert!(!frontend.is_ready());
|
||||
record_startup_warmup_status(&core, true, StatusCode::INTERNAL_SERVER_ERROR);
|
||||
assert!(!core.is_ready());
|
||||
|
||||
record_startup_warmup_status(&frontend, true, StatusCode::OK);
|
||||
assert!(frontend.is_ready());
|
||||
record_startup_warmup_status(&core, true, StatusCode::OK);
|
||||
assert!(core.is_ready());
|
||||
}
|
||||
}
|
||||
+17
-17
@@ -1,6 +1,6 @@
|
||||
//! Common control-plane endpoints — `/server_info`, `/get_model_info`
|
||||
//! (+ `/model_info` alias), backed by typed operations on
|
||||
//! [`crate::frontend::FrontendHandle`]. Data-plane endpoints (incl. `/health*`,
|
||||
//! [`crate::api_server::core::CoreHandle`]. Data-plane endpoints (incl. `/health*`,
|
||||
//! which round-trips a generate probe) live in the sibling `native_api` and
|
||||
//! `openai` modules; the shared `AppState` lives in the parent
|
||||
//! `api_server` module.
|
||||
@@ -15,14 +15,14 @@ use axum::{
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::app::AppState;
|
||||
use super::frontend_error_status;
|
||||
use super::native_api::native_error;
|
||||
use crate::frontend::{FrontendError, ServerInfo};
|
||||
use crate::api_server::core::CoreError;
|
||||
use crate::message::info::ServerInfo;
|
||||
|
||||
/// The routes this module owns, mounted by `api_server::serve`.
|
||||
pub(super) fn routes() -> Router<Arc<AppState>> {
|
||||
Router::new()
|
||||
// Control-plane: the frontend performs the typed operation and this
|
||||
// Control-plane: the core performs the typed operation and this
|
||||
// adapter renders its result as one non-streamed JSON response.
|
||||
.route("/server_info", get(server_info))
|
||||
// Static config, no scheduler round-trip. `/get_model_info` (+ `/model_info`
|
||||
@@ -31,23 +31,23 @@ pub(super) fn routes() -> Router<Arc<AppState>> {
|
||||
.route("/model_info", get(model_info))
|
||||
}
|
||||
|
||||
/// Await the frontend's complete semantic server-info result. HTTP status and
|
||||
/// Await the core's complete semantic server-info result. HTTP status and
|
||||
/// serialization stay here; scheduler control bytes stay behind the contract.
|
||||
async fn await_server_info(state: &AppState) -> Result<ServerInfo, Response> {
|
||||
match state.frontend.server_info().await {
|
||||
match state.core.server_info().await {
|
||||
Ok(server_info) => Ok(server_info),
|
||||
Err(FrontendError::Unavailable) => Err(native_error(
|
||||
Err(CoreError::Unavailable) => Err(native_error(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"service unavailable",
|
||||
false,
|
||||
)),
|
||||
// Preserve the established HTTP behavior for a scheduler-side control
|
||||
// stream that disappears, while non-HTTP adapters see Internal via
|
||||
// FrontendError::kind().
|
||||
Err(FrontendError::ResponseTruncated) => {
|
||||
// CoreError::kind().
|
||||
Err(CoreError::ResponseTruncated) => {
|
||||
Err((StatusCode::from_u16(499).unwrap(), "request aborted").into_response())
|
||||
}
|
||||
Err(error @ FrontendError::InvalidResponse(_)) => {
|
||||
Err(error @ CoreError::InvalidResponse(_)) => {
|
||||
tracing::error!(%error, "server_info: invalid runtime response");
|
||||
Err((
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
@@ -55,14 +55,14 @@ async fn await_server_info(state: &AppState) -> Result<ServerInfo, Response> {
|
||||
)
|
||||
.into_response())
|
||||
}
|
||||
Err(error) => Err((frontend_error_status(&error), error.to_string()).into_response()),
|
||||
Err(error) => Err((error.http_status(), error.to_string()).into_response()),
|
||||
}
|
||||
}
|
||||
|
||||
/// `GET /get_model_info` (+ `/model_info` alias) — serialize the shared static
|
||||
/// model metadata (no scheduler round-trip).
|
||||
async fn model_info(State(state): State<Arc<AppState>>) -> Response {
|
||||
let info = state.frontend.model_info();
|
||||
let info = state.core.model_info();
|
||||
(
|
||||
StatusCode::OK,
|
||||
[("content-type", "application/json")],
|
||||
@@ -104,7 +104,7 @@ mod tests {
|
||||
use tower::ServiceExt;
|
||||
|
||||
use super::*;
|
||||
use crate::frontend::{FrontendConfig, FrontendHandle, FrontendMetadata};
|
||||
use crate::api_server::core::{CoreConfig, CoreHandle, CoreMetadata};
|
||||
use crate::message::config::{DisaggregationMode, ModelConfig, ServerArgs};
|
||||
use crate::message::request::RequestKind;
|
||||
use crate::message::response::ResponseItem;
|
||||
@@ -120,21 +120,21 @@ mod tests {
|
||||
let server_args = Arc::new(server_args);
|
||||
let (intake_tx, intake_rx) = flume::unbounded();
|
||||
let (abort_tx, abort_rx) = flume::unbounded();
|
||||
let frontend = FrontendHandle::new(
|
||||
let core = CoreHandle::new(
|
||||
intake_tx,
|
||||
abort_tx,
|
||||
FrontendConfig {
|
||||
CoreConfig {
|
||||
response_capacity: 8,
|
||||
response_activity: Default::default(),
|
||||
startup_ready: true,
|
||||
is_disaggregation: false,
|
||||
mm_limits: Default::default(),
|
||||
metadata: FrontendMetadata::from(server_args.as_ref()),
|
||||
metadata: CoreMetadata::from(server_args.as_ref()),
|
||||
},
|
||||
);
|
||||
(
|
||||
Arc::new(AppState {
|
||||
frontend,
|
||||
core,
|
||||
server_args,
|
||||
chat_formatter: None,
|
||||
}),
|
||||
+130
-130
@@ -4,23 +4,20 @@
|
||||
//! `http_server.generate_request`) and `/health` + `/health_generate` (which
|
||||
//! round-trip a 1-token generate probe). Frame shaping (`meta_info`, logprob
|
||||
//! tuples, cumulative vs incremental streams) lives here; submission and
|
||||
//! request lifetime belong to the shared [`crate::frontend::FrontendHandle`].
|
||||
//! request lifetime belong to the shared [`crate::api_server::core::CoreHandle`].
|
||||
|
||||
use std::convert::Infallible;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use super::app::AppState;
|
||||
use super::frame::{
|
||||
OutputAccumulator, cumulative_frame_string, frame_value, stream_frame_string, tag_value,
|
||||
};
|
||||
use super::frontend_error_status;
|
||||
use crate::frontend::{
|
||||
FrontendCall, FrontendError, FrontendEvent, FrontendOutput, FrontendRequest, HealthStatus,
|
||||
use crate::api_server::core::{
|
||||
CoreCall, CoreError, CoreEvent, CoreOutput, HealthStatus, recv_indexed,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use crate::message::ids::Rid;
|
||||
use crate::message::request::GenerateBody;
|
||||
use crate::message::request::{GenerateRequest, into_requests};
|
||||
use crate::message::wire::merge_preferred_sampling;
|
||||
use crate::utils::{
|
||||
environ,
|
||||
response::{error_response, error_value},
|
||||
@@ -36,6 +33,7 @@ use axum::{
|
||||
},
|
||||
routing::{get, post},
|
||||
};
|
||||
use sglang_api_types::api::v1 as api;
|
||||
|
||||
/// API-local timing for one request.
|
||||
///
|
||||
@@ -89,8 +87,8 @@ pub(super) fn native_error(code: StatusCode, message: &str, stream: bool) -> Res
|
||||
error_response(code, error_value(code.as_u16(), message), stream)
|
||||
}
|
||||
|
||||
fn native_frontend_error(error: FrontendError, stream: bool) -> Response {
|
||||
native_error(frontend_error_status(&error), &error.to_string(), stream)
|
||||
fn native_core_error(error: CoreError, stream: bool) -> Response {
|
||||
native_error(error.http_status(), &error.to_string(), stream)
|
||||
}
|
||||
|
||||
/// `/health` + `/health_generate`. Both env knobs are resolved ONCE here, at
|
||||
@@ -116,7 +114,7 @@ fn health_routes() -> Router<Arc<AppState>> {
|
||||
}
|
||||
|
||||
async fn health_without_generation(State(state): State<Arc<AppState>>) -> Response {
|
||||
if state.frontend.is_ready() {
|
||||
if state.core.is_ready() {
|
||||
StatusCode::OK.into_response()
|
||||
} else {
|
||||
StatusCode::SERVICE_UNAVAILABLE.into_response()
|
||||
@@ -131,19 +129,19 @@ async fn health_without_generation(State(state): State<Arc<AppState>>) -> Respon
|
||||
///
|
||||
/// Fires a pre-tokenized 1-token probe (`input_ids = [0]`, skips the tokenizer) so
|
||||
/// an idle pipeline produces a frame, then watches the *global*
|
||||
/// frontend response-activity counter (not the probe's own rid) — so a busy
|
||||
/// core response-activity counter (not the probe's own rid) — so a busy
|
||||
/// server passes immediately and a backlog never false-503s (the analogue of
|
||||
/// Python's `last_receive_tstamp`).
|
||||
async fn health_generate(
|
||||
State(state): State<Arc<AppState>>,
|
||||
timeout: std::time::Duration,
|
||||
) -> Response {
|
||||
match state.frontend.probe_health(timeout).await {
|
||||
match state.core.probe_health(timeout).await {
|
||||
Ok(HealthStatus::Healthy) => StatusCode::OK.into_response(),
|
||||
Ok(HealthStatus::NotReady | HealthStatus::Stalled) => {
|
||||
StatusCode::SERVICE_UNAVAILABLE.into_response()
|
||||
}
|
||||
Err(FrontendError::Unavailable) => native_error(
|
||||
Err(CoreError::Unavailable) => native_error(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"service unavailable",
|
||||
false,
|
||||
@@ -162,30 +160,35 @@ async fn health_generate(
|
||||
/// message, instead of axum's default 422.
|
||||
async fn generate(
|
||||
State(state): State<Arc<AppState>>,
|
||||
body: Result<Json<GenerateBody>, JsonRejection>,
|
||||
body: Result<Json<serde_json::Value>, JsonRejection>,
|
||||
) -> Response {
|
||||
let mut body = match body {
|
||||
Ok(Json(body)) => body,
|
||||
// A body that fails to parse has no readable `stream` flag, so this one
|
||||
// can only answer unary — as Python's does (FastAPI rejects before its
|
||||
// can only answer unary -- as Python's does (FastAPI rejects before its
|
||||
// handler runs).
|
||||
Err(rejection) => {
|
||||
return native_error(StatusCode::BAD_REQUEST, &rejection.body_text(), false);
|
||||
}
|
||||
};
|
||||
let stream = body.stream;
|
||||
// Launch-time preferred sampling params fill in beneath the request's own
|
||||
// keys before decoding, so an explicit request value (even a default or
|
||||
// null) wins: Python TokenizerManager's precedence.
|
||||
if let Some(preferred) = &state.server_args.preferred_sampling_params
|
||||
&& let Err(error) = body.apply_preferred_sampling(&preferred.0)
|
||||
&& let Err(error) = merge_preferred_sampling(&mut body, &preferred.0)
|
||||
{
|
||||
return native_error(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
&error.to_string(),
|
||||
stream,
|
||||
);
|
||||
return native_error(StatusCode::INTERNAL_SERVER_ERROR, &error, false);
|
||||
}
|
||||
// The schema (sglang_api_types) is the contract: decode errors carry
|
||||
// serde's field-level text, as Python's 400s do.
|
||||
let body: api::GenerateRequest = match serde_json::from_value(body) {
|
||||
Ok(body) => body,
|
||||
Err(error) => return native_error(StatusCode::BAD_REQUEST, &error.to_string(), false),
|
||||
};
|
||||
let stream = body.stream.unwrap_or(false);
|
||||
// Fan `text`/`input_ids`/`sampling_params` (scalar or list) into per-request
|
||||
// payloads. `is_batch` = list form → the response is a JSON array.
|
||||
let (payloads, is_batch) = match body.into_requests() {
|
||||
// payloads. `is_batch` = list form -> the response is a JSON array.
|
||||
let (payloads, is_batch) = match into_requests(body) {
|
||||
Ok(v) => v,
|
||||
// Request normalization only returns validation failures.
|
||||
Err(e) => {
|
||||
@@ -197,7 +200,7 @@ async fn generate(
|
||||
// equivalent boundary: into_requests() has normalized the body, while prefetch
|
||||
// and every downstream stage are still ahead of us.
|
||||
let timing = RequestTiming::new();
|
||||
// Shared frontend admission performs media I/O (URL downloads, file reads)
|
||||
// Shared core admission performs media I/O (URL downloads, file reads)
|
||||
// on this async runtime before any request reaches the MM worker pool.
|
||||
if !is_batch {
|
||||
// `into_requests` guarantees exactly one payload for a non-batch body.
|
||||
@@ -215,15 +218,15 @@ async fn generate(
|
||||
/// SSE frames or fold to one unary response.
|
||||
async fn generate_single(
|
||||
state: &AppState,
|
||||
req: FrontendRequest,
|
||||
req: GenerateRequest,
|
||||
stream: bool,
|
||||
timing: RequestTiming,
|
||||
) -> Response {
|
||||
// `return_text_in_logprobs` is decoded on the detok shard into `*_txt`, so
|
||||
// `frame_value` just reads them — no tokenizer needed here.
|
||||
let mut call = match state.frontend.generate(req).await {
|
||||
// the frame builder just reads them — no tokenizer needed here.
|
||||
let mut call = match state.core.generate(req).await {
|
||||
Ok(call) => call,
|
||||
Err(error) => return native_frontend_error(error, stream),
|
||||
Err(error) => return native_core_error(error, stream),
|
||||
};
|
||||
let rid = call.public_id().to_owned();
|
||||
// Cumulative frames (SGLang default) vs per-step deltas.
|
||||
@@ -237,47 +240,59 @@ async fn generate_single(
|
||||
.map(|data| Ok::<_, Infallible>(Event::default().data(data)));
|
||||
Sse::new(s).into_response()
|
||||
} else {
|
||||
// Unary: fold to the terminal and respond once. `FrontendCall` disarms
|
||||
// Unary: fold to the terminal and respond once. `CoreCall` disarms
|
||||
// itself on a terminal item; truncation leaves it armed for drop cleanup.
|
||||
let (status, value) = drain_unary(&mut call, &rid, timing).await;
|
||||
(status, Json(value)).into_response()
|
||||
let (status, item) = drain_unary(&mut call, &rid, timing).await;
|
||||
(status, Json(item)).into_response()
|
||||
}
|
||||
}
|
||||
|
||||
/// One unary `/generate` result: the frame, or the item's own error body (a
|
||||
/// failed batch item is its own `{ "error": ... }` entry in the array).
|
||||
#[derive(Debug, serde::Serialize)]
|
||||
#[serde(untagged)]
|
||||
enum GenerateItem {
|
||||
// Boxed: a frame is ~300 bytes against the error's ~40, and a batch
|
||||
// array holds one per item.
|
||||
Frame(Box<api::GenerateResponse>),
|
||||
Error(api::GenerateStreamError),
|
||||
}
|
||||
|
||||
/// Fold a unary request to its terminal HTTP result. Shared by single + batch.
|
||||
async fn drain_unary(
|
||||
call: &mut FrontendCall,
|
||||
call: &mut CoreCall,
|
||||
rid_str: &str,
|
||||
mut timing: RequestTiming,
|
||||
) -> (StatusCode, serde_json::Value) {
|
||||
let mut acc = OutputAccumulator::default();
|
||||
) -> (StatusCode, GenerateItem) {
|
||||
let mut accumulated = CoreOutput::default();
|
||||
while let Some(item) = call.recv().await {
|
||||
match item {
|
||||
FrontendEvent::Delta(out) => {
|
||||
CoreEvent::Delta(out) => {
|
||||
timing.observe_first_output();
|
||||
acc.fold(&out);
|
||||
accumulated.append_delta(&out);
|
||||
}
|
||||
FrontendEvent::Finished(out) => {
|
||||
CoreEvent::Finished(out) => {
|
||||
timing.observe_first_output();
|
||||
timing.finish();
|
||||
acc.fold(&out);
|
||||
let final_out = acc.into_output();
|
||||
let mut value = frame_value(&final_out, rid_str);
|
||||
add_e2e_latency(&mut value, &timing);
|
||||
return (StatusCode::OK, value);
|
||||
accumulated.append_delta(&out);
|
||||
let frame = accumulated.frame(rid_str, None, Some(e2e_latency(&timing)));
|
||||
return (StatusCode::OK, GenerateItem::Frame(Box::new(frame)));
|
||||
}
|
||||
FrontendEvent::Failed(error) => {
|
||||
CoreEvent::Failed(error) => {
|
||||
timing.finish();
|
||||
let status = frontend_error_status(&error);
|
||||
return (status, error_value(status.as_u16(), &error.to_string()));
|
||||
return (
|
||||
error.http_status(),
|
||||
GenerateItem::Error(error.stream_error(None)),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Defensive fallback: FrontendCall normally turns premature runtime
|
||||
// Defensive fallback: CoreCall normally turns premature runtime
|
||||
// termination into a terminal Failed event before exposing stream end.
|
||||
let truncated = CoreError::ResponseTruncated;
|
||||
(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
error_value(500, "response truncated before completion"),
|
||||
truncated.http_status(),
|
||||
GenerateItem::Error(truncated.stream_error(None)),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -289,19 +304,19 @@ async fn drain_unary(
|
||||
/// own `{ "error": … }` entry; the batch response is 200.
|
||||
async fn generate_batch(
|
||||
state: &AppState,
|
||||
requests: Vec<FrontendRequest>,
|
||||
requests: Vec<GenerateRequest>,
|
||||
stream: bool,
|
||||
timing: RequestTiming,
|
||||
) -> Response {
|
||||
// No cross-item rid collision to worry about: `into_requests` rejected duplicate
|
||||
// rids within this batch, and `Rid::from_client` made each one unique against
|
||||
// every other in-flight request.
|
||||
let calls = match state.frontend.generate_batch(requests).await {
|
||||
let calls = match state.core.generate_batch(requests).await {
|
||||
Ok(calls) => calls
|
||||
.into_iter()
|
||||
.map(|call| (call, timing.clone()))
|
||||
.collect(),
|
||||
Err(error) => return native_frontend_error(error, stream),
|
||||
Err(error) => return native_core_error(error, stream),
|
||||
};
|
||||
|
||||
if stream {
|
||||
@@ -319,37 +334,20 @@ async fn generate_batch(
|
||||
let drained = futures::future::join_all(calls.into_iter().map(
|
||||
|(mut call, request_timing)| async move {
|
||||
let client_rid = call.public_id().to_owned();
|
||||
let (_status, value) = drain_unary(&mut call, &client_rid, request_timing).await;
|
||||
value
|
||||
let (_status, item) = drain_unary(&mut call, &client_rid, request_timing).await;
|
||||
item
|
||||
},
|
||||
))
|
||||
.await;
|
||||
(StatusCode::OK, Json(serde_json::Value::Array(drained))).into_response()
|
||||
(StatusCode::OK, Json(drained)).into_response()
|
||||
}
|
||||
}
|
||||
|
||||
/// Await the next event from `call`, then drain whatever queued behind it (so the
|
||||
/// caller can coalesce a backlog, as Python's `state.out_list` does), handing the
|
||||
/// call back for `FuturesUnordered` to re-poll. An empty result means the semantic
|
||||
/// stream was already exhausted.
|
||||
async fn recv_indexed(
|
||||
index: usize,
|
||||
mut call: FrontendCall,
|
||||
) -> (usize, FrontendCall, Vec<FrontendEvent>) {
|
||||
let mut items = Vec::new();
|
||||
match call.recv().await {
|
||||
Some(item) => items.push(item),
|
||||
None => return (index, call, items), // already exhausted
|
||||
}
|
||||
call.drain_ready(&mut items);
|
||||
(index, call, items)
|
||||
}
|
||||
|
||||
/// Multiplex calls into SSE `data` strings + a final `[DONE]`; `with_index` tags
|
||||
/// each frame (batch only), `incremental` = delta vs cumulative. Each call
|
||||
/// aborts itself if the stream is dropped while it is unfinished.
|
||||
fn generation_event_stream(
|
||||
calls: Vec<(FrontendCall, RequestTiming)>,
|
||||
calls: Vec<(CoreCall, RequestTiming)>,
|
||||
incremental: bool,
|
||||
with_index: bool,
|
||||
) -> impl futures::Stream<Item = String> {
|
||||
@@ -365,11 +363,10 @@ fn generation_event_stream(
|
||||
.iter()
|
||||
.map(|(_, timing)| timing.clone())
|
||||
.collect();
|
||||
let mut accs: Vec<OutputAccumulator> =
|
||||
(0..n).map(|_| OutputAccumulator::default()).collect();
|
||||
let mut accumulated: Vec<CoreOutput> = (0..n).map(|_| CoreOutput::default()).collect();
|
||||
|
||||
// Batch position, tagged onto every frame (a single request omits it).
|
||||
let idx = |i: usize| with_index.then_some(i);
|
||||
let idx = |i: usize| with_index.then_some(i as u32);
|
||||
|
||||
// Poll all receivers concurrently; re-arm a receiver's future after each
|
||||
// non-terminal frame so its stream keeps flowing.
|
||||
@@ -381,9 +378,9 @@ fn generation_event_stream(
|
||||
while let Some((i, call, items)) = futs.next().await {
|
||||
if items.is_empty() {
|
||||
// Defensive fallback: a premature runtime close is normally a
|
||||
// Failed event produced by FrontendCall. Keep this item's call
|
||||
// Failed event produced by CoreCall. Keep this item's call
|
||||
// armed if its semantic stream somehow ends unexpectedly.
|
||||
yield tag_value(error_value(500, "response truncated before completion"), idx(i));
|
||||
yield json_string(&CoreError::ResponseTruncated.stream_error(idx(i)));
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -395,22 +392,22 @@ fn generation_event_stream(
|
||||
|
||||
for item in items {
|
||||
match item {
|
||||
FrontendEvent::Delta(out) => {
|
||||
CoreEvent::Delta(out) => {
|
||||
timings[i].observe_first_output();
|
||||
accs[i].fold(&out);
|
||||
accumulated[i].append_delta(&out);
|
||||
if incremental {
|
||||
yield stream_frame_string(out, &accs[i], true, &rid_strs[i], idx(i));
|
||||
yield json_string(&delta_frame(out, &accumulated[i], &rid_strs[i], idx(i), None));
|
||||
} else {
|
||||
coalesced = true;
|
||||
}
|
||||
}
|
||||
FrontendEvent::Finished(out) => {
|
||||
CoreEvent::Finished(out) => {
|
||||
timings[i].observe_first_output();
|
||||
timings[i].finish();
|
||||
accs[i].fold(&out);
|
||||
accumulated[i].append_delta(&out);
|
||||
terminal = Some(out);
|
||||
}
|
||||
FrontendEvent::Failed(error) => {
|
||||
CoreEvent::Failed(error) => {
|
||||
timings[i].finish();
|
||||
failed = Some(error);
|
||||
}
|
||||
@@ -418,22 +415,24 @@ fn generation_event_stream(
|
||||
}
|
||||
|
||||
if let Some(error) = failed {
|
||||
let status = frontend_error_status(&error);
|
||||
yield tag_value(error_value(status.as_u16(), &error.to_string()), idx(i));
|
||||
yield json_string(&error.stream_error(idx(i)));
|
||||
} else if let Some(out) = terminal {
|
||||
// The final frame carries the full cumulative state, so any
|
||||
// coalesced non-terminal frames are moot.
|
||||
yield terminal_stream_frame_string(
|
||||
out,
|
||||
&accs[i],
|
||||
incremental,
|
||||
&rid_strs[i],
|
||||
idx(i),
|
||||
&timings[i],
|
||||
);
|
||||
let e2e_latency = Some(e2e_latency(&timings[i]));
|
||||
// The terminal frame is the accumulated output's last use, so
|
||||
// the cumulative form moves it out instead of cloning.
|
||||
let frame = if incremental {
|
||||
delta_frame(out, &accumulated[i], &rid_strs[i], idx(i), e2e_latency)
|
||||
} else {
|
||||
std::mem::take(&mut accumulated[i]).frame(&rid_strs[i], idx(i), e2e_latency)
|
||||
};
|
||||
yield json_string(&frame);
|
||||
} else {
|
||||
if coalesced {
|
||||
yield cumulative_frame_string(&accs[i], &rid_strs[i], idx(i));
|
||||
// An intermediate cumulative frame has to clone: the accumulated
|
||||
// output stays behind for the next one.
|
||||
yield json_string(&accumulated[i].clone().frame(&rid_strs[i], idx(i), None));
|
||||
}
|
||||
futs.push(recv_indexed(i, call)); // keep this item flowing
|
||||
}
|
||||
@@ -444,30 +443,33 @@ fn generation_event_stream(
|
||||
|
||||
/// Python's `e2e_latency` is `finished_time - created_time`, in seconds, and is
|
||||
/// attached only when the request finishes. The Rust native API owns the same
|
||||
/// lifecycle boundary, so it adds the value while handling the terminal egress
|
||||
/// item rather than putting API-only timing onto every shared frontend output.
|
||||
fn add_e2e_latency(value: &mut serde_json::Value, timing: &RequestTiming) {
|
||||
/// lifecycle boundary, so it sets the value while handling the terminal egress
|
||||
/// item rather than putting API-only timing onto every shared core output.
|
||||
fn e2e_latency(timing: &RequestTiming) -> f64 {
|
||||
let (time_to_first_token, e2e_latency) = timing
|
||||
.terminal_latencies()
|
||||
.expect("a successful terminal output has complete request timing");
|
||||
debug_assert!(time_to_first_token <= e2e_latency);
|
||||
value["meta_info"]["e2e_latency"] = serde_json::json!(e2e_latency.as_secs_f64());
|
||||
e2e_latency.as_secs_f64()
|
||||
}
|
||||
|
||||
/// Render a terminal streaming frame. Intermediate cumulative frames keep the
|
||||
/// memoized fast path; the one terminal frame uses the Value path so it can carry
|
||||
/// the request-local `e2e_latency`, exactly as Python does.
|
||||
fn terminal_stream_frame_string(
|
||||
out: FrontendOutput,
|
||||
acc: &OutputAccumulator,
|
||||
incremental: bool,
|
||||
/// One incremental streaming frame: this step's delta with the cumulative
|
||||
/// token count in `meta_info` (matching Python). Only the terminal frame
|
||||
/// carries `e2e_latency`.
|
||||
fn delta_frame(
|
||||
mut delta: CoreOutput,
|
||||
accumulated: &CoreOutput,
|
||||
rid_str: &str,
|
||||
index: Option<usize>,
|
||||
timing: &RequestTiming,
|
||||
) -> String {
|
||||
let mut value = super::frame::stream_frame_value(out, acc, incremental, rid_str);
|
||||
add_e2e_latency(&mut value, timing);
|
||||
tag_value(value, index)
|
||||
index: Option<u32>,
|
||||
e2e_latency: Option<f64>,
|
||||
) -> api::GenerateResponse {
|
||||
delta.completion_tokens = accumulated.completion_tokens;
|
||||
delta.frame(rid_str, index, e2e_latency)
|
||||
}
|
||||
|
||||
/// The SSE `data` text of one `api.v1` item (a frame or an error).
|
||||
fn json_string<T: serde::Serialize>(value: &T) -> String {
|
||||
serde_json::to_string(value).expect("api.v1 types serialize to JSON")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -509,7 +511,7 @@ mod tests {
|
||||
serde_json::from_str(s).expect("frame is JSON")
|
||||
}
|
||||
|
||||
fn timed_call(rid: u64, rx: mpsc::Receiver<ResponseItem>) -> (FrontendCall, RequestTiming) {
|
||||
fn timed_call(rid: u64, rx: mpsc::Receiver<ResponseItem>) -> (CoreCall, RequestTiming) {
|
||||
timed_call_with_abort(rid, rx, flume::unbounded().0)
|
||||
}
|
||||
|
||||
@@ -517,9 +519,9 @@ mod tests {
|
||||
rid: u64,
|
||||
rx: mpsc::Receiver<ResponseItem>,
|
||||
abort_tx: flume::Sender<crate::tokenizer_manager::wiring::AbortSource>,
|
||||
) -> (FrontendCall, RequestTiming) {
|
||||
) -> (CoreCall, RequestTiming) {
|
||||
(
|
||||
FrontendCall::from_test_generation_parts(Rid::from(rid.to_string()), rx, abort_tx),
|
||||
CoreCall::from_test_generation_parts(Rid::from(rid.to_string()), rx, abort_tx),
|
||||
RequestTiming {
|
||||
created_at: Instant::now() - Duration::from_millis(10),
|
||||
time_to_first_token: None,
|
||||
@@ -533,16 +535,16 @@ mod tests {
|
||||
let (intake_tx, intake_rx) = flume::unbounded();
|
||||
let (abort_tx, abort_rx) = flume::unbounded();
|
||||
let state = Arc::new(AppState {
|
||||
frontend: crate::frontend::FrontendHandle::new(
|
||||
core: crate::api_server::core::CoreHandle::new(
|
||||
intake_tx,
|
||||
abort_tx,
|
||||
crate::frontend::FrontendConfig {
|
||||
crate::api_server::core::CoreConfig {
|
||||
response_capacity: 8,
|
||||
response_activity: Default::default(),
|
||||
startup_ready: false,
|
||||
is_disaggregation: false,
|
||||
mm_limits: Default::default(),
|
||||
metadata: crate::frontend::FrontendMetadata::default(),
|
||||
metadata: crate::api_server::core::CoreMetadata::default(),
|
||||
},
|
||||
),
|
||||
server_args: Arc::new(crate::message::config::ServerArgs::default()),
|
||||
@@ -559,20 +561,20 @@ mod tests {
|
||||
async fn unary_http_generate_round_trips_through_frontend_handle() {
|
||||
let (intake_tx, intake_rx) = flume::unbounded();
|
||||
let (abort_tx, abort_rx) = flume::unbounded();
|
||||
let frontend = crate::frontend::FrontendHandle::new(
|
||||
let core = crate::api_server::core::CoreHandle::new(
|
||||
intake_tx,
|
||||
abort_tx,
|
||||
crate::frontend::FrontendConfig {
|
||||
crate::api_server::core::CoreConfig {
|
||||
response_capacity: 8,
|
||||
response_activity: Default::default(),
|
||||
startup_ready: true,
|
||||
is_disaggregation: false,
|
||||
mm_limits: Default::default(),
|
||||
metadata: crate::frontend::FrontendMetadata::default(),
|
||||
metadata: crate::api_server::core::CoreMetadata::default(),
|
||||
},
|
||||
);
|
||||
let state = Arc::new(AppState {
|
||||
frontend,
|
||||
core,
|
||||
server_args: Arc::new(crate::message::config::ServerArgs::default()),
|
||||
chat_formatter: None,
|
||||
});
|
||||
@@ -580,7 +582,7 @@ mod tests {
|
||||
let responder = tokio::spawn(async move {
|
||||
let TmEvent::Intake { request, admission } = intake_rx.recv_async().await.unwrap()
|
||||
else {
|
||||
panic!("HTTP generation must enter through frontend intake");
|
||||
panic!("HTTP generation must enter through core intake");
|
||||
};
|
||||
assert!(admission.try_accept());
|
||||
let RequestKind::Generate(generate) = &request.kind else {
|
||||
@@ -681,18 +683,16 @@ mod tests {
|
||||
}))
|
||||
.await
|
||||
.unwrap();
|
||||
let mut call = FrontendCall::from_test_generation_parts(
|
||||
"internal-rid".into(),
|
||||
rx,
|
||||
flume::unbounded().0,
|
||||
);
|
||||
let mut call =
|
||||
CoreCall::from_test_generation_parts("internal-rid".into(), rx, flume::unbounded().0);
|
||||
|
||||
let timing = RequestTiming {
|
||||
created_at: Instant::now() - Duration::from_millis(20),
|
||||
time_to_first_token: None,
|
||||
e2e_latency: None,
|
||||
};
|
||||
let (status, value) = drain_unary(&mut call, "client-rid", timing).await;
|
||||
let (status, item) = drain_unary(&mut call, "client-rid", timing).await;
|
||||
let value = serde_json::to_value(&item).unwrap();
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
assert_eq!(value["meta_info"]["id"], "client-rid");
|
||||
assert_eq!(value["meta_info"]["prompt_tokens"], 5);
|
||||
+17
-20
@@ -1,7 +1,7 @@
|
||||
//! OpenAI-compatible generation endpoints.
|
||||
//!
|
||||
//! The HTTP adapter stays deliberately thin: Dynamo owns the standard OpenAI
|
||||
//! request and response primitives. [`FrontendOutput`] remains the one backend
|
||||
//! request and response primitives. [`CoreOutput`] remains the one backend
|
||||
//! output type for both unary and streaming responses.
|
||||
|
||||
use axum::{Router, http::StatusCode, response::Response};
|
||||
@@ -21,12 +21,9 @@ mod tools;
|
||||
pub(super) use template::{ChatFormatter, ChatTemplateKwargs};
|
||||
|
||||
use super::app::AppState;
|
||||
use super::frame::OutputAccumulator;
|
||||
use super::frontend_error_status;
|
||||
use crate::frontend::{
|
||||
FrontendCall, FrontendError, FrontendEvent, FrontendOutput, FrontendRequest,
|
||||
};
|
||||
use crate::api_server::core::{CoreCall, CoreError, CoreEvent, CoreOutput};
|
||||
use crate::message::config::ServerArgs;
|
||||
use crate::message::request::GenerateRequest;
|
||||
use crate::tokenizer_manager::tokenizer;
|
||||
use crate::utils::response::error_response;
|
||||
|
||||
@@ -118,17 +115,17 @@ pub(super) fn openai_error(code: StatusCode, message: impl Into<String>, stream:
|
||||
/// Drain one submitted request to its terminal output, fold frames, and map
|
||||
/// semantic failures / truncation to `(status, message)` for the OpenAI error
|
||||
/// shape. The call owns cancellation and disarms itself.
|
||||
async fn collect_output(mut call: FrontendCall) -> Result<FrontendOutput, (StatusCode, String)> {
|
||||
let mut accumulator = OutputAccumulator::default();
|
||||
async fn collect_output(mut call: CoreCall) -> Result<CoreOutput, (StatusCode, String)> {
|
||||
let mut accumulated = CoreOutput::default();
|
||||
let output = loop {
|
||||
match call.recv().await {
|
||||
Some(FrontendEvent::Delta(output)) => accumulator.fold(&output),
|
||||
Some(FrontendEvent::Finished(output)) => {
|
||||
accumulator.fold(&output);
|
||||
break accumulator.into_output();
|
||||
Some(CoreEvent::Delta(output)) => accumulated.append_delta(&output),
|
||||
Some(CoreEvent::Finished(output)) => {
|
||||
accumulated.append_delta(&output);
|
||||
break accumulated;
|
||||
}
|
||||
Some(FrontendEvent::Failed(error)) => {
|
||||
let status = frontend_error_status(&error);
|
||||
Some(CoreEvent::Failed(error)) => {
|
||||
let status = error.http_status();
|
||||
return Err((status, error.to_string()));
|
||||
}
|
||||
None => {
|
||||
@@ -144,15 +141,15 @@ async fn collect_output(mut call: FrontendCall) -> Result<FrontendOutput, (Statu
|
||||
|
||||
async fn submit_generation(
|
||||
state: &AppState,
|
||||
request: FrontendRequest,
|
||||
request: GenerateRequest,
|
||||
stream: bool,
|
||||
) -> Result<FrontendCall, Response> {
|
||||
match state.frontend.generate(request).await {
|
||||
) -> Result<CoreCall, Response> {
|
||||
match state.core.generate(request).await {
|
||||
Ok(call) => Ok(call),
|
||||
// Same `error_response` rule: a committed stream gets 200 plus an
|
||||
// SSE error frame + `[DONE]`, not a unary 503 — but with the OpenAI
|
||||
// error shape, since this is the OpenAI frontend.
|
||||
Err(FrontendError::Unavailable) => Err(openai_error(
|
||||
Err(CoreError::Unavailable) => Err(openai_error(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"service unavailable",
|
||||
stream,
|
||||
@@ -167,8 +164,8 @@ async fn submit_generation(
|
||||
|
||||
fn indexed_decode_stream(
|
||||
index: usize,
|
||||
call: FrontendCall,
|
||||
) -> futures::stream::BoxStream<'static, (usize, FrontendEvent)> {
|
||||
call: CoreCall,
|
||||
) -> futures::stream::BoxStream<'static, (usize, CoreEvent)> {
|
||||
futures::stream::unfold((call, false), move |(mut call, finished)| async move {
|
||||
if finished {
|
||||
return None;
|
||||
+8
-8
@@ -36,10 +36,10 @@ use super::{
|
||||
AppState, ChatFormatter, ChatTemplateKwargs, collect_output, contains_media, error_payload,
|
||||
indexed_decode_stream, openai_error, submit_generation, unix_seconds_u32,
|
||||
};
|
||||
use crate::api_server::frontend_error_status;
|
||||
use crate::frontend::{FrontendCall, FrontendEvent, FrontendRequest};
|
||||
use crate::api_server::core::{CoreCall, CoreEvent};
|
||||
use crate::message::config::{DefaultSamplingParams, ServerArgs};
|
||||
use crate::message::ids::Rid;
|
||||
use crate::message::request::GenerateRequest;
|
||||
use crate::message::response::ChunkExtras;
|
||||
use crate::message::sampling::SamplingParams;
|
||||
use crate::message::types::OneOrMany;
|
||||
@@ -204,7 +204,7 @@ async fn chat_completions(
|
||||
.expect("chat prompt exists until the last choice")
|
||||
.clone()
|
||||
};
|
||||
let native = FrontendRequest {
|
||||
let native = GenerateRequest {
|
||||
rid: rid.clone(),
|
||||
text: Some(choice_prompt),
|
||||
// Rendered templates own their special tokens — the pool must not
|
||||
@@ -438,7 +438,7 @@ pub(super) fn chat_sampling_params(
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) async fn unary_chat(
|
||||
submitted: Vec<(usize, FrontendCall)>,
|
||||
submitted: Vec<(usize, CoreCall)>,
|
||||
response_id: String,
|
||||
model: String,
|
||||
created: u32,
|
||||
@@ -521,7 +521,7 @@ pub(super) async fn unary_chat(
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) fn chat_event_stream(
|
||||
submitted: Vec<(usize, FrontendCall)>,
|
||||
submitted: Vec<(usize, CoreCall)>,
|
||||
response_id: String,
|
||||
model: String,
|
||||
created: u32,
|
||||
@@ -582,14 +582,14 @@ pub(super) fn chat_event_stream(
|
||||
let mut events = futures::stream::select_all(streams);
|
||||
while let Some((index, event)) = events.next().await {
|
||||
let output = match event {
|
||||
FrontendEvent::Delta(output) | FrontendEvent::Finished(output) => output,
|
||||
FrontendEvent::Failed(error) => {
|
||||
CoreEvent::Delta(output) | CoreEvent::Finished(output) => output,
|
||||
CoreEvent::Failed(error) => {
|
||||
yield Annotated {
|
||||
data: None,
|
||||
id: None,
|
||||
event: None,
|
||||
comment: None,
|
||||
error: Some(error_payload(frontend_error_status(&error), error.to_string()).to_string()),
|
||||
error: Some(error_payload(error.http_status(), error.to_string()).to_string()),
|
||||
};
|
||||
continue;
|
||||
}
|
||||
+15
-17
@@ -24,12 +24,10 @@ use super::{
|
||||
AppState, MAX_OPENAI_CHOICES, collect_output, error_payload, indexed_decode_stream,
|
||||
openai_error, submit_generation, unix_seconds_u32,
|
||||
};
|
||||
use crate::api_server::frontend_error_status;
|
||||
use crate::frontend::{
|
||||
FrontendCall, FrontendError, FrontendEvent, FrontendOutput, FrontendRequest,
|
||||
};
|
||||
use crate::api_server::core::{CoreCall, CoreError, CoreEvent, CoreOutput};
|
||||
use crate::message::finish_reason::Matched;
|
||||
use crate::message::ids::Rid;
|
||||
use crate::message::request::GenerateRequest;
|
||||
use crate::message::response::ChunkExtras;
|
||||
use crate::message::sampling::SamplingParams;
|
||||
use crate::message::types::{OneOrMany, TokenIds};
|
||||
@@ -48,7 +46,7 @@ pub(super) struct SubmittedChoice {
|
||||
pub(super) index: usize,
|
||||
pub(super) prompt_index: usize,
|
||||
pub(super) echo: String,
|
||||
pub(super) call: FrontendCall,
|
||||
pub(super) call: CoreCall,
|
||||
}
|
||||
#[derive(Debug, Default)]
|
||||
pub(super) struct ChoiceExtensions {
|
||||
@@ -82,7 +80,7 @@ async fn completions(
|
||||
if request.prompt_embeds.is_some() {
|
||||
return openai_error(
|
||||
StatusCode::BAD_REQUEST,
|
||||
"prompt_embeds is not supported by the Rust frontend",
|
||||
"prompt_embeds is not supported by the Rust server",
|
||||
false,
|
||||
);
|
||||
}
|
||||
@@ -164,7 +162,7 @@ async fn completions(
|
||||
Err(response) => return response,
|
||||
};
|
||||
}
|
||||
let native = FrontendRequest {
|
||||
let native = GenerateRequest {
|
||||
rid: rid.clone(),
|
||||
text: text.clone(),
|
||||
input_ids: input_ids.clone(),
|
||||
@@ -230,29 +228,29 @@ async fn completions(
|
||||
}
|
||||
|
||||
/// Decode a token-id prompt back to text for `echo=true`, via a
|
||||
/// detokenize operation through the shared frontend handle — the
|
||||
/// detokenize operation through the shared core handle — the
|
||||
/// detok stage answers it with a single `Data` payload (the raw UTF-8 text),
|
||||
/// or an `Error` (e.g. out-of-range ids → `Validation` → 400).
|
||||
async fn decode_prompt_echo(state: &AppState, token_ids: TokenIds) -> Result<String, Response> {
|
||||
match state.frontend.detokenize(token_ids).await {
|
||||
match state.core.detokenize(token_ids).await {
|
||||
Ok(text) => Ok(text),
|
||||
// Same rule as `submit_generation`: build the refusal in the OpenAI
|
||||
// error shape rather than forwarding the native-shaped response.
|
||||
Err(FrontendError::Unavailable) => Err(openai_error(
|
||||
Err(CoreError::Unavailable) => Err(openai_error(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"service unavailable",
|
||||
false,
|
||||
)),
|
||||
Err(FrontendError::InvalidArgument(message)) => {
|
||||
Err(CoreError::InvalidArgument(message)) => {
|
||||
Err(openai_error(StatusCode::BAD_REQUEST, &message, false))
|
||||
}
|
||||
Err(FrontendError::InvalidResponse(message)) => Err(openai_error(
|
||||
Err(CoreError::InvalidResponse(message)) => Err(openai_error(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
message,
|
||||
false,
|
||||
)),
|
||||
Err(error) => {
|
||||
let status = frontend_error_status(&error);
|
||||
let status = error.http_status();
|
||||
Err(openai_error(
|
||||
status,
|
||||
format!("failed to decode prompt for echo: {error}"),
|
||||
@@ -406,7 +404,7 @@ pub(super) async fn unary_completion(
|
||||
fn completion_choice(
|
||||
index: usize,
|
||||
text: String,
|
||||
output: &FrontendOutput,
|
||||
output: &CoreOutput,
|
||||
want_logprobs: bool,
|
||||
include_input_logprobs: bool,
|
||||
) -> (Choice, ChoiceExtensions) {
|
||||
@@ -521,9 +519,9 @@ pub(super) fn completion_event_stream(
|
||||
|
||||
while let Some((index, event)) = events.next().await {
|
||||
let output = match event {
|
||||
FrontendEvent::Delta(output) | FrontendEvent::Finished(output) => output,
|
||||
FrontendEvent::Failed(error) => {
|
||||
yield error_payload(frontend_error_status(&error), error.to_string()).to_string();
|
||||
CoreEvent::Delta(output) | CoreEvent::Finished(output) => output,
|
||||
CoreEvent::Failed(error) => {
|
||||
yield error_payload(error.http_status(), error.to_string()).to_string();
|
||||
continue;
|
||||
}
|
||||
};
|
||||
+21
-24
@@ -3,7 +3,7 @@
|
||||
//! Submodule tests live next to the code they cover: `chat`, `completions`,
|
||||
//! `tools`, and `reasoning` each carry their own
|
||||
//! `#[cfg(test)] mod tests`. This module keeps the fixtures they all share —
|
||||
//! frontend/call fixtures (`frontend`, `chunk`, `submitted`, `chat_submitted`) and the
|
||||
//! core/call fixtures (`core_handle`, `chunk`, `submitted`, `chat_submitted`) and the
|
||||
//! full-router harness (`server_args`, `app_state`,
|
||||
//! `oneshot`, `post_json`, `body_json`) — plus the handler-level tests that
|
||||
//! exercise [`routes`] end to end. The helpers are `pub(super)` so sibling
|
||||
@@ -20,20 +20,20 @@ use serde_json::json;
|
||||
use tower::util::ServiceExt;
|
||||
|
||||
use super::{indexed_decode_stream, openai_error, routes};
|
||||
use crate::frontend::{FrontendCall, FrontendEvent, FrontendHandle};
|
||||
use crate::api_server::core::{CoreCall, CoreEvent, CoreHandle};
|
||||
use crate::message::config::ServerArgs;
|
||||
use crate::message::response::{ChunkEvent, ResponseItem};
|
||||
pub(super) fn frontend() -> FrontendHandle {
|
||||
FrontendHandle::new(
|
||||
pub(super) fn core_handle() -> CoreHandle {
|
||||
CoreHandle::new(
|
||||
flume::unbounded().0,
|
||||
flume::unbounded().0,
|
||||
crate::frontend::FrontendConfig {
|
||||
crate::api_server::core::CoreConfig {
|
||||
response_capacity: 8,
|
||||
response_activity: Default::default(),
|
||||
startup_ready: false,
|
||||
is_disaggregation: false,
|
||||
mm_limits: Default::default(),
|
||||
metadata: crate::frontend::FrontendMetadata::from(server_args().as_ref()),
|
||||
metadata: crate::api_server::core::CoreMetadata::from(server_args().as_ref()),
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -76,7 +76,7 @@ pub(super) fn submitted(
|
||||
index,
|
||||
prompt_index,
|
||||
echo: String::new(),
|
||||
call: FrontendCall::from_test_generation_parts(rid.into(), rx, flume::unbounded().0),
|
||||
call: CoreCall::from_test_generation_parts(rid.into(), rx, flume::unbounded().0),
|
||||
},
|
||||
tx,
|
||||
)
|
||||
@@ -86,15 +86,12 @@ pub(super) fn submitted(
|
||||
pub(super) fn chat_submitted(
|
||||
index: usize,
|
||||
rid: &str,
|
||||
) -> (
|
||||
(usize, FrontendCall),
|
||||
tokio::sync::mpsc::Sender<ResponseItem>,
|
||||
) {
|
||||
) -> ((usize, CoreCall), tokio::sync::mpsc::Sender<ResponseItem>) {
|
||||
let (tx, rx) = tokio::sync::mpsc::channel(8);
|
||||
(
|
||||
(
|
||||
index,
|
||||
FrontendCall::from_test_generation_parts(rid.into(), rx, flume::unbounded().0),
|
||||
CoreCall::from_test_generation_parts(rid.into(), rx, flume::unbounded().0),
|
||||
),
|
||||
tx,
|
||||
)
|
||||
@@ -107,31 +104,31 @@ pub(super) fn server_args() -> Arc<ServerArgs> {
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn app_state(frontend: FrontendHandle) -> Arc<super::AppState> {
|
||||
pub(super) fn app_state(core: CoreHandle) -> Arc<super::AppState> {
|
||||
Arc::new(super::AppState {
|
||||
frontend,
|
||||
core,
|
||||
server_args: server_args(),
|
||||
chat_formatter: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn frontend_closed() -> FrontendHandle {
|
||||
// Dropping the receivers makes frontend admission report the shutdown
|
||||
pub(super) fn frontend_closed() -> CoreHandle {
|
||||
// Dropping the receivers makes core admission report the shutdown
|
||||
// state as a 503.
|
||||
let (tm_tx, tm_rx) = flume::unbounded();
|
||||
drop(tm_rx);
|
||||
let (abort_tx, abort_rx) = flume::unbounded();
|
||||
drop(abort_rx);
|
||||
FrontendHandle::new(
|
||||
CoreHandle::new(
|
||||
tm_tx,
|
||||
abort_tx,
|
||||
crate::frontend::FrontendConfig {
|
||||
crate::api_server::core::CoreConfig {
|
||||
response_capacity: 8,
|
||||
response_activity: Default::default(),
|
||||
startup_ready: false,
|
||||
is_disaggregation: false,
|
||||
mm_limits: Default::default(),
|
||||
metadata: crate::frontend::FrontendMetadata::from(server_args().as_ref()),
|
||||
metadata: crate::api_server::core::CoreMetadata::from(server_args().as_ref()),
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -166,13 +163,13 @@ async fn dropping_indexed_stream_aborts_its_live_call() {
|
||||
|
||||
let (response_tx, response_rx) = tokio::sync::mpsc::channel(8);
|
||||
let (abort_tx, abort_rx) = flume::unbounded();
|
||||
let call = FrontendCall::from_test_generation_parts("live".into(), response_rx, abort_tx);
|
||||
let call = CoreCall::from_test_generation_parts("live".into(), response_rx, abort_tx);
|
||||
let mut stream = indexed_decode_stream(0, call);
|
||||
|
||||
response_tx.send(chunk("live", "x", false)).await.unwrap();
|
||||
assert!(matches!(
|
||||
stream.next().await,
|
||||
Some((0, FrontendEvent::Delta(_)))
|
||||
Some((0, CoreEvent::Delta(_)))
|
||||
));
|
||||
|
||||
drop(stream);
|
||||
@@ -217,7 +214,7 @@ async fn openai_error_response_covers_unary_and_sse() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn completions_handler_validates_before_submit() {
|
||||
let app = routes().with_state(app_state(frontend()));
|
||||
let app = routes().with_state(app_state(core_handle()));
|
||||
let cases = [
|
||||
(json!({"model": "other", "prompt": "hi"}), "unknown model"),
|
||||
(json!({"model": "model", "prompt": "hi", "n": 0}), "n=0"),
|
||||
@@ -265,7 +262,7 @@ async fn completions_handler_validates_before_submit() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn chat_handler_validates_before_submit() {
|
||||
let app = routes().with_state(app_state(frontend()));
|
||||
let app = routes().with_state(app_state(core_handle()));
|
||||
let cases = [
|
||||
(
|
||||
json!({"model": "other", "messages": [{"role": "user", "content": "hi"}]}),
|
||||
@@ -309,7 +306,7 @@ async fn chat_handler_validates_before_submit() {
|
||||
|
||||
#[tokio::test]
|
||||
async fn basic_openai_router_excludes_responses_api() {
|
||||
let app = routes().with_state(app_state(frontend()));
|
||||
let app = routes().with_state(app_state(core_handle()));
|
||||
let response = post_json(app, "/v1/responses", json!({"input": "hi"})).await;
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
+5
-5
@@ -36,7 +36,7 @@ use dynamo_protocols::types::{
|
||||
Role,
|
||||
};
|
||||
|
||||
use crate::frontend::FrontendOutput;
|
||||
use crate::api_server::core::CoreOutput;
|
||||
use crate::message::sampling::SamplingParams;
|
||||
|
||||
/// Canonicalize a tool-call parser name onto the dynamo-parsers registry keys.
|
||||
@@ -246,7 +246,7 @@ pub(super) async fn parse_chat_tool_calls(
|
||||
/// Map the scheduler's finish kind onto the OpenAI wire values. Length and
|
||||
/// content-filter keep their names; everything else (including a bare abort)
|
||||
/// reports as `stop`, matching Python's fallback.
|
||||
pub(super) fn chat_finish_reason(output: &FrontendOutput) -> Option<OpenAIFinishReason> {
|
||||
pub(super) fn chat_finish_reason(output: &CoreOutput) -> Option<OpenAIFinishReason> {
|
||||
let kind = output
|
||||
.finish_reason
|
||||
.as_ref()
|
||||
@@ -264,7 +264,7 @@ mod tests {
|
||||
apply_tool_constraint, chat_delta, chat_finish_reason, dynamo_parser_name,
|
||||
dynamo_tool_choice, parse_chat_tool_calls,
|
||||
};
|
||||
use crate::frontend::FrontendOutput;
|
||||
use crate::api_server::core::CoreOutput;
|
||||
use crate::message::sampling::SamplingParams;
|
||||
use dynamo_parsers::tool_calling::jail::{Annotated, apply_tool_calling_jail};
|
||||
use dynamo_parsers::{ToolChoice as DynamoToolChoice, ToolDefinition};
|
||||
@@ -766,7 +766,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn chat_finish_reason_maps_scheduler_kinds() {
|
||||
let output = |finish: serde_json::Value| FrontendOutput {
|
||||
let output = |finish: serde_json::Value| CoreOutput {
|
||||
text: "x".into(),
|
||||
token_ids: vec![1],
|
||||
prompt_tokens: 1,
|
||||
@@ -794,7 +794,7 @@ mod tests {
|
||||
Some(OpenAIFinishReason::Stop)
|
||||
);
|
||||
assert_eq!(
|
||||
chat_finish_reason(&FrontendOutput {
|
||||
chat_finish_reason(&CoreOutput {
|
||||
finish_reason: None,
|
||||
..Default::default()
|
||||
}),
|
||||
@@ -1,356 +0,0 @@
|
||||
//! Transport-neutral types shared by frontend adapters.
|
||||
//!
|
||||
//! These types describe what an inference operation means after a wire adapter
|
||||
//! has normalized its request. They intentionally contain no Axum, HTTP, SSE,
|
||||
//! Tonic, protobuf, or scheduler-channel details.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::message::config::{DisaggregationMode, PreferredSamplingParams};
|
||||
use crate::message::finish_reason::FinishReason;
|
||||
use crate::message::request::GenerateRequest;
|
||||
use crate::message::response::{ChunkEvent, ChunkExtras};
|
||||
use crate::message::types::TokenIds;
|
||||
|
||||
/// Canonical transport-neutral input for one generation operation.
|
||||
///
|
||||
/// The existing in-process request already contains semantic generation fields
|
||||
/// and has no Serde, Axum, Tonic, or protobuf representation attached to it.
|
||||
/// Re-exporting it through the frontend contract gives every adapter one public
|
||||
/// boundary without duplicating the runtime's request model.
|
||||
pub(crate) type FrontendRequest = GenerateRequest;
|
||||
|
||||
/// Canonical transport-neutral output payload for generation events.
|
||||
///
|
||||
/// The runtime correlation ID is deliberately absent: it is an implementation
|
||||
/// detail used for scheduler routing, while adapters obtain the client-visible
|
||||
/// identity from [`crate::frontend::FrontendCall::public_id`]. Runtime-only
|
||||
/// response variants such as raw control bytes are likewise hidden by
|
||||
/// [`crate::frontend::FrontendCall`].
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub(crate) struct FrontendOutput {
|
||||
pub(crate) token_ids: TokenIds,
|
||||
pub(crate) finish_reason: Option<FinishReason>,
|
||||
pub(crate) prompt_tokens: u32,
|
||||
pub(crate) text: String,
|
||||
pub(crate) completion_tokens: u64,
|
||||
pub(crate) extras: Option<Box<ChunkExtras>>,
|
||||
}
|
||||
|
||||
impl From<ChunkEvent> for FrontendOutput {
|
||||
fn from(output: ChunkEvent) -> Self {
|
||||
let ChunkEvent {
|
||||
rid: _,
|
||||
token_ids,
|
||||
finish_reason,
|
||||
prompt_tokens,
|
||||
text,
|
||||
completion_tokens,
|
||||
extras,
|
||||
} = output;
|
||||
Self {
|
||||
token_ids,
|
||||
finish_reason,
|
||||
prompt_tokens,
|
||||
text,
|
||||
completion_tokens,
|
||||
extras,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FrontendOutput {
|
||||
/// Fold one runtime delta into this cumulative native-generation output.
|
||||
///
|
||||
/// Runtime events are always incremental. Native HTTP and `runtime.v1`
|
||||
/// gRPC independently decide whether to expose those deltas or the
|
||||
/// cumulative result, so the fold itself lives at their shared semantic
|
||||
/// boundary rather than in either wire adapter.
|
||||
pub(crate) fn append_delta(&mut self, delta: &Self) {
|
||||
self.text.push_str(&delta.text);
|
||||
self.token_ids.extend_from_slice(&delta.token_ids);
|
||||
self.completion_tokens += delta.completion_tokens;
|
||||
self.prompt_tokens = delta.prompt_tokens;
|
||||
if delta.finish_reason.is_some() {
|
||||
self.finish_reason = delta.finish_reason.clone();
|
||||
}
|
||||
|
||||
let Some(delta_extras) = delta.extras.as_deref() else {
|
||||
return;
|
||||
};
|
||||
let extras = self
|
||||
.extras
|
||||
.get_or_insert_with(|| Box::new(ChunkExtras::default()));
|
||||
|
||||
extras
|
||||
.out_lp_val
|
||||
.extend_from_slice(&delta_extras.out_lp_val);
|
||||
extras
|
||||
.out_lp_idx
|
||||
.extend_from_slice(&delta_extras.out_lp_idx);
|
||||
extras
|
||||
.out_top_val
|
||||
.extend_from_slice(&delta_extras.out_top_val);
|
||||
extras
|
||||
.out_top_idx
|
||||
.extend_from_slice(&delta_extras.out_top_idx);
|
||||
extras
|
||||
.out_top_lens
|
||||
.extend_from_slice(&delta_extras.out_top_lens);
|
||||
extras
|
||||
.out_tid_val
|
||||
.extend_from_slice(&delta_extras.out_tid_val);
|
||||
extras
|
||||
.out_tid_idx
|
||||
.extend_from_slice(&delta_extras.out_tid_idx);
|
||||
extras
|
||||
.out_tid_lens
|
||||
.extend_from_slice(&delta_extras.out_tid_lens);
|
||||
extras
|
||||
.out_lp_txt
|
||||
.extend_from_slice(&delta_extras.out_lp_txt);
|
||||
extras
|
||||
.out_top_txt
|
||||
.extend_from_slice(&delta_extras.out_top_txt);
|
||||
extras
|
||||
.out_tid_txt
|
||||
.extend_from_slice(&delta_extras.out_tid_txt);
|
||||
|
||||
// Input logprobs arrive once (during prefill). A later non-empty value
|
||||
// replaces the earlier one, matching the existing native HTTP fold.
|
||||
if !delta_extras.in_lp_val.is_empty() {
|
||||
extras.in_lp_val = delta_extras.in_lp_val.clone();
|
||||
extras.in_lp_idx = delta_extras.in_lp_idx.clone();
|
||||
extras.in_lp_txt = delta_extras.in_lp_txt.clone();
|
||||
}
|
||||
if !delta_extras.in_top_lens.is_empty() {
|
||||
extras.in_top_val = delta_extras.in_top_val.clone();
|
||||
extras.in_top_idx = delta_extras.in_top_idx.clone();
|
||||
extras.in_top_lens = delta_extras.in_top_lens.clone();
|
||||
extras.in_top_txt = delta_extras.in_top_txt.clone();
|
||||
}
|
||||
if !delta_extras.in_tid_lens.is_empty() {
|
||||
extras.in_tid_val = delta_extras.in_tid_val.clone();
|
||||
extras.in_tid_idx = delta_extras.in_tid_idx.clone();
|
||||
extras.in_tid_lens = delta_extras.in_tid_lens.clone();
|
||||
extras.in_tid_txt = delta_extras.in_tid_txt.clone();
|
||||
}
|
||||
// Hidden states are non-cumulative: the latest complete set wins.
|
||||
if !delta_extras.hidden_lens.is_empty() {
|
||||
extras.hidden_val = delta_extras.hidden_val.clone();
|
||||
extras.hidden_lens = delta_extras.hidden_lens.clone();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Semantic result of a deep frontend health check.
|
||||
///
|
||||
/// Expected lifecycle states are values rather than transport errors so each
|
||||
/// adapter can render them according to its own protocol.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(crate) enum HealthStatus {
|
||||
Healthy,
|
||||
NotReady,
|
||||
Stalled,
|
||||
}
|
||||
|
||||
/// A semantic generation event before any transport-specific framing.
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum FrontendEvent {
|
||||
/// One incremental model-output update.
|
||||
Delta(FrontendOutput),
|
||||
/// The final model-output update for this request.
|
||||
Finished(FrontendOutput),
|
||||
/// A request-scoped failure. Other requests in a batch may continue.
|
||||
Failed(FrontendError),
|
||||
}
|
||||
|
||||
/// Protocol-independent error category used by transport adapters.
|
||||
///
|
||||
/// The set intentionally follows operation semantics rather than HTTP or gRPC
|
||||
/// status spaces. An adapter maps this category into its own wire status.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(crate) enum FrontendErrorKind {
|
||||
InvalidArgument,
|
||||
NotFound,
|
||||
FailedPrecondition,
|
||||
ResourceExhausted,
|
||||
Cancelled,
|
||||
DeadlineExceeded,
|
||||
Unavailable,
|
||||
Internal,
|
||||
}
|
||||
|
||||
impl FrontendEvent {
|
||||
pub(crate) fn is_terminal(&self) -> bool {
|
||||
matches!(self, Self::Finished(_) | Self::Failed(_))
|
||||
}
|
||||
}
|
||||
|
||||
/// A transport-neutral failure category.
|
||||
///
|
||||
/// HTTP and gRPC adapters map these categories into their own status spaces;
|
||||
/// neither adapter needs to understand errors emitted by runtime stages.
|
||||
#[derive(Clone, Debug, thiserror::Error)]
|
||||
pub(crate) enum FrontendError {
|
||||
#[error("{0}")]
|
||||
InvalidArgument(String),
|
||||
|
||||
/// The frontend intake loop has shut down. A caller may retry elsewhere.
|
||||
#[error("service unavailable")]
|
||||
Unavailable,
|
||||
|
||||
/// Runtime capacity was exhausted before the request could make progress.
|
||||
#[error("{0}")]
|
||||
Overloaded(String),
|
||||
|
||||
#[error("{0}")]
|
||||
Cancelled(String),
|
||||
|
||||
/// The internal producer disappeared before returning a terminal result.
|
||||
///
|
||||
/// This is a server-side failure, not evidence that the caller cancelled.
|
||||
#[error("response truncated before completion")]
|
||||
ResponseTruncated,
|
||||
|
||||
/// A scheduler-side abort that carried an error status.
|
||||
///
|
||||
/// The current Python scheduler communicates this category as an HTTP
|
||||
/// status code inside its finish reason. The semantic event is still
|
||||
/// `Failed`; retaining the legacy code only lets the HTTP adapter preserve
|
||||
/// existing behavior while another adapter maps it into its own status
|
||||
/// space. New frontend failures should use the semantic variants above.
|
||||
#[error("{message}")]
|
||||
RuntimeRejected {
|
||||
kind: FrontendErrorKind,
|
||||
message: String,
|
||||
legacy_http_status: u16,
|
||||
},
|
||||
|
||||
/// The runtime replied with an unexpected or malformed result.
|
||||
#[error("{0}")]
|
||||
InvalidResponse(String),
|
||||
|
||||
#[error("{0}")]
|
||||
Internal(String),
|
||||
}
|
||||
|
||||
impl FrontendError {
|
||||
/// Semantic status for adapters that do not use the scheduler's legacy
|
||||
/// HTTP code space (notably the future gRPC adapter).
|
||||
pub(crate) fn kind(&self) -> FrontendErrorKind {
|
||||
match self {
|
||||
Self::InvalidArgument(_) => FrontendErrorKind::InvalidArgument,
|
||||
Self::Unavailable => FrontendErrorKind::Unavailable,
|
||||
Self::Overloaded(_) => FrontendErrorKind::ResourceExhausted,
|
||||
Self::Cancelled(_) => FrontendErrorKind::Cancelled,
|
||||
Self::ResponseTruncated => FrontendErrorKind::Internal,
|
||||
Self::RuntimeRejected { kind, .. } => *kind,
|
||||
Self::InvalidResponse(_) | Self::Internal(_) => FrontendErrorKind::Internal,
|
||||
}
|
||||
}
|
||||
|
||||
/// Translate the scheduler's inherited HTTP-coded failure into a semantic
|
||||
/// category once, at the runtime boundary. HTTP can retain the exact legacy
|
||||
/// status while other adapters consume [`Self::kind`].
|
||||
pub(super) fn from_runtime_rejection(message: String, legacy_http_status: u16) -> Self {
|
||||
let kind = match legacy_http_status {
|
||||
404 => FrontendErrorKind::NotFound,
|
||||
408 | 504 => FrontendErrorKind::DeadlineExceeded,
|
||||
412 => FrontendErrorKind::FailedPrecondition,
|
||||
413 | 429 => FrontendErrorKind::ResourceExhausted,
|
||||
499 => FrontendErrorKind::Cancelled,
|
||||
502 | 503 => FrontendErrorKind::Unavailable,
|
||||
400..=499 => FrontendErrorKind::InvalidArgument,
|
||||
_ => FrontendErrorKind::Internal,
|
||||
};
|
||||
Self::RuntimeRejected {
|
||||
kind,
|
||||
message,
|
||||
legacy_http_status,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Static model metadata shared by every public transport.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub(crate) struct ModelInfo {
|
||||
pub(crate) model_path: String,
|
||||
pub(crate) served_model_name: String,
|
||||
pub(crate) tokenizer_path: String,
|
||||
pub(crate) is_generation: bool,
|
||||
pub(crate) preferred_sampling_params: Option<PreferredSamplingParams>,
|
||||
pub(crate) weight_version: Option<String>,
|
||||
pub(crate) load_format: Option<String>,
|
||||
pub(crate) reasoning_parser: Option<String>,
|
||||
pub(crate) tool_call_parser: Option<String>,
|
||||
pub(crate) disaggregation_mode: DisaggregationMode,
|
||||
}
|
||||
|
||||
/// Public server metadata plus scheduler-owned runtime metrics.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub(crate) struct ServerInfo {
|
||||
pub(crate) model_path: String,
|
||||
pub(crate) served_model_name: String,
|
||||
pub(crate) tokenizer_path: String,
|
||||
pub(crate) max_context_length: u64,
|
||||
pub(crate) max_total_num_tokens: u64,
|
||||
pub(crate) version: String,
|
||||
pub(crate) frontend: &'static str,
|
||||
pub(crate) internal_states: Vec<InternalState>,
|
||||
}
|
||||
|
||||
/// Scheduler-owned runtime metrics used by the public server-info operation.
|
||||
///
|
||||
/// Deserializing into this allowlisted shape is intentional: the scheduler's
|
||||
/// raw state also contains its full launch arguments, including credentials.
|
||||
/// Unknown fields are discarded before any transport receives the result.
|
||||
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
|
||||
#[serde(default)]
|
||||
pub(crate) struct InternalState {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) last_gen_throughput: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) memory_usage: Option<MemoryUsage>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) effective_max_running_requests_per_dp: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) avg_spec_accept_length: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) step_time_dict: Option<BTreeMap<u64, Vec<f64>>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) rust_mm_transport: Option<BTreeMap<String, u64>>,
|
||||
}
|
||||
|
||||
/// Public subset of scheduler memory metrics.
|
||||
///
|
||||
/// This nested allowlist is intentional: adding a scheduler field does not
|
||||
/// silently add different public data to HTTP before the gRPC contract can add
|
||||
/// the same field. New metrics should be promoted here deliberately.
|
||||
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
|
||||
#[serde(default)]
|
||||
pub(crate) struct MemoryUsage {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) weight: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) kvcache: Option<MemoryMeasurement>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) startup_available: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) token_capacity: Option<u64>,
|
||||
pub(crate) token_capacity_swa: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) graph: Option<BTreeMap<String, f64>>,
|
||||
}
|
||||
|
||||
/// The scheduler's KV-cache measurement can be a native float or a NumPy
|
||||
/// scalar stringified by the MessagePack bridge. Preserve that representation
|
||||
/// so extracting the typed frontend contract does not change public responses.
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub(crate) enum MemoryMeasurement {
|
||||
Number(f64),
|
||||
String(String),
|
||||
}
|
||||
@@ -1,531 +0,0 @@
|
||||
//! `runtime.v1` request conversion for the Rust frontend.
|
||||
//!
|
||||
//! The canonical protobuf is wider than the Rust frontend today. Fields with
|
||||
//! no internal representation are rejected explicitly instead of being
|
||||
//! silently ignored or expanding frontend behavior in this wiring change.
|
||||
|
||||
use sglang_grpc_types::sglang::runtime::v1 as proto;
|
||||
use tonic::Status;
|
||||
|
||||
use crate::frontend::FrontendRequest;
|
||||
use crate::message::config::PreferredSamplingParams;
|
||||
use crate::message::ids::Rid;
|
||||
use crate::message::sampling::SamplingParams;
|
||||
|
||||
type ConvertResult<T> = Result<T, ConvertError>;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub(super) enum ConvertError {
|
||||
#[error("{0}")]
|
||||
InvalidArgument(String),
|
||||
#[error("{0}")]
|
||||
Unsupported(String),
|
||||
#[error("{0}")]
|
||||
Internal(String),
|
||||
}
|
||||
|
||||
impl ConvertError {
|
||||
fn invalid_argument(message: impl Into<String>) -> Self {
|
||||
Self::InvalidArgument(message.into())
|
||||
}
|
||||
|
||||
fn internal(message: impl Into<String>) -> Self {
|
||||
Self::Internal(message.into())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<ConvertError> for Status {
|
||||
fn from(error: ConvertError) -> Self {
|
||||
match error {
|
||||
ConvertError::InvalidArgument(message) => Self::invalid_argument(message),
|
||||
ConvertError::Unsupported(message) => Self::unimplemented(message),
|
||||
ConvertError::Internal(message) => Self::internal(message),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn text_generate(
|
||||
request: proto::TextGenerateRequest,
|
||||
preferred: Option<&PreferredSamplingParams>,
|
||||
) -> ConvertResult<FrontendRequest> {
|
||||
let proto::TextGenerateRequest {
|
||||
text,
|
||||
sampling_params,
|
||||
stream,
|
||||
return_logprob,
|
||||
top_logprobs_num,
|
||||
logprob_start_len,
|
||||
return_text_in_logprobs,
|
||||
rid,
|
||||
lora_path,
|
||||
routing_key,
|
||||
routed_dp_rank,
|
||||
trace_headers,
|
||||
session_id,
|
||||
disaggregated_params,
|
||||
priority,
|
||||
require_reasoning,
|
||||
max_thinking_tokens,
|
||||
kv_hints,
|
||||
} = request;
|
||||
|
||||
reject_unsupported([
|
||||
("kv_hints", kv_hints.is_some()),
|
||||
("lora_path", lora_path.is_some()),
|
||||
("routing_key", routing_key.is_some()),
|
||||
("trace_headers", !trace_headers.is_empty()),
|
||||
("session_id", session_id.is_some()),
|
||||
("priority", priority.is_some()),
|
||||
("require_reasoning", require_reasoning.is_some()),
|
||||
("max_thinking_tokens", max_thinking_tokens.is_some()),
|
||||
])?;
|
||||
|
||||
let (bootstrap_host, bootstrap_port, bootstrap_room) =
|
||||
disaggregated_fields(disaggregated_params);
|
||||
Ok(FrontendRequest {
|
||||
rid: request_id(rid),
|
||||
text: Some(text),
|
||||
input_ids: None,
|
||||
skip_special_tokens: false,
|
||||
sampling_params: convert_sampling_params(sampling_params, preferred)?,
|
||||
stream: stream.unwrap_or(false),
|
||||
return_logprob: return_logprob.unwrap_or(false),
|
||||
logprob_start_len: logprob_start_len.map(i64::from).unwrap_or(-1),
|
||||
top_logprobs_num: top_logprobs_num.map(i64::from).unwrap_or(0),
|
||||
token_ids_logprob: None,
|
||||
return_sampling_mask: false,
|
||||
return_hidden_states: false,
|
||||
return_text_in_logprobs,
|
||||
bootstrap_host,
|
||||
bootstrap_port,
|
||||
bootstrap_room,
|
||||
bootstrap_pair_key: None,
|
||||
decode_tp_size: None,
|
||||
routed_dp_rank: routed_dp_rank.map(i64::from),
|
||||
disagg_prefill_dp_rank: None,
|
||||
mm: None,
|
||||
mm_buffers: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn generate(
|
||||
request: proto::GenerateRequest,
|
||||
preferred: Option<&PreferredSamplingParams>,
|
||||
) -> ConvertResult<FrontendRequest> {
|
||||
let proto::GenerateRequest {
|
||||
input_ids,
|
||||
sampling_params,
|
||||
stream,
|
||||
return_logprob,
|
||||
top_logprobs_num,
|
||||
logprob_start_len,
|
||||
rid,
|
||||
lora_path,
|
||||
routing_key,
|
||||
routed_dp_rank,
|
||||
trace_headers,
|
||||
session_id,
|
||||
disaggregated_params,
|
||||
priority,
|
||||
require_reasoning,
|
||||
max_thinking_tokens,
|
||||
kv_hints,
|
||||
} = request;
|
||||
|
||||
reject_unsupported([
|
||||
("kv_hints", kv_hints.is_some()),
|
||||
("lora_path", lora_path.is_some()),
|
||||
("routing_key", routing_key.is_some()),
|
||||
("trace_headers", !trace_headers.is_empty()),
|
||||
("session_id", session_id.is_some()),
|
||||
("priority", priority.is_some()),
|
||||
("require_reasoning", require_reasoning.is_some()),
|
||||
("max_thinking_tokens", max_thinking_tokens.is_some()),
|
||||
])?;
|
||||
if input_ids.is_empty() {
|
||||
return Err(ConvertError::invalid_argument("input_ids cannot be empty"));
|
||||
}
|
||||
|
||||
let (bootstrap_host, bootstrap_port, bootstrap_room) =
|
||||
disaggregated_fields(disaggregated_params);
|
||||
Ok(FrontendRequest {
|
||||
rid: request_id(rid),
|
||||
text: None,
|
||||
input_ids: Some(input_ids.into_iter().map(i64::from).collect()),
|
||||
skip_special_tokens: false,
|
||||
sampling_params: convert_sampling_params(sampling_params, preferred)?,
|
||||
stream: stream.unwrap_or(false),
|
||||
return_logprob: return_logprob.unwrap_or(false),
|
||||
logprob_start_len: logprob_start_len.map(i64::from).unwrap_or(-1),
|
||||
top_logprobs_num: top_logprobs_num.map(i64::from).unwrap_or(0),
|
||||
token_ids_logprob: None,
|
||||
return_sampling_mask: false,
|
||||
return_hidden_states: false,
|
||||
return_text_in_logprobs: None,
|
||||
bootstrap_host,
|
||||
bootstrap_port,
|
||||
bootstrap_room,
|
||||
bootstrap_pair_key: None,
|
||||
decode_tp_size: None,
|
||||
routed_dp_rank: routed_dp_rank.map(i64::from),
|
||||
disagg_prefill_dp_rank: None,
|
||||
mm: None,
|
||||
mm_buffers: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
fn request_id(rid: Option<String>) -> Rid {
|
||||
rid.map_or_else(Rid::new, |rid| Rid::from_client(&rid))
|
||||
}
|
||||
|
||||
fn reject_unsupported<const N: usize>(fields: [(&'static str, bool); N]) -> ConvertResult<()> {
|
||||
let unsupported = fields
|
||||
.into_iter()
|
||||
.filter_map(|(name, present)| present.then_some(name))
|
||||
.collect::<Vec<_>>();
|
||||
if unsupported.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
Err(ConvertError::Unsupported(format!(
|
||||
"the Rust frontend does not yet support runtime.v1 field(s): {}",
|
||||
unsupported.join(", ")
|
||||
)))
|
||||
}
|
||||
|
||||
fn disaggregated_fields(
|
||||
params: Option<proto::DisaggregatedParams>,
|
||||
) -> (Option<String>, Option<i64>, Option<i64>) {
|
||||
params.map_or((None, None, None), |params| {
|
||||
(
|
||||
Some(params.bootstrap_host),
|
||||
Some(i64::from(params.bootstrap_port)),
|
||||
Some(params.bootstrap_room),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// Merge launch-time preferred values first, then overwrite only fields that
|
||||
/// are explicitly present in protobuf. This is the same precedence used by
|
||||
/// the Python-backed gRPC path; proto3 repeated fields cannot distinguish an
|
||||
/// omitted list from an explicit empty list, so empty stop lists remain absent.
|
||||
#[allow(deprecated)]
|
||||
fn convert_sampling_params(
|
||||
params: Option<proto::SamplingParams>,
|
||||
preferred: Option<&PreferredSamplingParams>,
|
||||
) -> ConvertResult<SamplingParams> {
|
||||
let mut output = SamplingParams::default();
|
||||
|
||||
let Some(params) = params else {
|
||||
if let Some(preferred) = preferred {
|
||||
output.apply_preferred(&preferred.0).map_err(|error| {
|
||||
ConvertError::internal(format!("invalid preferred_sampling_params: {error}"))
|
||||
})?;
|
||||
}
|
||||
return Ok(output);
|
||||
};
|
||||
if params.guided_decoding.is_some() && (params.json_schema.is_some() || params.regex.is_some())
|
||||
{
|
||||
return Err(ConvertError::invalid_argument(
|
||||
"legacy json_schema/regex cannot be combined with guided_decoding",
|
||||
));
|
||||
}
|
||||
|
||||
if let Some(value) = params.temperature {
|
||||
output.temperature = f64::from(value);
|
||||
output.mark_explicit("temperature");
|
||||
}
|
||||
if let Some(value) = params.top_p {
|
||||
output.top_p = f64::from(value);
|
||||
output.mark_explicit("top_p");
|
||||
}
|
||||
if let Some(value) = params.top_k {
|
||||
output.top_k = i64::from(value);
|
||||
output.mark_explicit("top_k");
|
||||
}
|
||||
if let Some(value) = params.min_p {
|
||||
output.min_p = f64::from(value);
|
||||
output.mark_explicit("min_p");
|
||||
}
|
||||
if let Some(value) = params.frequency_penalty {
|
||||
output.frequency_penalty = f64::from(value);
|
||||
output.mark_explicit("frequency_penalty");
|
||||
}
|
||||
if let Some(value) = params.presence_penalty {
|
||||
output.presence_penalty = f64::from(value);
|
||||
output.mark_explicit("presence_penalty");
|
||||
}
|
||||
if let Some(value) = params.repetition_penalty {
|
||||
output.repetition_penalty = f64::from(value);
|
||||
output.mark_explicit("repetition_penalty");
|
||||
}
|
||||
if let Some(value) = params.max_new_tokens {
|
||||
output.max_new_tokens = Some(i64::from(value));
|
||||
output.mark_explicit("max_new_tokens");
|
||||
}
|
||||
if let Some(value) = params.min_new_tokens {
|
||||
output.min_new_tokens = i64::from(value);
|
||||
output.mark_explicit("min_new_tokens");
|
||||
}
|
||||
if !params.stop.is_empty() {
|
||||
output.stop = Some(crate::message::types::OneOrMany::Many(params.stop));
|
||||
output.mark_explicit("stop");
|
||||
}
|
||||
if !params.stop_token_ids.is_empty() {
|
||||
output.stop_token_ids = Some(params.stop_token_ids.into_iter().map(i64::from).collect());
|
||||
output.mark_explicit("stop_token_ids");
|
||||
}
|
||||
if let Some(value) = params.ignore_eos {
|
||||
output.ignore_eos = value;
|
||||
output.mark_explicit("ignore_eos");
|
||||
}
|
||||
if let Some(value) = params.n {
|
||||
output.n = i64::from(value);
|
||||
output.mark_explicit("n");
|
||||
}
|
||||
if let Some(value) = params.seed {
|
||||
output.sampling_seed = Some(value);
|
||||
output.mark_explicit("sampling_seed");
|
||||
}
|
||||
|
||||
if let Some(guided) = params.guided_decoding {
|
||||
apply_guided_decoding(&mut output, guided)?;
|
||||
} else {
|
||||
if let Some(value) = params.json_schema {
|
||||
if value.is_empty() {
|
||||
return Err(ConvertError::invalid_argument(
|
||||
"legacy json_schema must not be empty",
|
||||
));
|
||||
}
|
||||
output.json_schema = Some(value);
|
||||
output.mark_explicit("json_schema");
|
||||
}
|
||||
if let Some(value) = params.regex {
|
||||
if value.is_empty() {
|
||||
return Err(ConvertError::invalid_argument(
|
||||
"legacy regex must not be empty",
|
||||
));
|
||||
}
|
||||
output.regex = Some(value);
|
||||
output.mark_explicit("regex");
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(preferred) = preferred {
|
||||
output.apply_preferred(&preferred.0).map_err(|error| {
|
||||
ConvertError::internal(format!("invalid preferred_sampling_params: {error}"))
|
||||
})?;
|
||||
}
|
||||
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
fn apply_guided_decoding(
|
||||
output: &mut SamplingParams,
|
||||
guided: proto::GuidedDecoding,
|
||||
) -> ConvertResult<()> {
|
||||
use proto::guided_decoding::Constraint;
|
||||
|
||||
match guided.constraint {
|
||||
Some(Constraint::JsonSchema(value)) if !value.is_empty() => {
|
||||
output.json_schema = Some(value);
|
||||
output.mark_explicit("json_schema");
|
||||
}
|
||||
Some(Constraint::Regex(value)) if !value.is_empty() => {
|
||||
output.regex = Some(value);
|
||||
output.mark_explicit("regex");
|
||||
}
|
||||
Some(Constraint::Ebnf(value)) if !value.is_empty() => {
|
||||
output.ebnf = Some(value);
|
||||
output.mark_explicit("ebnf");
|
||||
}
|
||||
Some(Constraint::StructuralTag(value)) if !value.is_empty() => {
|
||||
output.structural_tag = Some(value);
|
||||
output.mark_explicit("structural_tag");
|
||||
}
|
||||
Some(Constraint::Choice(choice))
|
||||
if !choice.values.is_empty() && choice.values.iter().all(|value| !value.is_empty()) =>
|
||||
{
|
||||
let alternatives = choice
|
||||
.values
|
||||
.iter()
|
||||
.map(|value| regex_escape_literal(value))
|
||||
.collect::<Vec<_>>()
|
||||
.join("|");
|
||||
output.regex = Some(format!("(?:{alternatives})"));
|
||||
output.mark_explicit("regex");
|
||||
}
|
||||
Some(Constraint::Choice(_)) => {
|
||||
return Err(ConvertError::invalid_argument(
|
||||
"guided choice must contain only non-empty values",
|
||||
));
|
||||
}
|
||||
Some(_) => {
|
||||
return Err(ConvertError::invalid_argument(
|
||||
"guided decoding constraint must not be empty",
|
||||
));
|
||||
}
|
||||
None => {
|
||||
return Err(ConvertError::invalid_argument(
|
||||
"guided decoding constraint must be specified",
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn regex_escape_literal(value: &str) -> String {
|
||||
let mut escaped = String::with_capacity(value.len());
|
||||
for character in value.chars() {
|
||||
if matches!(
|
||||
character,
|
||||
'.' | '+' | '*' | '?' | '^' | '$' | '(' | ')' | '[' | ']' | '{' | '}' | '|' | '\\'
|
||||
) {
|
||||
escaped.push('\\');
|
||||
}
|
||||
escaped.push(character);
|
||||
}
|
||||
escaped
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tonic::Code;
|
||||
|
||||
#[test]
|
||||
fn text_request_maps_supported_fields_and_preferred_sampling() {
|
||||
let preferred = PreferredSamplingParams(serde_json::json!({
|
||||
"temperature": 0.2,
|
||||
"top_p": 0.7,
|
||||
"max_new_tokens": 64,
|
||||
}));
|
||||
let request = proto::TextGenerateRequest {
|
||||
text: "hello".into(),
|
||||
sampling_params: Some(proto::SamplingParams {
|
||||
temperature: Some(0.8),
|
||||
seed: Some(7),
|
||||
..Default::default()
|
||||
}),
|
||||
stream: Some(true),
|
||||
return_logprob: Some(true),
|
||||
top_logprobs_num: Some(3),
|
||||
logprob_start_len: Some(2),
|
||||
return_text_in_logprobs: Some(true),
|
||||
rid: Some("client-id".into()),
|
||||
routed_dp_rank: Some(4),
|
||||
disaggregated_params: Some(proto::DisaggregatedParams {
|
||||
bootstrap_host: "10.0.0.1".into(),
|
||||
bootstrap_port: 8998,
|
||||
bootstrap_room: 12,
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let request = text_generate(request, Some(&preferred)).unwrap();
|
||||
assert_eq!(request.rid.client_facing(), "client-id");
|
||||
assert_eq!(request.text.as_deref(), Some("hello"));
|
||||
assert!(request.input_ids.is_none());
|
||||
assert!(request.stream);
|
||||
assert!(request.return_logprob);
|
||||
assert_eq!(request.top_logprobs_num, 3);
|
||||
assert_eq!(request.logprob_start_len, 2);
|
||||
assert_eq!(request.return_text_in_logprobs, Some(true));
|
||||
assert_eq!(request.routed_dp_rank, Some(4));
|
||||
assert_eq!(request.bootstrap_host.as_deref(), Some("10.0.0.1"));
|
||||
assert_eq!(request.bootstrap_port, Some(8998));
|
||||
assert_eq!(request.bootstrap_room, Some(12));
|
||||
assert_eq!(request.sampling_params.temperature, f64::from(0.8_f32));
|
||||
assert_eq!(request.sampling_params.top_p, 0.7);
|
||||
assert_eq!(request.sampling_params.max_new_tokens, Some(64));
|
||||
assert_eq!(request.sampling_params.sampling_seed, Some(7));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn token_request_validates_input_and_rejects_unported_fields() {
|
||||
let empty = Status::from(generate(proto::GenerateRequest::default(), None).unwrap_err());
|
||||
assert_eq!(empty.code(), Code::InvalidArgument);
|
||||
|
||||
let unsupported = Status::from(
|
||||
generate(
|
||||
proto::GenerateRequest {
|
||||
input_ids: vec![1],
|
||||
routing_key: Some("worker-a".into()),
|
||||
priority: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
)
|
||||
.unwrap_err(),
|
||||
);
|
||||
assert_eq!(unsupported.code(), Code::Unimplemented);
|
||||
assert!(unsupported.message().contains("routing_key, priority"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absent_sampling_params_still_uses_preferred_values() {
|
||||
let preferred = PreferredSamplingParams(serde_json::json!({
|
||||
"temperature": 0.25,
|
||||
"max_new_tokens": 32,
|
||||
}));
|
||||
let request = generate(
|
||||
proto::GenerateRequest {
|
||||
input_ids: vec![1],
|
||||
..Default::default()
|
||||
},
|
||||
Some(&preferred),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(request.sampling_params.temperature, 0.25);
|
||||
assert_eq!(request.sampling_params.max_new_tokens, Some(32));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn guided_choice_preserves_existing_runtime_v1_mapping() {
|
||||
let request = generate(
|
||||
proto::GenerateRequest {
|
||||
input_ids: vec![1],
|
||||
sampling_params: Some(proto::SamplingParams {
|
||||
guided_decoding: Some(proto::GuidedDecoding {
|
||||
constraint: Some(proto::guided_decoding::Constraint::Choice(
|
||||
proto::ChoiceConstraint {
|
||||
values: vec!["a+b".into(), "x.y".into()],
|
||||
},
|
||||
)),
|
||||
}),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
request.sampling_params.regex.as_deref(),
|
||||
Some("(?:a\\+b|x\\.y)")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[allow(deprecated)]
|
||||
fn guided_and_legacy_constraints_cannot_be_combined() {
|
||||
let error = Status::from(
|
||||
generate(
|
||||
proto::GenerateRequest {
|
||||
input_ids: vec![1],
|
||||
sampling_params: Some(proto::SamplingParams {
|
||||
regex: Some("a".into()),
|
||||
guided_decoding: Some(proto::GuidedDecoding {
|
||||
constraint: Some(proto::guided_decoding::Constraint::Regex("b".into())),
|
||||
}),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
)
|
||||
.unwrap_err(),
|
||||
);
|
||||
assert_eq!(error.code(), Code::InvalidArgument);
|
||||
}
|
||||
}
|
||||
@@ -1,303 +0,0 @@
|
||||
//! Tonic adapter for the canonical `sglang.runtime.v1` API.
|
||||
//!
|
||||
//! This module translates protobuf requests and responses and mounts that thin
|
||||
//! adapter on Tonic. The shared [`FrontendHandle`] owns preprocessing,
|
||||
//! admission, runtime communication, and request cancellation.
|
||||
//!
|
||||
//! The thin Tonic-service structure, streamed response approach, and test
|
||||
//! strategy build on Rain Jiang's multi-protocol prototype in
|
||||
//! `sgl-project/sglang#36923`. This stack keeps those ideas while using the
|
||||
//! canonical `runtime.v1` contract and the narrower [`FrontendHandle`] seam.
|
||||
|
||||
use std::pin::Pin;
|
||||
use std::time::Duration;
|
||||
|
||||
use futures::Stream;
|
||||
use sglang_grpc_types::sglang::runtime::v1 as proto;
|
||||
use sglang_grpc_types::sglang::runtime::v1::sglang_service_server::SglangService;
|
||||
use tonic::{Request, Response, Status};
|
||||
|
||||
use crate::frontend::FrontendHandle;
|
||||
use crate::message::config::{PreferredSamplingParams, ServerArgs};
|
||||
|
||||
mod convert;
|
||||
mod response;
|
||||
mod server;
|
||||
|
||||
pub(crate) use server::serve;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
const DEFAULT_RESPONSE_TIMEOUT: Duration = Duration::from_secs(300);
|
||||
|
||||
type ResponseStream<T> = Pin<Box<dyn Stream<Item = Result<T, Status>> + Send + 'static>>;
|
||||
|
||||
/// Narrow snapshot of launch policy required to preserve the existing
|
||||
/// `runtime.v1` generation behavior. It intentionally excludes listener,
|
||||
/// authentication, TLS, and lifecycle configuration.
|
||||
#[derive(Clone)]
|
||||
struct AdapterConfig {
|
||||
preferred_sampling_params: Option<PreferredSamplingParams>,
|
||||
incremental_streaming_output: bool,
|
||||
response_timeout: Duration,
|
||||
}
|
||||
|
||||
/// Tonic-facing implementation backed by the transport-neutral Rust frontend.
|
||||
pub(crate) struct GrpcService {
|
||||
frontend: FrontendHandle,
|
||||
config: AdapterConfig,
|
||||
}
|
||||
|
||||
impl GrpcService {
|
||||
pub(crate) fn new(frontend: FrontendHandle, server_args: &ServerArgs) -> Self {
|
||||
Self {
|
||||
frontend,
|
||||
config: AdapterConfig {
|
||||
preferred_sampling_params: server_args.preferred_sampling_params.clone(),
|
||||
incremental_streaming_output: server_args.incremental_streaming_output,
|
||||
response_timeout: DEFAULT_RESPONSE_TIMEOUT,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn for_test(
|
||||
frontend: FrontendHandle,
|
||||
preferred_sampling_params: Option<PreferredSamplingParams>,
|
||||
incremental_streaming_output: bool,
|
||||
response_timeout: Duration,
|
||||
) -> Self {
|
||||
Self {
|
||||
frontend,
|
||||
config: AdapterConfig {
|
||||
preferred_sampling_params,
|
||||
incremental_streaming_output,
|
||||
response_timeout,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn unimplemented_rpc(name: &'static str) -> Status {
|
||||
Status::unimplemented(format!(
|
||||
"{name} is not implemented by the Rust frontend gRPC adapter"
|
||||
))
|
||||
}
|
||||
|
||||
#[tonic::async_trait]
|
||||
impl SglangService for GrpcService {
|
||||
async fn text_generate(
|
||||
&self,
|
||||
request: Request<proto::TextGenerateRequest>,
|
||||
) -> Result<Response<ResponseStream<proto::TextGenerateResponse>>, Status> {
|
||||
let request = convert::text_generate(
|
||||
request.into_inner(),
|
||||
self.config.preferred_sampling_params.as_ref(),
|
||||
)
|
||||
.map_err(Status::from)?;
|
||||
let call = self
|
||||
.frontend
|
||||
.generate(request)
|
||||
.await
|
||||
.map_err(response::status)?;
|
||||
Ok(Response::new(response::text_generate_stream(
|
||||
call,
|
||||
self.config.incremental_streaming_output,
|
||||
self.config.response_timeout,
|
||||
)))
|
||||
}
|
||||
|
||||
async fn generate(
|
||||
&self,
|
||||
request: Request<proto::GenerateRequest>,
|
||||
) -> Result<Response<ResponseStream<proto::GenerateResponse>>, Status> {
|
||||
let request = convert::generate(
|
||||
request.into_inner(),
|
||||
self.config.preferred_sampling_params.as_ref(),
|
||||
)
|
||||
.map_err(Status::from)?;
|
||||
let call = self
|
||||
.frontend
|
||||
.generate(request)
|
||||
.await
|
||||
.map_err(response::status)?;
|
||||
Ok(Response::new(response::generate_stream(
|
||||
call,
|
||||
self.config.incremental_streaming_output,
|
||||
self.config.response_timeout,
|
||||
)))
|
||||
}
|
||||
|
||||
// runtime.v1 is shared with the broader Python-backed service, so its
|
||||
// generated trait contains more operations than this initial Rust adapter.
|
||||
// Keep that support boundary explicit: adding an RPC is a deliberate future
|
||||
// change, never an accidental empty success.
|
||||
async fn text_embed(
|
||||
&self,
|
||||
_request: Request<proto::TextEmbedRequest>,
|
||||
) -> Result<Response<proto::TextEmbedResponse>, Status> {
|
||||
Err(unimplemented_rpc("text_embed"))
|
||||
}
|
||||
|
||||
async fn embed(
|
||||
&self,
|
||||
_request: Request<proto::EmbedRequest>,
|
||||
) -> Result<Response<proto::EmbedResponse>, Status> {
|
||||
Err(unimplemented_rpc("embed"))
|
||||
}
|
||||
|
||||
async fn classify(
|
||||
&self,
|
||||
_request: Request<proto::ClassifyRequest>,
|
||||
) -> Result<Response<proto::ClassifyResponse>, Status> {
|
||||
Err(unimplemented_rpc("classify"))
|
||||
}
|
||||
|
||||
async fn tokenize(
|
||||
&self,
|
||||
_request: Request<proto::TokenizeRequest>,
|
||||
) -> Result<Response<proto::TokenizeResponse>, Status> {
|
||||
Err(unimplemented_rpc("tokenize"))
|
||||
}
|
||||
|
||||
async fn detokenize(
|
||||
&self,
|
||||
_request: Request<proto::DetokenizeRequest>,
|
||||
) -> Result<Response<proto::DetokenizeResponse>, Status> {
|
||||
Err(unimplemented_rpc("detokenize"))
|
||||
}
|
||||
|
||||
async fn health_check(
|
||||
&self,
|
||||
_request: Request<proto::HealthCheckRequest>,
|
||||
) -> Result<Response<proto::HealthCheckResponse>, Status> {
|
||||
Err(unimplemented_rpc("health_check"))
|
||||
}
|
||||
|
||||
async fn get_model_info(
|
||||
&self,
|
||||
_request: Request<proto::GetModelInfoRequest>,
|
||||
) -> Result<Response<proto::GetModelInfoResponse>, Status> {
|
||||
Err(unimplemented_rpc("get_model_info"))
|
||||
}
|
||||
|
||||
async fn get_server_info(
|
||||
&self,
|
||||
_request: Request<proto::GetServerInfoRequest>,
|
||||
) -> Result<Response<proto::GetServerInfoResponse>, Status> {
|
||||
Err(unimplemented_rpc("get_server_info"))
|
||||
}
|
||||
|
||||
async fn list_models(
|
||||
&self,
|
||||
_request: Request<proto::ListModelsRequest>,
|
||||
) -> Result<Response<proto::ListModelsResponse>, Status> {
|
||||
Err(unimplemented_rpc("list_models"))
|
||||
}
|
||||
|
||||
async fn get_load(
|
||||
&self,
|
||||
_request: Request<proto::GetLoadRequest>,
|
||||
) -> Result<Response<proto::GetLoadResponse>, Status> {
|
||||
Err(unimplemented_rpc("get_load"))
|
||||
}
|
||||
|
||||
async fn abort(
|
||||
&self,
|
||||
_request: Request<proto::AbortRequest>,
|
||||
) -> Result<Response<proto::AbortResponse>, Status> {
|
||||
Err(unimplemented_rpc("abort"))
|
||||
}
|
||||
|
||||
async fn flush_cache(
|
||||
&self,
|
||||
_request: Request<proto::FlushCacheRequest>,
|
||||
) -> Result<Response<proto::FlushCacheResponse>, Status> {
|
||||
Err(unimplemented_rpc("flush_cache"))
|
||||
}
|
||||
|
||||
async fn pause_generation(
|
||||
&self,
|
||||
_request: Request<proto::PauseGenerationRequest>,
|
||||
) -> Result<Response<proto::PauseGenerationResponse>, Status> {
|
||||
Err(unimplemented_rpc("pause_generation"))
|
||||
}
|
||||
|
||||
async fn watch_engine_state(
|
||||
&self,
|
||||
_request: Request<proto::WatchEngineStateRequest>,
|
||||
) -> Result<Response<ResponseStream<proto::EngineStateSnapshot>>, Status> {
|
||||
Err(unimplemented_rpc("watch_engine_state"))
|
||||
}
|
||||
|
||||
async fn continue_generation(
|
||||
&self,
|
||||
_request: Request<proto::ContinueGenerationRequest>,
|
||||
) -> Result<Response<proto::ContinueGenerationResponse>, Status> {
|
||||
Err(unimplemented_rpc("continue_generation"))
|
||||
}
|
||||
|
||||
async fn chat_complete(
|
||||
&self,
|
||||
_request: Request<proto::OpenAiRequest>,
|
||||
) -> Result<Response<ResponseStream<proto::OpenAiStreamChunk>>, Status> {
|
||||
Err(unimplemented_rpc("chat_complete"))
|
||||
}
|
||||
|
||||
async fn complete(
|
||||
&self,
|
||||
_request: Request<proto::OpenAiRequest>,
|
||||
) -> Result<Response<ResponseStream<proto::OpenAiStreamChunk>>, Status> {
|
||||
Err(unimplemented_rpc("complete"))
|
||||
}
|
||||
|
||||
async fn open_ai_embed(
|
||||
&self,
|
||||
_request: Request<proto::OpenAiRequest>,
|
||||
) -> Result<Response<proto::OpenAiResponse>, Status> {
|
||||
Err(unimplemented_rpc("open_ai_embed"))
|
||||
}
|
||||
|
||||
async fn open_ai_classify(
|
||||
&self,
|
||||
_request: Request<proto::OpenAiRequest>,
|
||||
) -> Result<Response<proto::OpenAiResponse>, Status> {
|
||||
Err(unimplemented_rpc("open_ai_classify"))
|
||||
}
|
||||
|
||||
async fn score(
|
||||
&self,
|
||||
_request: Request<proto::OpenAiRequest>,
|
||||
) -> Result<Response<proto::OpenAiResponse>, Status> {
|
||||
Err(unimplemented_rpc("score"))
|
||||
}
|
||||
|
||||
async fn rerank(
|
||||
&self,
|
||||
_request: Request<proto::OpenAiRequest>,
|
||||
) -> Result<Response<proto::OpenAiResponse>, Status> {
|
||||
Err(unimplemented_rpc("rerank"))
|
||||
}
|
||||
|
||||
async fn start_profile(
|
||||
&self,
|
||||
_request: Request<proto::StartProfileRequest>,
|
||||
) -> Result<Response<proto::StartProfileResponse>, Status> {
|
||||
Err(unimplemented_rpc("start_profile"))
|
||||
}
|
||||
|
||||
async fn stop_profile(
|
||||
&self,
|
||||
_request: Request<proto::StopProfileRequest>,
|
||||
) -> Result<Response<proto::StopProfileResponse>, Status> {
|
||||
Err(unimplemented_rpc("stop_profile"))
|
||||
}
|
||||
|
||||
async fn update_weights_from_disk(
|
||||
&self,
|
||||
_request: Request<proto::UpdateWeightsRequest>,
|
||||
) -> Result<Response<proto::UpdateWeightsResponse>, Status> {
|
||||
Err(unimplemented_rpc("update_weights_from_disk"))
|
||||
}
|
||||
}
|
||||
@@ -1,200 +0,0 @@
|
||||
//! Semantic frontend events to `runtime.v1` stream items and statuses.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::time::Duration;
|
||||
|
||||
use sglang_grpc_types::sglang::runtime::v1 as proto;
|
||||
use tonic::{Code, Status};
|
||||
|
||||
use super::ResponseStream;
|
||||
use crate::frontend::{
|
||||
FrontendCall, FrontendError, FrontendErrorKind, FrontendEvent, FrontendOutput,
|
||||
};
|
||||
use crate::native_generation::meta_info_value;
|
||||
|
||||
pub(super) fn text_generate_stream(
|
||||
call: FrontendCall,
|
||||
incremental: bool,
|
||||
response_timeout: Duration,
|
||||
) -> ResponseStream<proto::TextGenerateResponse> {
|
||||
generation_stream(
|
||||
call,
|
||||
incremental,
|
||||
response_timeout,
|
||||
|output, meta_info, finished| {
|
||||
Ok(proto::TextGenerateResponse {
|
||||
text: output.text.clone(),
|
||||
meta_info,
|
||||
finished,
|
||||
})
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn generate_stream(
|
||||
call: FrontendCall,
|
||||
incremental: bool,
|
||||
response_timeout: Duration,
|
||||
) -> ResponseStream<proto::GenerateResponse> {
|
||||
generation_stream(
|
||||
call,
|
||||
incremental,
|
||||
response_timeout,
|
||||
|output, meta_info, finished| {
|
||||
Ok(proto::GenerateResponse {
|
||||
output_ids: output
|
||||
.token_ids
|
||||
.iter()
|
||||
.copied()
|
||||
.map(i32::try_from)
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.map_err(|_| {
|
||||
Status::internal("output token ID exceeds runtime.v1 int32 range")
|
||||
})?,
|
||||
meta_info,
|
||||
finished,
|
||||
})
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn generation_stream<T, F>(
|
||||
mut call: FrontendCall,
|
||||
incremental: bool,
|
||||
response_timeout: Duration,
|
||||
build: F,
|
||||
) -> ResponseStream<T>
|
||||
where
|
||||
T: Send + 'static,
|
||||
F: Fn(&FrontendOutput, HashMap<String, String>, bool) -> Result<T, Status> + Send + 'static,
|
||||
{
|
||||
let public_id = call.public_id().to_owned();
|
||||
let stream = async_stream::stream! {
|
||||
let mut cumulative = FrontendOutput::default();
|
||||
let mut cumulative_completion_tokens = 0;
|
||||
loop {
|
||||
let event = match tokio::time::timeout(response_timeout, call.recv()).await {
|
||||
Ok(Some(event)) => event,
|
||||
// The loop stops on terminal events, so this is defensive only.
|
||||
Ok(None) => break,
|
||||
Err(_) => {
|
||||
yield Err(Status::deadline_exceeded("Stream chunk timed out"));
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
let (mut delta, finished) = match event {
|
||||
FrontendEvent::Delta(output) => (output, false),
|
||||
FrontendEvent::Finished(output) => (output, true),
|
||||
FrontendEvent::Failed(error) => {
|
||||
yield Err(status(error));
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
let output = if incremental {
|
||||
// Python's incremental native stream still reports the cumulative
|
||||
// completion-token count in every frame. Do not retain the full
|
||||
// generated output just to calculate that one cumulative field.
|
||||
cumulative_completion_tokens += delta.completion_tokens;
|
||||
delta.completion_tokens = cumulative_completion_tokens;
|
||||
&delta
|
||||
} else {
|
||||
cumulative.append_delta(&delta);
|
||||
&cumulative
|
||||
};
|
||||
let meta_info = grpc_meta_info(output, &public_id);
|
||||
match build(output, meta_info, finished) {
|
||||
Ok(response) => yield Ok(response),
|
||||
Err(error) => {
|
||||
yield Err(error);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if finished {
|
||||
break;
|
||||
}
|
||||
}
|
||||
// `call` remains owned by this stream. Dropping a non-terminal stream or
|
||||
// timing out drops the armed FrontendCall and uses the core abort lane;
|
||||
// observing a terminal event has already disarmed it.
|
||||
};
|
||||
Box::pin(stream)
|
||||
}
|
||||
|
||||
/// runtime.v1 represents heterogeneous metadata as strings. Match the existing
|
||||
/// Python-backed server by JSON-encoding each value independently—including
|
||||
/// string values such as the request ID.
|
||||
fn grpc_meta_info(output: &FrontendOutput, public_id: &str) -> HashMap<String, String> {
|
||||
let serde_json::Value::Object(meta_info) = meta_info_value(output, public_id) else {
|
||||
unreachable!("native meta_info is always a JSON object");
|
||||
};
|
||||
meta_info
|
||||
.into_iter()
|
||||
.map(|(key, value)| (key, value.to_string()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(super) fn status(error: FrontendError) -> Status {
|
||||
let code = match error.kind() {
|
||||
FrontendErrorKind::InvalidArgument => Code::InvalidArgument,
|
||||
FrontendErrorKind::NotFound => Code::NotFound,
|
||||
FrontendErrorKind::FailedPrecondition => Code::FailedPrecondition,
|
||||
FrontendErrorKind::ResourceExhausted => Code::ResourceExhausted,
|
||||
FrontendErrorKind::Cancelled => Code::Cancelled,
|
||||
FrontendErrorKind::DeadlineExceeded => Code::DeadlineExceeded,
|
||||
FrontendErrorKind::Unavailable => Code::Unavailable,
|
||||
FrontendErrorKind::Internal => Code::Internal,
|
||||
};
|
||||
Status::new(code, error.to_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn semantic_errors_map_to_canonical_grpc_codes() {
|
||||
let cases = [
|
||||
(
|
||||
FrontendError::InvalidArgument("bad".into()),
|
||||
Code::InvalidArgument,
|
||||
),
|
||||
(
|
||||
FrontendError::RuntimeRejected {
|
||||
kind: FrontendErrorKind::NotFound,
|
||||
message: "missing".into(),
|
||||
legacy_http_status: 404,
|
||||
},
|
||||
Code::NotFound,
|
||||
),
|
||||
(
|
||||
FrontendError::RuntimeRejected {
|
||||
kind: FrontendErrorKind::FailedPrecondition,
|
||||
message: "not ready".into(),
|
||||
legacy_http_status: 412,
|
||||
},
|
||||
Code::FailedPrecondition,
|
||||
),
|
||||
(
|
||||
FrontendError::Overloaded("full".into()),
|
||||
Code::ResourceExhausted,
|
||||
),
|
||||
(FrontendError::Cancelled("gone".into()), Code::Cancelled),
|
||||
(
|
||||
FrontendError::RuntimeRejected {
|
||||
kind: FrontendErrorKind::DeadlineExceeded,
|
||||
message: "late".into(),
|
||||
legacy_http_status: 504,
|
||||
},
|
||||
Code::DeadlineExceeded,
|
||||
),
|
||||
(FrontendError::Unavailable, Code::Unavailable),
|
||||
(FrontendError::Internal("bug".into()), Code::Internal),
|
||||
];
|
||||
|
||||
for (error, expected) in cases {
|
||||
assert_eq!(status(error).code(), expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,361 +0,0 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use futures::StreamExt;
|
||||
use sglang_grpc_types::sglang::runtime::v1 as proto;
|
||||
use sglang_grpc_types::sglang::runtime::v1::sglang_service_server::SglangService;
|
||||
use tonic::{Code, Request};
|
||||
|
||||
use super::GrpcService;
|
||||
use crate::frontend::{FrontendConfig, FrontendHandle, FrontendMetadata};
|
||||
use crate::message::finish_reason::FinishReason;
|
||||
use crate::message::ids::Rid;
|
||||
use crate::message::request::{GenerateRequest, Request as RuntimeRequest, RequestKind};
|
||||
use crate::message::response::{ChunkEvent, ResponseItem, ResponseSink};
|
||||
use crate::tokenizer_manager::wiring::{AbortSource, RequestAdmission, TmEvent};
|
||||
|
||||
struct Harness {
|
||||
service: GrpcService,
|
||||
intake_rx: flume::Receiver<TmEvent>,
|
||||
abort_rx: flume::Receiver<AbortSource>,
|
||||
}
|
||||
|
||||
struct GenerationIntake {
|
||||
request: Box<GenerateRequest>,
|
||||
rid: Rid,
|
||||
sink: ResponseSink,
|
||||
admission: RequestAdmission,
|
||||
}
|
||||
|
||||
impl Harness {
|
||||
fn new(response_capacity: usize, incremental: bool, response_timeout: Duration) -> Self {
|
||||
let (intake_tx, intake_rx) = flume::unbounded();
|
||||
let (abort_tx, abort_rx) = flume::unbounded();
|
||||
let frontend = FrontendHandle::new(
|
||||
intake_tx,
|
||||
abort_tx,
|
||||
FrontendConfig {
|
||||
response_capacity,
|
||||
response_activity: Default::default(),
|
||||
startup_ready: true,
|
||||
is_disaggregation: false,
|
||||
mm_limits: Default::default(),
|
||||
metadata: FrontendMetadata::default(),
|
||||
},
|
||||
);
|
||||
Self {
|
||||
service: GrpcService::for_test(frontend, None, incremental, response_timeout),
|
||||
intake_rx,
|
||||
abort_rx,
|
||||
}
|
||||
}
|
||||
|
||||
async fn next_generation(&self) -> GenerationIntake {
|
||||
let TmEvent::Intake { request, admission } = self
|
||||
.intake_rx
|
||||
.recv_async()
|
||||
.await
|
||||
.expect("request submitted")
|
||||
else {
|
||||
panic!("expected generation intake");
|
||||
};
|
||||
let RuntimeRequest {
|
||||
rid,
|
||||
sink,
|
||||
kind,
|
||||
state: _,
|
||||
} = request;
|
||||
let RequestKind::Generate(request) = kind else {
|
||||
panic!("expected generation request");
|
||||
};
|
||||
GenerationIntake {
|
||||
request,
|
||||
rid,
|
||||
sink,
|
||||
admission,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn chunk(rid: &Rid, text: &str, token_id: i64, finished: bool) -> ChunkEvent {
|
||||
ChunkEvent {
|
||||
rid: rid.clone(),
|
||||
token_ids: vec![token_id],
|
||||
finish_reason: finished.then(stop_reason),
|
||||
prompt_tokens: 3,
|
||||
text: text.into(),
|
||||
completion_tokens: 1,
|
||||
extras: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn stop_reason() -> FinishReason {
|
||||
serde_json::from_value(serde_json::json!({"type": "stop"})).unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn text_generate_maps_request_and_streams_cumulative_responses() {
|
||||
let harness = Harness::new(2, false, Duration::from_secs(1));
|
||||
let response = harness
|
||||
.service
|
||||
.text_generate(Request::new(proto::TextGenerateRequest {
|
||||
text: "prompt".into(),
|
||||
stream: Some(true),
|
||||
rid: Some("request-1".into()),
|
||||
..Default::default()
|
||||
}))
|
||||
.await
|
||||
.unwrap();
|
||||
let mut stream = response.into_inner();
|
||||
|
||||
let intake = harness.next_generation().await;
|
||||
assert!(intake.admission.try_accept());
|
||||
assert_eq!(intake.request.text.as_deref(), Some("prompt"));
|
||||
assert!(intake.request.input_ids.is_none());
|
||||
assert!(intake.request.stream);
|
||||
assert_eq!(intake.rid.client_facing(), "request-1");
|
||||
intake
|
||||
.sink
|
||||
.try_send(ResponseItem::Frame(chunk(&intake.rid, "Hel", 1, false)))
|
||||
.unwrap();
|
||||
intake
|
||||
.sink
|
||||
.try_send(ResponseItem::Done(chunk(&intake.rid, "lo", 2, true)))
|
||||
.unwrap();
|
||||
|
||||
let first = stream.next().await.unwrap().unwrap();
|
||||
assert_eq!(first.text, "Hel");
|
||||
assert!(!first.finished);
|
||||
assert_eq!(first.meta_info["id"], r#""request-1""#);
|
||||
assert_eq!(first.meta_info["prompt_tokens"], "3");
|
||||
assert_eq!(first.meta_info["completion_tokens"], "1");
|
||||
assert_eq!(first.meta_info["finish_reason"], "null");
|
||||
|
||||
let finished = stream.next().await.unwrap().unwrap();
|
||||
assert_eq!(finished.text, "Hello");
|
||||
assert!(finished.finished);
|
||||
assert_eq!(finished.meta_info["completion_tokens"], "2");
|
||||
assert!(finished.meta_info["finish_reason"].contains(r#""type":"stop""#));
|
||||
assert!(stream.next().await.is_none());
|
||||
assert!(harness.abort_rx.try_recv().is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn generate_streams_incremental_token_ids_with_cumulative_count() {
|
||||
let harness = Harness::new(2, true, Duration::from_secs(1));
|
||||
let response = harness
|
||||
.service
|
||||
.generate(Request::new(proto::GenerateRequest {
|
||||
input_ids: vec![4, 5],
|
||||
stream: Some(true),
|
||||
rid: Some("tokens".into()),
|
||||
..Default::default()
|
||||
}))
|
||||
.await
|
||||
.unwrap();
|
||||
let mut stream = response.into_inner();
|
||||
|
||||
let intake = harness.next_generation().await;
|
||||
assert!(intake.admission.try_accept());
|
||||
assert_eq!(intake.request.input_ids.as_deref(), Some(&[4, 5][..]));
|
||||
assert!(intake.request.text.is_none());
|
||||
intake
|
||||
.sink
|
||||
.try_send(ResponseItem::Frame(chunk(&intake.rid, "A", 10, false)))
|
||||
.unwrap();
|
||||
intake
|
||||
.sink
|
||||
.try_send(ResponseItem::Done(chunk(&intake.rid, "B", 11, true)))
|
||||
.unwrap();
|
||||
|
||||
let first = stream.next().await.unwrap().unwrap();
|
||||
let finished = stream.next().await.unwrap().unwrap();
|
||||
assert_eq!(first.output_ids, vec![10]);
|
||||
assert_eq!(finished.output_ids, vec![11]);
|
||||
assert_eq!(finished.meta_info["completion_tokens"], "2");
|
||||
assert!(finished.finished);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stream_drop_aborts_admitted_request() {
|
||||
let harness = Harness::new(1, false, Duration::from_secs(1));
|
||||
let mut stream = harness
|
||||
.service
|
||||
.generate(Request::new(proto::GenerateRequest {
|
||||
input_ids: vec![1],
|
||||
rid: Some("cancel-me".into()),
|
||||
..Default::default()
|
||||
}))
|
||||
.await
|
||||
.unwrap()
|
||||
.into_inner();
|
||||
let intake = harness.next_generation().await;
|
||||
assert!(intake.admission.try_accept());
|
||||
intake
|
||||
.sink
|
||||
.try_send(ResponseItem::Frame(chunk(&intake.rid, "partial", 1, false)))
|
||||
.unwrap();
|
||||
assert!(stream.next().await.unwrap().is_ok());
|
||||
drop(stream);
|
||||
let abort = harness.abort_rx.recv_async().await.unwrap();
|
||||
assert_eq!(abort.rid().client_facing(), "cancel-me");
|
||||
assert!(harness.abort_rx.try_recv().is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn runtime_failure_is_an_in_stream_status_and_disarms_abort() {
|
||||
let harness = Harness::new(1, false, Duration::from_secs(1));
|
||||
let mut stream = harness
|
||||
.service
|
||||
.generate(Request::new(proto::GenerateRequest {
|
||||
input_ids: vec![1],
|
||||
..Default::default()
|
||||
}))
|
||||
.await
|
||||
.unwrap()
|
||||
.into_inner();
|
||||
let intake = harness.next_generation().await;
|
||||
assert!(intake.admission.try_accept());
|
||||
intake
|
||||
.sink
|
||||
.try_send(ResponseItem::Error(crate::utils::error::Error::Validation(
|
||||
"bad sampling".into(),
|
||||
)))
|
||||
.unwrap();
|
||||
|
||||
let error = stream.next().await.unwrap().unwrap_err();
|
||||
assert_eq!(error.code(), Code::InvalidArgument);
|
||||
drop(stream);
|
||||
assert!(harness.abort_rx.try_recv().is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn per_chunk_timeout_after_progress_returns_deadline_exceeded_and_aborts() {
|
||||
let harness = Harness::new(1, false, Duration::from_millis(10));
|
||||
let mut stream = harness
|
||||
.service
|
||||
.generate(Request::new(proto::GenerateRequest {
|
||||
input_ids: vec![1],
|
||||
rid: Some("too-slow".into()),
|
||||
..Default::default()
|
||||
}))
|
||||
.await
|
||||
.unwrap()
|
||||
.into_inner();
|
||||
let intake = harness.next_generation().await;
|
||||
assert!(intake.admission.try_accept());
|
||||
intake
|
||||
.sink
|
||||
.try_send(ResponseItem::Frame(chunk(&intake.rid, "partial", 1, false)))
|
||||
.unwrap();
|
||||
assert!(stream.next().await.unwrap().is_ok());
|
||||
|
||||
let error = stream.next().await.unwrap().unwrap_err();
|
||||
assert_eq!(error.code(), Code::DeadlineExceeded);
|
||||
drop(stream);
|
||||
let abort = harness.abort_rx.recv_async().await.unwrap();
|
||||
assert_eq!(abort.rid().client_facing(), "too-slow");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn closed_intake_is_a_top_level_unavailable_status() {
|
||||
let (intake_tx, intake_rx) = flume::unbounded();
|
||||
drop(intake_rx);
|
||||
let (abort_tx, _abort_rx) = flume::unbounded();
|
||||
let frontend = FrontendHandle::new(
|
||||
intake_tx,
|
||||
abort_tx,
|
||||
FrontendConfig {
|
||||
response_capacity: 1,
|
||||
response_activity: Default::default(),
|
||||
startup_ready: true,
|
||||
is_disaggregation: false,
|
||||
mm_limits: Default::default(),
|
||||
metadata: FrontendMetadata::default(),
|
||||
},
|
||||
);
|
||||
let service = GrpcService::for_test(frontend, None, false, Duration::from_secs(1));
|
||||
|
||||
let result = service
|
||||
.generate(Request::new(proto::GenerateRequest {
|
||||
input_ids: vec![1],
|
||||
..Default::default()
|
||||
}))
|
||||
.await;
|
||||
let error = match result {
|
||||
Ok(_) => panic!("closed intake must reject the RPC"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert_eq!(error.code(), Code::Unavailable);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn new_proto_fields_and_engine_watch_are_explicitly_unsupported() {
|
||||
let harness = Harness::new(1, false, Duration::from_secs(1));
|
||||
let text_error = harness
|
||||
.service
|
||||
.text_generate(Request::new(proto::TextGenerateRequest {
|
||||
text: "hello".into(),
|
||||
kv_hints: Some(Default::default()),
|
||||
..Default::default()
|
||||
}))
|
||||
.await
|
||||
.err()
|
||||
.expect("KV hints must not be silently ignored");
|
||||
assert_eq!(text_error.code(), Code::Unimplemented);
|
||||
let token_error = harness
|
||||
.service
|
||||
.generate(Request::new(proto::GenerateRequest {
|
||||
input_ids: vec![1],
|
||||
kv_hints: Some(Default::default()),
|
||||
..Default::default()
|
||||
}))
|
||||
.await
|
||||
.err()
|
||||
.expect("KV hints must not be silently ignored");
|
||||
assert_eq!(token_error.code(), Code::Unimplemented);
|
||||
let watch_error = harness
|
||||
.service
|
||||
.watch_engine_state(Request::new(proto::WatchEngineStateRequest {}))
|
||||
.await
|
||||
.err()
|
||||
.expect("engine watch is outside the generation adapter");
|
||||
assert_eq!(watch_error.code(), Code::Unimplemented);
|
||||
assert!(harness.intake_rx.try_recv().is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn output_token_overflow_returns_error_instead_of_truncating() {
|
||||
let harness = Harness::new(1, true, Duration::from_secs(1));
|
||||
let mut stream = harness
|
||||
.service
|
||||
.generate(Request::new(proto::GenerateRequest {
|
||||
input_ids: vec![i32::MAX],
|
||||
..Default::default()
|
||||
}))
|
||||
.await
|
||||
.unwrap()
|
||||
.into_inner();
|
||||
let intake = harness.next_generation().await;
|
||||
assert!(intake.admission.try_accept());
|
||||
assert_eq!(
|
||||
intake.request.input_ids.as_deref(),
|
||||
Some(&[i64::from(i32::MAX)][..])
|
||||
);
|
||||
intake
|
||||
.sink
|
||||
.try_send(ResponseItem::Frame(chunk(
|
||||
&intake.rid,
|
||||
"",
|
||||
i64::from(i32::MAX) + 1,
|
||||
false,
|
||||
)))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
stream.next().await.unwrap().unwrap_err().code(),
|
||||
Code::Internal
|
||||
);
|
||||
assert!(stream.next().await.is_none());
|
||||
assert!(
|
||||
matches!(harness.abort_rx.recv_async().await.unwrap(), AbortSource::Guard(rid) if rid == intake.rid)
|
||||
);
|
||||
}
|
||||
@@ -12,11 +12,8 @@
|
||||
//! a `PyObject`.
|
||||
|
||||
mod api_server;
|
||||
mod frontend;
|
||||
mod grpc;
|
||||
mod message;
|
||||
mod multi_modality;
|
||||
mod native_generation;
|
||||
mod tokenizer_manager;
|
||||
mod utils;
|
||||
|
||||
@@ -25,7 +22,7 @@ pub use message::config::{
|
||||
RustServerServerArgs, ServerArgs,
|
||||
};
|
||||
pub use message::multimodal::MmItem;
|
||||
pub use message::request::{MmData, ProcessorExtensions};
|
||||
pub use message::request::MmData;
|
||||
pub use message::types::TokenIds;
|
||||
pub use multi_modality::encoded::{
|
||||
MRope, MmEncodedEntry, MmEncodedItem, MmMetaValue, MmModality, MmTokenIds,
|
||||
|
||||
@@ -7,9 +7,11 @@ pub mod config;
|
||||
pub mod detok;
|
||||
pub mod finish_reason;
|
||||
pub mod ids;
|
||||
pub mod info;
|
||||
pub mod io_struct;
|
||||
pub mod multimodal;
|
||||
pub mod request;
|
||||
pub mod response;
|
||||
pub mod sampling;
|
||||
pub mod types;
|
||||
pub mod wire;
|
||||
|
||||
@@ -572,7 +572,11 @@ impl ServerArgs {
|
||||
}
|
||||
}
|
||||
if let Some(preferred) = &self.preferred_sampling_params {
|
||||
super::sampling::SamplingParamsInput::from_preferred(&preferred.0)
|
||||
// The wire schema is the contract: a preferred value the decoder
|
||||
// rejects would 400 every request, so reject it at boot instead.
|
||||
serde_json::from_value::<sglang_api_types::api::v1::SamplingParams>(
|
||||
preferred.0.clone(),
|
||||
)
|
||||
.map_err(|e| format!("invalid preferred_sampling_params: {e}"))?;
|
||||
}
|
||||
Ok(())
|
||||
|
||||
@@ -98,7 +98,7 @@ impl FinishReason {
|
||||
}
|
||||
|
||||
/// `Some((status, message))` when this is an abort carrying the scheduler's
|
||||
/// legacy HTTP status — a request failure the frontend boundary converts
|
||||
/// legacy HTTP status — a request failure the core boundary converts
|
||||
/// into a semantic error event. A plain abort (no code) reads as `None`.
|
||||
pub fn abort_status(&self) -> Option<(u16, &str)> {
|
||||
match self {
|
||||
@@ -128,7 +128,7 @@ mod tests {
|
||||
Some((code, message.to_string()))
|
||||
}
|
||||
|
||||
/// The classifier the frontend boundary uses: a validation abort yields its
|
||||
/// The classifier the core boundary uses: a validation abort yields its
|
||||
/// `(code, message)` so it can become a semantic failure event; anything
|
||||
/// else yields `None`.
|
||||
#[test]
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
//! Public model and server metadata: the typed results of `/get_model_info`
|
||||
//! and `/server_info` on every transport.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::config::{DisaggregationMode, PreferredSamplingParams};
|
||||
|
||||
/// Static model metadata shared by every public transport.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub(crate) struct ModelInfo {
|
||||
pub(crate) model_path: String,
|
||||
pub(crate) served_model_name: String,
|
||||
pub(crate) tokenizer_path: String,
|
||||
pub(crate) is_generation: bool,
|
||||
pub(crate) preferred_sampling_params: Option<PreferredSamplingParams>,
|
||||
pub(crate) weight_version: Option<String>,
|
||||
pub(crate) load_format: Option<String>,
|
||||
pub(crate) reasoning_parser: Option<String>,
|
||||
pub(crate) tool_call_parser: Option<String>,
|
||||
pub(crate) disaggregation_mode: DisaggregationMode,
|
||||
}
|
||||
|
||||
/// Public server metadata plus scheduler-owned runtime metrics.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub(crate) struct ServerInfo {
|
||||
pub(crate) model_path: String,
|
||||
pub(crate) served_model_name: String,
|
||||
pub(crate) tokenizer_path: String,
|
||||
pub(crate) max_context_length: u64,
|
||||
pub(crate) max_total_num_tokens: u64,
|
||||
pub(crate) version: String,
|
||||
pub(crate) frontend: &'static str,
|
||||
pub(crate) internal_states: Vec<InternalState>,
|
||||
}
|
||||
|
||||
/// Scheduler-owned runtime metrics used by the public server-info operation.
|
||||
///
|
||||
/// Deserializing into this allowlisted shape is intentional: the scheduler's
|
||||
/// raw state also contains its full launch arguments, including credentials.
|
||||
/// Unknown fields are discarded before any transport receives the result.
|
||||
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
|
||||
#[serde(default)]
|
||||
pub(crate) struct InternalState {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) last_gen_throughput: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) memory_usage: Option<MemoryUsage>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) effective_max_running_requests_per_dp: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) avg_spec_accept_length: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) step_time_dict: Option<BTreeMap<u64, Vec<f64>>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) rust_mm_transport: Option<BTreeMap<String, u64>>,
|
||||
}
|
||||
|
||||
/// Public subset of scheduler memory metrics.
|
||||
///
|
||||
/// This nested allowlist is intentional: adding a scheduler field does not
|
||||
/// silently add different public data to HTTP before the gRPC contract can add
|
||||
/// the same field. New metrics should be promoted here deliberately.
|
||||
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
|
||||
#[serde(default)]
|
||||
pub(crate) struct MemoryUsage {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) weight: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) kvcache: Option<MemoryMeasurement>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) startup_available: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) token_capacity: Option<u64>,
|
||||
pub(crate) token_capacity_swa: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) graph: Option<BTreeMap<String, f64>>,
|
||||
}
|
||||
|
||||
/// The scheduler's KV-cache measurement can be a native float or a NumPy
|
||||
/// scalar stringified by the MessagePack bridge. Preserve that representation
|
||||
/// so extracting the typed contract does not change public responses.
|
||||
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub(crate) enum MemoryMeasurement {
|
||||
Number(f64),
|
||||
String(String),
|
||||
}
|
||||
@@ -1,12 +1,6 @@
|
||||
//! Typed multimodal inputs of the `/generate` body — the Rust form of Python
|
||||
//! `MultimodalDataInputFormat` (`io_struct.py`) — and their per-request fan-out.
|
||||
|
||||
use std::fmt;
|
||||
|
||||
use serde::de::value::{MapAccessDeserializer, SeqAccessDeserializer};
|
||||
use serde::de::{MapAccess, SeqAccess, Visitor};
|
||||
use serde::{Deserialize, Deserializer};
|
||||
|
||||
use super::request::{HeapBytes, check_broadcast_budget};
|
||||
use crate::utils::error::Error;
|
||||
|
||||
@@ -22,19 +16,13 @@ pub enum MmItem {
|
||||
/// are read by model families this pipeline does not run, and Python's
|
||||
/// `load_image` itself reduces the item to `.url`.
|
||||
Ref { url: String },
|
||||
/// A preprocessed item (`{"format": "processor_output" | "precomputed_embedding", …}`).
|
||||
/// Parsed only far enough to be rejected by name at the MM stage; Python
|
||||
/// ignores it the same way on a text-only model.
|
||||
Preprocessed { format: String },
|
||||
}
|
||||
|
||||
impl MmItem {
|
||||
/// The raw source string for the modality pipeline, `None` for a
|
||||
/// preprocessed item.
|
||||
/// The raw source string for the modality pipeline.
|
||||
pub fn source(&self) -> Option<&str> {
|
||||
match self {
|
||||
MmItem::Source(source) | MmItem::Ref { url: source } => Some(source),
|
||||
MmItem::Preprocessed { .. } => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -42,66 +30,9 @@ impl MmItem {
|
||||
impl HeapBytes for MmItem {
|
||||
fn heap_bytes(&self) -> usize {
|
||||
match self {
|
||||
MmItem::Source(s) | MmItem::Ref { url: s } | MmItem::Preprocessed { format: s } => {
|
||||
s.len()
|
||||
MmItem::Source(s) | MmItem::Ref { url: s } => s.len(),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The object form of an item, as Python's `Dict[str, Any]`: `format` marks a
|
||||
/// preprocessed item (checked first, as `glm4v` does), `url` an `ImageData`.
|
||||
#[derive(Deserialize)]
|
||||
struct ItemObject {
|
||||
#[serde(default)]
|
||||
url: Option<String>,
|
||||
#[serde(default)]
|
||||
format: Option<String>,
|
||||
}
|
||||
|
||||
impl TryFrom<ItemObject> for MmItem {
|
||||
type Error = &'static str;
|
||||
|
||||
fn try_from(object: ItemObject) -> Result<Self, Self::Error> {
|
||||
match (object.format, object.url) {
|
||||
(Some(format), _) => Ok(MmItem::Preprocessed { format }),
|
||||
(None, Some(url)) => Ok(MmItem::Ref { url }),
|
||||
(None, None) => Err("a multimodal item object needs a `url` or a `format` key"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Hand-written rather than `#[serde(untagged)]` so a bad item is reported as
|
||||
/// what it is ("expected a source string or an item object"), not as "did not
|
||||
/// match any variant".
|
||||
impl<'de> Deserialize<'de> for MmItem {
|
||||
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
struct ItemVisitor;
|
||||
|
||||
impl<'de> Visitor<'de> for ItemVisitor {
|
||||
type Value = MmItem;
|
||||
|
||||
fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str("a media source string or a multimodal item object")
|
||||
}
|
||||
|
||||
fn visit_str<E: serde::de::Error>(self, value: &str) -> Result<Self::Value, E> {
|
||||
Ok(MmItem::Source(value.to_owned()))
|
||||
}
|
||||
|
||||
fn visit_string<E: serde::de::Error>(self, value: String) -> Result<Self::Value, E> {
|
||||
Ok(MmItem::Source(value))
|
||||
}
|
||||
|
||||
fn visit_map<A: MapAccess<'de>>(self, map: A) -> Result<Self::Value, A::Error> {
|
||||
ItemObject::deserialize(MapAccessDeserializer::new(map))?
|
||||
.try_into()
|
||||
.map_err(serde::de::Error::custom)
|
||||
}
|
||||
}
|
||||
|
||||
deserializer.deserialize_any(ItemVisitor)
|
||||
}
|
||||
}
|
||||
|
||||
/// One `image_data` / `video_data` / `audio_data` field as sent: Python
|
||||
@@ -117,119 +48,6 @@ pub enum MmDataInput {
|
||||
Nested(Vec<Option<Vec<Option<MmItem>>>>),
|
||||
}
|
||||
|
||||
/// One element of a list-form field, before the list is known to be flat or nested.
|
||||
enum ListElement {
|
||||
Null,
|
||||
Item(MmItem),
|
||||
List(Vec<Option<MmItem>>),
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for ListElement {
|
||||
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
struct ElementVisitor;
|
||||
|
||||
impl<'de> Visitor<'de> for ElementVisitor {
|
||||
type Value = ListElement;
|
||||
|
||||
fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str("null, a media source string, an item object, or a list of items")
|
||||
}
|
||||
|
||||
fn visit_unit<E: serde::de::Error>(self) -> Result<Self::Value, E> {
|
||||
Ok(ListElement::Null)
|
||||
}
|
||||
|
||||
fn visit_none<E: serde::de::Error>(self) -> Result<Self::Value, E> {
|
||||
Ok(ListElement::Null)
|
||||
}
|
||||
|
||||
fn visit_str<E: serde::de::Error>(self, value: &str) -> Result<Self::Value, E> {
|
||||
Ok(ListElement::Item(MmItem::Source(value.to_owned())))
|
||||
}
|
||||
|
||||
fn visit_string<E: serde::de::Error>(self, value: String) -> Result<Self::Value, E> {
|
||||
Ok(ListElement::Item(MmItem::Source(value)))
|
||||
}
|
||||
|
||||
fn visit_map<A: MapAccess<'de>>(self, map: A) -> Result<Self::Value, A::Error> {
|
||||
ItemObject::deserialize(MapAccessDeserializer::new(map))?
|
||||
.try_into()
|
||||
.map(ListElement::Item)
|
||||
.map_err(serde::de::Error::custom)
|
||||
}
|
||||
|
||||
fn visit_seq<A: SeqAccess<'de>>(self, seq: A) -> Result<Self::Value, A::Error> {
|
||||
Vec::<Option<MmItem>>::deserialize(SeqAccessDeserializer::new(seq))
|
||||
.map(ListElement::List)
|
||||
}
|
||||
}
|
||||
|
||||
deserializer.deserialize_any(ElementVisitor)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for MmDataInput {
|
||||
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
struct InputVisitor;
|
||||
|
||||
impl<'de> Visitor<'de> for InputVisitor {
|
||||
type Value = MmDataInput;
|
||||
|
||||
fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str("a media item, a list of items, or a list of item lists")
|
||||
}
|
||||
|
||||
fn visit_str<E: serde::de::Error>(self, value: &str) -> Result<Self::Value, E> {
|
||||
Ok(MmDataInput::One(MmItem::Source(value.to_owned())))
|
||||
}
|
||||
|
||||
fn visit_string<E: serde::de::Error>(self, value: String) -> Result<Self::Value, E> {
|
||||
Ok(MmDataInput::One(MmItem::Source(value)))
|
||||
}
|
||||
|
||||
fn visit_map<A: MapAccess<'de>>(self, map: A) -> Result<Self::Value, A::Error> {
|
||||
ItemObject::deserialize(MapAccessDeserializer::new(map))?
|
||||
.try_into()
|
||||
.map(MmDataInput::One)
|
||||
.map_err(serde::de::Error::custom)
|
||||
}
|
||||
|
||||
fn visit_seq<A: SeqAccess<'de>>(self, seq: A) -> Result<Self::Value, A::Error> {
|
||||
let elements = Vec::<ListElement>::deserialize(SeqAccessDeserializer::new(seq))?;
|
||||
let nested = elements
|
||||
.iter()
|
||||
.any(|element| matches!(element, ListElement::List(_)));
|
||||
if !nested {
|
||||
return Ok(MmDataInput::Many(
|
||||
elements
|
||||
.into_iter()
|
||||
.map(|element| match element {
|
||||
ListElement::Item(item) => Some(item),
|
||||
ListElement::Null => None,
|
||||
ListElement::List(_) => unreachable!("checked above"),
|
||||
})
|
||||
.collect(),
|
||||
));
|
||||
}
|
||||
elements
|
||||
.into_iter()
|
||||
.map(|element| match element {
|
||||
ListElement::List(items) => Ok(Some(items)),
|
||||
ListElement::Null => Ok(None),
|
||||
ListElement::Item(_) => Err(serde::de::Error::custom(
|
||||
"a nested list cannot mix bare items with item lists",
|
||||
)),
|
||||
})
|
||||
.collect::<Result<_, _>>()
|
||||
.map(MmDataInput::Nested)
|
||||
}
|
||||
}
|
||||
|
||||
deserializer.deserialize_any(InputVisitor)
|
||||
}
|
||||
}
|
||||
|
||||
/// The items of one modality for one request, `null` entries dropped.
|
||||
fn present(items: Vec<Option<MmItem>>) -> Vec<MmItem> {
|
||||
items.into_iter().flatten().collect()
|
||||
}
|
||||
@@ -300,8 +118,12 @@ fn check_len(len: usize, n: usize, name: &str) -> Result<(), Error> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn parse(json: &str) -> Result<MmDataInput, serde_json::Error> {
|
||||
serde_json::from_str(json)
|
||||
/// Parsed as the wire type (the schema's `MediaInput`), then converted;
|
||||
/// either step's rejection is an `Err`.
|
||||
fn parse(json: &str) -> Result<MmDataInput, String> {
|
||||
let wire: sglang_api_types::api::v1::MediaInput =
|
||||
serde_json::from_str(json).map_err(|e| e.to_string())?;
|
||||
crate::message::wire::media_input(wire).map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
fn src(s: &str) -> MmItem {
|
||||
@@ -324,39 +146,28 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// Object items: `format` wins over `url` (a preprocessed item may carry
|
||||
/// both), and an object with neither is named in the error.
|
||||
/// Object items are typed `MediaRef`s: only `url` reaches this pipeline, a
|
||||
/// hint the schema does not name is an error, and a preprocessed-input
|
||||
/// dict is not a wire shape at all (its values are tensors; Engine-only).
|
||||
#[test]
|
||||
fn parses_item_objects() {
|
||||
assert_eq!(
|
||||
parse(r#"{"url": "u", "detail": "high"}"#).unwrap(),
|
||||
MmDataInput::One(MmItem::Ref { url: "u".into() })
|
||||
);
|
||||
assert_eq!(
|
||||
parse(r#"[{"format": "processor_output", "url": "u", "pixel_values": [1]}]"#).unwrap(),
|
||||
MmDataInput::Many(vec![Some(MmItem::Preprocessed {
|
||||
format: "processor_output".into()
|
||||
})])
|
||||
);
|
||||
let err = parse(r#"{"detail": "high"}"#).unwrap_err().to_string();
|
||||
assert!(err.contains("`url` or a `format`"), "{err}");
|
||||
let err = parse(r#"{"url": "u", "detai": "high"}"#)
|
||||
.unwrap_err()
|
||||
.to_string(); // codespell:ignore detai
|
||||
assert!(err.contains("unknown field"), "{err}");
|
||||
assert!(parse(r#"[{"format": "processor_output", "url": "u"}]"#).is_err());
|
||||
assert!(parse(r#"{"detail": "high"}"#).is_err(), "a ref needs a url");
|
||||
}
|
||||
|
||||
/// Anything Python's item union does not cover is rejected up front, with
|
||||
/// the expected shape in the message.
|
||||
/// Anything Python's item union does not cover is rejected up front.
|
||||
#[test]
|
||||
fn rejects_non_items() {
|
||||
for (json, expect) in [
|
||||
("5", "expected a media item, a list of items"),
|
||||
(r#"["a", 5]"#, "expected null, a media source string"),
|
||||
(r#"["a", ["b"]]"#, "cannot mix"),
|
||||
(
|
||||
r#"[[["a"]]]"#,
|
||||
"expected a media source string or a multimodal item object",
|
||||
),
|
||||
] {
|
||||
let err = parse(json).unwrap_err().to_string();
|
||||
assert!(err.contains(expect), "{json}: {err}");
|
||||
for json in ["5", r#"["a", 5]"#, r#"["a", ["b"]]"#, r#"[[["a"]]]"#] {
|
||||
assert!(parse(json).is_err(), "{json}");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,24 +1,23 @@
|
||||
//! The `/generate` request path: the HTTP body and its per-request fan-out
|
||||
//! ([`GenerateBody`] → [`GenerateRequest`]s).
|
||||
|
||||
use std::collections::{BTreeMap, HashSet};
|
||||
use std::fmt;
|
||||
use std::collections::HashSet;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use bytes::Bytes;
|
||||
use itertools::izip;
|
||||
use serde::de::{DeserializeOwned, SeqAccess, Visitor, value::SeqAccessDeserializer};
|
||||
use serde::{Deserialize, Deserializer};
|
||||
|
||||
use super::buffers::Buffer;
|
||||
use super::io_struct::{ControlRequest, TokenizedGenerateReqInput};
|
||||
use super::multimodal::{self, MmDataInput, MmItem};
|
||||
use super::multimodal::{self, MmItem};
|
||||
use super::response::ResponseSink;
|
||||
use super::sampling::{SamplingParams, SamplingParamsInput};
|
||||
use super::sampling::SamplingParams;
|
||||
use super::types::{OneOrMany, OneOrManyItem, TokenIds};
|
||||
use super::wire;
|
||||
use crate::message::ids::Rid;
|
||||
use crate::utils::fsm::RequestState;
|
||||
use crate::utils::{environ::env_i64, error::Error};
|
||||
use sglang_api_types::api::v1 as api_v1;
|
||||
|
||||
/// Hard cap on how many scheduler requests one `/generate` HTTP call may expand
|
||||
/// into. Every column below is allocated per item before anything is dispatched,
|
||||
@@ -47,222 +46,17 @@ const MAX_BROADCAST_CLONE_BYTES: usize = 64 << 20;
|
||||
/// the wire form does not); 8 is the ceiling of that range, not a worst case.
|
||||
const JSON_TO_HEAP_FACTOR: usize = 8;
|
||||
|
||||
/// Top-level fields in this namespace belong to the selected multimodal
|
||||
/// processor. Everything else unknown to [`GenerateBody`] keeps Python's
|
||||
/// accepted-but-ignored behavior.
|
||||
const PROCESSOR_EXTENSION_PREFIX: &str = "multimodal_";
|
||||
|
||||
/// Model-owned request fields. The shared server preserves and batches their
|
||||
/// MessagePack value representation; the selected processor deserializes that
|
||||
/// map into its own concrete schema.
|
||||
#[derive(Debug, Clone, Default, Deserialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct ProcessorExtensions(BTreeMap<String, rmpv::Value>);
|
||||
|
||||
impl ProcessorExtensions {
|
||||
/// Deserialize the model-agnostic value tree directly into the selected
|
||||
/// processor's schema. This does not encode or decode MessagePack bytes.
|
||||
pub fn deserialize<T: DeserializeOwned>(self) -> Result<T, String> {
|
||||
let fields = self
|
||||
.0
|
||||
.into_iter()
|
||||
.map(|(name, value)| (rmpv::Value::from(name), value))
|
||||
.collect();
|
||||
rmpv::ext::from_value(rmpv::Value::Map(fields))
|
||||
.map_err(|error| format!("invalid processor extensions: {error}"))
|
||||
}
|
||||
|
||||
pub(crate) fn is_empty(&self) -> bool {
|
||||
self.0.is_empty()
|
||||
}
|
||||
|
||||
fn values(&self) -> impl Iterator<Item = &rmpv::Value> {
|
||||
self.0.values()
|
||||
}
|
||||
}
|
||||
|
||||
impl FromIterator<(String, rmpv::Value)> for ProcessorExtensions {
|
||||
fn from_iter<T: IntoIterator<Item = (String, rmpv::Value)>>(iter: T) -> Self {
|
||||
Self(iter.into_iter().collect())
|
||||
}
|
||||
}
|
||||
|
||||
/// The `/generate` wire body before batch splitting: `text`/`input_ids`/`sampling_params`
|
||||
/// each scalar-or-list, fanned into per-request [`GenerateRequest`]s by
|
||||
/// [`into_requests`](GenerateBody::into_requests).
|
||||
/// Validate, normalize and fan the `/generate` wire request into one
|
||||
/// [`GenerateRequest`] per prompt + `is_batch` (list form -- a 1-element list
|
||||
/// is still a batch -> JSON array response). The Rust counterpart of Python
|
||||
/// `GenerateReqInput.normalize_batch_and_arguments`; an invalid/inconsistent
|
||||
/// batch is [`Error::Validation`], which the HTTP adapter surfaces as 400.
|
||||
///
|
||||
/// Unknown keys are ignored, matching Python, except `multimodal_*` fields. Those
|
||||
/// are opaque processor extensions: this layer only fans them out with the
|
||||
/// request batch and passes them to the selected multimodal processor.
|
||||
#[derive(Debug, Clone, Default, Deserialize)]
|
||||
pub struct GenerateBody {
|
||||
/// Optional client-supplied request id(s): a single string (a batch fans it
|
||||
/// out as `{rid}_{i}`, mirroring Python `_normalize_batch`) or one per item.
|
||||
pub rid: Option<OneOrMany<String>>,
|
||||
pub text: Option<OneOrMany<String>>,
|
||||
#[serde(default, deserialize_with = "deserialize_input_ids")]
|
||||
pub input_ids: Option<OneOrMany<TokenIds>>,
|
||||
#[serde(default)]
|
||||
pub stream: bool,
|
||||
/// One params object (broadcast) or a list of them (per item); see
|
||||
/// [`SamplingParamsInput`].
|
||||
pub sampling_params: Option<SamplingParamsInput>,
|
||||
/// Logprob / hidden-state options: a scalar broadcasts to every prompt, a
|
||||
/// list is per-prompt (Python `_normalize_logprob_params`).
|
||||
pub return_logprob: Option<OneOrMany<bool>>,
|
||||
pub logprob_start_len: Option<OneOrMany<i64>>,
|
||||
pub top_logprobs_num: Option<OneOrMany<i64>>,
|
||||
/// Token ids to report logprobs for: one list (broadcast to every prompt) or
|
||||
/// one list per prompt, mirroring Python's
|
||||
/// `Union[List[int], List[List[int]]]` fan-out in `_normalize_batch`.
|
||||
pub token_ids_logprob: Option<OneOrMany<TokenIds>>,
|
||||
pub return_hidden_states: Option<OneOrMany<bool>>,
|
||||
/// Scalar-only in Python too (`return_text_in_logprobs: bool`).
|
||||
pub return_text_in_logprobs: Option<bool>,
|
||||
// PD-disaggregation routing, injected per request by the PD router
|
||||
// (mini_lb / sgl-model-gateway): a scalar for a single prompt, one-per-item
|
||||
// lists for a batch. Elements are nullable (`List[Optional[...]]` in
|
||||
// Python) — the router sends `bootstrap_port: [null, …]` when deferring to
|
||||
// the scheduler's `--disaggregation-bootstrap-port` default.
|
||||
pub bootstrap_host: Option<OneOrMany<Option<String>>>,
|
||||
pub bootstrap_port: Option<OneOrMany<Option<i64>>>,
|
||||
/// `bootstrap_room` fits in i64: the PD routers draw it from `[0, 2^63)`.
|
||||
pub bootstrap_room: Option<OneOrMany<Option<i64>>>,
|
||||
pub bootstrap_pair_key: Option<OneOrMany<Option<String>>>,
|
||||
pub decode_tp_size: Option<OneOrMany<Option<i64>>>,
|
||||
/// DP routing hints — per-request scalars even for batches, as in Python.
|
||||
pub routed_dp_rank: Option<i64>,
|
||||
pub disagg_prefill_dp_rank: Option<i64>,
|
||||
// Multimodal inputs (Python `MultimodalDataInputFormat`), fanned out per
|
||||
// request by `multimodal::fan_out`.
|
||||
pub image_data: Option<MmDataInput>,
|
||||
/// Caller-supplied per-item content hashes (hex) overriding the computed
|
||||
/// ones, so an external router's keys align with the prefix cache. Single
|
||||
/// requests only: Python declares the batched (nested) shape but
|
||||
/// `__getitem__` never forwards it, so a batch is rejected here rather than
|
||||
/// answered with hashes it did not ask for.
|
||||
pub mm_hashes: Option<OneOrMany<Vec<String>>>,
|
||||
pub video_data: Option<MmDataInput>,
|
||||
pub audio_data: Option<MmDataInput>,
|
||||
/// Model-specific multimodal fields, retained without teaching the shared
|
||||
/// request schema their contents. Other unknown fields remain ignored.
|
||||
#[serde(flatten)]
|
||||
processor_extensions: ProcessorExtensions,
|
||||
}
|
||||
|
||||
/// Decode `input_ids` directly into token vectors. The first element selects
|
||||
/// flat vs. batched input, avoiding the untagged enum's intermediate value tree.
|
||||
fn deserialize_input_ids<'de, D>(deserializer: D) -> Result<Option<OneOrMany<TokenIds>>, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
struct InputIds(OneOrMany<TokenIds>);
|
||||
|
||||
// Token width follows `TokenIds` (i64 in this crate); main's parser was
|
||||
// written against its i32 alias and is ported here.
|
||||
enum FirstElement {
|
||||
Token(i64),
|
||||
Tokens(TokenIds),
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for FirstElement {
|
||||
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
struct FirstVisitor;
|
||||
|
||||
impl<'de> Visitor<'de> for FirstVisitor {
|
||||
type Value = FirstElement;
|
||||
|
||||
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
|
||||
formatter.write_str("a token id or a list of token ids")
|
||||
}
|
||||
|
||||
fn visit_i64<E: serde::de::Error>(self, value: i64) -> Result<Self::Value, E> {
|
||||
Ok(FirstElement::Token(value))
|
||||
}
|
||||
|
||||
fn visit_u64<E: serde::de::Error>(self, value: u64) -> Result<Self::Value, E> {
|
||||
i64::try_from(value)
|
||||
.map(FirstElement::Token)
|
||||
.map_err(E::custom)
|
||||
}
|
||||
|
||||
fn visit_seq<A: SeqAccess<'de>>(
|
||||
self,
|
||||
sequence: A,
|
||||
) -> Result<Self::Value, A::Error> {
|
||||
TokenIds::deserialize(SeqAccessDeserializer::new(sequence))
|
||||
.map(FirstElement::Tokens)
|
||||
}
|
||||
}
|
||||
|
||||
deserializer.deserialize_any(FirstVisitor)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for InputIds {
|
||||
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
struct TokensVisitor;
|
||||
|
||||
impl<'de> Visitor<'de> for TokensVisitor {
|
||||
type Value = InputIds;
|
||||
|
||||
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
|
||||
formatter.write_str("a token list or a list of token lists")
|
||||
}
|
||||
|
||||
fn visit_seq<A: SeqAccess<'de>>(
|
||||
self,
|
||||
mut sequence: A,
|
||||
) -> Result<Self::Value, A::Error> {
|
||||
let tokens = match sequence.next_element::<FirstElement>()? {
|
||||
// Preserve OneOrMany's first-variant choice for [].
|
||||
None => OneOrMany::One(Vec::new()),
|
||||
Some(FirstElement::Token(first)) => {
|
||||
let mut tokens = vec![first];
|
||||
while let Some(token) = sequence.next_element::<i64>()? {
|
||||
tokens.push(token);
|
||||
}
|
||||
OneOrMany::One(tokens)
|
||||
}
|
||||
Some(FirstElement::Tokens(first)) => {
|
||||
let mut prompts = vec![first];
|
||||
while let Some(tokens) = sequence.next_element::<TokenIds>()? {
|
||||
prompts.push(tokens);
|
||||
}
|
||||
OneOrMany::Many(prompts)
|
||||
}
|
||||
};
|
||||
Ok(InputIds(tokens))
|
||||
}
|
||||
}
|
||||
|
||||
deserializer.deserialize_seq(TokensVisitor)
|
||||
}
|
||||
}
|
||||
|
||||
Option::<InputIds>::deserialize(deserializer).map(|value| value.map(|tokens| tokens.0))
|
||||
}
|
||||
|
||||
impl GenerateBody {
|
||||
/// Merge operator-provided sampling defaults beneath request values,
|
||||
/// matching Python TokenizerManager's preferred/request precedence.
|
||||
pub fn apply_preferred_sampling(&mut self, preferred: &serde_json::Value) -> Result<(), Error> {
|
||||
match &mut self.sampling_params {
|
||||
Some(params) => params.apply_preferred(preferred),
|
||||
None => SamplingParamsInput::from_preferred(preferred).map(|params| {
|
||||
self.sampling_params = Some(params);
|
||||
}),
|
||||
}
|
||||
.map_err(|e| Error::Validation(format!("invalid preferred_sampling_params: {e}")))
|
||||
}
|
||||
|
||||
/// Validate, normalize and fan the body into one [`GenerateRequest`] per
|
||||
/// prompt + `is_batch` (list form — a 1-element list is still a batch → JSON
|
||||
/// array response). The Rust counterpart of Python
|
||||
/// `GenerateReqInput.normalize_batch_and_arguments`; an invalid/inconsistent
|
||||
/// batch is [`Error::Validation`], which the HTTP adapter surfaces as 400.
|
||||
pub fn into_requests(self) -> Result<(Vec<GenerateRequest>, bool), Error> {
|
||||
let GenerateBody {
|
||||
/// The wire type is the generated `sglang_api_types` request (the schema is
|
||||
/// the contract); its carriers become the internal ones through `message::wire`
|
||||
/// on the way in.
|
||||
pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec<GenerateRequest>, bool), Error> {
|
||||
let api_v1::GenerateRequest {
|
||||
rid,
|
||||
text,
|
||||
input_ids,
|
||||
@@ -282,11 +76,33 @@ impl GenerateBody {
|
||||
routed_dp_rank,
|
||||
disagg_prefill_dp_rank,
|
||||
image_data,
|
||||
mm_hashes,
|
||||
video_data,
|
||||
audio_data,
|
||||
mm_hashes,
|
||||
processor_extensions,
|
||||
} = self;
|
||||
} = req;
|
||||
let rid = rid.and_then(wire::string_or_list);
|
||||
let text = text.and_then(wire::string_or_list);
|
||||
let input_ids = input_ids.and_then(wire::token_ids_or_list);
|
||||
let stream = stream.unwrap_or(false);
|
||||
let sampling_params = sampling_params
|
||||
.map(wire::sampling_params_or_list)
|
||||
.transpose()
|
||||
.map_err(Error::Validation)?
|
||||
.flatten();
|
||||
let return_logprob = return_logprob.and_then(wire::bool_or_list);
|
||||
let logprob_start_len = logprob_start_len.and_then(wire::int64_or_list);
|
||||
let top_logprobs_num = top_logprobs_num.and_then(wire::int64_or_list);
|
||||
let token_ids_logprob = token_ids_logprob.and_then(wire::token_ids_or_list);
|
||||
let return_hidden_states = return_hidden_states.and_then(wire::bool_or_list);
|
||||
let bootstrap_host = bootstrap_host.and_then(wire::optional_string_or_list);
|
||||
let bootstrap_port = bootstrap_port.and_then(wire::optional_int64_or_list);
|
||||
let bootstrap_room = bootstrap_room.and_then(wire::optional_int64_or_list);
|
||||
let bootstrap_pair_key = bootstrap_pair_key.and_then(wire::optional_string_or_list);
|
||||
let decode_tp_size = decode_tp_size.and_then(wire::optional_int64_or_list);
|
||||
let image_data = image_data.map(wire::media_input).transpose()?;
|
||||
let video_data = video_data.map(wire::media_input).transpose()?;
|
||||
let audio_data = audio_data.map(wire::media_input).transpose()?;
|
||||
let mm_hashes = mm_hashes.and_then(wire::string_list_or_list);
|
||||
|
||||
// Cap the batch BEFORE the columns below allocate anything. Reading the
|
||||
// declared length off the input costs nothing; the previous placement (after
|
||||
@@ -353,7 +169,7 @@ impl GenerateBody {
|
||||
// A list is per-item; a single object broadcasts to every item.
|
||||
let sps: Vec<SamplingParams> = match sampling_params {
|
||||
None => vec![SamplingParams::default(); n],
|
||||
Some(SamplingParamsInput::Many(v)) => {
|
||||
Some(wire::SamplingInput::Many(v)) => {
|
||||
if v.len() != n {
|
||||
return Err(Error::Validation(format!(
|
||||
"sampling_params list length {} does not match batch size {n}",
|
||||
@@ -362,7 +178,7 @@ impl GenerateBody {
|
||||
}
|
||||
v
|
||||
}
|
||||
Some(SamplingParamsInput::One(sp)) => {
|
||||
Some(wire::SamplingInput::One(sp)) => {
|
||||
// Broadcasting deep-clones the client's params once per prompt,
|
||||
// heap and all — `stop`, `logit_bias` and `custom_params` (arbitrary
|
||||
// JSON) are still unnormalized client data here. The blow-up is
|
||||
@@ -465,8 +281,7 @@ impl GenerateBody {
|
||||
}
|
||||
other => flatten_column(fan_out(other, n, "bootstrap_room")?),
|
||||
};
|
||||
let bootstrap_pair_keys =
|
||||
flatten_column(fan_out(bootstrap_pair_key, n, "bootstrap_pair_key")?);
|
||||
let bootstrap_pair_keys = flatten_column(fan_out(bootstrap_pair_key, n, "bootstrap_pair_key")?);
|
||||
let decode_tp_sizes = flatten_column(fan_out(decode_tp_size, n, "decode_tp_size")?);
|
||||
// `mm_hashes` has no batch form: honoring it only here would give the two
|
||||
// servers different prefix-cache keys for the same body. Reject instead of
|
||||
@@ -477,8 +292,7 @@ impl GenerateBody {
|
||||
Some(OneOrMany::One(hashes)) if hashes.is_empty() => Vec::new(),
|
||||
Some(_) if is_batch => {
|
||||
return Err(Error::Validation(
|
||||
"mm_hashes is not supported for batch requests; send one request per prompt"
|
||||
.into(),
|
||||
"mm_hashes is not supported for batch requests; send one request per prompt".into(),
|
||||
));
|
||||
}
|
||||
Some(OneOrMany::One(hashes)) => hashes,
|
||||
@@ -492,7 +306,6 @@ impl GenerateBody {
|
||||
let images = multimodal::fan_out(image_data, n, is_batch, "image_data")?;
|
||||
let videos = multimodal::fan_out(video_data, n, is_batch, "video_data")?;
|
||||
let audios = multimodal::fan_out(audio_data, n, is_batch, "audio_data")?;
|
||||
let processor_extensions = split_extension_columns(processor_extensions, n, is_batch)?;
|
||||
|
||||
// Every column above is exactly `n` long, so zip them by value: each
|
||||
// request takes ownership of its cell, with no indexing or bounds checks.
|
||||
@@ -514,7 +327,6 @@ impl GenerateBody {
|
||||
images,
|
||||
videos,
|
||||
audios,
|
||||
processor_extensions,
|
||||
)
|
||||
.map(
|
||||
|(
|
||||
@@ -535,7 +347,6 @@ impl GenerateBody {
|
||||
image_data,
|
||||
video_data,
|
||||
audio_data,
|
||||
processor_extensions,
|
||||
)| GenerateRequest {
|
||||
rid,
|
||||
text,
|
||||
@@ -562,7 +373,7 @@ impl GenerateBody {
|
||||
decode_tp_size,
|
||||
routed_dp_rank,
|
||||
disagg_prefill_dp_rank,
|
||||
mm: pack_mm(image_data, video_data, audio_data, processor_extensions),
|
||||
mm: pack_mm(image_data, video_data, audio_data),
|
||||
mm_buffers: Vec::new(),
|
||||
},
|
||||
)
|
||||
@@ -573,7 +384,6 @@ impl GenerateBody {
|
||||
mm.mm_hashes = mm_hashes;
|
||||
}
|
||||
Ok((requests, is_batch))
|
||||
}
|
||||
}
|
||||
|
||||
/// Box the per-item mm values, `None` when the item has none — the common
|
||||
@@ -582,78 +392,24 @@ fn pack_mm(
|
||||
image_data: Vec<MmItem>,
|
||||
video_data: Vec<MmItem>,
|
||||
audio_data: Vec<MmItem>,
|
||||
processor_extensions: ProcessorExtensions,
|
||||
) -> Option<Box<MmData>> {
|
||||
if image_data.is_empty()
|
||||
&& video_data.is_empty()
|
||||
&& audio_data.is_empty()
|
||||
&& processor_extensions.is_empty()
|
||||
{
|
||||
if image_data.is_empty() && video_data.is_empty() && audio_data.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(Box::new(MmData {
|
||||
image_data,
|
||||
video_data,
|
||||
audio_data,
|
||||
processor_extensions,
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
|
||||
fn split_extension_columns(
|
||||
fields: ProcessorExtensions,
|
||||
n: usize,
|
||||
is_batch: bool,
|
||||
) -> Result<Vec<ProcessorExtensions>, Error> {
|
||||
let mut requests = vec![ProcessorExtensions::default(); n];
|
||||
for (name, value) in fields.0 {
|
||||
if !name.starts_with(PROCESSOR_EXTENSION_PREFIX) || value.is_nil() {
|
||||
continue;
|
||||
}
|
||||
if !is_batch {
|
||||
requests[0].0.insert(name, value);
|
||||
continue;
|
||||
}
|
||||
let rmpv::Value::Array(values) = value else {
|
||||
return Err(Error::Validation(format!(
|
||||
"{name} must be a list for batch processing"
|
||||
)));
|
||||
};
|
||||
if values.is_empty() {
|
||||
for request in &mut requests {
|
||||
request
|
||||
.0
|
||||
.insert(name.clone(), rmpv::Value::Array(Vec::new()));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if values.len() != n {
|
||||
return Err(Error::Validation(format!(
|
||||
"{name} list length {} does not match batch size {n}",
|
||||
values.len()
|
||||
)));
|
||||
}
|
||||
for (request, value) in requests.iter_mut().zip(values) {
|
||||
request.0.insert(name.clone(), value);
|
||||
}
|
||||
}
|
||||
Ok(requests)
|
||||
}
|
||||
|
||||
fn extension_value_present(value: &rmpv::Value) -> bool {
|
||||
match value {
|
||||
rmpv::Value::Nil => false,
|
||||
rmpv::Value::Array(values) => values.iter().any(extension_value_present),
|
||||
_ => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// The owned request as it travels request stages (single owner, so `state` is
|
||||
/// mutated lock-free). Common fields here; variant data in [`RequestKind`].
|
||||
#[derive(Debug)]
|
||||
pub struct Request {
|
||||
/// Runtime correlation ID. Client-provided IDs carry a private incarnation
|
||||
/// suffix; adapters recover the public ID through `FrontendCall::public_id`.
|
||||
/// suffix; adapters recover the public ID through `CoreCall::public_id`.
|
||||
pub rid: Rid,
|
||||
pub state: RequestState,
|
||||
/// Back-channel to the client connection for response frames.
|
||||
@@ -679,7 +435,7 @@ pub enum RequestKind {
|
||||
/// Generation: tokenize (if needed) then push a `TokenizedGenerateReqInput`.
|
||||
Generate(Box<GenerateRequest>),
|
||||
/// Internal runtime control operation: no tokenization, with one serialized
|
||||
/// result that the frontend boundary converts into a typed response.
|
||||
/// result that the core boundary converts into a typed response.
|
||||
Control(Box<ControlRequest>),
|
||||
/// Internal service call: decode a complete token-id sequence to text. Walks
|
||||
/// the same FSM as every request (validate → register → Queued), but the
|
||||
@@ -692,7 +448,7 @@ pub enum RequestKind {
|
||||
|
||||
/// A single in-flight generation request (one item after adapter fan-out),
|
||||
/// serialized to the scheduler wire once tokenized (see `to_header_msgpack`). Not a
|
||||
/// wire type — built by frontend adapters, never (de)serialized; `input_ids` is
|
||||
/// wire type — built by the API adapters, never (de)serialized; `input_ids` is
|
||||
/// supplied by an adapter or filled by the Tokenizer stage.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct GenerateRequest {
|
||||
@@ -775,16 +531,15 @@ pub struct GenerateRequest {
|
||||
/// processor's input: the MM worker moves it out of the request whole, and
|
||||
/// `payload::to_mm_input` converts it to the driver's.
|
||||
///
|
||||
/// Constructed directly only by tests: `frontend::prefetch` fills its
|
||||
/// Constructed directly only by tests: `core::prefetch` fills its
|
||||
/// `prefetched` field, everything else gets it packed inside a `GenerateRequest`.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct MmData {
|
||||
pub image_data: Vec<MmItem>,
|
||||
pub video_data: Vec<MmItem>,
|
||||
pub audio_data: Vec<MmItem>,
|
||||
pub processor_extensions: ProcessorExtensions,
|
||||
/// Bytes of `image_data`'s I/O-backed sources, resolved by
|
||||
/// `frontend::prefetch` in `payload::io_sources` order so MM workers
|
||||
/// `core::prefetch` in `payload::io_sources` order so MM workers
|
||||
/// never block on I/O. Out-of-band: the values above stay as the client
|
||||
/// sent them.
|
||||
pub prefetched: Vec<bytes::Bytes>,
|
||||
@@ -802,13 +557,7 @@ impl GenerateRequest {
|
||||
/// Python `GenerateReqInput.contains_mm_input()`.
|
||||
pub fn has_multimodal(&self) -> bool {
|
||||
self.mm.as_ref().is_some_and(|mm| {
|
||||
!mm.image_data.is_empty()
|
||||
|| !mm.video_data.is_empty()
|
||||
|| !mm.audio_data.is_empty()
|
||||
|| mm
|
||||
.processor_extensions
|
||||
.values()
|
||||
.any(extension_value_present)
|
||||
!mm.image_data.is_empty() || !mm.video_data.is_empty() || !mm.audio_data.is_empty()
|
||||
})
|
||||
}
|
||||
|
||||
@@ -920,26 +669,14 @@ fn fan_out<T: OneOrManyItem + Clone + HeapBytes>(
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[derive(Debug, Deserialize, PartialEq)]
|
||||
struct TestProcessorExtensions {
|
||||
multimodal_custom: TestProcessorExtension,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, PartialEq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct TestProcessorExtension {
|
||||
value: i64,
|
||||
}
|
||||
use crate::message::multimodal::MmDataInput;
|
||||
|
||||
/// Vocab size for tests that aren't about the vocab bound (see
|
||||
/// `sampling::tests::TEST_VOCAB`).
|
||||
const TEST_VOCAB: u64 = 1000;
|
||||
|
||||
fn requests(body: &str) -> Result<(Vec<GenerateRequest>, bool), Error> {
|
||||
serde_json::from_str::<GenerateBody>(body)
|
||||
.unwrap()
|
||||
.into_requests()
|
||||
into_requests(serde_json::from_str::<api_v1::GenerateRequest>(body).unwrap())
|
||||
}
|
||||
|
||||
/// Scalar `text` → one item, not a batch (response stays a single object).
|
||||
@@ -1000,72 +737,33 @@ mod tests {
|
||||
assert_eq!(ps[1].input_ids, Some(vec![3]));
|
||||
}
|
||||
|
||||
/// A bad element inside `input_ids` is reported against the field with
|
||||
/// serde's type text (the wire decoder settles the list form from its
|
||||
/// first element, so the element index is not part of the path), and a
|
||||
/// malformed body is a rejection, not a panic.
|
||||
#[test]
|
||||
fn input_ids_deserialization_matches_untagged() {
|
||||
fn input_ids_rejections_name_the_field() {
|
||||
for body in [
|
||||
r#"{"input_ids":[true]}"#,
|
||||
r#"{"input_ids":[[1,true]]}"#,
|
||||
r#"{"input_ids":[[1],[2,true]]}"#,
|
||||
] {
|
||||
let error =
|
||||
axum::Json::<api_v1::GenerateRequest>::from_bytes(body.as_bytes()).unwrap_err();
|
||||
let text = error.body_text();
|
||||
assert!(
|
||||
serde_json::from_str::<GenerateBody>(r#"{"text":"hi"}"#)
|
||||
.unwrap()
|
||||
.input_ids
|
||||
.is_none()
|
||||
text.contains("input_ids") && text.contains("boolean"),
|
||||
"{text}"
|
||||
);
|
||||
for value in [
|
||||
"null",
|
||||
"[]",
|
||||
"[0]",
|
||||
"[-2147483648,2147483647]",
|
||||
"[[]]",
|
||||
"[[],[1,-2]]",
|
||||
"[[1,2],[3]]",
|
||||
"[[-2147483648],[2147483647]]",
|
||||
"1",
|
||||
"true",
|
||||
"{}",
|
||||
r#""1""#,
|
||||
"[null]",
|
||||
"[true]",
|
||||
r#"["1"]"#,
|
||||
"[1.0]",
|
||||
"[1e0]",
|
||||
"[-0]",
|
||||
"[2147483648]",
|
||||
"[-2147483649]",
|
||||
"[18446744073709551616]",
|
||||
"[0,2147483648]",
|
||||
"[0,-0]",
|
||||
"[1,[2]]",
|
||||
"[[1],2]",
|
||||
"[[[1]]]",
|
||||
"[[1.0]]",
|
||||
"[[0],[2147483648]]",
|
||||
] {
|
||||
let expected = serde_json::from_str::<Option<OneOrMany<TokenIds>>>(value);
|
||||
let body = format!(r#"{{"input_ids":{value}}}"#);
|
||||
let actual = serde_json::from_str::<GenerateBody>(&body).map(|body| body.input_ids);
|
||||
match (expected, actual) {
|
||||
(Ok(expected), Ok(actual)) => assert_eq!(actual, expected, "{value}"),
|
||||
(Err(_), Err(_)) => {}
|
||||
(expected, actual) => panic!("{value}: expected {expected:?}, got {actual:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
for (body, path) in [
|
||||
(r#"{"input_ids":[true]}"#, "input_ids[0]"),
|
||||
(r#"{"input_ids":[[1,true]]}"#, "input_ids[0][1]"),
|
||||
(r#"{"input_ids":[[1],[2,true]]}"#, "input_ids[1][1]"),
|
||||
] {
|
||||
let error = axum::Json::<GenerateBody>::from_bytes(body.as_bytes()).unwrap_err();
|
||||
assert!(error.body_text().contains(path), "{error}");
|
||||
}
|
||||
for body in [
|
||||
r#"{"input_ids":null,"input_ids":[1]}"#,
|
||||
r#"{"input_ids":[1],"input_ids":[2]}"#,
|
||||
r#"{"input_ids":[1]} {}"#,
|
||||
] {
|
||||
assert!(axum::Json::<GenerateBody>::from_bytes(body.as_bytes()).is_err());
|
||||
assert!(axum::Json::<api_v1::GenerateRequest>::from_bytes(body.as_bytes()).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
/// Both / neither of text+input_ids is a 400.
|
||||
#[test]
|
||||
fn split_validates_inputs() {
|
||||
assert!(requests(r#"{"text": "a", "input_ids": [1]}"#).is_err());
|
||||
@@ -1218,92 +916,6 @@ mod tests {
|
||||
assert!(ps[1].has_multimodal());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multimodal_extensions_follow_request_batch_shape() {
|
||||
let single = r#"{"input_ids":[9],"image_data":"u","multimodal_placeholders":[{"type":"image","token_index":0,"item_index":0}]}"#;
|
||||
let (reqs, is_batch) = requests(single).unwrap();
|
||||
assert!(!is_batch);
|
||||
let value = reqs[0]
|
||||
.mm
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.processor_extensions
|
||||
.0
|
||||
.get("multimodal_placeholders")
|
||||
.unwrap();
|
||||
assert_eq!(value.as_array().unwrap().len(), 1);
|
||||
|
||||
let batched = r#"{"input_ids":[[9],[8]],"image_data":["u","v"],"multimodal_placeholders":[[{"type":"image","token_index":0,"item_index":0}],[{"type":"image","token_index":0,"item_index":0}]]}"#;
|
||||
let (reqs, is_batch) = requests(batched).unwrap();
|
||||
assert!(is_batch);
|
||||
assert_eq!(reqs.len(), 2);
|
||||
assert!(reqs.iter().all(GenerateRequest::has_multimodal));
|
||||
assert!(reqs.iter().all(|request| {
|
||||
request
|
||||
.mm
|
||||
.as_ref()
|
||||
.and_then(|mm| mm.processor_extensions.0.get("multimodal_placeholders"))
|
||||
.and_then(rmpv::Value::as_array)
|
||||
.is_some_and(|placeholders| placeholders.len() == 1)
|
||||
}));
|
||||
|
||||
let invalid = r#"{"input_ids":[[9],[8]],"image_data":["u","v"],"multimodal_placeholders":[{"type":"image","token_index":0,"item_index":0}]}"#;
|
||||
assert!(requests(invalid).is_err());
|
||||
|
||||
let generic = r#"{"input_ids":[[9],[8]],"image_data":["u","v"],"multimodal_custom":[{"value":1},{"value":2}]}"#;
|
||||
let (reqs, _) = requests(generic).unwrap();
|
||||
assert_eq!(
|
||||
reqs[1]
|
||||
.mm
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.processor_extensions
|
||||
.0
|
||||
.get("multimodal_custom")
|
||||
.unwrap()
|
||||
.as_map()
|
||||
.unwrap()[0]
|
||||
.1
|
||||
.as_i64(),
|
||||
Some(2)
|
||||
);
|
||||
|
||||
let extensions: TestProcessorExtensions =
|
||||
requests(r#"{"input_ids":[9],"multimodal_custom":{"value":3}}"#)
|
||||
.unwrap()
|
||||
.0
|
||||
.pop()
|
||||
.unwrap()
|
||||
.mm
|
||||
.unwrap()
|
||||
.processor_extensions
|
||||
.deserialize()
|
||||
.unwrap();
|
||||
assert_eq!(extensions.multimodal_custom.value, 3);
|
||||
|
||||
for fields in [
|
||||
r#"{"multimodal_custom":{"value":true}}"#,
|
||||
r#"{"multimodal_custom":{"value":"3"}}"#,
|
||||
r#"{"multimodal_custom":{"value":3,"unknown":0}}"#,
|
||||
r#"{"multimodal_custom":{}}"#,
|
||||
] {
|
||||
let extensions: ProcessorExtensions = serde_json::from_str(fields).unwrap();
|
||||
assert!(
|
||||
extensions.deserialize::<TestProcessorExtensions>().is_err(),
|
||||
"{fields}"
|
||||
);
|
||||
}
|
||||
|
||||
let (reqs, _) = requests(r#"{"text":"hi","totally_made_up":1}"#).unwrap();
|
||||
assert!(reqs[0].mm.is_none());
|
||||
|
||||
let (reqs, _) = requests(r#"{"input_ids":[9],"multimodal_custom":null}"#).unwrap();
|
||||
assert!(!reqs[0].has_multimodal());
|
||||
}
|
||||
|
||||
/// A scalar broadcast is budget-checked before the deep clones (16 MiB ×
|
||||
/// 4096 prompts would be 64 GiB and an abort); per-item lists clone nothing
|
||||
/// and are never charged.
|
||||
#[test]
|
||||
fn oversized_mm_broadcast_rejected() {
|
||||
let big = MmItem::Source("x".repeat(MAX_BROADCAST_CLONE_BYTES / 2 + 1));
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! The runtime response direction: per-request channels into the frontend
|
||||
//! The runtime response direction: per-request channels into the core
|
||||
//! boundary ([`ResponseSink`] / [`ResponseItem`]), response frame encodings,
|
||||
//! and columnar batch decoding into per-request [`ChunkEvent`]s.
|
||||
|
||||
@@ -12,7 +12,7 @@ use crate::message::types::TokenIds;
|
||||
use crate::utils::error::Error;
|
||||
|
||||
/// Per-request back-channel the detok shard writes runtime outputs to and the
|
||||
/// frontend boundary drains; bounded, and receiver-drop cancels the producer.
|
||||
/// core boundary drains; bounded, and receiver-drop cancels the producer.
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum ResponseSink {
|
||||
Local(mpsc::Sender<ResponseItem>),
|
||||
@@ -38,12 +38,12 @@ impl ResponseSink {
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)] // the receiver half is owned inside the frontend boundary.
|
||||
#[allow(dead_code)] // the receiver half is owned inside the core boundary.
|
||||
pub type ResponseSource = mpsc::Receiver<ResponseItem>;
|
||||
|
||||
/// Runtime-facing output delivered to the frontend boundary: a detok-decoded
|
||||
/// Runtime-facing output delivered to the core boundary: a detok-decoded
|
||||
/// [`ChunkEvent`], a serialized internal-operation result, or a stage error.
|
||||
/// [`crate::frontend::FrontendHandle`] translates these variants before a
|
||||
/// [`crate::api_server::core::CoreHandle`] translates these variants before a
|
||||
/// transport adapter sees them.
|
||||
#[derive(Debug)]
|
||||
pub enum ResponseItem {
|
||||
@@ -54,10 +54,10 @@ pub enum ResponseItem {
|
||||
/// A control-request result encoded by the Python scheduler.
|
||||
Control(Bytes),
|
||||
/// Reply to an internal service request (`RequestKind::Detokenize`): raw
|
||||
/// bytes for the frontend to decode (e.g. detokenized text), not a
|
||||
/// bytes for the core to decode (e.g. detokenized text), not a
|
||||
/// client-facing wire payload and not a generation frame.
|
||||
Data(Bytes),
|
||||
/// Terminal runtime failure; the frontend maps it to a semantic error.
|
||||
/// Terminal runtime failure; the core maps it to a semantic error.
|
||||
Error(Error),
|
||||
}
|
||||
|
||||
@@ -558,7 +558,7 @@ pub fn frame_error(rid: &str, message: &str) -> Bytes {
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct ChunkEvent {
|
||||
/// Runtime correlation ID. Moved out of the frame header (which owns it and
|
||||
/// drops it), so carrying it costs no allocation. The frontend strips this
|
||||
/// drops it), so carrying it costs no allocation. The core strips this
|
||||
/// field before exposing semantic output; the shard routes it via `Rid::shard`.
|
||||
pub rid: Rid,
|
||||
/// New token ids for this step, widened from the scheduler's int32 wire
|
||||
|
||||
@@ -3,12 +3,9 @@
|
||||
//! `__post_init__` → `normalize` → `verify` pipeline (run in that order, as
|
||||
//! `TokenizerManager._create_tokenized_object` does).
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::fmt;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use serde::de::value::{MapAccessDeserializer, SeqAccessDeserializer};
|
||||
use serde::de::{MapAccess, SeqAccess, Visitor};
|
||||
use serde::{Deserialize, Deserializer, Serialize};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::types::OneOrMany;
|
||||
use crate::utils::{error::Error, regex::RegexPattern};
|
||||
@@ -241,90 +238,6 @@ pub struct SamplingParams {
|
||||
/// strip that wrapper from its own grammar, so it is a pipeline output only.
|
||||
#[serde(skip_deserializing)]
|
||||
pub ebnf_full_assistant: bool,
|
||||
/// API fields present in the request object. Serde defaults erase this
|
||||
/// distinction, but preferred sampling parameters must not overwrite an
|
||||
/// explicit request value, including an explicit default or null.
|
||||
#[serde(skip)]
|
||||
pub(crate) explicit_fields: BTreeSet<String>,
|
||||
}
|
||||
|
||||
/// The `/generate` body's `sampling_params`: one object (broadcast to every
|
||||
/// prompt) or a list of them (one per prompt), fanned out by `GenerateBody::into_requests`.
|
||||
///
|
||||
/// Hand-written `Deserialize` rather than `#[serde(untagged)]`: untagged buffers
|
||||
/// the input and, on failure, reports only "data did not match any variant" —
|
||||
/// losing the field-level message ("unknown field `temperature`, expected one of
|
||||
/// …") that makes a typo actionable. Object-vs-list is unambiguous here, so a
|
||||
/// single `deserialize_any` dispatch keeps the inner error verbatim.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum SamplingParamsInput {
|
||||
/// Boxed: `SamplingParams` is ~440 bytes, so an inline variant would make
|
||||
/// every `GenerateBody` that big regardless of which form arrived.
|
||||
One(Box<SamplingParams>),
|
||||
Many(Vec<SamplingParams>),
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for SamplingParamsInput {
|
||||
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
struct InputVisitor;
|
||||
|
||||
impl<'de> Visitor<'de> for InputVisitor {
|
||||
type Value = SamplingParamsInput;
|
||||
|
||||
fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str("a sampling_params object, or a list of them (one per prompt)")
|
||||
}
|
||||
|
||||
fn visit_map<A: MapAccess<'de>>(self, map: A) -> Result<Self::Value, A::Error> {
|
||||
let value = serde_json::Value::deserialize(MapAccessDeserializer::new(map))?;
|
||||
sampling_params_from_value(value)
|
||||
.map(|p| SamplingParamsInput::One(Box::new(p)))
|
||||
.map_err(serde::de::Error::custom)
|
||||
}
|
||||
|
||||
fn visit_seq<A: SeqAccess<'de>>(self, seq: A) -> Result<Self::Value, A::Error> {
|
||||
let values =
|
||||
Vec::<serde_json::Value>::deserialize(SeqAccessDeserializer::new(seq))?;
|
||||
values
|
||||
.into_iter()
|
||||
.map(sampling_params_from_value)
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.map(SamplingParamsInput::Many)
|
||||
.map_err(serde::de::Error::custom)
|
||||
}
|
||||
}
|
||||
|
||||
deserializer.deserialize_any(InputVisitor)
|
||||
}
|
||||
}
|
||||
|
||||
fn sampling_params_from_value(value: serde_json::Value) -> Result<SamplingParams, String> {
|
||||
let explicit_fields = value
|
||||
.as_object()
|
||||
.ok_or_else(|| "sampling_params must be an object".to_string())?
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect();
|
||||
let mut params: SamplingParams = serde_json::from_value(value).map_err(|e| e.to_string())?;
|
||||
params.explicit_fields = explicit_fields;
|
||||
Ok(params)
|
||||
}
|
||||
|
||||
impl SamplingParamsInput {
|
||||
/// Merge launch-time preferred params beneath request params. A request key
|
||||
/// wins even when it explicitly carries the type's default or null.
|
||||
pub fn apply_preferred(&mut self, preferred: &serde_json::Value) -> Result<(), String> {
|
||||
match self {
|
||||
Self::One(params) => params.apply_preferred(preferred),
|
||||
Self::Many(params) => params
|
||||
.iter_mut()
|
||||
.try_for_each(|params| params.apply_preferred(preferred)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_preferred(preferred: &serde_json::Value) -> Result<Self, String> {
|
||||
sampling_params_from_value(preferred.clone()).map(|params| Self::One(Box::new(params)))
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for SamplingParams {
|
||||
@@ -366,39 +279,11 @@ impl Default for SamplingParams {
|
||||
stop_regex_max_len: 0,
|
||||
is_normalized: false,
|
||||
ebnf_full_assistant: false,
|
||||
explicit_fields: BTreeSet::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SamplingParams {
|
||||
/// Record a field supplied by a non-Serde adapter. Preferred launch values
|
||||
/// are merged underneath these fields just as they are underneath keys
|
||||
/// present in an HTTP sampling object.
|
||||
pub(crate) fn mark_explicit(&mut self, field: &'static str) {
|
||||
self.explicit_fields.insert(field.to_owned());
|
||||
}
|
||||
|
||||
/// Merge operator-provided sampling defaults beneath explicitly supplied
|
||||
/// request fields. HTTP and gRPC adapters use the same precedence policy.
|
||||
pub(crate) fn apply_preferred(&mut self, preferred: &serde_json::Value) -> Result<(), String> {
|
||||
let mut merged = preferred
|
||||
.as_object()
|
||||
.ok_or_else(|| "preferred_sampling_params must be a JSON object".to_string())?
|
||||
.clone();
|
||||
let request_value = serde_json::to_value(&*self).map_err(|e| e.to_string())?;
|
||||
let request = request_value
|
||||
.as_object()
|
||||
.ok_or_else(|| "SamplingParams did not serialize as an object".to_string())?;
|
||||
for field in &self.explicit_fields {
|
||||
if let Some(value) = request.get(field) {
|
||||
merged.insert(field.clone(), value.clone());
|
||||
}
|
||||
}
|
||||
*self = sampling_params_from_value(serde_json::Value::Object(merged))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `__post_init__` → `normalize` → `verify`, the order
|
||||
/// `TokenizerManager._create_tokenized_object` runs them in. `Err` is a
|
||||
/// request-local 400. `skip_tokenizer_init` stands in for Python's
|
||||
@@ -1220,35 +1105,4 @@ mod tests {
|
||||
let err = norm_err(&json).to_string();
|
||||
assert!(err.contains("at most"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preferred_params_fill_only_omitted_request_fields() {
|
||||
let preferred = serde_json::json!({
|
||||
"temperature": 0.25,
|
||||
"top_p": 0.75,
|
||||
"max_new_tokens": 4096
|
||||
});
|
||||
let mut input: SamplingParamsInput =
|
||||
serde_json::from_str(r#"{"temperature": 1.0, "top_p": null}"#).unwrap();
|
||||
input.apply_preferred(&preferred).unwrap();
|
||||
let SamplingParamsInput::One(params) = input else {
|
||||
panic!("expected scalar params")
|
||||
};
|
||||
assert_eq!(params.temperature, 1.0, "explicit default wins");
|
||||
assert_eq!(params.top_p, 1.0, "explicit null keeps the type default");
|
||||
assert_eq!(params.max_new_tokens, Some(4096), "omitted uses preferred");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preferred_params_apply_to_every_batched_object() {
|
||||
let preferred = serde_json::json!({"temperature": 0.25, "top_p": 0.75});
|
||||
let mut input: SamplingParamsInput =
|
||||
serde_json::from_str(r#"[{"temperature": 0.5}, {"top_p": 0.9}]"#).unwrap();
|
||||
input.apply_preferred(&preferred).unwrap();
|
||||
let SamplingParamsInput::Many(params) = input else {
|
||||
panic!("expected batched params")
|
||||
};
|
||||
assert_eq!((params[0].temperature, params[0].top_p), (0.5, 0.75));
|
||||
assert_eq!((params[1].temperature, params[1].top_p), (0.25, 0.9));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,410 @@
|
||||
//! The `/generate` wire contract is the generated `sglang_api_types` request
|
||||
//! (the proto schema is the single source of truth for both servers). This
|
||||
//! module is the one place its carriers become the internal request types:
|
||||
//! scalar-or-list columns into [`OneOrMany`], media containers into
|
||||
//! [`MmDataInput`], and the wire `SamplingParams` into the internal one that
|
||||
//! carries normalization state. Nothing here validates batch shape; that is
|
||||
//! [`into_requests`](super::request::into_requests).
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use sglang_api_types::api::v1 as api;
|
||||
|
||||
use super::multimodal::{MmDataInput, MmItem};
|
||||
use super::sampling::{CustomParamValue, SamplingParams};
|
||||
use super::types::{OneOrMany, TokenIds};
|
||||
use crate::utils::error::Error;
|
||||
|
||||
pub fn string_or_list(v: api::StringOrList) -> Option<OneOrMany<String>> {
|
||||
use api::string_or_list::Value;
|
||||
v.value.map(|v| match v {
|
||||
Value::One(s) => OneOrMany::One(s),
|
||||
Value::Many(l) => OneOrMany::Many(l.items),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn int64_or_list(v: api::Int64OrList) -> Option<OneOrMany<i64>> {
|
||||
use api::int64_or_list::Value;
|
||||
v.value.map(|v| match v {
|
||||
Value::One(x) => OneOrMany::One(x),
|
||||
Value::Many(l) => OneOrMany::Many(l.items),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn bool_or_list(v: api::BoolOrList) -> Option<OneOrMany<bool>> {
|
||||
use api::bool_or_list::Value;
|
||||
v.value.map(|v| match v {
|
||||
Value::One(x) => OneOrMany::One(x),
|
||||
Value::Many(l) => OneOrMany::Many(l.items),
|
||||
})
|
||||
}
|
||||
|
||||
fn token_ids(v: api::TokenIds) -> TokenIds {
|
||||
v.ids
|
||||
}
|
||||
|
||||
pub fn token_ids_or_list(v: api::TokenIdsOrList) -> Option<OneOrMany<TokenIds>> {
|
||||
use api::token_ids_or_list::Value;
|
||||
v.value.map(|v| match v {
|
||||
Value::One(ids) => OneOrMany::One(token_ids(ids)),
|
||||
Value::Many(l) => OneOrMany::Many(l.items.into_iter().map(token_ids).collect()),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn optional_string_or_list(v: api::OptionalStringOrList) -> Option<OneOrMany<Option<String>>> {
|
||||
use api::optional_string_or_list::Value;
|
||||
v.value.map(|v| match v {
|
||||
Value::One(s) => OneOrMany::One(s.value),
|
||||
Value::Many(l) => OneOrMany::Many(l.items.into_iter().map(|s| s.value).collect()),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn optional_int64_or_list(v: api::OptionalInt64OrList) -> Option<OneOrMany<Option<i64>>> {
|
||||
use api::optional_int64_or_list::Value;
|
||||
v.value.map(|v| match v {
|
||||
Value::One(x) => OneOrMany::One(x.value),
|
||||
Value::Many(l) => OneOrMany::Many(l.items.into_iter().map(|x| x.value).collect()),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn string_list_or_list(v: api::StringListOrList) -> Option<OneOrMany<Vec<String>>> {
|
||||
use api::string_list_or_list::Value;
|
||||
v.value.map(|v| match v {
|
||||
Value::One(l) => OneOrMany::One(l.items),
|
||||
Value::Many(ll) => OneOrMany::Many(ll.items.into_iter().map(|l| l.items).collect()),
|
||||
})
|
||||
}
|
||||
|
||||
/// A media item: a bare source, or a ref whose `url` must be present. proto3
|
||||
/// strings carry no presence, so an object without `url` decodes as an empty
|
||||
/// one and is rejected here (Python's `load_image` fails on it the same way).
|
||||
pub fn media_item(item: api::MediaItem) -> Result<MmItem, Error> {
|
||||
use api::media_item::Value;
|
||||
match item.value {
|
||||
// Only `url` reaches the Rust MM pipeline: the hints are read by model
|
||||
// families it does not run (Python's `load_image` reduces an item to
|
||||
// `.url` the same way).
|
||||
Some(Value::Ref(r)) if !r.url.is_empty() => Ok(MmItem::Ref { url: r.url }),
|
||||
Some(Value::Ref(_)) => Err(Error::Validation(
|
||||
"a multimodal item object needs a non-empty `url`".into(),
|
||||
)),
|
||||
Some(Value::Source(s)) => Ok(MmItem::Source(s)),
|
||||
// Unreachable from JSON (the untagged union always sets an arm).
|
||||
None => Err(Error::Validation("empty multimodal item".into())),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn media_input(input: api::MediaInput) -> Result<MmDataInput, Error> {
|
||||
use api::media_input::Value;
|
||||
let item = |i: api::OptionalMediaItem| i.item.map(media_item).transpose();
|
||||
Ok(match input.value {
|
||||
Some(Value::Item(i)) => MmDataInput::One(media_item(i)?),
|
||||
Some(Value::Many(l)) => {
|
||||
MmDataInput::Many(l.items.into_iter().map(item).collect::<Result<_, _>>()?)
|
||||
}
|
||||
Some(Value::Nested(ll)) => MmDataInput::Nested(
|
||||
ll.items
|
||||
.into_iter()
|
||||
.map(|entry| {
|
||||
entry
|
||||
.items
|
||||
.map(|l| l.items.into_iter().map(item).collect::<Result<_, _>>())
|
||||
.transpose()
|
||||
})
|
||||
.collect::<Result<_, _>>()?,
|
||||
),
|
||||
None => MmDataInput::Many(Vec::new()),
|
||||
})
|
||||
}
|
||||
|
||||
/// The wire params, with the schema defaults already applied by the generated
|
||||
/// decoder (an absent key reads its default), become the internal params. Only
|
||||
/// `custom_params` needs parsing: it crosses the wire as JSON text.
|
||||
impl TryFrom<api::SamplingParams> for SamplingParams {
|
||||
type Error = String;
|
||||
|
||||
fn try_from(p: api::SamplingParams) -> Result<Self, String> {
|
||||
let defaults = SamplingParams::default();
|
||||
let custom_params: Option<BTreeMap<String, CustomParamValue>> = match p.custom_params {
|
||||
None => None,
|
||||
Some(text) => {
|
||||
Some(serde_json::from_str(&text).map_err(|e| format!("custom_params: {e}"))?)
|
||||
}
|
||||
};
|
||||
Ok(SamplingParams {
|
||||
max_new_tokens: p.max_new_tokens,
|
||||
stop: p.stop.and_then(string_or_list),
|
||||
stop_token_ids: p.stop_token_ids.map(|l| l.items),
|
||||
stop_regex: p.stop_regex.and_then(string_or_list),
|
||||
temperature: p.temperature.unwrap_or(defaults.temperature),
|
||||
top_p: p.top_p.unwrap_or(defaults.top_p),
|
||||
top_k: p.top_k.unwrap_or(defaults.top_k),
|
||||
min_p: p.min_p.unwrap_or(defaults.min_p),
|
||||
frequency_penalty: p.frequency_penalty.unwrap_or(defaults.frequency_penalty),
|
||||
presence_penalty: p.presence_penalty.unwrap_or(defaults.presence_penalty),
|
||||
repetition_penalty: p.repetition_penalty.unwrap_or(defaults.repetition_penalty),
|
||||
min_new_tokens: p.min_new_tokens.unwrap_or(defaults.min_new_tokens),
|
||||
n: p.n.unwrap_or(defaults.n),
|
||||
beam_width: p.beam_width,
|
||||
json_schema: p.json_schema,
|
||||
regex: p.regex,
|
||||
ebnf: p.ebnf,
|
||||
structural_tag: p.structural_tag,
|
||||
ignore_eos: p.ignore_eos.unwrap_or(defaults.ignore_eos),
|
||||
skip_special_tokens: p
|
||||
.skip_special_tokens
|
||||
.unwrap_or(defaults.skip_special_tokens),
|
||||
spaces_between_special_tokens: p
|
||||
.spaces_between_special_tokens
|
||||
.unwrap_or(defaults.spaces_between_special_tokens),
|
||||
no_stop_trim: p.no_stop_trim.unwrap_or(defaults.no_stop_trim),
|
||||
stream_interval: p.stream_interval,
|
||||
logit_bias: (!p.logit_bias.is_empty()).then(|| p.logit_bias.into_iter().collect()),
|
||||
sampling_seed: p.sampling_seed,
|
||||
custom_params,
|
||||
..defaults
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// `sampling_params` as sent: one object (a broadcast in a batch) or one per
|
||||
/// prompt. Not a [`OneOrMany`]: that carrier is sealed to wire-shaped items,
|
||||
/// and the internal params are not one.
|
||||
pub enum SamplingInput {
|
||||
/// Boxed: the params are ~450 bytes, so an inline variant would make
|
||||
/// every input that big regardless of which form arrived.
|
||||
One(Box<SamplingParams>),
|
||||
Many(Vec<SamplingParams>),
|
||||
}
|
||||
|
||||
pub fn sampling_params_or_list(
|
||||
v: api::SamplingParamsOrList,
|
||||
) -> Result<Option<SamplingInput>, String> {
|
||||
use api::sampling_params_or_list::Value;
|
||||
Ok(match v.value {
|
||||
None => None,
|
||||
Some(Value::One(p)) => Some(SamplingInput::One(Box::new(p.try_into()?))),
|
||||
Some(Value::Many(l)) => Some(SamplingInput::Many(
|
||||
l.items
|
||||
.into_iter()
|
||||
.map(SamplingParams::try_from)
|
||||
.collect::<Result<_, _>>()?,
|
||||
)),
|
||||
})
|
||||
}
|
||||
|
||||
/// Launch-time preferred sampling params fill the keys a request did not
|
||||
/// send, at the JSON level, before the body is decoded: `{**preferred,
|
||||
/// **request}` per object, which is Python TokenizerManager's precedence. A
|
||||
/// request key wins even when it carries the type's default or null, which
|
||||
/// a decoded struct can no longer tell apart from an absent key.
|
||||
pub fn merge_preferred_sampling(
|
||||
body: &mut serde_json::Value,
|
||||
preferred: &serde_json::Value,
|
||||
) -> Result<(), String> {
|
||||
let preferred = preferred
|
||||
.as_object()
|
||||
.ok_or_else(|| "preferred_sampling_params must be a JSON object".to_string())?;
|
||||
let Some(body) = body.as_object_mut() else {
|
||||
return Ok(()); // not an object: the decoder reports it
|
||||
};
|
||||
let merge_into = |request: &mut serde_json::Value| {
|
||||
if let Some(obj) = request.as_object_mut() {
|
||||
for (k, v) in preferred {
|
||||
obj.entry(k.clone()).or_insert_with(|| v.clone());
|
||||
}
|
||||
}
|
||||
};
|
||||
match body.get_mut("sampling_params") {
|
||||
None | Some(serde_json::Value::Null) => {
|
||||
body.insert(
|
||||
"sampling_params".into(),
|
||||
serde_json::Value::Object(preferred.clone()),
|
||||
);
|
||||
}
|
||||
Some(serde_json::Value::Array(items)) => items.iter_mut().for_each(merge_into),
|
||||
Some(one) => merge_into(one),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The same fill for a request that arrived already decoded (gRPC): the
|
||||
/// carrier round-trips through its JSON form so protobuf and HTTP share one
|
||||
/// precedence rule. Protobuf has no null, so an unset field is an absent key
|
||||
/// (the schema's null-emitting fields are stripped before the merge), never
|
||||
/// the explicit null a JSON client can send.
|
||||
pub fn fill_preferred_sampling(
|
||||
params: Option<api::SamplingParamsOrList>,
|
||||
preferred: &serde_json::Value,
|
||||
) -> Result<Option<api::SamplingParamsOrList>, String> {
|
||||
let mut body = serde_json::json!({ "sampling_params": params });
|
||||
let strip_nulls = |object: &mut serde_json::Value| {
|
||||
if let Some(map) = object.as_object_mut() {
|
||||
map.retain(|_, value| !value.is_null());
|
||||
}
|
||||
};
|
||||
match &mut body["sampling_params"] {
|
||||
serde_json::Value::Array(items) => items.iter_mut().for_each(strip_nulls),
|
||||
one => strip_nulls(one),
|
||||
}
|
||||
merge_preferred_sampling(&mut body, preferred)?;
|
||||
serde_json::from_value(body["sampling_params"].take()).map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn decode(body: &str, preferred: serde_json::Value) -> Vec<SamplingParams> {
|
||||
let mut value: serde_json::Value = serde_json::from_str(body).unwrap();
|
||||
merge_preferred_sampling(&mut value, &preferred).unwrap();
|
||||
let req: api::GenerateRequest = serde_json::from_value(value).unwrap();
|
||||
match sampling_params_or_list(req.sampling_params.unwrap())
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
{
|
||||
SamplingInput::One(p) => vec![*p],
|
||||
SamplingInput::Many(v) => v,
|
||||
}
|
||||
}
|
||||
|
||||
/// A request key wins over the preferred value, even as an explicit
|
||||
/// default or null; only omitted keys take the preferred value.
|
||||
#[test]
|
||||
fn preferred_params_fill_only_omitted_request_fields() {
|
||||
let preferred =
|
||||
serde_json::json!({"temperature": 0.25, "top_p": 0.75, "max_new_tokens": 4096});
|
||||
let p = &decode(
|
||||
r#"{"sampling_params": {"temperature": 1.0, "top_p": null}}"#,
|
||||
preferred.clone(),
|
||||
)[0];
|
||||
assert_eq!(p.temperature, 1.0, "explicit default wins");
|
||||
assert_eq!(p.top_p, 1.0, "explicit null keeps the type default");
|
||||
assert_eq!(p.max_new_tokens, Some(4096), "omitted uses preferred");
|
||||
let p = &decode(r#"{"text": "hi"}"#, preferred)[0];
|
||||
assert_eq!(
|
||||
(p.temperature, p.max_new_tokens),
|
||||
(0.25, Some(4096)),
|
||||
"absent object takes all"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preferred_params_apply_to_every_batched_object() {
|
||||
let preferred = serde_json::json!({"temperature": 0.25, "top_p": 0.75});
|
||||
let ps = decode(
|
||||
r#"{"sampling_params": [{"temperature": 0.5}, {"top_p": 0.9}]}"#,
|
||||
preferred,
|
||||
);
|
||||
assert_eq!((ps[0].temperature, ps[0].top_p), (0.5, 0.75));
|
||||
assert_eq!((ps[1].temperature, ps[1].top_p), (0.25, 0.9));
|
||||
}
|
||||
|
||||
/// The generated `TokenIdsOrList` and the crate's own `OneOrMany<TokenIds>`
|
||||
/// accept and reject the same shapes (the first-element rule), so the
|
||||
/// swap to the wire type changed no input_ids contract.
|
||||
#[test]
|
||||
fn token_ids_decoding_matches_one_or_many() {
|
||||
for value in [
|
||||
"null",
|
||||
"[]",
|
||||
"[0]",
|
||||
"[-2147483648,2147483647]",
|
||||
"[[]]",
|
||||
"[[],[1,-2]]",
|
||||
"[[1,2],[3]]",
|
||||
"1",
|
||||
"true",
|
||||
"{}",
|
||||
r#""1""#,
|
||||
"[null]",
|
||||
"[true]",
|
||||
r#"["1"]"#,
|
||||
"[1.0]",
|
||||
"[-0]",
|
||||
"[2147483648]",
|
||||
"[0,2147483648]",
|
||||
"[1,[2]]",
|
||||
"[[1],2]",
|
||||
"[[[1]]]",
|
||||
"[[1.0]]",
|
||||
] {
|
||||
let expected = serde_json::from_str::<Option<OneOrMany<TokenIds>>>(value);
|
||||
let actual = serde_json::from_str::<Option<api::TokenIdsOrList>>(value)
|
||||
.map(|v| v.and_then(token_ids_or_list));
|
||||
match (expected, actual) {
|
||||
(Ok(e), Ok(a)) => assert_eq!(a, e, "{value}"),
|
||||
(Err(_), Err(_)) => {}
|
||||
(e, a) => panic!("{value}: expected {e:?}, got {a:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn media_input_converts_every_container_form() {
|
||||
let conv = |json: &str| -> MmDataInput {
|
||||
media_input(serde_json::from_str::<api::MediaInput>(json).unwrap()).unwrap()
|
||||
};
|
||||
let src = |s: &str| MmItem::Source(s.to_owned());
|
||||
assert_eq!(conv(r#""u""#), MmDataInput::One(src("u")));
|
||||
assert_eq!(
|
||||
conv(r#"{"url": "u", "detail": "high"}"#),
|
||||
MmDataInput::One(MmItem::Ref { url: "u".into() })
|
||||
);
|
||||
assert_eq!(
|
||||
conv(r#"["a", null, "b"]"#),
|
||||
MmDataInput::Many(vec![Some(src("a")), None, Some(src("b"))])
|
||||
);
|
||||
assert_eq!(conv("[]"), MmDataInput::Many(vec![]));
|
||||
assert_eq!(
|
||||
conv(r#"[["a", null], null, []]"#),
|
||||
MmDataInput::Nested(vec![Some(vec![Some(src("a")), None]), None, Some(vec![])])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn custom_params_cross_as_json_text() {
|
||||
let p: api::SamplingParams =
|
||||
serde_json::from_str(r#"{"custom_params": {"k": [1, "x"]}, "logit_bias": {"5": 1.5}}"#)
|
||||
.unwrap();
|
||||
let p = SamplingParams::try_from(p).unwrap();
|
||||
assert!(
|
||||
p.custom_params
|
||||
.as_ref()
|
||||
.is_some_and(|m| m.contains_key("k"))
|
||||
);
|
||||
assert_eq!(
|
||||
p.logit_bias.as_ref().and_then(|m| m.get("5")).copied(),
|
||||
Some(1.5)
|
||||
);
|
||||
assert_eq!(p.temperature, 1.0, "schema default applied by the decoder");
|
||||
}
|
||||
|
||||
/// The protobuf entry fills the same keys the JSON entry does: a set field
|
||||
/// wins, an unset one takes the preferred value, and an absent carrier
|
||||
/// takes them all.
|
||||
#[test]
|
||||
fn preferred_params_fill_unset_protobuf_fields() {
|
||||
use api::sampling_params_or_list::Value;
|
||||
let preferred = serde_json::json!({"temperature": 0.25, "max_new_tokens": 4096});
|
||||
let sent = api::SamplingParamsOrList {
|
||||
value: Some(Value::One(api::SamplingParams {
|
||||
temperature: Some(1.0),
|
||||
..Default::default()
|
||||
})),
|
||||
};
|
||||
let filled = fill_preferred_sampling(Some(sent), &preferred)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let Some(Value::One(p)) = filled.value else {
|
||||
panic!("one object in, one object out");
|
||||
};
|
||||
assert_eq!((p.temperature, p.max_new_tokens), (Some(1.0), Some(4096)));
|
||||
|
||||
let filled = fill_preferred_sampling(None, &preferred).unwrap().unwrap();
|
||||
let Some(Value::One(p)) = filled.value else {
|
||||
panic!("an absent carrier becomes the preferred object");
|
||||
};
|
||||
assert_eq!((p.temperature, p.max_new_tokens), (Some(0.25), Some(4096)));
|
||||
}
|
||||
}
|
||||
@@ -10,7 +10,7 @@ use sglang_mm::common::fetch::{ByteBudget, fetch_bytes_budgeted};
|
||||
use sglang_mm::driver::{ImageSource, MmInput};
|
||||
|
||||
use crate::message::multimodal::MmItem;
|
||||
use crate::message::request::{MmData, ProcessorExtensions};
|
||||
use crate::message::request::MmData;
|
||||
use crate::message::types::TokenIds;
|
||||
|
||||
/// Fully resolved media for a multimodal processor. I/O sources were
|
||||
@@ -23,9 +23,6 @@ pub struct ResolvedMediaWork {
|
||||
pub images: Vec<Bytes>,
|
||||
pub videos: Vec<Bytes>,
|
||||
pub audios: Vec<Bytes>,
|
||||
/// Request fields owned by the selected processor rather than this shared
|
||||
/// payload layer.
|
||||
pub processor_extensions: ProcessorExtensions,
|
||||
}
|
||||
|
||||
/// Resolve all modality fields in the fixed image/video/audio prefetch order.
|
||||
@@ -42,7 +39,6 @@ fn resolve_media_work_with_budget(
|
||||
image_data,
|
||||
video_data,
|
||||
audio_data,
|
||||
processor_extensions,
|
||||
prefetched,
|
||||
mm_hashes: _,
|
||||
} = mm;
|
||||
@@ -59,7 +55,6 @@ fn resolve_media_work_with_budget(
|
||||
images,
|
||||
videos,
|
||||
audios,
|
||||
processor_extensions,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -83,16 +78,13 @@ fn collect_media(
|
||||
fetch_bytes_budgeted(&source, budget).map(Bytes::from)
|
||||
}
|
||||
}
|
||||
MmItem::Preprocessed { format } => Err(format!(
|
||||
"unsupported {field} item: preprocessed `{format}` input"
|
||||
)),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// True for sources the API layer must resolve before MM dispatch: I/O — network
|
||||
/// *or* disk, since a network mount can hang past any HTTP timeout — never runs
|
||||
/// on the fixed MM worker pool (see `frontend::prefetch`). `data:` and bare
|
||||
/// on the fixed MM worker pool (see `core::prefetch`). `data:` and bare
|
||||
/// base64 are pure CPU and stay on the worker. Lives next to [`image_source`]
|
||||
/// so the prefetch walk and the parse walk cannot drift.
|
||||
pub fn is_io_source(src: &str) -> bool {
|
||||
@@ -120,16 +112,12 @@ pub fn to_mm_input(input_ids: TokenIds, mm: MmData) -> Result<MmInput, String> {
|
||||
image_data,
|
||||
video_data,
|
||||
audio_data,
|
||||
processor_extensions,
|
||||
prefetched,
|
||||
mm_hashes: _,
|
||||
} = mm;
|
||||
if !video_data.is_empty() || !audio_data.is_empty() {
|
||||
return Err("unsupported modality: video/audio input".into());
|
||||
}
|
||||
if !processor_extensions.is_empty() {
|
||||
return Err("unsupported generate extensions for this processor".into());
|
||||
}
|
||||
let mut prefetched = prefetched.iter();
|
||||
let images = image_data
|
||||
.into_iter()
|
||||
@@ -155,9 +143,6 @@ fn image_source(
|
||||
.map(|bytes| ImageSource::Bytes(bytes.to_vec()))
|
||||
.ok_or_else(|| "I/O-backed image source was not prefetched".to_string())
|
||||
}
|
||||
MmItem::Preprocessed { format } => Err(format!(
|
||||
"unsupported image_data item: preprocessed `{format}` input"
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -207,31 +192,6 @@ mod tests {
|
||||
.unwrap()
|
||||
.contains("video/audio")
|
||||
);
|
||||
|
||||
let err = to_mm_input(
|
||||
IDS.to_vec(),
|
||||
image_work(vec![MmItem::Preprocessed {
|
||||
format: "processor_output".into(),
|
||||
}]),
|
||||
)
|
||||
.err()
|
||||
.unwrap();
|
||||
assert!(err.contains("preprocessed `processor_output`"), "{err}");
|
||||
|
||||
let extension = MmData {
|
||||
processor_extensions: std::iter::once((
|
||||
"multimodal_custom".to_owned(),
|
||||
rmpv::Value::Boolean(true),
|
||||
))
|
||||
.collect(),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(
|
||||
to_mm_input(IDS.to_vec(), extension)
|
||||
.err()
|
||||
.unwrap()
|
||||
.contains("unsupported generate extensions")
|
||||
);
|
||||
}
|
||||
|
||||
/// I/O-backed sources (URLs, file paths) take their prefetched bytes in walk
|
||||
|
||||
@@ -1,160 +0,0 @@
|
||||
//! Shared wire semantics for SGLang-native generation responses.
|
||||
//!
|
||||
//! Native HTTP places these values in a JSON `meta_info` object, while
|
||||
//! `runtime.v1` gRPC JSON-encodes each value into `map<string, string>`. Keeping
|
||||
//! the logical shape here prevents the sibling adapters from drifting without
|
||||
//! making either adapter depend on the other.
|
||||
|
||||
use crate::frontend::FrontendOutput;
|
||||
|
||||
/// The text slot of a `[logprob, token_id, text]` tuple: the decoded token when
|
||||
/// `return_text_in_logprobs` supplied a text buffer, else `null`.
|
||||
pub(crate) fn text_slot(texts: Option<&[String]>, j: usize) -> serde_json::Value {
|
||||
texts
|
||||
.and_then(|texts| texts.get(j))
|
||||
.map(|text| serde_json::json!(text))
|
||||
.unwrap_or(serde_json::Value::Null)
|
||||
}
|
||||
|
||||
/// A decoded-text column becomes the tuples' text source only when populated.
|
||||
pub(crate) fn opt_texts(texts: &[String]) -> Option<&[String]> {
|
||||
(!texts.is_empty()).then_some(texts)
|
||||
}
|
||||
|
||||
/// The logprob slot of a tuple: a finite value, or `null` for the `NaN` sentinel.
|
||||
pub(crate) fn lp_value(value: f32) -> serde_json::Value {
|
||||
if value.is_nan() {
|
||||
serde_json::Value::Null
|
||||
} else {
|
||||
serde_json::json!(value)
|
||||
}
|
||||
}
|
||||
|
||||
/// SGLang logprob shape: a list of `[logprob, token_id, text]` tuples.
|
||||
pub(crate) fn logprob_tuples(
|
||||
values: &[f32],
|
||||
token_ids: &[i32],
|
||||
texts: Option<&[String]>,
|
||||
) -> serde_json::Value {
|
||||
let tuples = values
|
||||
.iter()
|
||||
.zip(token_ids.iter())
|
||||
.enumerate()
|
||||
.map(|(index, (&value, &token_id))| {
|
||||
serde_json::json!([lp_value(value), token_id, text_slot(texts, index)])
|
||||
})
|
||||
.collect();
|
||||
serde_json::Value::Array(tuples)
|
||||
}
|
||||
|
||||
/// Ragged top-k / requested-token shape: one entry per position, or `null`
|
||||
/// where that position's length is zero.
|
||||
pub(crate) fn ragged_logprob_tuples(
|
||||
values: &[f32],
|
||||
token_ids: &[i32],
|
||||
lengths: &[u32],
|
||||
texts: Option<&[String]>,
|
||||
) -> serde_json::Value {
|
||||
let mut positions = Vec::with_capacity(lengths.len());
|
||||
let mut offset = 0usize;
|
||||
for &length in lengths {
|
||||
let length = length as usize;
|
||||
if length == 0 {
|
||||
positions.push(serde_json::Value::Null);
|
||||
} else {
|
||||
// Bounds checks turn malformed parallel columns into a shortened
|
||||
// row instead of panicking a transport worker.
|
||||
let tuples = (offset..offset + length)
|
||||
.filter_map(|index| {
|
||||
Some(serde_json::json!([
|
||||
lp_value(*values.get(index)?),
|
||||
*token_ids.get(index)?,
|
||||
text_slot(texts, index)
|
||||
]))
|
||||
})
|
||||
.collect();
|
||||
positions.push(serde_json::Value::Array(tuples));
|
||||
}
|
||||
offset += length;
|
||||
}
|
||||
serde_json::Value::Array(positions)
|
||||
}
|
||||
|
||||
/// Reshape flat hidden-state values and per-row lengths into nested rows.
|
||||
pub(crate) fn hidden_states_rows(values: &[f32], lengths: &[u32]) -> serde_json::Value {
|
||||
let mut rows = Vec::with_capacity(lengths.len());
|
||||
let mut offset = 0usize;
|
||||
for &length in lengths {
|
||||
let length = length as usize;
|
||||
rows.push(serde_json::json!(
|
||||
values.get(offset..offset + length).unwrap_or(&[])
|
||||
));
|
||||
offset += length;
|
||||
}
|
||||
serde_json::Value::Array(rows)
|
||||
}
|
||||
|
||||
/// Build the heterogeneous native `meta_info` object before either adapter
|
||||
/// chooses its wire encoding.
|
||||
pub(crate) fn meta_info_value(out: &FrontendOutput, rid: &str) -> serde_json::Value {
|
||||
let mut meta = serde_json::json!({
|
||||
"id": rid,
|
||||
"prompt_tokens": out.prompt_tokens,
|
||||
"completion_tokens": out.completion_tokens,
|
||||
"finish_reason": out.finish_reason,
|
||||
});
|
||||
let Some(extras) = out.extras.as_deref() else {
|
||||
return meta;
|
||||
};
|
||||
|
||||
// Python always sets input and output token logprobs together, including
|
||||
// the empty input list emitted by a disaggregated decode node.
|
||||
if !extras.out_lp_val.is_empty() || !extras.in_lp_val.is_empty() {
|
||||
meta["output_token_logprobs"] = logprob_tuples(
|
||||
&extras.out_lp_val,
|
||||
&extras.out_lp_idx,
|
||||
opt_texts(&extras.out_lp_txt),
|
||||
);
|
||||
meta["input_token_logprobs"] = logprob_tuples(
|
||||
&extras.in_lp_val,
|
||||
&extras.in_lp_idx,
|
||||
opt_texts(&extras.in_lp_txt),
|
||||
);
|
||||
}
|
||||
if !extras.out_top_lens.is_empty() {
|
||||
meta["output_top_logprobs"] = ragged_logprob_tuples(
|
||||
&extras.out_top_val,
|
||||
&extras.out_top_idx,
|
||||
&extras.out_top_lens,
|
||||
opt_texts(&extras.out_top_txt),
|
||||
);
|
||||
}
|
||||
if !extras.in_top_lens.is_empty() {
|
||||
meta["input_top_logprobs"] = ragged_logprob_tuples(
|
||||
&extras.in_top_val,
|
||||
&extras.in_top_idx,
|
||||
&extras.in_top_lens,
|
||||
opt_texts(&extras.in_top_txt),
|
||||
);
|
||||
}
|
||||
if !extras.out_tid_lens.is_empty() {
|
||||
meta["output_token_ids_logprobs"] = ragged_logprob_tuples(
|
||||
&extras.out_tid_val,
|
||||
&extras.out_tid_idx,
|
||||
&extras.out_tid_lens,
|
||||
opt_texts(&extras.out_tid_txt),
|
||||
);
|
||||
}
|
||||
if !extras.in_tid_lens.is_empty() {
|
||||
meta["input_token_ids_logprobs"] = ragged_logprob_tuples(
|
||||
&extras.in_tid_val,
|
||||
&extras.in_tid_idx,
|
||||
&extras.in_tid_lens,
|
||||
opt_texts(&extras.in_tid_txt),
|
||||
);
|
||||
}
|
||||
if !extras.hidden_lens.is_empty() {
|
||||
meta["hidden_states"] = hidden_states_rows(&extras.hidden_val, &extras.hidden_lens);
|
||||
}
|
||||
meta
|
||||
}
|
||||
@@ -184,7 +184,7 @@ impl Runnable for DetokenizerWorker {
|
||||
|
||||
// Plain `recv`: exits when the `DetokMsg` channel closes (every `Senders`
|
||||
// clone gone). On shutdown that happens once to-scheduler/from-scheduler
|
||||
// exit — frontend calls own only their abort sender, not detok channels.
|
||||
// exit — core calls own only their abort sender, not detok channels.
|
||||
while let Ok(msg) = self.rx.recv() {
|
||||
match msg {
|
||||
DetokMsg::Register {
|
||||
@@ -395,7 +395,7 @@ fn handle_chunk(
|
||||
}
|
||||
let _ = st.fsm.apply(Event::Disconnect);
|
||||
// Abort ONLY when the sink is full. `Closed` means the handler future is
|
||||
// already gone, so its `FrontendCall` has queued the abort; do not emit
|
||||
// already gone, so its `CoreCall` has queued the abort; do not emit
|
||||
// the same lifecycle event a second time. A full sink still has a live
|
||||
// handler, so the detokenizer must initiate cleanup itself.
|
||||
if matches!(e, SinkError::Full) {
|
||||
|
||||
@@ -102,7 +102,7 @@ fn ids_len(req: &SchedulerRequest) -> usize {
|
||||
/// further chunk can be delivered, and tell the scheduler to stop generating.
|
||||
///
|
||||
/// Neither releases anything, and nothing needs them to. Release ordering used
|
||||
/// to be the delicate part here — `FrontendCall::drop` releasing a rid right
|
||||
/// to be the delicate part here — `CoreCall::drop` releasing a rid right
|
||||
/// after enqueuing the abort ordered the SEND, not the EFFECT, so a retry of
|
||||
/// the same rid could `Register` ahead of the stale abort and be torn down by
|
||||
/// it. `Rid::from_client` removes the premise: a retry carries a different
|
||||
|
||||
@@ -45,12 +45,12 @@ const ADMISSION_PENDING: u8 = 0;
|
||||
const ADMISSION_ACCEPTED: u8 = 1;
|
||||
const ADMISSION_CANCELLED: u8 = 2;
|
||||
|
||||
/// Cancellation-safe ownership handoff between a frontend call and Intake.
|
||||
/// Cancellation-safe ownership handoff between a core call and Intake.
|
||||
///
|
||||
/// A bounded `send_async` can enqueue a request and wake its sender without the
|
||||
/// sender being polled again. If that task is then cancelled, this token decides
|
||||
/// exactly one outcome: either Intake observes the prior cancellation and drops
|
||||
/// the request, or the frontend observes prior admission and emits an abort.
|
||||
/// the request, or the core observes prior admission and emits an abort.
|
||||
#[derive(Clone)]
|
||||
pub struct RequestAdmission {
|
||||
state: Arc<AtomicU8>,
|
||||
@@ -91,7 +91,7 @@ impl RequestAdmission {
|
||||
/// cannot be tangled up with an abort still in flight for the original.
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum AbortSource {
|
||||
/// From an in-flight frontend call drop. Owns the release.
|
||||
/// From an in-flight core call drop. Owns the release.
|
||||
Guard(Rid),
|
||||
/// From a detokenizer terminal path. Aborts the scheduler work.
|
||||
Detok(Rid),
|
||||
|
||||
@@ -301,16 +301,16 @@ pub fn start(cfg: RuntimeConfig) -> Result<Runtime, String> {
|
||||
|
||||
// One transport-neutral entrance to the runtime. Each configured listener
|
||||
// receives a clone; no listener owns or reconstructs scheduler wiring.
|
||||
let frontend = crate::frontend::FrontendHandle::new(
|
||||
let core = crate::api_server::core::CoreHandle::new(
|
||||
senders.tok_manager_tx.clone(),
|
||||
senders.abort_tx.clone(),
|
||||
crate::frontend::FrontendConfig {
|
||||
crate::api_server::core::CoreConfig {
|
||||
response_capacity: cfg.rust_server_args.stage_channel_cap,
|
||||
response_activity: response_activity.clone(),
|
||||
startup_ready: cfg.server_args.skip_server_warmup,
|
||||
is_disaggregation: cfg.server_args.is_disaggregation(),
|
||||
mm_limits: cfg.server_args.limit_mm_data_per_request.clone(),
|
||||
metadata: crate::frontend::FrontendMetadata::from(cfg.server_args.as_ref()),
|
||||
metadata: crate::api_server::core::CoreMetadata::from(cfg.server_args.as_ref()),
|
||||
},
|
||||
);
|
||||
|
||||
@@ -320,7 +320,8 @@ pub fn start(cfg: RuntimeConfig) -> Result<Runtime, String> {
|
||||
let api_cores = plan.as_ref().map(|p| p.api.clone());
|
||||
let shutdown_rx = shutdown_rx.clone();
|
||||
let grpc_server = grpc_listener.map(|listener| {
|
||||
let service = crate::grpc::GrpcService::new(frontend.clone(), &cfg.server_args);
|
||||
let service =
|
||||
api_server::grpc::service::GrpcService::new(core.clone(), &cfg.server_args);
|
||||
(listener, service)
|
||||
});
|
||||
let handle = std::thread::Builder::new()
|
||||
@@ -341,14 +342,17 @@ pub fn start(cfg: RuntimeConfig) -> Result<Runtime, String> {
|
||||
}
|
||||
let rt = builder.build().expect("build api runtime");
|
||||
rt.block_on(async move {
|
||||
let http = api_server::app::serve(
|
||||
let http = api_server::http::app::serve(
|
||||
http_listener,
|
||||
frontend,
|
||||
core,
|
||||
cfg.server_args.clone(),
|
||||
shutdown_rx.clone(),
|
||||
);
|
||||
if let Some((listener, service)) = grpc_server {
|
||||
tokio::join!(http, crate::grpc::serve(listener, service, shutdown_rx));
|
||||
tokio::join!(
|
||||
http,
|
||||
api_server::grpc::app::serve(listener, service, shutdown_rx)
|
||||
);
|
||||
} else {
|
||||
http.await;
|
||||
}
|
||||
@@ -375,8 +379,8 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::message::config::{RuntimeConfig, RustServerServerArgs, ServerArgs};
|
||||
use crate::message::response::{BatchHeader, frame_decode_batch_cols};
|
||||
use sglang_grpc_types::sglang::runtime::v1 as proto;
|
||||
use sglang_grpc_types::sglang::runtime::v1::sglang_service_client::SglangServiceClient;
|
||||
use sglang_api_types::api::v1 as proto;
|
||||
use sglang_api_types::api::v1::sglang_service_client::SglangServiceClient;
|
||||
|
||||
fn free_loopback_addrs() -> (std::net::SocketAddr, std::net::SocketAddr) {
|
||||
// Hold both probes at once so the OS cannot return the same ephemeral
|
||||
@@ -412,6 +416,21 @@ mod tests {
|
||||
SglangServiceClient::new(endpoint.connect().await.expect("connect gRPC client"))
|
||||
}
|
||||
|
||||
fn grpc_ids_request(ids: &[i64], rid: &str) -> proto::GenerateRequest {
|
||||
proto::GenerateRequest {
|
||||
input_ids: Some(proto::TokenIdsOrList {
|
||||
value: Some(proto::token_ids_or_list::Value::One(proto::TokenIds {
|
||||
ids: ids.to_vec(),
|
||||
})),
|
||||
}),
|
||||
rid: Some(proto::StringOrList {
|
||||
value: Some(proto::string_or_list::Value::One(rid.into())),
|
||||
}),
|
||||
stream: Some(true),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn scheduler_rid(header: &[u8]) -> String {
|
||||
let value = rmpv::decode::read_value(&mut std::io::Cursor::new(header)).unwrap();
|
||||
value
|
||||
@@ -643,9 +662,9 @@ mod tests {
|
||||
drop(rebound);
|
||||
}
|
||||
|
||||
/// End to end over the configured socket: the generated runtime.v1 client
|
||||
/// End to end over the configured socket: the generated api.v1 client
|
||||
/// reaches the existing scheduler ring, and an existing scheduler frame is
|
||||
/// translated back into a protobuf response. Shutdown then closes both
|
||||
/// translated back into a protobuf frame. Shutdown then closes both
|
||||
/// transport listeners.
|
||||
#[test]
|
||||
fn configured_grpc_serves_generation_and_closes_with_http() {
|
||||
@@ -663,12 +682,7 @@ mod tests {
|
||||
let mut client = connect_grpc(grpc_addr).await;
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
client.generate(proto::GenerateRequest {
|
||||
input_ids: vec![1, 2, 3],
|
||||
stream: Some(true),
|
||||
rid: Some("network-generation".into()),
|
||||
..Default::default()
|
||||
}),
|
||||
client.generate(grpc_ids_request(&[1, 2, 3], "network-generation")),
|
||||
)
|
||||
.await
|
||||
.expect("timed out starting Generate RPC")
|
||||
@@ -694,25 +708,29 @@ mod tests {
|
||||
.push(terminal_token_frame(rid, &[9, 10]))
|
||||
);
|
||||
|
||||
let response = client_runtime
|
||||
let item = client_runtime
|
||||
.block_on(async {
|
||||
tokio::time::timeout(Duration::from_secs(2), stream.message()).await
|
||||
})
|
||||
.expect("timed out waiting for gRPC response")
|
||||
.expect("gRPC stream error")
|
||||
.expect("gRPC stream ended without a response");
|
||||
assert_eq!(response.output_ids, vec![9, 10]);
|
||||
assert!(response.finished);
|
||||
assert_eq!(response.meta_info["id"], r#""network-generation""#);
|
||||
let Some(proto::generate_stream_item::Item::Frame(response)) = item.item else {
|
||||
panic!("expected a frame, got {item:?}");
|
||||
};
|
||||
assert_eq!(response.output_ids.map(|ids| ids.ids), Some(vec![9, 10]));
|
||||
let meta_info = response.meta_info.expect("frame meta_info");
|
||||
assert_eq!(meta_info.id, "network-generation");
|
||||
assert!(meta_info.finish_reason.is_some());
|
||||
|
||||
rt.request_shutdown();
|
||||
assert!(std::net::TcpStream::connect(http_addr).is_err());
|
||||
assert!(std::net::TcpStream::connect(grpc_addr).is_err());
|
||||
}
|
||||
|
||||
/// The existing runtime.v1 listener accepts bodies above Tonic's 4 MiB
|
||||
/// default. Keep that transport policy when mounting the same protocol on
|
||||
/// the Rust frontend; reaching adapter validation proves the body decoded.
|
||||
/// The Python-backed gRPC listener accepts bodies above Tonic's 4 MiB
|
||||
/// default. Keep that transport policy on the Rust frontend; reaching
|
||||
/// adapter validation proves the body decoded.
|
||||
#[test]
|
||||
fn grpc_listener_preserves_existing_large_message_limit() {
|
||||
use tonic::Code;
|
||||
@@ -729,19 +747,23 @@ mod tests {
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_secs(10),
|
||||
client.generate(proto::GenerateRequest {
|
||||
input_ids: vec![1],
|
||||
// This field is intentionally unsupported. An
|
||||
// UNIMPLEMENTED status proves the >4 MiB protobuf reached
|
||||
// the adapter instead of Tonic rejecting it on size.
|
||||
routing_key: Some("x".repeat(5 * 1024 * 1024)),
|
||||
..Default::default()
|
||||
// `text` and `input_ids` together is a normalization
|
||||
// error: an INVALID_ARGUMENT status proves the >4 MiB
|
||||
// protobuf reached the adapter instead of Tonic rejecting
|
||||
// it on size.
|
||||
text: Some(proto::StringOrList {
|
||||
value: Some(proto::string_or_list::Value::One(
|
||||
"x".repeat(5 * 1024 * 1024),
|
||||
)),
|
||||
}),
|
||||
..grpc_ids_request(&[1], "large")
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.expect("timed out sending large Generate RPC")
|
||||
.expect_err("unsupported routing_key must fail")
|
||||
.expect_err("text and input_ids together must fail")
|
||||
});
|
||||
assert_eq!(error.code(), Code::Unimplemented);
|
||||
assert_eq!(error.code(), Code::InvalidArgument);
|
||||
|
||||
rt.request_shutdown();
|
||||
assert!(std::net::TcpStream::connect(grpc_addr).is_err());
|
||||
@@ -765,12 +787,7 @@ mod tests {
|
||||
let mut client = connect_grpc(grpc_addr).await;
|
||||
let stream = tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
client.generate(proto::GenerateRequest {
|
||||
input_ids: vec![1, 2, 3],
|
||||
stream: Some(true),
|
||||
rid: Some("in-flight".into()),
|
||||
..Default::default()
|
||||
}),
|
||||
client.generate(grpc_ids_request(&[1, 2, 3], "in-flight")),
|
||||
)
|
||||
.await
|
||||
.expect("timed out starting Generate RPC")
|
||||
|
||||
@@ -11,7 +11,7 @@ from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
BUILD_AND_RUN_TIMEOUT_S = 900
|
||||
RUST_WORKSPACE = Path(__file__).resolve().parents[3] / "rust"
|
||||
GRPC_TYPES_PACKAGE = "sglang-grpc-types"
|
||||
GRPC_TYPES_PACKAGE = "sglang-api-types"
|
||||
register_cpu_ci(est_time=170, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
|
||||
@@ -590,21 +590,23 @@ crate-type = ["cdylib"]
|
||||
self.assertNotIn(module_name, sys.modules)
|
||||
|
||||
def test_checked_in_crates_are_discovered_from_wheel_metadata(self):
|
||||
grpc_proto = (rust_extension._RUST_WORKSPACE.parent / "proto").resolve()
|
||||
# No crate reads `proto/` at build time: the generated bindings are
|
||||
# checked in under rust/sglang-api-types (`cargo gen-api`), so the
|
||||
# workspace is every extension's only source input.
|
||||
for python_module, package, library, features, source_inputs in (
|
||||
(
|
||||
"sglang.srt.rust_extensions._server",
|
||||
"sglang-server",
|
||||
"sglang_server",
|
||||
(),
|
||||
(grpc_proto,),
|
||||
(),
|
||||
),
|
||||
(
|
||||
"sglang.srt.rust_extensions._grpc",
|
||||
"sglang-grpc",
|
||||
"sglang_grpc_core",
|
||||
(),
|
||||
(grpc_proto,),
|
||||
(),
|
||||
),
|
||||
(
|
||||
"sglang.srt.rust_extensions._multimodal",
|
||||
|
||||
@@ -0,0 +1,325 @@
|
||||
"""JSON-contract tests for the generated Python API types (sglang.api.v1).
|
||||
|
||||
Mirrors rust/sglang-api-types/tests/json_contract.rs case for case (minus the
|
||||
protobuf-arrival accessors, which have no Python counterpart), so the Rust and
|
||||
Python emitters of sglang-api-codegen are held to one contract.
|
||||
"""
|
||||
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
import json
|
||||
import unittest
|
||||
|
||||
from sglang.api.v1.api_types import (
|
||||
FinishAbort,
|
||||
FinishLength,
|
||||
FinishStop,
|
||||
GenerateMetaInfo,
|
||||
GenerateRequest,
|
||||
GenerateResponse,
|
||||
JsonContractError,
|
||||
LogprobEntry,
|
||||
MediaRef,
|
||||
SamplingParams,
|
||||
decode_FinishReason,
|
||||
decode_MediaInput,
|
||||
decode_OptionalInt64OrList,
|
||||
decode_OptionalStringOrList,
|
||||
decode_SamplingParamsOrList,
|
||||
decode_StringListOrList,
|
||||
decode_StringOrList,
|
||||
decode_TokenIdsOrList,
|
||||
encode_FinishReason,
|
||||
encode_MediaInput,
|
||||
encode_OptionalStringOrList,
|
||||
encode_StringOrList,
|
||||
)
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
|
||||
def compact(value) -> str:
|
||||
"""serde_json::to_string parity: no whitespace."""
|
||||
return json.dumps(value, separators=(",", ":"))
|
||||
|
||||
|
||||
def sp(text: str) -> SamplingParams:
|
||||
return SamplingParams.from_json_value(json.loads(text))
|
||||
|
||||
|
||||
class TestSamplingParams(CustomTestCase):
|
||||
def test_defaults_match_hand_written(self):
|
||||
"""Absent keys yield the schema defaults."""
|
||||
p = sp("{}")
|
||||
self.assertEqual(p.max_new_tokens, 128)
|
||||
self.assertEqual(p.temperature, 1.0)
|
||||
self.assertEqual(p.top_p, 1.0)
|
||||
self.assertEqual(p.top_k, 1 << 30)
|
||||
self.assertEqual(p.min_p, 0.0)
|
||||
self.assertEqual(p.repetition_penalty, 1.0)
|
||||
self.assertEqual(p.min_new_tokens, 0)
|
||||
self.assertEqual(p.n, 1)
|
||||
self.assertIs(p.ignore_eos, False)
|
||||
self.assertIs(p.skip_special_tokens, True)
|
||||
self.assertIs(p.spaces_between_special_tokens, True)
|
||||
self.assertIs(p.no_stop_trim, False)
|
||||
|
||||
def test_constructor_defaults_match_absent_keys(self):
|
||||
"""A dataclass built by hand carries the same schema defaults."""
|
||||
self.assertEqual(SamplingParams(), sp("{}"))
|
||||
|
||||
def test_null_resets_default(self):
|
||||
"""null_resets_default: an explicit null is the default, not an error."""
|
||||
p = sp('{"temperature": null, "top_k": null, "ignore_eos": null}')
|
||||
self.assertEqual(p.temperature, 1.0)
|
||||
self.assertEqual(p.top_k, 1 << 30)
|
||||
self.assertIs(p.ignore_eos, False)
|
||||
|
||||
def test_max_new_tokens_absent_vs_null(self):
|
||||
"""null_is_none: absent = 128, null = unbounded."""
|
||||
self.assertEqual(sp("{}").max_new_tokens, 128)
|
||||
self.assertIsNone(sp('{"max_new_tokens": null}').max_new_tokens)
|
||||
self.assertEqual(sp('{"max_new_tokens": 7}').max_new_tokens, 7)
|
||||
|
||||
def test_denies_unknown_keys(self):
|
||||
"""DENY policy with serde's own error text."""
|
||||
with self.assertRaises(JsonContractError) as ctx:
|
||||
sp('{"temperatur": 0.5}') # codespell:ignore temperatur
|
||||
self.assertTrue(
|
||||
str(ctx.exception).startswith(
|
||||
"unknown field `temperatur`, expected one of" # codespell:ignore temperatur
|
||||
),
|
||||
str(ctx.exception),
|
||||
)
|
||||
|
||||
def test_type_mismatch_uses_serde_text(self):
|
||||
with self.assertRaises(JsonContractError) as ctx:
|
||||
sp('{"temperature": "hot"}')
|
||||
self.assertEqual(str(ctx.exception), 'invalid type: string "hot", expected f64')
|
||||
|
||||
def test_or_list_keeps_field_errors(self):
|
||||
"""one_or_many over SamplingParams keeps field-level error texts."""
|
||||
with self.assertRaises(JsonContractError) as ctx:
|
||||
decode_SamplingParamsOrList(
|
||||
{"temperatur": 1} # codespell:ignore temperatur
|
||||
)
|
||||
self.assertTrue(
|
||||
str(ctx.exception).startswith(
|
||||
"unknown field `temperatur`" # codespell:ignore temperatur
|
||||
),
|
||||
str(ctx.exception),
|
||||
)
|
||||
many = decode_SamplingParamsOrList([{"n": 2}, {}])
|
||||
self.assertEqual([p.n for p in many], [2, 1])
|
||||
|
||||
def test_raw_json_passthrough(self):
|
||||
"""raw_json: custom_params crosses untouched, integers included."""
|
||||
p = sp('{"custom_params": {"k": 3}}')
|
||||
self.assertEqual(p.custom_params, {"k": 3})
|
||||
self.assertEqual(p.to_json_value()["custom_params"], {"k": 3})
|
||||
|
||||
def test_int_key_map(self):
|
||||
p = sp('{"logit_bias": {"42": -1}}')
|
||||
self.assertEqual(p.logit_bias, {"42": -1.0})
|
||||
|
||||
|
||||
class TestGenerateRequest(CustomTestCase):
|
||||
def test_ignores_unknown_keys(self):
|
||||
"""IGNORE policy: unported Python fields must not 400."""
|
||||
req = GenerateRequest.from_json_value(
|
||||
{
|
||||
"text": "hi",
|
||||
"priority": 3,
|
||||
"session_id": "s",
|
||||
"custom_logit_processor": "x",
|
||||
}
|
||||
)
|
||||
self.assertEqual(req.text, "hi")
|
||||
|
||||
def test_media_input_shapes_dispatch(self):
|
||||
"""Typed media: the value's shape picks the container form, nulls are
|
||||
kept per element, and a ref object decodes to a MediaRef."""
|
||||
req = GenerateRequest.from_json_value(
|
||||
{"image_data": [["data:image/png;base64,xx"], None, []]}
|
||||
)
|
||||
self.assertEqual(req.image_data, [["data:image/png;base64,xx"], None, []])
|
||||
flat = decode_MediaInput(["a", None, {"url": "b", "detail": "high"}])
|
||||
self.assertEqual(flat[:2], ["a", None])
|
||||
self.assertIsInstance(flat[2], MediaRef)
|
||||
self.assertEqual((flat[2].url, flat[2].detail), ("b", "high"))
|
||||
self.assertEqual(decode_MediaInput("u"), "u")
|
||||
self.assertEqual(decode_MediaInput([]), [])
|
||||
for value in (
|
||||
["a", None, {"url": "b", "detail": "high"}],
|
||||
[["a", None], None, []],
|
||||
"u",
|
||||
):
|
||||
self.assertEqual(encode_MediaInput(decode_MediaInput(value)), value)
|
||||
|
||||
def test_media_ref_is_typed_and_strict(self):
|
||||
with self.assertRaises(JsonContractError) as ctx:
|
||||
decode_MediaInput(
|
||||
{"url": "u", "max_dynamic_patc": 6}
|
||||
) # codespell:ignore patc
|
||||
self.assertIn("unknown field", str(ctx.exception))
|
||||
with self.assertRaises(JsonContractError):
|
||||
decode_MediaInput([{"format": "processor_output"}])
|
||||
with self.assertRaises(JsonContractError):
|
||||
decode_MediaInput(["a", ["b"]])
|
||||
|
||||
def test_string_list_or_list_shape_dispatch(self):
|
||||
self.assertEqual(decode_StringListOrList(["a", "b"]), ["a", "b"])
|
||||
self.assertEqual(
|
||||
decode_StringListOrList([["a"], ["b", "c"]]), [["a"], ["b", "c"]]
|
||||
)
|
||||
self.assertEqual(decode_StringListOrList([]), [])
|
||||
|
||||
def test_stream_null_resets_default(self):
|
||||
self.assertIs(GenerateRequest.from_json_value({"stream": None}).stream, False)
|
||||
self.assertIs(GenerateRequest.from_json_value({"stream": True}).stream, True)
|
||||
|
||||
def test_round_trip_batch(self):
|
||||
body = {
|
||||
"text": ["a", "b"],
|
||||
"sampling_params": [{"n": 2}, {"temperature": 0.5}],
|
||||
"return_logprob": True,
|
||||
"bootstrap_host": [None, "h"],
|
||||
}
|
||||
req = GenerateRequest.from_json_value(body)
|
||||
back = req.to_json_value()
|
||||
self.assertEqual(back["text"], ["a", "b"])
|
||||
self.assertEqual(back["return_logprob"], True)
|
||||
self.assertEqual(back["bootstrap_host"], [None, "h"])
|
||||
self.assertEqual(back["sampling_params"][1]["temperature"], 0.5)
|
||||
|
||||
|
||||
class TestCarriers(CustomTestCase):
|
||||
def test_one_or_many_flattens(self):
|
||||
self.assertEqual(decode_StringOrList("a"), "a")
|
||||
self.assertEqual(compact(encode_StringOrList("a")), '"a"')
|
||||
self.assertEqual(decode_StringOrList(["a", "b"]), ["a", "b"])
|
||||
self.assertEqual(compact(encode_StringOrList(["a", "b"])), '["a","b"]')
|
||||
with self.assertRaises(JsonContractError) as ctx:
|
||||
decode_StringOrList({"x": 1})
|
||||
self.assertEqual(
|
||||
str(ctx.exception),
|
||||
"invalid type: map, expected a value or an array of values for StringOrList",
|
||||
)
|
||||
|
||||
def test_token_ids_or_list_shape_dispatch(self):
|
||||
"""[1,2] is one id list, [[1],[2,3]] is a batch, [] is one empty list."""
|
||||
self.assertEqual(decode_TokenIdsOrList([1, 2]), [1, 2])
|
||||
self.assertEqual(decode_TokenIdsOrList([[1], [2, 3]]), [[1], [2, 3]])
|
||||
self.assertEqual(decode_TokenIdsOrList([]), [])
|
||||
|
||||
def test_optional_carriers_keep_element_nulls(self):
|
||||
"""PD bootstrap columns: [null, "h"] keeps per-item presence."""
|
||||
v = decode_OptionalStringOrList([None, "h"])
|
||||
self.assertEqual(v, [None, "h"])
|
||||
self.assertEqual(compact(encode_OptionalStringOrList(v)), '[null,"h"]')
|
||||
self.assertEqual(decode_OptionalInt64OrList(17000), 17000)
|
||||
|
||||
|
||||
class TestFinishReason(CustomTestCase):
|
||||
def test_round_trips(self):
|
||||
"""Tagged by "type"; matched keeps its wire shape (str / int / int list)."""
|
||||
for text in [
|
||||
'{"type":"stop","matched":"</s>"}',
|
||||
'{"type":"stop","matched":7}',
|
||||
'{"type":"stop","matched":[1,2]}',
|
||||
'{"type":"length","length":2}',
|
||||
]:
|
||||
want = json.loads(text)
|
||||
parsed = decode_FinishReason(want)
|
||||
self.assertEqual(encode_FinishReason(parsed), want, text)
|
||||
|
||||
def test_variant_classes(self):
|
||||
self.assertIsInstance(decode_FinishReason({"type": "stop"}), FinishStop)
|
||||
self.assertIsInstance(
|
||||
decode_FinishReason({"type": "length", "length": 2}), FinishLength
|
||||
)
|
||||
self.assertIsInstance(decode_FinishReason({"type": "abort"}), FinishAbort)
|
||||
|
||||
def test_abort_emits_null_keys(self):
|
||||
"""An abort carries all three keys, nulls included."""
|
||||
parsed = decode_FinishReason({"type": "abort", "message": "m"})
|
||||
back = encode_FinishReason(parsed)
|
||||
self.assertEqual(
|
||||
back,
|
||||
{"type": "abort", "message": "m", "status_code": None, "err_type": None},
|
||||
)
|
||||
|
||||
def test_unknown_type_passes_through(self):
|
||||
"""The forward-compat escape hatch: unknown tags round-trip unchanged."""
|
||||
raw = {"type": "paused", "step": 3}
|
||||
parsed = decode_FinishReason(raw)
|
||||
self.assertIsInstance(parsed, dict)
|
||||
self.assertEqual(encode_FinishReason(parsed), raw)
|
||||
|
||||
def test_malformed_known_type_passes_through(self):
|
||||
parsed = decode_FinishReason({"type": "stop", "matched": {"bad": "shape"}})
|
||||
self.assertIsInstance(parsed, dict)
|
||||
|
||||
|
||||
class TestFrames(CustomTestCase):
|
||||
def test_logprob_tuple_shape(self):
|
||||
"""[logprob, token_id, text|null]; the logprob slot is nullable."""
|
||||
e = LogprobEntry(logprob=-0.25, token_id=7, text=None)
|
||||
self.assertEqual(compact(e.to_json_value()), "[-0.25,7,null]")
|
||||
parsed = LogprobEntry.from_json_value([-0.5, 9, "x"])
|
||||
self.assertEqual(parsed.token_id, 9)
|
||||
self.assertEqual(parsed.text, "x")
|
||||
first_prefill = LogprobEntry.from_json_value([None, 10, None])
|
||||
self.assertIsNone(first_prefill.logprob)
|
||||
self.assertEqual(compact(first_prefill.to_json_value()), "[null,10,null]")
|
||||
with self.assertRaises(JsonContractError) as ctx:
|
||||
LogprobEntry.from_json_value([1.0, 2])
|
||||
self.assertEqual(
|
||||
str(ctx.exception), "invalid length 2, expected a 3-element tuple"
|
||||
)
|
||||
|
||||
def test_generate_response_matches_frame_value_shape(self):
|
||||
"""Key order (text, meta_info, output_ids), conditional keys, and
|
||||
finish_reason always present (null pre-terminal)."""
|
||||
frame = GenerateResponse(
|
||||
text="ok",
|
||||
output_ids=[7, 8],
|
||||
meta_info=GenerateMetaInfo(
|
||||
id="client-rid",
|
||||
prompt_tokens=5,
|
||||
completion_tokens=2,
|
||||
finish_reason=None,
|
||||
),
|
||||
index=None,
|
||||
)
|
||||
self.assertEqual(
|
||||
compact(frame.to_json_value()),
|
||||
'{"text":"ok","meta_info":{"id":"client-rid","prompt_tokens":5,"completion_tokens":2,"finish_reason":null},"output_ids":[7,8]}',
|
||||
)
|
||||
|
||||
def test_frame_round_trip_with_logprobs(self):
|
||||
wire = {
|
||||
"text": "ok",
|
||||
"meta_info": {
|
||||
"id": "r",
|
||||
"prompt_tokens": 1,
|
||||
"completion_tokens": 1,
|
||||
"finish_reason": {"type": "stop", "matched": 2},
|
||||
"output_token_logprobs": [[-0.1, 5, "x"]],
|
||||
"output_top_logprobs": [None, [[-0.2, 6, None]]],
|
||||
"hidden_states": [[0.5, 0.25]],
|
||||
"e2e_latency": 0.01,
|
||||
},
|
||||
"output_ids": [5],
|
||||
"index": 0,
|
||||
}
|
||||
frame = GenerateResponse.from_json_value(wire)
|
||||
self.assertIsInstance(frame.meta_info.finish_reason, FinishStop)
|
||||
self.assertEqual(frame.meta_info.output_top_logprobs[0], None)
|
||||
self.assertEqual(frame.meta_info.output_top_logprobs[1][0].token_id, 6)
|
||||
self.assertEqual(frame.to_json_value(), wire)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,250 @@
|
||||
"""Parity of the three `/generate` vocabularies: the hand-written Python
|
||||
request structs (`io_struct.GenerateReqInput`, `SamplingParams`), the
|
||||
generated Python types (`sglang.api.v1.api_types`), and the generated Rust
|
||||
types (`rust/sglang-api-types`).
|
||||
|
||||
The cross-language half runs through `rust/sglang-api-types/testdata/
|
||||
json_parity.json`: both generators must turn each fixture input into the same
|
||||
canonical bytes (`tests/json_parity.rs` checks the Rust side). The Python half
|
||||
pins the field sets against the hand-written structs and proves a schema
|
||||
request still normalizes through the dataclass the Python server runs.
|
||||
"""
|
||||
|
||||
import dataclasses
|
||||
import json
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel
|
||||
|
||||
maybe_stub_sgl_kernel() # must precede any import that pulls in sgl_kernel
|
||||
|
||||
from sglang.api.v1 import api_types
|
||||
from sglang.api.v1.api_types import (
|
||||
GenerateMetaInfo,
|
||||
GenerateRequest,
|
||||
GenerateResponse,
|
||||
SamplingParams,
|
||||
)
|
||||
from sglang.srt.managers.io_struct import GenerateReqInput
|
||||
from sglang.srt.sampling.sampling_params import SamplingParams as PySamplingParams
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[5]
|
||||
FIXTURES = REPO_ROOT / "rust" / "sglang-api-types" / "testdata" / "json_parity.json"
|
||||
|
||||
# `GenerateReqInput` fields with no schema counterpart. Each is either
|
||||
# Engine-only (tensors, callbacks), set by the server after admission, or a
|
||||
# feature the HTTP contract has not adopted. A new dataclass field must be
|
||||
# added to the proto or listed here, so a client-facing field cannot slip
|
||||
# into the Python server without the Rust server and the contract seeing it.
|
||||
PYTHON_ONLY_REQUEST_FIELDS = frozenset(
|
||||
{
|
||||
"background",
|
||||
"cache_salt",
|
||||
"conversation_id",
|
||||
"custom_labels",
|
||||
"custom_logit_processor",
|
||||
"data_parallel_rank",
|
||||
"encoder_urls",
|
||||
"external_trace_header",
|
||||
"extra_key",
|
||||
"http_worker_ipc",
|
||||
"image_max_dynamic_patch",
|
||||
"images_config",
|
||||
"input_embeds",
|
||||
"kv_hints",
|
||||
"log_metrics",
|
||||
"lora_id",
|
||||
"lora_path",
|
||||
"max_dynamic_patch",
|
||||
"max_thinking_tokens",
|
||||
"min_dynamic_patch",
|
||||
"mm_content_hashes",
|
||||
"modalities",
|
||||
"multi_item_delimiter_indices",
|
||||
"need_wait_for_mm_inputs",
|
||||
"no_logs",
|
||||
"num_items_assigned",
|
||||
"positional_embed_overrides",
|
||||
"priority",
|
||||
"received_time",
|
||||
"require_reasoning",
|
||||
"return_bytes",
|
||||
"return_entropy",
|
||||
"return_flat_raw_top_logprobs",
|
||||
"return_flat_raw_top_logprobs_b64",
|
||||
"return_indexer_topk",
|
||||
"return_prompt_token_ids",
|
||||
"return_routed_experts",
|
||||
"return_sampling_mask",
|
||||
"routed_experts_start_len",
|
||||
"routing_key",
|
||||
"sampling_logprobs_mode",
|
||||
"session_id",
|
||||
"session_params",
|
||||
"token_indices_to_pool",
|
||||
"use_audio_in_video",
|
||||
"video_config",
|
||||
"video_max_dynamic_patch",
|
||||
}
|
||||
)
|
||||
|
||||
# `SamplingParams` members that `normalize()` derives; never on the wire.
|
||||
DERIVED_SAMPLING_FIELDS = frozenset(
|
||||
{
|
||||
"stop_strs",
|
||||
"stop_regex_strs",
|
||||
"stop_str_max_len",
|
||||
"stop_regex_max_len",
|
||||
"is_normalized",
|
||||
"ebnf_full_assistant",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def compact(value) -> str:
|
||||
"""serde_json::to_string parity: no whitespace."""
|
||||
return json.dumps(value, separators=(",", ":"))
|
||||
|
||||
|
||||
def load_fixtures():
|
||||
with open(FIXTURES) as f:
|
||||
return json.load(f)["cases"]
|
||||
|
||||
|
||||
class TestCrossLanguageFixtures(CustomTestCase):
|
||||
def test_fixtures_decode_to_their_canonical_bytes(self):
|
||||
"""The Python emitter agrees with the Rust emitter on every fixture:
|
||||
same keys, same order, same presence and null rules, and the
|
||||
canonical form is a fixed point."""
|
||||
cases = load_fixtures()
|
||||
self.assertTrue(cases)
|
||||
for case in cases:
|
||||
with self.subTest(case=f"{case['type']}: {case['name']}"):
|
||||
cls = getattr(api_types, case["type"])
|
||||
decoded = cls.from_json_value(case["input"])
|
||||
self.assertEqual(compact(decoded.to_json_value()), case["canonical"])
|
||||
again = cls.from_json_value(json.loads(case["canonical"]))
|
||||
self.assertEqual(compact(again.to_json_value()), case["canonical"])
|
||||
|
||||
def test_every_fixture_type_is_a_generated_struct(self):
|
||||
"""A fixture naming a type the Python module lacks would be skipped
|
||||
silently by a lookup-by-name; make it a failure."""
|
||||
for case in load_fixtures():
|
||||
self.assertTrue(hasattr(api_types, case["type"]), case["type"])
|
||||
|
||||
|
||||
class TestRequestFieldParity(CustomTestCase):
|
||||
def test_schema_request_fields_are_a_subset_of_the_dataclass(self):
|
||||
"""Every schema field is a `GenerateReqInput` field, so a schema body
|
||||
constructs the dataclass as-is, and the dataclass-only fields are
|
||||
exactly the pinned list."""
|
||||
schema = set(GenerateRequest.__struct_fields__)
|
||||
dataclass_fields = {f.name for f in dataclasses.fields(GenerateReqInput)}
|
||||
self.assertEqual(schema - dataclass_fields, set())
|
||||
self.assertEqual(dataclass_fields - schema, PYTHON_ONLY_REQUEST_FIELDS)
|
||||
|
||||
def test_schema_sampling_fields_match_the_python_api(self):
|
||||
"""The schema's SamplingParams is the Python struct minus the fields
|
||||
`normalize()` derives, in the same order."""
|
||||
python_api = [
|
||||
name
|
||||
for name in PySamplingParams.__struct_fields__
|
||||
if name not in DERIVED_SAMPLING_FIELDS
|
||||
]
|
||||
self.assertEqual(set(python_api), set(SamplingParams.__struct_fields__))
|
||||
self.assertEqual(
|
||||
set(PySamplingParams.__struct_fields__) - set(python_api),
|
||||
DERIVED_SAMPLING_FIELDS,
|
||||
)
|
||||
|
||||
def test_request_fixtures_normalize_through_the_dataclass(self):
|
||||
"""A schema-decoded body, re-encoded, is a valid `GenerateReqInput`
|
||||
and survives `normalize_batch_and_arguments` with its prompt, batch
|
||||
shape, stream flag, and sampling values intact."""
|
||||
for case in load_fixtures():
|
||||
if case["type"] != "GenerateRequest":
|
||||
continue
|
||||
with self.subTest(case=case["name"]):
|
||||
req = GenerateRequest.from_json_value(case["input"])
|
||||
wire = req.to_json_value()
|
||||
dataclass_req = GenerateReqInput(**wire)
|
||||
dataclass_req.normalize_batch_and_arguments()
|
||||
self.assertEqual(dataclass_req.stream, wire["stream"])
|
||||
if "text" in wire:
|
||||
self.assertEqual(dataclass_req.text, wire["text"])
|
||||
batch = isinstance(wire["text"], list)
|
||||
else:
|
||||
self.assertEqual(dataclass_req.input_ids, wire["input_ids"])
|
||||
batch = isinstance(wire["input_ids"][0], list)
|
||||
self.assertEqual(dataclass_req.is_single, not batch)
|
||||
if "sampling_params" in wire:
|
||||
sent = wire["sampling_params"]
|
||||
got = dataclass_req.sampling_params
|
||||
if isinstance(sent, list):
|
||||
self.assertEqual(len(got), len(sent))
|
||||
for sent_item, got_item in zip(sent, got):
|
||||
self.assertEqual({**got_item, **sent_item}, got_item)
|
||||
else:
|
||||
self.assertEqual({**got, **sent}, got)
|
||||
if "rid" in wire:
|
||||
self.assertEqual(dataclass_req.rid, wire["rid"])
|
||||
|
||||
|
||||
class TestResponseParity(CustomTestCase):
|
||||
def test_python_server_meta_info_decodes_to_the_schema_subset(self):
|
||||
"""The Python server's `meta_info` carries keys the schema does not
|
||||
model (weight_version, num_retractions, spec stats, ...). The
|
||||
generated types ignore them and re-emit the schema subset, so a
|
||||
Rust-server client and a Python-server client see one shape."""
|
||||
python_frame = {
|
||||
"text": "ok",
|
||||
"meta_info": {
|
||||
"id": "rid",
|
||||
"finish_reason": {"type": "stop", "matched": 2},
|
||||
"prompt_tokens": 3,
|
||||
"weight_version": "default",
|
||||
"num_retractions": 0,
|
||||
"completion_tokens": 2,
|
||||
"cached_tokens": 1,
|
||||
"spec_accept_rate": 0.5,
|
||||
"e2e_latency": 0.25,
|
||||
},
|
||||
}
|
||||
frame = GenerateResponse.from_json_value(python_frame)
|
||||
back = frame.to_json_value()
|
||||
self.assertEqual(
|
||||
list(back["meta_info"]),
|
||||
[
|
||||
"id",
|
||||
"prompt_tokens",
|
||||
"completion_tokens",
|
||||
"finish_reason",
|
||||
"e2e_latency",
|
||||
],
|
||||
)
|
||||
self.assertEqual(
|
||||
back["meta_info"]["finish_reason"], {"type": "stop", "matched": 2}
|
||||
)
|
||||
self.assertNotIn("output_ids", back)
|
||||
|
||||
def test_response_key_order_is_the_proto_field_order(self):
|
||||
"""The Rust HTTP server's memoized cumulative frame writes these keys
|
||||
by hand in this order (`Accumulated::frame_json`); the generated
|
||||
serializers must keep matching it."""
|
||||
self.assertEqual(
|
||||
list(GenerateResponse.__struct_fields__),
|
||||
["text", "meta_info", "output_ids", "index"],
|
||||
)
|
||||
self.assertEqual(
|
||||
list(GenerateMetaInfo.__struct_fields__)[:4],
|
||||
["id", "prompt_tokens", "completion_tokens", "finish_reason"],
|
||||
)
|
||||
self.assertEqual(list(GenerateMetaInfo.__struct_fields__)[-1], "e2e_latency")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,73 @@
|
||||
"""The `/generate` schema gate (`entrypoints/api_contract.py`): the typed media
|
||||
contract that replaced the raw_json passthrough, checked without a server."""
|
||||
|
||||
import unittest
|
||||
|
||||
from sglang.srt.entrypoints.api_contract import generate_contract_error
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=3, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class TestGenerateContract(CustomTestCase):
|
||||
def test_every_media_container_form_is_accepted(self):
|
||||
for image_data in (
|
||||
"u",
|
||||
{"url": "u", "detail": "high"},
|
||||
["u", None, {"url": "v"}],
|
||||
[],
|
||||
[["a", "b"], None, []],
|
||||
):
|
||||
with self.subTest(image_data=image_data):
|
||||
self.assertIsNone(
|
||||
generate_contract_error({"text": "hi", "image_data": image_data})
|
||||
)
|
||||
self.assertIsNone(
|
||||
generate_contract_error({"text": "hi", "mm_hashes": ["a1b2", "0xff"]})
|
||||
)
|
||||
self.assertIsNone(
|
||||
generate_contract_error(
|
||||
{
|
||||
"text": "hi",
|
||||
"video_data": {"url": "v", "fps": 2.0, "use_audio": True},
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
def test_unknown_media_hint_is_rejected(self):
|
||||
"""A hint the schema does not name used to be silently dropped."""
|
||||
error = generate_contract_error(
|
||||
{
|
||||
"text": "hi",
|
||||
"image_data": {"url": "u", "max_dynamic_patc": 4},
|
||||
} # codespell:ignore patc
|
||||
)
|
||||
self.assertIsNotNone(error)
|
||||
self.assertIn("unknown field", error)
|
||||
|
||||
def test_preprocessed_inputs_are_not_an_http_shape(self):
|
||||
"""processor_output / precomputed_embedding carry tensors: Engine-only."""
|
||||
error = generate_contract_error(
|
||||
{"input_ids": [1, 2], "image_data": [{"format": "processor_output"}]}
|
||||
)
|
||||
self.assertIsNotNone(error)
|
||||
|
||||
def test_mixed_list_shapes_are_rejected(self):
|
||||
error = generate_contract_error({"text": "hi", "image_data": ["a", ["b"]]})
|
||||
self.assertIsNotNone(error)
|
||||
|
||||
def test_sampling_params_follow_the_schema(self):
|
||||
self.assertIsNone(
|
||||
generate_contract_error(
|
||||
{"text": "hi", "sampling_params": {"beam_width": 2, "n": 2}}
|
||||
)
|
||||
)
|
||||
bad = {"temperatur": 1} # codespell:ignore temperatur
|
||||
error = generate_contract_error({"text": "hi", "sampling_params": bad})
|
||||
self.assertIsNotNone(error)
|
||||
self.assertIn("unknown field", error)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -484,7 +484,8 @@ class TestSamplingParamsMsgspecStruct(CustomTestCase):
|
||||
)
|
||||
source = rust_path.read_text()
|
||||
start = source.index("pub struct SamplingParams {")
|
||||
end = source.index("\n}\n\n/// The `/generate`", start)
|
||||
# The struct's own closing brace: the first one back at column 0.
|
||||
end = source.index("\n}\n", start)
|
||||
rust_fields = tuple(
|
||||
re.findall(
|
||||
r"^\s*pub ([a-z][a-z0-9_]*):",
|
||||
|
||||
Reference in New Issue
Block a user