From c52dcc5a01744b3ba236c44fd3eb995142cb1919 Mon Sep 17 00:00:00 2001 From: Rain Jiang <96632942+rainj-me@users.noreply.github.com> Date: Wed, 7 Oct 2026 01:55:01 -0700 Subject: [PATCH] The sgalng api specs (#41585) --- .codespellrc | 2 +- .pre-commit-config.yaml | 5 +- docker/Dockerfile | 6 +- docker/Dockerfile.cu134 | 4 +- proto/Cargo.toml | 26 - proto/README.md | 15 - proto/sglang/api/v1/common.proto | 134 + proto/sglang/api/v1/finish.proto | 60 + proto/sglang/api/v1/generate.proto | 235 ++ proto/sglang/api/v1/sampling.proto | 100 + proto/sglang/api/v1/service.proto | 72 + proto/sglang/json/v1/options.proto | 156 + proto/src/lib.rs | 13 - python/sglang/api/v1/api_types.py | 1528 ++++++++ python/sglang/srt/entrypoints/api_contract.py | 18 + python/sglang/srt/entrypoints/http_server.py | 6 + rust/.cargo/config.toml | 5 + rust/Cargo.lock | 270 +- rust/Cargo.toml | 15 +- rust/sglang-api-codegen/Cargo.toml | 19 + rust/sglang-api-codegen/src/emit.rs | 1081 ++++++ rust/sglang-api-codegen/src/emit_py.rs | 1172 +++++++ rust/sglang-api-codegen/src/main.rs | 179 + rust/sglang-api-codegen/src/model.rs | 550 +++ rust/sglang-api-types/Cargo.toml | 25 + rust/sglang-api-types/src/ext.rs | 63 + .../src/generated/sglang.api.v1.rs | 657 ++++ .../src/generated/sglang.api.v1.serde.rs | 3115 +++++++++++++++++ .../src/generated/sglang.api.v1.tonic.rs | 489 +++ .../src/generated/sglang.runtime.v1.rs | 457 +++ .../src/generated/sglang.runtime.v1.tonic.rs | 1922 ++++++++++ rust/sglang-api-types/src/lib.rs | 28 + .../testdata/json_parity.json | 373 ++ rust/sglang-api-types/tests/json_contract.rs | 346 ++ rust/sglang-api-types/tests/json_parity.rs | 50 + rust/sglang-grpc/Cargo.toml | 9 +- rust/sglang-grpc/pyproject.toml | 6 +- rust/sglang-grpc/src/server.rs | 2 +- rust/sglang-grpc/src/server/tests.rs | 4 +- rust/sglang-grpc/src/utils/request_utils.rs | 2 +- rust/sglang-server/Cargo.toml | 6 +- rust/sglang-server/src/api_server.rs | 89 +- rust/sglang-server/src/api_server/core.rs | 18 + .../src/api_server/core/error.rs | 186 + .../src/api_server/core/frame.rs | 457 +++ .../core/handle.rs} | 218 +- .../{frontend => api_server/core}/prefetch.rs | 2 +- rust/sglang-server/src/api_server/frame.rs | 697 ---- rust/sglang-server/src/api_server/grpc.rs | 15 + .../server.rs => api_server/grpc/app.rs} | 15 +- .../sglang-server/src/api_server/grpc/info.rs | 68 + .../src/api_server/grpc/native_api.rs | 215 ++ .../src/api_server/grpc/service.rs | 718 ++++ rust/sglang-server/src/api_server/http.rs | 4 + .../src/api_server/{ => http}/app.rs | 51 +- .../api_server/{common.rs => http/info.rs} | 34 +- .../src/api_server/{ => http}/native_api.rs | 260 +- .../src/api_server/{ => http}/openai.rs | 37 +- .../src/api_server/{ => http}/openai/chat.rs | 16 +- .../{ => http}/openai/completions.rs | 32 +- .../api_server/{ => http}/openai/models.rs | 0 .../api_server/{ => http}/openai/reasoning.rs | 0 .../api_server/{ => http}/openai/template.rs | 0 .../{ => http}/openai/template_builtins.rs | 0 .../{ => http}/openai/template_legacy.rs | 0 .../{ => http}/openai/template_loader.rs | 0 .../{ => http}/openai/test_utils.rs | 45 +- .../src/api_server/{ => http}/openai/tools.rs | 10 +- rust/sglang-server/src/frontend/contract.rs | 356 -- rust/sglang-server/src/grpc/convert.rs | 531 --- rust/sglang-server/src/grpc/mod.rs | 303 -- rust/sglang-server/src/grpc/response.rs | 200 -- rust/sglang-server/src/grpc/tests.rs | 361 -- rust/sglang-server/src/lib.rs | 5 +- rust/sglang-server/src/message.rs | 2 + rust/sglang-server/src/message/config.rs | 8 +- .../src/message/finish_reason.rs | 4 +- rust/sglang-server/src/message/info.rs | 89 + rust/sglang-server/src/message/multimodal.rs | 229 +- rust/sglang-server/src/message/request.rs | 1060 ++---- rust/sglang-server/src/message/response.rs | 16 +- rust/sglang-server/src/message/sampling.rs | 150 +- rust/sglang-server/src/message/wire.rs | 410 +++ .../src/multi_modality/payload.rs | 44 +- rust/sglang-server/src/native_generation.rs | 160 - .../src/tokenizer_manager/detokenizer.rs | 4 +- .../tokenizer_manager/to_scheduler_tests.rs | 2 +- .../src/tokenizer_manager/wiring.rs | 6 +- rust/sglang-server/src/utils/runtime.rs | 93 +- test/registered/rust/test_run_rust_tests.py | 2 +- test/registered/rust/test_rust_extension.py | 8 +- .../unit/api/v1/test_json_contract.py | 325 ++ test/registered/unit/api/v1/test_parity.py | 250 ++ .../entrypoints/test_generate_api_contract.py | 73 + .../unit/sampling/test_sampling_params.py | 3 +- 95 files changed, 16648 insertions(+), 4458 deletions(-) delete mode 100644 proto/Cargo.toml delete mode 100644 proto/README.md create mode 100644 proto/sglang/api/v1/common.proto create mode 100644 proto/sglang/api/v1/finish.proto create mode 100644 proto/sglang/api/v1/generate.proto create mode 100644 proto/sglang/api/v1/sampling.proto create mode 100644 proto/sglang/api/v1/service.proto create mode 100644 proto/sglang/json/v1/options.proto delete mode 100644 proto/src/lib.rs create mode 100644 python/sglang/api/v1/api_types.py create mode 100644 python/sglang/srt/entrypoints/api_contract.py create mode 100644 rust/.cargo/config.toml create mode 100644 rust/sglang-api-codegen/Cargo.toml create mode 100644 rust/sglang-api-codegen/src/emit.rs create mode 100644 rust/sglang-api-codegen/src/emit_py.rs create mode 100644 rust/sglang-api-codegen/src/main.rs create mode 100644 rust/sglang-api-codegen/src/model.rs create mode 100644 rust/sglang-api-types/Cargo.toml create mode 100644 rust/sglang-api-types/src/ext.rs create mode 100644 rust/sglang-api-types/src/generated/sglang.api.v1.rs create mode 100644 rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs create mode 100644 rust/sglang-api-types/src/generated/sglang.api.v1.tonic.rs create mode 100644 rust/sglang-api-types/src/generated/sglang.runtime.v1.rs create mode 100644 rust/sglang-api-types/src/generated/sglang.runtime.v1.tonic.rs create mode 100644 rust/sglang-api-types/src/lib.rs create mode 100644 rust/sglang-api-types/testdata/json_parity.json create mode 100644 rust/sglang-api-types/tests/json_contract.rs create mode 100644 rust/sglang-api-types/tests/json_parity.rs create mode 100644 rust/sglang-server/src/api_server/core.rs create mode 100644 rust/sglang-server/src/api_server/core/error.rs create mode 100644 rust/sglang-server/src/api_server/core/frame.rs rename rust/sglang-server/src/{frontend.rs => api_server/core/handle.rs} (84%) rename rust/sglang-server/src/{frontend => api_server/core}/prefetch.rs (99%) delete mode 100644 rust/sglang-server/src/api_server/frame.rs create mode 100644 rust/sglang-server/src/api_server/grpc.rs rename rust/sglang-server/src/{grpc/server.rs => api_server/grpc/app.rs} (85%) create mode 100644 rust/sglang-server/src/api_server/grpc/info.rs create mode 100644 rust/sglang-server/src/api_server/grpc/native_api.rs create mode 100644 rust/sglang-server/src/api_server/grpc/service.rs create mode 100644 rust/sglang-server/src/api_server/http.rs rename rust/sglang-server/src/api_server/{ => http}/app.rs (79%) rename rust/sglang-server/src/api_server/{common.rs => http/info.rs} (92%) rename rust/sglang-server/src/api_server/{ => http}/native_api.rs (81%) rename rust/sglang-server/src/api_server/{ => http}/openai.rs (84%) rename rust/sglang-server/src/api_server/{ => http}/openai/chat.rs (98%) rename rust/sglang-server/src/api_server/{ => http}/openai/completions.rs (96%) rename rust/sglang-server/src/api_server/{ => http}/openai/models.rs (100%) rename rust/sglang-server/src/api_server/{ => http}/openai/reasoning.rs (100%) rename rust/sglang-server/src/api_server/{ => http}/openai/template.rs (100%) rename rust/sglang-server/src/api_server/{ => http}/openai/template_builtins.rs (100%) rename rust/sglang-server/src/api_server/{ => http}/openai/template_legacy.rs (100%) rename rust/sglang-server/src/api_server/{ => http}/openai/template_loader.rs (100%) rename rust/sglang-server/src/api_server/{ => http}/openai/test_utils.rs (89%) rename rust/sglang-server/src/api_server/{ => http}/openai/tools.rs (99%) delete mode 100644 rust/sglang-server/src/frontend/contract.rs delete mode 100644 rust/sglang-server/src/grpc/convert.rs delete mode 100644 rust/sglang-server/src/grpc/mod.rs delete mode 100644 rust/sglang-server/src/grpc/response.rs delete mode 100644 rust/sglang-server/src/grpc/tests.rs create mode 100644 rust/sglang-server/src/message/info.rs create mode 100644 rust/sglang-server/src/message/wire.rs delete mode 100644 rust/sglang-server/src/native_generation.rs create mode 100644 test/registered/unit/api/v1/test_json_contract.py create mode 100644 test/registered/unit/api/v1/test_parity.py create mode 100644 test/registered/unit/entrypoints/test_generate_api_contract.py diff --git a/.codespellrc b/.codespellrc index 715171d21d..19eb54e660 100644 --- a/.codespellrc +++ b/.codespellrc @@ -1,3 +1,3 @@ [codespell] -ignore-words-list = ans,als,hel,boostrap,childs,te,vas,hsa,ment,cann,thi,makro,wil,rouge,PRIS,ather,MIS,medias,allready,inout,nd,fo,visibles,nothink,renderD,ond,tbe,CopyIn,notin,subtile,subtiles,dout,IST,kInf,datas,abl,valu +ignore-words-list = ans,als,hel,boostrap,childs,te,vas,hsa,ment,cann,thi,makro,wil,rouge,PRIS,ather,MIS,medias,allready,inout,nd,fo,visibles,nothink,renderD,ond,tbe,CopyIn,notin,subtile,subtiles,dout,IST,kInf,datas,abl,valu,ser skip = *.json, *.jsonl, *.patch, *.txt, *.lock diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index af88cf5af5..394459516f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,7 +29,7 @@ repos: rev: 7.0.0 hooks: - id: isort - exclude: '^python/sglang/srt/grpc/.*_pb2\.py$|^python/sglang/srt/grpc/.*_pb2_grpc\.py$|^python/sglang/srt/grpc/.*_pb2\.pyi$|^python/sglang/srt/grpc/.*_pb2_grpc\.pyi$' + exclude: '^python/sglang/srt/grpc/.*_pb2\.py$|^python/sglang/srt/grpc/.*_pb2_grpc\.py$|^python/sglang/srt/grpc/.*_pb2\.pyi$|^python/sglang/srt/grpc/.*_pb2_grpc\.pyi$|^python/sglang/(api|json)/v1/.*\.py$' - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.15.1 hooks: @@ -47,9 +47,10 @@ repos: python/sglang/srt/grpc/.*_pb2_grpc\.py$| python/sglang/srt/grpc/.*_pb2\.pyi$| python/sglang/srt/grpc/.*_pb2_grpc\.pyi$| + python/sglang/(api|json)/v1/.*\.py$| )$ - id: ruff-format - exclude: '^python/sglang/srt/grpc/.*_pb2\.py$|^python/sglang/srt/grpc/.*_pb2_grpc\.py$|^python/sglang/srt/grpc/.*_pb2\.pyi$|^python/sglang/srt/grpc/.*_pb2_grpc\.pyi$' + exclude: '^python/sglang/srt/grpc/.*_pb2\.py$|^python/sglang/srt/grpc/.*_pb2_grpc\.py$|^python/sglang/srt/grpc/.*_pb2\.pyi$|^python/sglang/srt/grpc/.*_pb2_grpc\.pyi$|^python/sglang/(api|json)/v1/.*\.py$' - repo: https://github.com/codespell-project/codespell rev: v2.4.1 hooks: diff --git a/docker/Dockerfile b/docker/Dockerfile index 43dc0907db..69b7297bed 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -202,13 +202,13 @@ RUN --mount=type=cache,target=/root/.cache/uv \ # --no-deps prevents uv from pulling torch from default PyPI && uv pip install sglang-kernel==${SGL_KERNEL_VERSION} --reinstall --no-deps -# Copy the dependency spec, Rust extension sources, and shared protocol crate. -# Split from the dependency install so changes to these build inputs invalidate the +# Copy the dependency spec, Rust extension sources, and shared API types. +# Split from the pip install so changes to these build inputs invalidate the # dependency layer, while unrelated Python source changes do not. COPY python/pyproject.toml /tmp/sglang_deps/python/pyproject.toml COPY rust/sglang-grpc /tmp/sglang_deps/rust/sglang-grpc COPY rust/sglang-mm /tmp/sglang_deps/rust/sglang-mm -COPY proto /tmp/sglang_deps/proto +COPY rust/sglang-api-types /tmp/sglang_deps/rust/sglang-api-types # Install sglang dependencies (torch, transformers, etc.). # Generate constraints.txt to prevent reinstalling these deps in later stages. diff --git a/docker/Dockerfile.cu134 b/docker/Dockerfile.cu134 index 6a01a79458..67c75c6aef 100644 --- a/docker/Dockerfile.cu134 +++ b/docker/Dockerfile.cu134 @@ -422,13 +422,13 @@ RUN --mount=type=cache,target=/root/.cache/pip \ ; \ fi -# Copy the dependency spec, Rust extension sources, and shared protocol crate. +# Copy the dependency spec, Rust extension sources, and shared API types. # Split from the pip install so changes to these build inputs invalidate the # dependency layer, while unrelated Python source changes do not. COPY python/pyproject.toml /tmp/sglang_deps/python/pyproject.toml COPY rust/sglang-grpc /tmp/sglang_deps/rust/sglang-grpc COPY rust/sglang-mm /tmp/sglang_deps/rust/sglang-mm -COPY proto /tmp/sglang_deps/proto +COPY rust/sglang-api-types /tmp/sglang_deps/rust/sglang-api-types # Install sglang dependencies (torch, transformers, etc.). # Generate constraints.txt to prevent reinstalling these deps in later stages. diff --git a/proto/Cargo.toml b/proto/Cargo.toml deleted file mode 100644 index 18ac4bde20..0000000000 --- a/proto/Cargo.toml +++ /dev/null @@ -1,26 +0,0 @@ -[package] -name = "sglang-grpc-types" -description = "Generated Rust bindings for the SGLang runtime.v1 API" -version = "0.1.0" -edition.workspace = true -rust-version = "1.92" -license.workspace = true -repository = "https://github.com/sgl-project/sglang" -readme = "README.md" -workspace = "../rust" -include = [ - "build.rs", - "README.md", - "src/**", - "sglang/**/*.proto", -] - -[dependencies] -prost = { workspace = true } -prost-types = { workspace = true } -tonic = { workspace = true } - -[build-dependencies] -prost-build = { workspace = true } -protoc-bin-vendored = { workspace = true } -tonic-build = { workspace = true } diff --git a/proto/README.md b/proto/README.md deleted file mode 100644 index 7e13b8baa5..0000000000 --- a/proto/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# SGLang gRPC types - -This crate provides the generated Prost messages and Tonic client/server -interfaces for SGLang's canonical `sglang.runtime.v1` API. - -The language-independent contract remains in -`sglang/runtime/v1/sglang.proto`. Rust bindings are generated from that schema -at build time with a vendored `protoc`, so consumers do not need a system -protobuf compiler. - -The generated API currently uses Tonic 0.12 to remain compatible with -SGLang's existing gRPC server. - -This crate contains protocol types only. SGLang's request handling and server -implementation remain in their respective frontend crates. diff --git a/proto/sglang/api/v1/common.proto b/proto/sglang/api/v1/common.proto new file mode 100644 index 0000000000..cd7f8008ee --- /dev/null +++ b/proto/sglang/api/v1/common.proto @@ -0,0 +1,134 @@ +// Shared carriers for the native SGLang API. The ground truth for the JSON +// shapes is the sglang.json.v1 options on each field; the protobuf shapes are +// the explicit oneofs/messages themselves. + +syntax = "proto3"; + +package sglang.api.v1; + +import "sglang/json/v1/options.proto"; + +// A token-id sequence. int64 matches the servers' token id width (Rust +// `TokenIds = Vec`, Python `array("q")`). +message TokenIds { + option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST; + repeated int64 ids = 1; +} + +// Scalar-or-list carriers: `/generate` accepts every per-request field as one +// value (single request) or a list (batch). JSON flattens via the one_or_many +// option; gRPC clients pick the variant explicitly. +message StringOrList { + oneof value { + option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_ONE_OR_MANY; + string one = 1; + StringList many = 2; + } +} + +message StringList { + option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST; + repeated string items = 1; +} + +// A flat string list for one request, or one list per batch entry. JSON: +// `["a", "b"]` is `one`; `[["a"], ["b", "c"]]` is `many` (first element +// decides, `[]` is an empty `one`). +message StringListOrList { + oneof value { + option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_ONE_OR_MANY; + StringList one = 1; + StringListList many = 2; + } +} + +message StringListList { + option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST; + repeated StringList items = 1; +} + +message BoolOrList { + oneof value { + option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_ONE_OR_MANY; + bool one = 1; + BoolList many = 2; + } +} + +message BoolList { + option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST; + repeated bool items = 1; +} + +message Int64OrList { + oneof value { + option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_ONE_OR_MANY; + int64 one = 1; + Int64List many = 2; + } +} + +message Int64List { + option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST; + repeated int64 items = 1; +} + +message TokenIdsOrList { + oneof value { + option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_ONE_OR_MANY; + TokenIds one = 1; + TokenIdsList many = 2; + } +} + +message TokenIdsList { + option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST; + repeated TokenIds items = 1; +} + +// List variants whose ELEMENTS may be JSON null (PD bootstrap columns: a +// batch mixes bootstrapped and plain requests). Proto carries element +// presence as a per-element oneof-free wrapper. +message OptionalString { + option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_TRANSPARENT; + optional string value = 1; +} + +message OptionalStringOrList { + oneof value { + option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_ONE_OR_MANY; + OptionalString one = 1; + OptionalStringList many = 2; + } +} + +message OptionalStringList { + option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST; + repeated OptionalString items = 1; +} + +message OptionalInt64 { + option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_TRANSPARENT; + optional int64 value = 1; +} + +message OptionalInt64OrList { + oneof value { + option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_ONE_OR_MANY; + OptionalInt64 one = 1; + OptionalInt64List many = 2; + } +} + +message OptionalInt64List { + option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST; + repeated OptionalInt64 items = 1; +} + +// The native error body: `{"error": {"message", "code"}}` (Python +// `http_server.generate_request` parity). Transports frame it themselves +// (unary status + body, SSE error frame, gRPC status detail). +message ErrorBody { + string message = 1; + uint32 code = 2; +} diff --git a/proto/sglang/api/v1/finish.proto b/proto/sglang/api/v1/finish.proto new file mode 100644 index 0000000000..c28e58ef7a --- /dev/null +++ b/proto/sglang/api/v1/finish.proto @@ -0,0 +1,60 @@ +// The terminal finish reason — Python's `FinishReasonDict` +// (`BaseFinishReason.to_json()` in schedule_batch.py), echoed verbatim as +// `meta_info.finish_reason`. JSON is internally tagged by "type" with the +// variant payload inlined; a type this build doesn't know round-trips +// unchanged (unknown_variant_passthrough) so a Python-side addition never +// fails a frame decode. + +syntax = "proto3"; + +package sglang.api.v1; + +import "sglang/json/v1/options.proto"; + +// Python's `matched` key, `Union[str, int, List[int]]`: the three wire shapes +// are disjoint, so shape alone picks the arm (untagged). +message Matched { + oneof value { + option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_UNTAGGED; + int64 token = 1; + string str = 2; + MatchedTokens tokens = 3; + } +} + +message MatchedTokens { + option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST; + repeated int64 ids = 1; +} + +// `FINISH_MATCHED_TOKEN` / `FINISH_MATCHED_STR` / `FINISHED_MATCHED_REGEX` — +// all three report `type: "stop"`, matching the OpenAI API's value. +message FinishStop { + optional Matched matched = 1; +} + +// `FINISH_LENGTH` — hit `max_new_tokens` (or the context limit). +message FinishLength { + optional uint64 length = 1; +} + +// The `FINISH_ABORT` payload. Python emits all three keys, nulls included. +message FinishAbort { + optional string message = 1 [(sglang.json.v1.field_json).emit_null_when_absent = true]; + optional uint32 status_code = 2 [(sglang.json.v1.field_json).emit_null_when_absent = true]; + optional string err_type = 3 [(sglang.json.v1.field_json).emit_null_when_absent = true]; +} + +message FinishReason { + oneof kind { + option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_TAGGED; + option (sglang.json.v1.oneof_json).tag = "type"; + option (sglang.json.v1.oneof_json).unknown_variant_passthrough = true; + FinishStop stop = 1; + FinishLength length = 2; + FinishAbort abort = 3; + // The passthrough arm: an unrecognized `type`, kept as its raw map + // (JSON text — Struct would coerce integer values to doubles). + string unknown = 4 [(sglang.json.v1.field_json).raw_json = true]; + } +} diff --git a/proto/sglang/api/v1/generate.proto b/proto/sglang/api/v1/generate.proto new file mode 100644 index 0000000000..446cbc38df --- /dev/null +++ b/proto/sglang/api/v1/generate.proto @@ -0,0 +1,235 @@ +// The native `/generate` surface — the ground truth for +// rust/sglang-server/src/message/request.rs's GenerateBody (request) and +// api_core/frame.rs's frame shape (response). Unknown JSON keys on the +// request are IGNORED: `deny_unknown_fields` broke real clients that send +// Python-only `GenerateReqInput` fields (priority, extra_key, session_id, +// ...), pinned by request.rs's unported-fields test. + +syntax = "proto3"; + +package sglang.api.v1; + +import "sglang/api/v1/common.proto"; +import "sglang/api/v1/finish.proto"; +import "sglang/api/v1/sampling.proto"; +import "sglang/json/v1/options.proto"; + +message GenerateRequest { + // (message_json).unknown_fields defaults to IGNORE — stated here as schema + // intent, load-bearing for client compat. + + StringOrList rid = 1; + StringOrList text = 2; + TokenIdsOrList input_ids = 3; + optional bool stream = 4 [ + (sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT, + (sglang.json.v1.field_json).default_bool = false + ]; + // One set of params, or one per batch item. JSON errors keep serde's + // field-level texts (the hand-written SamplingParamsInput contract). + SamplingParamsOrList sampling_params = 5; + BoolOrList return_logprob = 6; + Int64OrList logprob_start_len = 7; + Int64OrList top_logprobs_num = 8; + TokenIdsOrList token_ids_logprob = 9; + BoolOrList return_hidden_states = 10; + optional bool return_text_in_logprobs = 11; + // PD bootstrap columns: a batch mixes bootstrapped and plain requests, so + // elements are individually nullable. + OptionalStringOrList bootstrap_host = 12; + OptionalInt64OrList bootstrap_port = 13; + OptionalInt64OrList bootstrap_room = 14; + OptionalStringOrList bootstrap_pair_key = 15; + OptionalInt64OrList decode_tp_size = 16; + optional int64 routed_dp_rank = 17; + optional int64 disagg_prefill_dp_rank = 18; + // Multimodal payloads pass through untyped (data URIs, URLs, file paths, or + // nested batch lists) to the MM pipeline, which owns their validation. + // Media inputs, one container per modality (see MediaInput). The item + // hints a modality does not support are rejected at the processor. + optional MediaInput image_data = 19; + // Per-item content hashes overriding the computed ones, in item order; a + // flat list for one request (the batch form is rejected downstream). + optional StringListOrList mm_hashes = 20; + optional MediaInput video_data = 21; + optional MediaInput audio_data = 22; +} + +message SamplingParamsOrList { + oneof value { + option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_ONE_OR_MANY; + SamplingParams one = 1; + SamplingParamsList many = 2; + } +} + +message SamplingParamsList { + option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST; + repeated SamplingParams items = 1; +} + +// One `[logprob, token_id, text|null]` entry (text present when +// `return_text_in_logprobs`, decoded on the detok shard). The logprob slot is +// optional: Python emits null for a position with no logprob (the first +// prefill token; Rust maps the scheduler's NaN to the same null). +message LogprobEntry { + option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_TUPLE; + optional double logprob = 1; + int64 token_id = 2; + optional string text = 3; +} + +message LogprobEntries { + option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST; + repeated LogprobEntry entries = 1; +} + +// One decode position's top-k alternatives. +message TopLogprobRow { + option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST; + repeated LogprobEntry entries = 1; +} + +// A position's row, or null: a position that recorded no alternatives is +// null on the wire (Python parity), never []. +message NullableTopLogprobs { + option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_TRANSPARENT; + TopLogprobRow row = 1; +} + +message HiddenStateRow { + option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST; + repeated double values = 1; +} + +message GenerateMetaInfo { + // The client's rid, byte-for-byte (`Rid::client_facing`). + string id = 1; + uint32 prompt_tokens = 2; + uint64 completion_tokens = 3; + // Always present in JSON, null until a terminal frame. + FinishReason finish_reason = 4 [(sglang.json.v1.field_json).emit_null_when_absent = true]; + // input+output token logprobs are set together, empty lists included: a PD + // decode node never receives input logprobs, but the PD router keys its + // merge of prefill's on the key's presence. + LogprobEntries output_token_logprobs = 5; + LogprobEntries input_token_logprobs = 6; + repeated NullableTopLogprobs output_top_logprobs = 7; + repeated NullableTopLogprobs input_top_logprobs = 8; + repeated NullableTopLogprobs output_token_ids_logprobs = 9; + repeated NullableTopLogprobs input_token_ids_logprobs = 10; + repeated HiddenStateRow hidden_states = 11; + // Seconds from normalized-request acceptance to the terminal output; only + // on the terminal frame. + optional double e2e_latency = 12; +} + +message GenerateResponse { + // JSON key order is FIELD-NUMBER order: text, meta_info, then the + // conditionally-present output_ids and index — matching frame_value's + // insertion order byte-for-byte. + string text = 1; + GenerateMetaInfo meta_info = 2; + // Omitted while empty (a text-only frame). + TokenIds output_ids = 3; + // Batch position; only on batch responses/frames. + optional uint32 index = 4; +} + +// One request's failure inside a Generate stream; carries its batch position +// (the HTTP SSE shape puts `index` beside the error object the same way). +message GenerateStreamError { + ErrorBody error = 1; + optional uint32 index = 2; +} + +// One item of the gRPC Generate stream: a frame, or one request's failure +// (the stream continues — other batch items may still be running). On HTTP +// the same failure is an in-stream SSE error frame. +message GenerateStreamItem { + oneof item { + GenerateResponse frame = 1; + GenerateStreamError error = 2; + } +} + +// ---- Media inputs ---------------------------------------------------------- +// +// Every form Python's `/generate` accepts, typed. Preprocessed inputs +// (`format: "processor_output"` / `"precomputed_embedding"`) are NOT part of +// the HTTP contract: their values are tensors, which JSON cannot carry; they +// exist for the in-process Engine only. + +// One media item's source and per-item hints. Unknown keys are an error: a +// hint this schema does not name would otherwise be silently dropped. +message MediaRef { + option (sglang.json.v1.message_json).unknown_fields = UNKNOWN_FIELDS_DENY; + // URL, `file://` / absolute path, `data:` URI, or bare base64. + string url = 1; + // Image: "auto" | "low" | "high" (the OpenAI detail hint). + optional string detail = 2; + // Image/video: dynamic-patch budget (InternVL-style processors). + optional int64 max_dynamic_patch = 3; + // Image: a caller-supplied content hash that overrides the computed one. + optional string content_hash = 4; + // Video/image budgets read by the GLM and MiMo processors. + optional int64 min_pixels = 5; + optional int64 max_pixels = 6; + optional int64 total_max_pixels = 7; + optional double fps = 8; + optional int64 num_frames = 9; + optional int64 min_frames = 10; + optional int64 max_frames = 11; + optional int64 max_tokens_per_frame = 12; + optional int64 max_image_tokens = 13; + // Video: also decode the audio track. + optional bool use_audio = 14; +} + +// One media item. JSON: a bare string is the source; an object is a MediaRef. +message MediaItem { + oneof value { + option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_UNTAGGED; + string source = 1; + MediaRef ref = 2; + } +} + +// A list element that may be JSON null (Python's `[img, None]`). +message OptionalMediaItem { + option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_TRANSPARENT; + optional MediaItem item = 1; +} + +// One request's items (`[a, null, b]`), or one batch entry's items. +message MediaItems { + option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST; + repeated OptionalMediaItem items = 1; +} + +// A batch entry whose item list may be JSON null. +message OptionalMediaItems { + option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_TRANSPARENT; + optional MediaItems items = 1; +} + +// One item list per batch entry (`[[a, b], null, []]`). +message MediaItemsList { + option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST; + repeated OptionalMediaItems items = 1; +} + +// The three container forms Python's `_normalize_image_data` distinguishes. +// JSON: a string or object is `item` (one item; a broadcast in a batch); an +// array whose first element is not an array is `many` (a flat list; `[]` is +// an empty `many`); an array whose first element is an array is `nested` +// (one list per batch entry). Which forms a request may use depends on +// whether it is a batch, and is checked downstream, not here. +message MediaInput { + oneof value { + option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_UNTAGGED; + MediaItem item = 1; + MediaItems many = 2; + MediaItemsList nested = 3; + } +} diff --git a/proto/sglang/api/v1/sampling.proto b/proto/sglang/api/v1/sampling.proto new file mode 100644 index 0000000000..f821d2a9bf --- /dev/null +++ b/proto/sglang/api/v1/sampling.proto @@ -0,0 +1,100 @@ +// The API face of SamplingParams — the ground truth for +// rust/sglang-server/src/message/sampling.rs's 25 API fields (its 5 +// normalize-internal fields and the msgpack scheduler encoding are IPC, not +// API, and stay hand-written there). Unknown JSON keys are DENIED: parity with +// Python's `SamplingParams(**kwargs)` TypeError. + +syntax = "proto3"; + +package sglang.api.v1; + +import "sglang/api/v1/common.proto"; +import "sglang/json/v1/options.proto"; + +message SamplingParams { + option (sglang.json.v1.message_json).unknown_fields = UNKNOWN_FIELDS_DENY; + + // Defaulted fields are `optional`: proto3 gives plain scalars no presence, + // so an unset double would arrive as 0 instead of its schema default (the + // top_p=0 bug the first live gRPC call found). JSON deserialization bakes + // the default on absent/null; protobuf arrival reads it through the + // generated *_or_default accessors. + + // JSON: absent = 128, explicit null = no limit (Python `Optional[int] = + // 128`). Protobuf has no null: unset = 128 (apply_absent_defaults at the + // gRPC entry); no limit needs an explicit large value. + optional int64 max_new_tokens = 1 [ + (sglang.json.v1.field_json).null_policy = NULL_POLICY_IS_NONE, + (sglang.json.v1.field_json).default_i64 = 128 + ]; + StringOrList stop = 2; + Int64List stop_token_ids = 3; + StringOrList stop_regex = 4; + optional double temperature = 5 [ + (sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT, + (sglang.json.v1.field_json).default_f64 = 1.0 + ]; + optional double top_p = 6 [ + (sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT, + (sglang.json.v1.field_json).default_f64 = 1.0 + ]; + // Default 1 << 30 (TOP_K_ALL): "consider the whole vocabulary". + optional int64 top_k = 7 [ + (sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT, + (sglang.json.v1.field_json).default_i64 = 1073741824 + ]; + optional double min_p = 8 [ + (sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT, + (sglang.json.v1.field_json).default_f64 = 0.0 + ]; + optional double frequency_penalty = 9 [ + (sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT, + (sglang.json.v1.field_json).default_f64 = 0.0 + ]; + optional double presence_penalty = 10 [ + (sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT, + (sglang.json.v1.field_json).default_f64 = 0.0 + ]; + optional double repetition_penalty = 11 [ + (sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT, + (sglang.json.v1.field_json).default_f64 = 1.0 + ]; + optional int64 min_new_tokens = 12 [ + (sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT, + (sglang.json.v1.field_json).default_i64 = 0 + ]; + optional int64 n = 13 [ + (sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT, + (sglang.json.v1.field_json).default_i64 = 1 + ]; + optional string json_schema = 14; + optional string regex = 15; + optional string ebnf = 16; + optional string structural_tag = 17; + optional bool ignore_eos = 18 [ + (sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT, + (sglang.json.v1.field_json).default_bool = false + ]; + optional bool skip_special_tokens = 19 [ + (sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT, + (sglang.json.v1.field_json).default_bool = true + ]; + optional bool spaces_between_special_tokens = 20 [ + (sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT, + (sglang.json.v1.field_json).default_bool = true + ]; + optional bool no_stop_trim = 21 [ + (sglang.json.v1.field_json).null_policy = NULL_POLICY_RESETS_DEFAULT, + (sglang.json.v1.field_json).default_bool = false + ]; + optional int64 stream_interval = 22; + // Token id (numeric-string key, Python parity) -> bias. + map logit_bias = 23 [(sglang.json.v1.field_json).int_key_map = true]; + optional int64 sampling_seed = 24; + // Opaque JSON forwarded to a custom logit processor; never inspected here. + optional string custom_params = 25 [(sglang.json.v1.field_json).raw_json = true]; + + // beam_width > 1 turns the request into a beam search request; `n` is then + // the number of returned sequences (n <= beam_width). + optional int64 beam_width = 26; +} diff --git a/proto/sglang/api/v1/service.proto b/proto/sglang/api/v1/service.proto new file mode 100644 index 0000000000..11adf88796 --- /dev/null +++ b/proto/sglang/api/v1/service.proto @@ -0,0 +1,72 @@ +// The native SGLang gRPC service served by rust/sglang-server's tonic +// transport, over the same api_core the HTTP endpoints use. Covers exactly the +// surface the native server implements (the legacy sglang.runtime.v1 bridge's +// embed/classify/tokenize RPCs have no native backend here and are not +// carried over). + +syntax = "proto3"; + +package sglang.api.v1; + +import "sglang/api/v1/generate.proto"; +import "sglang/json/v1/options.proto"; + +service SglangService { + // `/generate`: always server-streaming (a unary request is a one-frame + // stream); per-batch-item failures ride the stream as GenerateStreamItem + // errors, pre-submit failures end the RPC with a gRPC status. + rpc Generate(GenerateRequest) returns (stream GenerateStreamItem); + + // `/health_generate`: fires a 1-token probe and watches the response + // heartbeat. UNAVAILABLE on a stalled pipeline. + rpc HealthCheck(HealthCheckRequest) returns (HealthCheckResponse); + + // `/get_model_info`: static model metadata, no scheduler round-trip. + rpc GetModelInfo(GetModelInfoRequest) returns (GetModelInfoResponse); + + // `/server_info`: one control round-trip, shaped through the runtime-metric + // allowlist (never the raw server-args dump, which embeds keys). + rpc GetServerInfo(GetServerInfoRequest) returns (GetServerInfoResponse); +} + +message HealthCheckRequest {} + +message HealthCheckResponse { + bool healthy = 1; +} + +message GetModelInfoRequest {} + +message GetModelInfoResponse { + string model_path = 1; + string served_model_name = 2; + string tokenizer_path = 3; + bool is_generation = 4; + // The launch-time `--preferred-sampling-params` object, or null when unset. + // JSON carries the object itself (the HTTP `/get_model_info` shape); the + // protobuf field holds it as JSON text. + optional string preferred_sampling_params = 5 [ + (sglang.json.v1.field_json).raw_json = true, + (sglang.json.v1.field_json).emit_null_when_absent = true + ]; + optional string weight_version = 6 [(sglang.json.v1.field_json).emit_null_when_absent = true]; + optional string load_format = 7 [(sglang.json.v1.field_json).emit_null_when_absent = true]; + optional string reasoning_parser = 8 [(sglang.json.v1.field_json).emit_null_when_absent = true]; + optional string tool_call_parser = 9 [(sglang.json.v1.field_json).emit_null_when_absent = true]; +} + +message GetServerInfoRequest {} + +message GetServerInfoResponse { + string model_path = 1; + string served_model_name = 2; + string tokenizer_path = 3; + uint32 max_context_length = 4; + optional uint64 max_total_num_tokens = 5; + string version = 6; + // The allowlisted runtime metrics (last_gen_throughput, memory_usage, + // effective_max_running_requests_per_dp, avg_spec_accept_length, + // step_time_dict), untyped because the scheduler owns their shapes; JSON + // passes them through verbatim. + repeated string internal_states = 7 [(sglang.json.v1.field_json).raw_json = true]; +} diff --git a/proto/sglang/json/v1/options.proto b/proto/sglang/json/v1/options.proto new file mode 100644 index 0000000000..e3bf5e0f33 --- /dev/null +++ b/proto/sglang/json/v1/options.proto @@ -0,0 +1,156 @@ +// The JSON-fidelity option vocabulary: how a message, field, or oneof behaves +// on the HTTP JSON wire, declared in the schema itself. The sglang-api-codegen +// generator reads these options from the FileDescriptorSet and emits the serde +// implementations; proto3's canonical JSON mapping is NOT used. Each option is +// one generator template with its own unit tests. +// +// gRPC never sees these options: the protobuf wire format is untouched. + +syntax = "proto3"; + +package sglang.json.v1; + +import "google/protobuf/descriptor.proto"; + +// How a message treats unknown JSON keys on deserialization. +enum UnknownFields { + // Silently dropped (the default): parity with endpoints that accept and + // ignore unported fields, where rejecting would break existing clients. + UNKNOWN_FIELDS_IGNORE = 0; + // A 400 with serde's field-level message: parity with Python constructors + // that TypeError on unexpected kwargs. + UNKNOWN_FIELDS_DENY = 1; +} + +// How a message is shaped on the JSON wire. One shape per message: an enum, +// not independent flags, so two cannot be set at once. +enum MessageShape { + // A plain JSON object keyed by field name (the default). + MESSAGE_SHAPE_OBJECT = 0; + + // A single-`repeated`-field carrier (TokenIds, StringList, ...): its JSON + // is the bare array, not an {items: [...]} object, wherever it appears. + MESSAGE_SHAPE_BARE_LIST = 1; + + // A positional tuple: the fields, in declaration order, form a bare array + // (e.g. a logprob entry's [logprob, token_id, text|null]). Optional members + // serialize as null. + MESSAGE_SHAPE_TUPLE = 2; + + // A single-field message that is invisible in JSON: it serializes as its + // sole field's value (null when unset). Exists to give proto a presence + // wrapper for nullable LIST ELEMENTS (OptionalString, OptionalInt64). + MESSAGE_SHAPE_TRANSPARENT = 3; +} + +// Attached to a message: `option (sglang.json.v1.message_json). = ...;` +// in its body. Decides the JSON form of the message as a whole, for both +// directions of the HTTP wire: `shape` says what the serializer writes and +// what the deserializer expects (object, bare array, tuple, or the sole +// field's value); `unknown_fields` says what the deserializer does with keys +// the message does not declare. Serialization never emits unknown keys. +message MessageJson { + UnknownFields unknown_fields = 1; + MessageShape shape = 2; +} + +// How an absent key or an explicit JSON null reads into a field. One policy +// per field: an enum, not independent flags, so two cannot be set at once. +// Deserialization only; `emit_null_when_absent` is the serialization side. +enum NullPolicy { + // Absent is unset; an explicit null is a type error (the default). + NULL_POLICY_DEFAULT = 0; + + // Explicit null (and an absent key) yields the stated default instead of + // an error: parity with Python dataclasses whose __post_init__ maps None + // back to the default. + NULL_POLICY_RESETS_DEFAULT = 1; + + // An ABSENT key yields the stated default, but an explicit null clears the + // field to no-value (e.g. max_new_tokens: absent = 128, null = no limit). + NULL_POLICY_IS_NONE = 2; +} + +// Attached to one field: `type name = N [(sglang.json.v1.field_json). +// = ...];`. Decides how that key behaves, with the read and write sides kept +// apart. Deserialization: `null_policy` (with a `default_*`) maps an absent +// key or an explicit null onto a value. Serialization: `emit_null_when_absent` +// writes the key as null instead of omitting it. Both directions: `raw_json` +// carries the value through as JSON text, and `int_key_map` spells numeric +// map keys as strings. +message FieldJson { + NullPolicy null_policy = 1; + // The default a non-DEFAULT policy yields; exactly one must accompany it. + optional double default_f64 = 2; + optional int64 default_i64 = 3; + optional bool default_bool = 4; + optional string default_string = 5; + + // The key is always serialized, an unset value as an explicit null (e.g. + // reasoning_content in chat deltas: Python always emits the key). + bool emit_null_when_absent = 6; + + // On a string (or repeated string) field: the string holds JSON text, and + // HTTP (de)serialization passes the VALUE through — integers, nesting, and + // element shapes intact (google.protobuf.Struct cannot do this: it coerces + // integers to doubles). gRPC carries the JSON text. + bool raw_json = 7; + + // On a map field whose keys are numeric strings (Python + // parity, e.g. logit_bias token ids). + bool int_key_map = 8; +} + +// How a oneof (a union) is spelled on the JSON wire. One shape per union. +enum OneofShape { + // Unset: the message owning the oneof is a plain object. + ONEOF_SHAPE_UNSPECIFIED = 0; + + // Untagged: the value's shape picks the variant, and variant mismatch + // errors keep serde's field-level texts (never "data did not match any + // variant"). + ONEOF_SHAPE_UNTAGGED = 1; + + // On a oneof of exactly { T one; TList many; }: a JSON scalar deserializes + // as `one`, a JSON array as `many`; serialization flattens the same way. + ONEOF_SHAPE_ONE_OR_MANY = 2; + + // Internally tagged by the key in `tag` (e.g. finish_reason's + // {"type": "stop", ...}); variant payload fields inline into the same + // object, with each variant's tag value being its field name. + ONEOF_SHAPE_TAGGED = 3; +} + +// Attached to a oneof: `option (sglang.json.v1.oneof_json). = ...;` in +// its body. A oneof is a tagged union on the protobuf wire (the field number +// is the tag); JSON has no such tag, so `shape` picks how the union is +// spelled there and how the deserializer recovers the variant: by value +// shape (untagged), scalar versus array (one_or_many), or a discriminator key +// named by `tag` (tagged). `unknown_variant_passthrough` lets a tagged union +// round-trip a variant this schema does not know. +message OneofJson { + OneofShape shape = 1; + + // The discriminator key; required with ONEOF_SHAPE_TAGGED, meaningless + // otherwise. + string tag = 2; + + // With ONEOF_SHAPE_TAGGED: JSON input matching no variant is captured + // verbatim by the designated catch-all variant (a raw_json Struct member) + // and round-trips unchanged -- the forward-compatibility escape hatch for + // values another component may extend (e.g. finish_reason). + bool unknown_variant_passthrough = 3; +} + +extend google.protobuf.MessageOptions { + // Field numbers from the internal-use extension range (50000-99999). + MessageJson message_json = 50100; +} + +extend google.protobuf.FieldOptions { + FieldJson field_json = 50100; +} + +extend google.protobuf.OneofOptions { + OneofJson oneof_json = 50100; +} diff --git a/proto/src/lib.rs b/proto/src/lib.rs deleted file mode 100644 index 1e1e198008..0000000000 --- a/proto/src/lib.rs +++ /dev/null @@ -1,13 +0,0 @@ -//! Rust bindings for SGLang's canonical `sglang.runtime.v1` protobuf API. -//! -//! The generated messages and Tonic client/server interfaces intentionally -//! contain no frontend implementation or transport policy. - -/// Protobuf packages exposed with the same namespace as the wire contract. -pub mod sglang { - pub mod runtime { - pub mod v1 { - tonic::include_proto!("sglang.runtime.v1"); - } - } -} diff --git a/python/sglang/api/v1/api_types.py b/python/sglang/api/v1/api_types.py new file mode 100644 index 0000000000..686c9c73ef --- /dev/null +++ b/python/sglang/api/v1/api_types.py @@ -0,0 +1,1528 @@ +# Generated by sglang-api-codegen from proto/sglang -- DO NOT EDIT. +# The sglang.json.v1 options in the schema are the JSON contract; edit the +# .proto files and `cargo gen-api`. +"""Generated SGLang API types: the HTTP JSON interface of package sglang.api.v1. + +Frozen msgspec Structs for JSON objects and tuples, type aliases for the other +JSON shapes, and `decode_*` / `encode_*` codecs implementing the sglang.json.v1 +options exactly as the Rust server does. Decode with +`X.from_json_value(json.loads(s))` and encode with `json.dumps(x.to_json_value())`. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Tuple, Union + +import msgspec + +class JsonContractError(ValueError): + """The JSON value violates the schema contract (an HTTP 400 in the server).""" + + +def _is_number(v: Any) -> bool: + return isinstance(v, (int, float)) and not isinstance(v, bool) + + +def _is_int(v: Any) -> bool: + return isinstance(v, int) and not isinstance(v, bool) + + +def _unexpected(v: Any) -> str: + """serde's `Unexpected` display, so both languages report one text.""" + if v is None: + return "null" + if isinstance(v, bool): + return f"boolean `{'true' if v else 'false'}`" + if isinstance(v, int): + return f"integer `{v}`" + if isinstance(v, float): + return f"floating point `{v}`" + if isinstance(v, str): + return f'string "{v}"' + if isinstance(v, list): + return "sequence" + if isinstance(v, dict): + return "map" + return type(v).__name__ + + +def _invalid_type(v: Any, expected: str) -> JsonContractError: + return JsonContractError(f"invalid type: {_unexpected(v)}, expected {expected}") + + +def _expect_bool(v: Any, expected: str) -> bool: + if isinstance(v, bool): + return v + raise _invalid_type(v, expected=expected) + + +def _expect_int(v: Any, expected: str) -> int: + if _is_int(v): + return v + raise _invalid_type(v, expected=expected) + + +def _expect_uint(v: Any, expected: str) -> int: + if _is_int(v): + if v < 0: + raise JsonContractError(f"invalid value: integer `{v}`, expected {expected}") + return v + raise _invalid_type(v, expected=expected) + + +def _expect_float(v: Any, expected: str) -> float: + if _is_number(v): + return float(v) + raise _invalid_type(v, expected=expected) + + +def _expect_str(v: Any, expected: str) -> str: + if isinstance(v, str): + return v + raise _invalid_type(v, expected=expected) + + +def _expect_seq(v: Any, expected: str) -> List[Any]: + if isinstance(v, list): + return v + raise _invalid_type(v, expected=expected) + + +def _expect_map(v: Any, expected: str) -> Dict[str, Any]: + if isinstance(v, dict): + return v + raise _invalid_type(v, expected=expected) + + +def _unknown_field(key: str, fields: Tuple[str, ...]) -> JsonContractError: + """serde's `unknown_field` text: the DENY policy's 400 body.""" + if not fields: + return JsonContractError(f"unknown field `{key}`, there are no fields") + if len(fields) == 1: + return JsonContractError(f"unknown field `{key}`, expected `{fields[0]}`") + if len(fields) == 2: + return JsonContractError( + f"unknown field `{key}`, expected `{fields[0]}` or `{fields[1]}`" + ) + listed = ", ".join(f"`{f}`" for f in fields) + return JsonContractError(f"unknown field `{key}`, expected one of {listed}") + + +# ---- Structs: JSON objects and positional tuples ---- + + +class ErrorBody(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.ErrorBody: JSON object (unknown keys ignored).""" + + message: str = "" + code: int = 0 + + @classmethod + def from_json_value(cls, value: Any) -> "ErrorBody": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_ErrorBody(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_ErrorBody(self) + + +class FinishStop(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.FinishStop: JSON object (unknown keys ignored).""" + + matched: Optional[Matched] = None + + @classmethod + def from_json_value(cls, value: Any) -> "FinishStop": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_FinishStop(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_FinishStop(self) + + +class FinishLength(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.FinishLength: JSON object (unknown keys ignored).""" + + length: Optional[int] = None + + @classmethod + def from_json_value(cls, value: Any) -> "FinishLength": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_FinishLength(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_FinishLength(self) + + +class FinishAbort(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.FinishAbort: JSON object (unknown keys ignored).""" + + message: Optional[str] = None + status_code: Optional[int] = None + err_type: Optional[str] = None + + @classmethod + def from_json_value(cls, value: Any) -> "FinishAbort": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_FinishAbort(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_FinishAbort(self) + + +class SamplingParams(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.SamplingParams: JSON object (unknown keys rejected).""" + + max_new_tokens: Optional[int] = 128 + stop: Optional[StringOrList] = None + stop_token_ids: Optional[Int64List] = None + stop_regex: Optional[StringOrList] = None + temperature: Optional[float] = 1.0 + top_p: Optional[float] = 1.0 + top_k: Optional[int] = 1073741824 + min_p: Optional[float] = 0.0 + frequency_penalty: Optional[float] = 0.0 + presence_penalty: Optional[float] = 0.0 + repetition_penalty: Optional[float] = 1.0 + min_new_tokens: Optional[int] = 0 + n: Optional[int] = 1 + json_schema: Optional[str] = None + regex: Optional[str] = None + ebnf: Optional[str] = None + structural_tag: Optional[str] = None + ignore_eos: Optional[bool] = False + skip_special_tokens: Optional[bool] = True + spaces_between_special_tokens: Optional[bool] = True + no_stop_trim: Optional[bool] = False + stream_interval: Optional[int] = None + logit_bias: Dict[str, float] = {} + sampling_seed: Optional[int] = None + custom_params: Optional[Any] = None + beam_width: Optional[int] = None + + @classmethod + def from_json_value(cls, value: Any) -> "SamplingParams": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_SamplingParams(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_SamplingParams(self) + + +class GenerateRequest(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.GenerateRequest: JSON object (unknown keys ignored).""" + + rid: Optional[StringOrList] = None + text: Optional[StringOrList] = None + input_ids: Optional[TokenIdsOrList] = None + stream: Optional[bool] = False + sampling_params: Optional[SamplingParamsOrList] = None + return_logprob: Optional[BoolOrList] = None + logprob_start_len: Optional[Int64OrList] = None + top_logprobs_num: Optional[Int64OrList] = None + token_ids_logprob: Optional[TokenIdsOrList] = None + return_hidden_states: Optional[BoolOrList] = None + return_text_in_logprobs: Optional[bool] = None + bootstrap_host: Optional[OptionalStringOrList] = None + bootstrap_port: Optional[OptionalInt64OrList] = None + bootstrap_room: Optional[OptionalInt64OrList] = None + bootstrap_pair_key: Optional[OptionalStringOrList] = None + decode_tp_size: Optional[OptionalInt64OrList] = None + routed_dp_rank: Optional[int] = None + disagg_prefill_dp_rank: Optional[int] = None + image_data: Optional[MediaInput] = None + mm_hashes: Optional[StringListOrList] = None + video_data: Optional[MediaInput] = None + audio_data: Optional[MediaInput] = None + + @classmethod + def from_json_value(cls, value: Any) -> "GenerateRequest": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_GenerateRequest(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_GenerateRequest(self) + + +class LogprobEntry(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.LogprobEntry: JSON positional array [logprob, token_id, text].""" + + logprob: Optional[float] = None + token_id: int = 0 + text: Optional[str] = None + + @classmethod + def from_json_value(cls, value: Any) -> "LogprobEntry": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_LogprobEntry(value) + + def to_json_value(self) -> List[Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_LogprobEntry(self) + + +class GenerateMetaInfo(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.GenerateMetaInfo: JSON object (unknown keys ignored).""" + + id: str = "" + prompt_tokens: int = 0 + completion_tokens: int = 0 + finish_reason: Optional[FinishReason] = None + output_token_logprobs: Optional[LogprobEntries] = None + input_token_logprobs: Optional[LogprobEntries] = None + output_top_logprobs: List[NullableTopLogprobs] = [] + input_top_logprobs: List[NullableTopLogprobs] = [] + output_token_ids_logprobs: List[NullableTopLogprobs] = [] + input_token_ids_logprobs: List[NullableTopLogprobs] = [] + hidden_states: List[HiddenStateRow] = [] + e2e_latency: Optional[float] = None + + @classmethod + def from_json_value(cls, value: Any) -> "GenerateMetaInfo": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_GenerateMetaInfo(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_GenerateMetaInfo(self) + + +class GenerateResponse(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.GenerateResponse: JSON object (unknown keys ignored).""" + + text: str = "" + meta_info: Optional[GenerateMetaInfo] = None + output_ids: Optional[TokenIds] = None + index: Optional[int] = None + + @classmethod + def from_json_value(cls, value: Any) -> "GenerateResponse": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_GenerateResponse(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_GenerateResponse(self) + + +class GenerateStreamError(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.GenerateStreamError: JSON object (unknown keys ignored).""" + + error: Optional[ErrorBody] = None + index: Optional[int] = None + + @classmethod + def from_json_value(cls, value: Any) -> "GenerateStreamError": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_GenerateStreamError(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_GenerateStreamError(self) + + +class MediaRef(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.MediaRef: JSON object (unknown keys rejected).""" + + url: str = "" + detail: Optional[str] = None + max_dynamic_patch: Optional[int] = None + content_hash: Optional[str] = None + min_pixels: Optional[int] = None + max_pixels: Optional[int] = None + total_max_pixels: Optional[int] = None + fps: Optional[float] = None + num_frames: Optional[int] = None + min_frames: Optional[int] = None + max_frames: Optional[int] = None + max_tokens_per_frame: Optional[int] = None + max_image_tokens: Optional[int] = None + use_audio: Optional[bool] = None + + @classmethod + def from_json_value(cls, value: Any) -> "MediaRef": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_MediaRef(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_MediaRef(self) + + +class HealthCheckRequest(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.HealthCheckRequest: JSON object (unknown keys ignored).""" + + @classmethod + def from_json_value(cls, value: Any) -> "HealthCheckRequest": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_HealthCheckRequest(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_HealthCheckRequest(self) + + +class HealthCheckResponse(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.HealthCheckResponse: JSON object (unknown keys ignored).""" + + healthy: bool = False + + @classmethod + def from_json_value(cls, value: Any) -> "HealthCheckResponse": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_HealthCheckResponse(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_HealthCheckResponse(self) + + +class GetModelInfoRequest(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.GetModelInfoRequest: JSON object (unknown keys ignored).""" + + @classmethod + def from_json_value(cls, value: Any) -> "GetModelInfoRequest": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_GetModelInfoRequest(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_GetModelInfoRequest(self) + + +class GetModelInfoResponse(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.GetModelInfoResponse: JSON object (unknown keys ignored).""" + + model_path: str = "" + served_model_name: str = "" + tokenizer_path: str = "" + is_generation: bool = False + preferred_sampling_params: Optional[Any] = None + weight_version: Optional[str] = None + load_format: Optional[str] = None + reasoning_parser: Optional[str] = None + tool_call_parser: Optional[str] = None + + @classmethod + def from_json_value(cls, value: Any) -> "GetModelInfoResponse": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_GetModelInfoResponse(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_GetModelInfoResponse(self) + + +class GetServerInfoRequest(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.GetServerInfoRequest: JSON object (unknown keys ignored).""" + + @classmethod + def from_json_value(cls, value: Any) -> "GetServerInfoRequest": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_GetServerInfoRequest(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_GetServerInfoRequest(self) + + +class GetServerInfoResponse(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.GetServerInfoResponse: JSON object (unknown keys ignored).""" + + model_path: str = "" + served_model_name: str = "" + tokenizer_path: str = "" + max_context_length: int = 0 + max_total_num_tokens: Optional[int] = None + version: str = "" + internal_states: List[Any] = [] + + @classmethod + def from_json_value(cls, value: Any) -> "GetServerInfoResponse": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_GetServerInfoResponse(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_GetServerInfoResponse(self) + + +# ---- type aliases: JSON shapes without an object of their own ---- + +# sglang.api.v1.TokenIds: JSON bare array. +TokenIds = List[int] + +# sglang.api.v1.StringList: JSON bare array. +StringList = List[str] + +# sglang.api.v1.StringListList: JSON bare array. +StringListList = List[StringList] + +# sglang.api.v1.BoolList: JSON bare array. +BoolList = List[bool] + +# sglang.api.v1.Int64List: JSON bare array. +Int64List = List[int] + +# sglang.api.v1.TokenIdsList: JSON bare array. +TokenIdsList = List[TokenIds] + +# sglang.api.v1.OptionalString: JSON: the wrapped value itself, null when unset. +OptionalString = Optional[str] + +# sglang.api.v1.OptionalStringList: JSON bare array. +OptionalStringList = List[OptionalString] + +# sglang.api.v1.OptionalInt64: JSON: the wrapped value itself, null when unset. +OptionalInt64 = Optional[int] + +# sglang.api.v1.OptionalInt64List: JSON bare array. +OptionalInt64List = List[OptionalInt64] + +# sglang.api.v1.MatchedTokens: JSON bare array. +MatchedTokens = List[int] + +# sglang.api.v1.FinishReason: JSON object tagged by its type key; unknown tags pass through as a dict. +FinishReason = Union[FinishStop, FinishLength, FinishAbort, Dict[str, Any]] + +# sglang.api.v1.SamplingParamsList: JSON bare array. +SamplingParamsList = List[SamplingParams] + +# sglang.api.v1.LogprobEntries: JSON bare array. +LogprobEntries = List[LogprobEntry] + +# sglang.api.v1.TopLogprobRow: JSON bare array. +TopLogprobRow = List[LogprobEntry] + +# sglang.api.v1.NullableTopLogprobs: JSON: the wrapped value itself, null when unset. +NullableTopLogprobs = Optional[TopLogprobRow] + +# sglang.api.v1.HiddenStateRow: JSON bare array. +HiddenStateRow = List[float] + +# sglang.api.v1.MediaItem: JSON: the value's shape picks the arm. +MediaItem = Union[str, MediaRef] + +# sglang.api.v1.OptionalMediaItem: JSON: the wrapped value itself, null when unset. +OptionalMediaItem = Optional[MediaItem] + +# sglang.api.v1.MediaItems: JSON bare array. +MediaItems = List[OptionalMediaItem] + +# sglang.api.v1.OptionalMediaItems: JSON: the wrapped value itself, null when unset. +OptionalMediaItems = Optional[MediaItems] + +# sglang.api.v1.MediaItemsList: JSON bare array. +MediaItemsList = List[OptionalMediaItems] + +# sglang.api.v1.MediaInput: JSON: the value's shape picks the arm. +MediaInput = Union[MediaItem, MediaItems, MediaItemsList] + +# sglang.api.v1.StringOrList: JSON: one value or an array of values. +StringOrList = Union[str, StringList] + +# sglang.api.v1.StringListOrList: JSON: one value or an array of values. +StringListOrList = Union[StringList, StringListList] + +# sglang.api.v1.BoolOrList: JSON: one value or an array of values. +BoolOrList = Union[bool, BoolList] + +# sglang.api.v1.Int64OrList: JSON: one value or an array of values. +Int64OrList = Union[int, Int64List] + +# sglang.api.v1.TokenIdsOrList: JSON: one value or an array of values. +TokenIdsOrList = Union[TokenIds, TokenIdsList] + +# sglang.api.v1.OptionalStringOrList: JSON: one value or an array of values. +OptionalStringOrList = Union[OptionalString, OptionalStringList] + +# sglang.api.v1.OptionalInt64OrList: JSON: one value or an array of values. +OptionalInt64OrList = Union[OptionalInt64, OptionalInt64List] + +# sglang.api.v1.Matched: JSON: the value's shape picks the arm. +Matched = Union[int, str, MatchedTokens] + +# sglang.api.v1.SamplingParamsOrList: JSON: one value or an array of values. +SamplingParamsOrList = Union[SamplingParams, SamplingParamsList] + + +# ---- codecs: parsed JSON value <-> Python value ---- + + +def decode_TokenIds(value: Any) -> TokenIds: + return [_expect_int(e, expected="i64") for e in _expect_seq(value, expected="a sequence")] + + +def encode_TokenIds(v: TokenIds) -> List[Any]: + return list(v) + + +def decode_StringOrList(value: Any) -> StringOrList: + if isinstance(value, list): + return decode_StringList(value) + return _expect_str(value, expected="a value or an array of values for StringOrList") + + +def encode_StringOrList(v: StringOrList) -> Any: + if isinstance(v, list): + return encode_StringList(v) + return v + + +def decode_StringList(value: Any) -> StringList: + return [_expect_str(e, expected="a string") for e in _expect_seq(value, expected="a sequence")] + + +def encode_StringList(v: StringList) -> List[Any]: + return list(v) + + +def decode_StringListOrList(value: Any) -> StringListOrList: + if isinstance(value, list): + # [] and [scalar, ...] are one flat list; anything else is a batch. + if not value or isinstance(value[0], str): + return decode_StringList(value) + return decode_StringListList(value) + raise _invalid_type(value, expected="a value or an array of values for StringListOrList") + + +def encode_StringListOrList(v: StringListOrList) -> Any: + if v and isinstance(v[0], list): + return encode_StringListList(v) + return encode_StringList(v) + + +def decode_StringListList(value: Any) -> StringListList: + return [decode_StringList(e) for e in _expect_seq(value, expected="a sequence")] + + +def encode_StringListList(v: StringListList) -> List[Any]: + return [encode_StringList(e) for e in v] + + +def decode_BoolOrList(value: Any) -> BoolOrList: + if isinstance(value, list): + return decode_BoolList(value) + return _expect_bool(value, expected="a value or an array of values for BoolOrList") + + +def encode_BoolOrList(v: BoolOrList) -> Any: + if isinstance(v, list): + return encode_BoolList(v) + return v + + +def decode_BoolList(value: Any) -> BoolList: + return [_expect_bool(e, expected="a boolean") for e in _expect_seq(value, expected="a sequence")] + + +def encode_BoolList(v: BoolList) -> List[Any]: + return list(v) + + +def decode_Int64OrList(value: Any) -> Int64OrList: + if isinstance(value, list): + return decode_Int64List(value) + return _expect_int(value, expected="a value or an array of values for Int64OrList") + + +def encode_Int64OrList(v: Int64OrList) -> Any: + if isinstance(v, list): + return encode_Int64List(v) + return v + + +def decode_Int64List(value: Any) -> Int64List: + return [_expect_int(e, expected="i64") for e in _expect_seq(value, expected="a sequence")] + + +def encode_Int64List(v: Int64List) -> List[Any]: + return list(v) + + +def decode_TokenIdsOrList(value: Any) -> TokenIdsOrList: + if isinstance(value, list): + # [] and [scalar, ...] are one flat list; anything else is a batch. + if not value or _is_number(value[0]): + return decode_TokenIds(value) + return decode_TokenIdsList(value) + raise _invalid_type(value, expected="a value or an array of values for TokenIdsOrList") + + +def encode_TokenIdsOrList(v: TokenIdsOrList) -> Any: + if v and isinstance(v[0], list): + return encode_TokenIdsList(v) + return encode_TokenIds(v) + + +def decode_TokenIdsList(value: Any) -> TokenIdsList: + return [decode_TokenIds(e) for e in _expect_seq(value, expected="a sequence")] + + +def encode_TokenIdsList(v: TokenIdsList) -> List[Any]: + return [encode_TokenIds(e) for e in v] + + +def decode_OptionalString(value: Any) -> OptionalString: + return None if value is None else _expect_str(value, expected="a string") + + +def encode_OptionalString(v: OptionalString) -> Any: + return None if v is None else v + + +def decode_OptionalStringOrList(value: Any) -> OptionalStringOrList: + if isinstance(value, list): + return decode_OptionalStringList(value) + return _expect_str(value, expected="a value or an array of values for OptionalStringOrList") + + +def encode_OptionalStringOrList(v: OptionalStringOrList) -> Any: + if isinstance(v, list): + return encode_OptionalStringList(v) + return v + + +def decode_OptionalStringList(value: Any) -> OptionalStringList: + return [decode_OptionalString(e) for e in _expect_seq(value, expected="a sequence")] + + +def encode_OptionalStringList(v: OptionalStringList) -> List[Any]: + return [encode_OptionalString(e) for e in v] + + +def decode_OptionalInt64(value: Any) -> OptionalInt64: + return None if value is None else _expect_int(value, expected="i64") + + +def encode_OptionalInt64(v: OptionalInt64) -> Any: + return None if v is None else v + + +def decode_OptionalInt64OrList(value: Any) -> OptionalInt64OrList: + if isinstance(value, list): + return decode_OptionalInt64List(value) + return _expect_int(value, expected="a value or an array of values for OptionalInt64OrList") + + +def encode_OptionalInt64OrList(v: OptionalInt64OrList) -> Any: + if isinstance(v, list): + return encode_OptionalInt64List(v) + return v + + +def decode_OptionalInt64List(value: Any) -> OptionalInt64List: + return [decode_OptionalInt64(e) for e in _expect_seq(value, expected="a sequence")] + + +def encode_OptionalInt64List(v: OptionalInt64List) -> List[Any]: + return [encode_OptionalInt64(e) for e in v] + + +def decode_ErrorBody(value: Any) -> ErrorBody: + m = _expect_map(value, expected="struct ErrorBody") + kw: Dict[str, Any] = {} + for key, x in m.items(): + if key == "message": + kw["message"] = _expect_str(x, expected="a string") + elif key == "code": + kw["code"] = _expect_uint(x, expected="u32") + return ErrorBody(**kw) + + +def encode_ErrorBody(v: ErrorBody) -> Dict[str, Any]: + d: Dict[str, Any] = {} + d["message"] = v.message + d["code"] = v.code + return d + + +def decode_Matched(value: Any) -> Matched: + if isinstance(value, str): + return value + if _is_int(value): + return value + if isinstance(value, list): + return decode_MatchedTokens(value) + raise _invalid_type(value, expected="one of the Matched shapes") + + +def encode_Matched(v: Matched) -> Any: + if isinstance(v, (int,)): + return v + if isinstance(v, (str,)): + return v + if isinstance(v, (list,)): + return encode_MatchedTokens(v) + return v + + +def decode_MatchedTokens(value: Any) -> MatchedTokens: + return [_expect_int(e, expected="i64") for e in _expect_seq(value, expected="a sequence")] + + +def encode_MatchedTokens(v: MatchedTokens) -> List[Any]: + return list(v) + + +def decode_FinishStop(value: Any) -> FinishStop: + m = _expect_map(value, expected="struct FinishStop") + kw: Dict[str, Any] = {} + for key, x in m.items(): + if key == "matched": + kw["matched"] = None if x is None else decode_Matched(x) + return FinishStop(**kw) + + +def encode_FinishStop(v: FinishStop) -> Dict[str, Any]: + d: Dict[str, Any] = {} + if v.matched is not None: + d["matched"] = encode_Matched(v.matched) + return d + + +def decode_FinishLength(value: Any) -> FinishLength: + m = _expect_map(value, expected="struct FinishLength") + kw: Dict[str, Any] = {} + for key, x in m.items(): + if key == "length": + kw["length"] = None if x is None else _expect_uint(x, expected="u64") + return FinishLength(**kw) + + +def encode_FinishLength(v: FinishLength) -> Dict[str, Any]: + d: Dict[str, Any] = {} + if v.length is not None: + d["length"] = v.length + return d + + +def decode_FinishAbort(value: Any) -> FinishAbort: + m = _expect_map(value, expected="struct FinishAbort") + kw: Dict[str, Any] = {} + for key, x in m.items(): + if key == "message": + kw["message"] = None if x is None else _expect_str(x, expected="a string") + elif key == "status_code": + kw["status_code"] = None if x is None else _expect_uint(x, expected="u32") + elif key == "err_type": + kw["err_type"] = None if x is None else _expect_str(x, expected="a string") + return FinishAbort(**kw) + + +def encode_FinishAbort(v: FinishAbort) -> Dict[str, Any]: + d: Dict[str, Any] = {} + d["message"] = v.message + d["status_code"] = v.status_code + d["err_type"] = v.err_type + return d + + +def decode_FinishReason(value: Any) -> FinishReason: + raw = _expect_map(value, expected="map") + tag = raw.get("type") + if tag == "stop": + try: + return FinishStop( + matched=None if raw.get("matched") is None else decode_Matched(raw["matched"]), + ) + except JsonContractError: + return dict(raw) + elif tag == "length": + try: + return FinishLength( + length=None if raw.get("length") is None else _expect_uint(raw["length"], expected="u64"), + ) + except JsonContractError: + return dict(raw) + elif tag == "abort": + try: + return FinishAbort( + message=None if raw.get("message") is None else _expect_str(raw["message"], expected="a string"), + status_code=None if raw.get("status_code") is None else _expect_uint(raw["status_code"], expected="u32"), + err_type=None if raw.get("err_type") is None else _expect_str(raw["err_type"], expected="a string"), + ) + except JsonContractError: + return dict(raw) + return dict(raw) + + +def encode_FinishReason(v: FinishReason) -> Any: + if isinstance(v, FinishStop): + d: Dict[str, Any] = {"type": "stop"} + d["matched"] = None if v.matched is None else encode_Matched(v.matched) + return d + if isinstance(v, FinishLength): + d: Dict[str, Any] = {"type": "length"} + d["length"] = v.length + return d + if isinstance(v, FinishAbort): + d: Dict[str, Any] = {"type": "abort"} + d["message"] = v.message + d["status_code"] = v.status_code + d["err_type"] = v.err_type + return d + if isinstance(v, dict): + return dict(v) + raise TypeError(f"FinishReason: unsupported value {type(v).__name__}") + + +_FIELDS_SamplingParams = ("max_new_tokens", "stop", "stop_token_ids", "stop_regex", "temperature", "top_p", "top_k", "min_p", "frequency_penalty", "presence_penalty", "repetition_penalty", "min_new_tokens", "n", "json_schema", "regex", "ebnf", "structural_tag", "ignore_eos", "skip_special_tokens", "spaces_between_special_tokens", "no_stop_trim", "stream_interval", "logit_bias", "sampling_seed", "custom_params", "beam_width",) + + +def decode_SamplingParams(value: Any) -> SamplingParams: + m = _expect_map(value, expected="struct SamplingParams") + kw: Dict[str, Any] = {} + for key, x in m.items(): + if key == "max_new_tokens": + kw["max_new_tokens"] = None if x is None else _expect_int(x, expected="i64") + elif key == "stop": + kw["stop"] = None if x is None else decode_StringOrList(x) + elif key == "stop_token_ids": + kw["stop_token_ids"] = None if x is None else decode_Int64List(x) + elif key == "stop_regex": + kw["stop_regex"] = None if x is None else decode_StringOrList(x) + elif key == "temperature": + kw["temperature"] = 1.0 if x is None else _expect_float(x, expected="f64") + elif key == "top_p": + kw["top_p"] = 1.0 if x is None else _expect_float(x, expected="f64") + elif key == "top_k": + kw["top_k"] = 1073741824 if x is None else _expect_int(x, expected="i64") + elif key == "min_p": + kw["min_p"] = 0.0 if x is None else _expect_float(x, expected="f64") + elif key == "frequency_penalty": + kw["frequency_penalty"] = 0.0 if x is None else _expect_float(x, expected="f64") + elif key == "presence_penalty": + kw["presence_penalty"] = 0.0 if x is None else _expect_float(x, expected="f64") + elif key == "repetition_penalty": + kw["repetition_penalty"] = 1.0 if x is None else _expect_float(x, expected="f64") + elif key == "min_new_tokens": + kw["min_new_tokens"] = 0 if x is None else _expect_int(x, expected="i64") + elif key == "n": + kw["n"] = 1 if x is None else _expect_int(x, expected="i64") + elif key == "json_schema": + kw["json_schema"] = None if x is None else _expect_str(x, expected="a string") + elif key == "regex": + kw["regex"] = None if x is None else _expect_str(x, expected="a string") + elif key == "ebnf": + kw["ebnf"] = None if x is None else _expect_str(x, expected="a string") + elif key == "structural_tag": + kw["structural_tag"] = None if x is None else _expect_str(x, expected="a string") + elif key == "ignore_eos": + kw["ignore_eos"] = False if x is None else _expect_bool(x, expected="a boolean") + elif key == "skip_special_tokens": + kw["skip_special_tokens"] = True if x is None else _expect_bool(x, expected="a boolean") + elif key == "spaces_between_special_tokens": + kw["spaces_between_special_tokens"] = True if x is None else _expect_bool(x, expected="a boolean") + elif key == "no_stop_trim": + kw["no_stop_trim"] = False if x is None else _expect_bool(x, expected="a boolean") + elif key == "stream_interval": + kw["stream_interval"] = None if x is None else _expect_int(x, expected="i64") + elif key == "logit_bias": + kw["logit_bias"] = {} if x is None else {k: _expect_float(e, expected="f64") for k, e in _expect_map(x, expected="a map").items()} + elif key == "sampling_seed": + kw["sampling_seed"] = None if x is None else _expect_int(x, expected="i64") + elif key == "custom_params": + kw["custom_params"] = x + elif key == "beam_width": + kw["beam_width"] = None if x is None else _expect_int(x, expected="i64") + else: + raise _unknown_field(key, fields=_FIELDS_SamplingParams) + return SamplingParams(**kw) + + +def encode_SamplingParams(v: SamplingParams) -> Dict[str, Any]: + d: Dict[str, Any] = {} + d["max_new_tokens"] = v.max_new_tokens + if v.stop is not None: + d["stop"] = encode_StringOrList(v.stop) + if v.stop_token_ids is not None: + d["stop_token_ids"] = encode_Int64List(v.stop_token_ids) + if v.stop_regex is not None: + d["stop_regex"] = encode_StringOrList(v.stop_regex) + if v.temperature is not None: + d["temperature"] = float(v.temperature) + if v.top_p is not None: + d["top_p"] = float(v.top_p) + if v.top_k is not None: + d["top_k"] = v.top_k + if v.min_p is not None: + d["min_p"] = float(v.min_p) + if v.frequency_penalty is not None: + d["frequency_penalty"] = float(v.frequency_penalty) + if v.presence_penalty is not None: + d["presence_penalty"] = float(v.presence_penalty) + if v.repetition_penalty is not None: + d["repetition_penalty"] = float(v.repetition_penalty) + if v.min_new_tokens is not None: + d["min_new_tokens"] = v.min_new_tokens + if v.n is not None: + d["n"] = v.n + if v.json_schema is not None: + d["json_schema"] = v.json_schema + if v.regex is not None: + d["regex"] = v.regex + if v.ebnf is not None: + d["ebnf"] = v.ebnf + if v.structural_tag is not None: + d["structural_tag"] = v.structural_tag + if v.ignore_eos is not None: + d["ignore_eos"] = v.ignore_eos + if v.skip_special_tokens is not None: + d["skip_special_tokens"] = v.skip_special_tokens + if v.spaces_between_special_tokens is not None: + d["spaces_between_special_tokens"] = v.spaces_between_special_tokens + if v.no_stop_trim is not None: + d["no_stop_trim"] = v.no_stop_trim + if v.stream_interval is not None: + d["stream_interval"] = v.stream_interval + if v.logit_bias: + d["logit_bias"] = {k: float(e) for k, e in v.logit_bias.items()} + if v.sampling_seed is not None: + d["sampling_seed"] = v.sampling_seed + if v.custom_params is not None: + d["custom_params"] = v.custom_params + if v.beam_width is not None: + d["beam_width"] = v.beam_width + return d + + +def decode_GenerateRequest(value: Any) -> GenerateRequest: + m = _expect_map(value, expected="struct GenerateRequest") + kw: Dict[str, Any] = {} + for key, x in m.items(): + if key == "rid": + kw["rid"] = None if x is None else decode_StringOrList(x) + elif key == "text": + kw["text"] = None if x is None else decode_StringOrList(x) + elif key == "input_ids": + kw["input_ids"] = None if x is None else decode_TokenIdsOrList(x) + elif key == "stream": + kw["stream"] = False if x is None else _expect_bool(x, expected="a boolean") + elif key == "sampling_params": + kw["sampling_params"] = None if x is None else decode_SamplingParamsOrList(x) + elif key == "return_logprob": + kw["return_logprob"] = None if x is None else decode_BoolOrList(x) + elif key == "logprob_start_len": + kw["logprob_start_len"] = None if x is None else decode_Int64OrList(x) + elif key == "top_logprobs_num": + kw["top_logprobs_num"] = None if x is None else decode_Int64OrList(x) + elif key == "token_ids_logprob": + kw["token_ids_logprob"] = None if x is None else decode_TokenIdsOrList(x) + elif key == "return_hidden_states": + kw["return_hidden_states"] = None if x is None else decode_BoolOrList(x) + elif key == "return_text_in_logprobs": + kw["return_text_in_logprobs"] = None if x is None else _expect_bool(x, expected="a boolean") + elif key == "bootstrap_host": + kw["bootstrap_host"] = None if x is None else decode_OptionalStringOrList(x) + elif key == "bootstrap_port": + kw["bootstrap_port"] = None if x is None else decode_OptionalInt64OrList(x) + elif key == "bootstrap_room": + kw["bootstrap_room"] = None if x is None else decode_OptionalInt64OrList(x) + elif key == "bootstrap_pair_key": + kw["bootstrap_pair_key"] = None if x is None else decode_OptionalStringOrList(x) + elif key == "decode_tp_size": + kw["decode_tp_size"] = None if x is None else decode_OptionalInt64OrList(x) + elif key == "routed_dp_rank": + kw["routed_dp_rank"] = None if x is None else _expect_int(x, expected="i64") + elif key == "disagg_prefill_dp_rank": + kw["disagg_prefill_dp_rank"] = None if x is None else _expect_int(x, expected="i64") + elif key == "image_data": + kw["image_data"] = None if x is None else decode_MediaInput(x) + elif key == "mm_hashes": + kw["mm_hashes"] = None if x is None else decode_StringListOrList(x) + elif key == "video_data": + kw["video_data"] = None if x is None else decode_MediaInput(x) + elif key == "audio_data": + kw["audio_data"] = None if x is None else decode_MediaInput(x) + return GenerateRequest(**kw) + + +def encode_GenerateRequest(v: GenerateRequest) -> Dict[str, Any]: + d: Dict[str, Any] = {} + if v.rid is not None: + d["rid"] = encode_StringOrList(v.rid) + if v.text is not None: + d["text"] = encode_StringOrList(v.text) + if v.input_ids is not None: + d["input_ids"] = encode_TokenIdsOrList(v.input_ids) + if v.stream is not None: + d["stream"] = v.stream + if v.sampling_params is not None: + d["sampling_params"] = encode_SamplingParamsOrList(v.sampling_params) + if v.return_logprob is not None: + d["return_logprob"] = encode_BoolOrList(v.return_logprob) + if v.logprob_start_len is not None: + d["logprob_start_len"] = encode_Int64OrList(v.logprob_start_len) + if v.top_logprobs_num is not None: + d["top_logprobs_num"] = encode_Int64OrList(v.top_logprobs_num) + if v.token_ids_logprob is not None: + d["token_ids_logprob"] = encode_TokenIdsOrList(v.token_ids_logprob) + if v.return_hidden_states is not None: + d["return_hidden_states"] = encode_BoolOrList(v.return_hidden_states) + if v.return_text_in_logprobs is not None: + d["return_text_in_logprobs"] = v.return_text_in_logprobs + if v.bootstrap_host is not None: + d["bootstrap_host"] = encode_OptionalStringOrList(v.bootstrap_host) + if v.bootstrap_port is not None: + d["bootstrap_port"] = encode_OptionalInt64OrList(v.bootstrap_port) + if v.bootstrap_room is not None: + d["bootstrap_room"] = encode_OptionalInt64OrList(v.bootstrap_room) + if v.bootstrap_pair_key is not None: + d["bootstrap_pair_key"] = encode_OptionalStringOrList(v.bootstrap_pair_key) + if v.decode_tp_size is not None: + d["decode_tp_size"] = encode_OptionalInt64OrList(v.decode_tp_size) + if v.routed_dp_rank is not None: + d["routed_dp_rank"] = v.routed_dp_rank + if v.disagg_prefill_dp_rank is not None: + d["disagg_prefill_dp_rank"] = v.disagg_prefill_dp_rank + if v.image_data is not None: + d["image_data"] = encode_MediaInput(v.image_data) + if v.mm_hashes is not None: + d["mm_hashes"] = encode_StringListOrList(v.mm_hashes) + if v.video_data is not None: + d["video_data"] = encode_MediaInput(v.video_data) + if v.audio_data is not None: + d["audio_data"] = encode_MediaInput(v.audio_data) + return d + + +def decode_SamplingParamsOrList(value: Any) -> SamplingParamsOrList: + if isinstance(value, list): + return decode_SamplingParamsList(value) + return decode_SamplingParams(_expect_map(value, expected="a value or an array of values for SamplingParamsOrList")) + + +def encode_SamplingParamsOrList(v: SamplingParamsOrList) -> Any: + if isinstance(v, list): + return encode_SamplingParamsList(v) + return encode_SamplingParams(v) + + +def decode_SamplingParamsList(value: Any) -> SamplingParamsList: + return [decode_SamplingParams(e) for e in _expect_seq(value, expected="a sequence")] + + +def encode_SamplingParamsList(v: SamplingParamsList) -> List[Any]: + return [encode_SamplingParams(e) for e in v] + + +def decode_LogprobEntry(value: Any) -> LogprobEntry: + s = _expect_seq(value, expected="a 3-element tuple") + if len(s) != 3: + raise JsonContractError(f"invalid length {len(s)}, expected a 3-element tuple") + return LogprobEntry( + logprob=None if s[0] is None else _expect_float(s[0], expected="f64"), + token_id=_expect_int(s[1], expected="i64"), + text=None if s[2] is None else _expect_str(s[2], expected="a string"), + ) + + +def encode_LogprobEntry(v: LogprobEntry) -> List[Any]: + return [ + None if v.logprob is None else float(v.logprob), + v.token_id, + v.text, + ] + + +def decode_LogprobEntries(value: Any) -> LogprobEntries: + return [decode_LogprobEntry(e) for e in _expect_seq(value, expected="a sequence")] + + +def encode_LogprobEntries(v: LogprobEntries) -> List[Any]: + return [encode_LogprobEntry(e) for e in v] + + +def decode_TopLogprobRow(value: Any) -> TopLogprobRow: + return [decode_LogprobEntry(e) for e in _expect_seq(value, expected="a sequence")] + + +def encode_TopLogprobRow(v: TopLogprobRow) -> List[Any]: + return [encode_LogprobEntry(e) for e in v] + + +def decode_NullableTopLogprobs(value: Any) -> NullableTopLogprobs: + return None if value is None else decode_TopLogprobRow(value) + + +def encode_NullableTopLogprobs(v: NullableTopLogprobs) -> Any: + return None if v is None else encode_TopLogprobRow(v) + + +def decode_HiddenStateRow(value: Any) -> HiddenStateRow: + return [_expect_float(e, expected="f64") for e in _expect_seq(value, expected="a sequence")] + + +def encode_HiddenStateRow(v: HiddenStateRow) -> List[Any]: + return [float(e) for e in v] + + +def decode_GenerateMetaInfo(value: Any) -> GenerateMetaInfo: + m = _expect_map(value, expected="struct GenerateMetaInfo") + kw: Dict[str, Any] = {} + for key, x in m.items(): + if key == "id": + kw["id"] = _expect_str(x, expected="a string") + elif key == "prompt_tokens": + kw["prompt_tokens"] = _expect_uint(x, expected="u32") + elif key == "completion_tokens": + kw["completion_tokens"] = _expect_uint(x, expected="u64") + elif key == "finish_reason": + kw["finish_reason"] = None if x is None else decode_FinishReason(x) + elif key == "output_token_logprobs": + kw["output_token_logprobs"] = None if x is None else decode_LogprobEntries(x) + elif key == "input_token_logprobs": + kw["input_token_logprobs"] = None if x is None else decode_LogprobEntries(x) + elif key == "output_top_logprobs": + kw["output_top_logprobs"] = [] if x is None else [decode_NullableTopLogprobs(e) for e in _expect_seq(x, expected="a sequence")] + elif key == "input_top_logprobs": + kw["input_top_logprobs"] = [] if x is None else [decode_NullableTopLogprobs(e) for e in _expect_seq(x, expected="a sequence")] + elif key == "output_token_ids_logprobs": + kw["output_token_ids_logprobs"] = [] if x is None else [decode_NullableTopLogprobs(e) for e in _expect_seq(x, expected="a sequence")] + elif key == "input_token_ids_logprobs": + kw["input_token_ids_logprobs"] = [] if x is None else [decode_NullableTopLogprobs(e) for e in _expect_seq(x, expected="a sequence")] + elif key == "hidden_states": + kw["hidden_states"] = [] if x is None else [decode_HiddenStateRow(e) for e in _expect_seq(x, expected="a sequence")] + elif key == "e2e_latency": + kw["e2e_latency"] = None if x is None else _expect_float(x, expected="f64") + return GenerateMetaInfo(**kw) + + +def encode_GenerateMetaInfo(v: GenerateMetaInfo) -> Dict[str, Any]: + d: Dict[str, Any] = {} + d["id"] = v.id + d["prompt_tokens"] = v.prompt_tokens + d["completion_tokens"] = v.completion_tokens + d["finish_reason"] = None if v.finish_reason is None else encode_FinishReason(v.finish_reason) + if v.output_token_logprobs is not None: + d["output_token_logprobs"] = encode_LogprobEntries(v.output_token_logprobs) + if v.input_token_logprobs is not None: + d["input_token_logprobs"] = encode_LogprobEntries(v.input_token_logprobs) + if v.output_top_logprobs: + d["output_top_logprobs"] = [encode_NullableTopLogprobs(e) for e in v.output_top_logprobs] + if v.input_top_logprobs: + d["input_top_logprobs"] = [encode_NullableTopLogprobs(e) for e in v.input_top_logprobs] + if v.output_token_ids_logprobs: + d["output_token_ids_logprobs"] = [encode_NullableTopLogprobs(e) for e in v.output_token_ids_logprobs] + if v.input_token_ids_logprobs: + d["input_token_ids_logprobs"] = [encode_NullableTopLogprobs(e) for e in v.input_token_ids_logprobs] + if v.hidden_states: + d["hidden_states"] = [encode_HiddenStateRow(e) for e in v.hidden_states] + if v.e2e_latency is not None: + d["e2e_latency"] = float(v.e2e_latency) + return d + + +def decode_GenerateResponse(value: Any) -> GenerateResponse: + m = _expect_map(value, expected="struct GenerateResponse") + kw: Dict[str, Any] = {} + for key, x in m.items(): + if key == "text": + kw["text"] = _expect_str(x, expected="a string") + elif key == "meta_info": + kw["meta_info"] = None if x is None else decode_GenerateMetaInfo(x) + elif key == "output_ids": + kw["output_ids"] = None if x is None else decode_TokenIds(x) + elif key == "index": + kw["index"] = None if x is None else _expect_uint(x, expected="u32") + return GenerateResponse(**kw) + + +def encode_GenerateResponse(v: GenerateResponse) -> Dict[str, Any]: + d: Dict[str, Any] = {} + d["text"] = v.text + if v.meta_info is not None: + d["meta_info"] = encode_GenerateMetaInfo(v.meta_info) + if v.output_ids is not None: + d["output_ids"] = encode_TokenIds(v.output_ids) + if v.index is not None: + d["index"] = v.index + return d + + +def decode_GenerateStreamError(value: Any) -> GenerateStreamError: + m = _expect_map(value, expected="struct GenerateStreamError") + kw: Dict[str, Any] = {} + for key, x in m.items(): + if key == "error": + kw["error"] = None if x is None else decode_ErrorBody(x) + elif key == "index": + kw["index"] = None if x is None else _expect_uint(x, expected="u32") + return GenerateStreamError(**kw) + + +def encode_GenerateStreamError(v: GenerateStreamError) -> Dict[str, Any]: + d: Dict[str, Any] = {} + if v.error is not None: + d["error"] = encode_ErrorBody(v.error) + if v.index is not None: + d["index"] = v.index + return d + + +_FIELDS_MediaRef = ("url", "detail", "max_dynamic_patch", "content_hash", "min_pixels", "max_pixels", "total_max_pixels", "fps", "num_frames", "min_frames", "max_frames", "max_tokens_per_frame", "max_image_tokens", "use_audio",) + + +def decode_MediaRef(value: Any) -> MediaRef: + m = _expect_map(value, expected="struct MediaRef") + kw: Dict[str, Any] = {} + for key, x in m.items(): + if key == "url": + kw["url"] = _expect_str(x, expected="a string") + elif key == "detail": + kw["detail"] = None if x is None else _expect_str(x, expected="a string") + elif key == "max_dynamic_patch": + kw["max_dynamic_patch"] = None if x is None else _expect_int(x, expected="i64") + elif key == "content_hash": + kw["content_hash"] = None if x is None else _expect_str(x, expected="a string") + elif key == "min_pixels": + kw["min_pixels"] = None if x is None else _expect_int(x, expected="i64") + elif key == "max_pixels": + kw["max_pixels"] = None if x is None else _expect_int(x, expected="i64") + elif key == "total_max_pixels": + kw["total_max_pixels"] = None if x is None else _expect_int(x, expected="i64") + elif key == "fps": + kw["fps"] = None if x is None else _expect_float(x, expected="f64") + elif key == "num_frames": + kw["num_frames"] = None if x is None else _expect_int(x, expected="i64") + elif key == "min_frames": + kw["min_frames"] = None if x is None else _expect_int(x, expected="i64") + elif key == "max_frames": + kw["max_frames"] = None if x is None else _expect_int(x, expected="i64") + elif key == "max_tokens_per_frame": + kw["max_tokens_per_frame"] = None if x is None else _expect_int(x, expected="i64") + elif key == "max_image_tokens": + kw["max_image_tokens"] = None if x is None else _expect_int(x, expected="i64") + elif key == "use_audio": + kw["use_audio"] = None if x is None else _expect_bool(x, expected="a boolean") + else: + raise _unknown_field(key, fields=_FIELDS_MediaRef) + return MediaRef(**kw) + + +def encode_MediaRef(v: MediaRef) -> Dict[str, Any]: + d: Dict[str, Any] = {} + d["url"] = v.url + if v.detail is not None: + d["detail"] = v.detail + if v.max_dynamic_patch is not None: + d["max_dynamic_patch"] = v.max_dynamic_patch + if v.content_hash is not None: + d["content_hash"] = v.content_hash + if v.min_pixels is not None: + d["min_pixels"] = v.min_pixels + if v.max_pixels is not None: + d["max_pixels"] = v.max_pixels + if v.total_max_pixels is not None: + d["total_max_pixels"] = v.total_max_pixels + if v.fps is not None: + d["fps"] = float(v.fps) + if v.num_frames is not None: + d["num_frames"] = v.num_frames + if v.min_frames is not None: + d["min_frames"] = v.min_frames + if v.max_frames is not None: + d["max_frames"] = v.max_frames + if v.max_tokens_per_frame is not None: + d["max_tokens_per_frame"] = v.max_tokens_per_frame + if v.max_image_tokens is not None: + d["max_image_tokens"] = v.max_image_tokens + if v.use_audio is not None: + d["use_audio"] = v.use_audio + return d + + +def decode_MediaItem(value: Any) -> MediaItem: + if isinstance(value, str): + return value + if isinstance(value, dict): + return decode_MediaRef(value) + raise _invalid_type(value, expected="one of the MediaItem shapes") + + +def encode_MediaItem(v: MediaItem) -> Any: + if isinstance(v, (str,)): + return v + if isinstance(v, (MediaRef,)): + return encode_MediaRef(v) + return v + + +def decode_OptionalMediaItem(value: Any) -> OptionalMediaItem: + return None if value is None else decode_MediaItem(value) + + +def encode_OptionalMediaItem(v: OptionalMediaItem) -> Any: + return None if v is None else encode_MediaItem(v) + + +def decode_MediaItems(value: Any) -> MediaItems: + return [decode_OptionalMediaItem(e) for e in _expect_seq(value, expected="a sequence")] + + +def encode_MediaItems(v: MediaItems) -> List[Any]: + return [encode_OptionalMediaItem(e) for e in v] + + +def decode_OptionalMediaItems(value: Any) -> OptionalMediaItems: + return None if value is None else decode_MediaItems(value) + + +def encode_OptionalMediaItems(v: OptionalMediaItems) -> Any: + return None if v is None else encode_MediaItems(v) + + +def decode_MediaItemsList(value: Any) -> MediaItemsList: + return [decode_OptionalMediaItems(e) for e in _expect_seq(value, expected="a sequence")] + + +def encode_MediaItemsList(v: MediaItemsList) -> List[Any]: + return [encode_OptionalMediaItems(e) for e in v] + + +def decode_MediaInput(value: Any) -> MediaInput: + if isinstance(value, str): + return decode_MediaItem(value) + if isinstance(value, dict): + return decode_MediaItem(value) + if isinstance(value, list): + if value and isinstance(value[0], list): + return decode_MediaItemsList(value) + return decode_MediaItems(value) + raise _invalid_type(value, expected="one of the MediaInput shapes") + + +def encode_MediaInput(v: MediaInput) -> Any: + if isinstance(v, (str, MediaRef,)): + return encode_MediaItem(v) + if isinstance(v, list) and not (v and isinstance(v[0], list)): + return encode_MediaItems(v) + if isinstance(v, list) and (v and isinstance(v[0], list)): + return encode_MediaItemsList(v) + return v + + +def decode_HealthCheckRequest(value: Any) -> HealthCheckRequest: + m = _expect_map(value, expected="struct HealthCheckRequest") + return HealthCheckRequest() + + +def encode_HealthCheckRequest(v: HealthCheckRequest) -> Dict[str, Any]: + d: Dict[str, Any] = {} + return d + + +def decode_HealthCheckResponse(value: Any) -> HealthCheckResponse: + m = _expect_map(value, expected="struct HealthCheckResponse") + kw: Dict[str, Any] = {} + for key, x in m.items(): + if key == "healthy": + kw["healthy"] = _expect_bool(x, expected="a boolean") + return HealthCheckResponse(**kw) + + +def encode_HealthCheckResponse(v: HealthCheckResponse) -> Dict[str, Any]: + d: Dict[str, Any] = {} + d["healthy"] = v.healthy + return d + + +def decode_GetModelInfoRequest(value: Any) -> GetModelInfoRequest: + m = _expect_map(value, expected="struct GetModelInfoRequest") + return GetModelInfoRequest() + + +def encode_GetModelInfoRequest(v: GetModelInfoRequest) -> Dict[str, Any]: + d: Dict[str, Any] = {} + return d + + +def decode_GetModelInfoResponse(value: Any) -> GetModelInfoResponse: + m = _expect_map(value, expected="struct GetModelInfoResponse") + kw: Dict[str, Any] = {} + for key, x in m.items(): + if key == "model_path": + kw["model_path"] = _expect_str(x, expected="a string") + elif key == "served_model_name": + kw["served_model_name"] = _expect_str(x, expected="a string") + elif key == "tokenizer_path": + kw["tokenizer_path"] = _expect_str(x, expected="a string") + elif key == "is_generation": + kw["is_generation"] = _expect_bool(x, expected="a boolean") + elif key == "preferred_sampling_params": + kw["preferred_sampling_params"] = x + elif key == "weight_version": + kw["weight_version"] = None if x is None else _expect_str(x, expected="a string") + elif key == "load_format": + kw["load_format"] = None if x is None else _expect_str(x, expected="a string") + elif key == "reasoning_parser": + kw["reasoning_parser"] = None if x is None else _expect_str(x, expected="a string") + elif key == "tool_call_parser": + kw["tool_call_parser"] = None if x is None else _expect_str(x, expected="a string") + return GetModelInfoResponse(**kw) + + +def encode_GetModelInfoResponse(v: GetModelInfoResponse) -> Dict[str, Any]: + d: Dict[str, Any] = {} + d["model_path"] = v.model_path + d["served_model_name"] = v.served_model_name + d["tokenizer_path"] = v.tokenizer_path + d["is_generation"] = v.is_generation + d["preferred_sampling_params"] = v.preferred_sampling_params + d["weight_version"] = v.weight_version + d["load_format"] = v.load_format + d["reasoning_parser"] = v.reasoning_parser + d["tool_call_parser"] = v.tool_call_parser + return d + + +def decode_GetServerInfoRequest(value: Any) -> GetServerInfoRequest: + m = _expect_map(value, expected="struct GetServerInfoRequest") + return GetServerInfoRequest() + + +def encode_GetServerInfoRequest(v: GetServerInfoRequest) -> Dict[str, Any]: + d: Dict[str, Any] = {} + return d + + +def decode_GetServerInfoResponse(value: Any) -> GetServerInfoResponse: + m = _expect_map(value, expected="struct GetServerInfoResponse") + kw: Dict[str, Any] = {} + for key, x in m.items(): + if key == "model_path": + kw["model_path"] = _expect_str(x, expected="a string") + elif key == "served_model_name": + kw["served_model_name"] = _expect_str(x, expected="a string") + elif key == "tokenizer_path": + kw["tokenizer_path"] = _expect_str(x, expected="a string") + elif key == "max_context_length": + kw["max_context_length"] = _expect_uint(x, expected="u32") + elif key == "max_total_num_tokens": + kw["max_total_num_tokens"] = None if x is None else _expect_uint(x, expected="u64") + elif key == "version": + kw["version"] = _expect_str(x, expected="a string") + elif key == "internal_states": + kw["internal_states"] = [] if x is None else list(_expect_seq(x, expected="a sequence")) + return GetServerInfoResponse(**kw) + + +def encode_GetServerInfoResponse(v: GetServerInfoResponse) -> Dict[str, Any]: + d: Dict[str, Any] = {} + d["model_path"] = v.model_path + d["served_model_name"] = v.served_model_name + d["tokenizer_path"] = v.tokenizer_path + d["max_context_length"] = v.max_context_length + if v.max_total_num_tokens is not None: + d["max_total_num_tokens"] = v.max_total_num_tokens + d["version"] = v.version + if v.internal_states: + d["internal_states"] = list(v.internal_states) + return d diff --git a/python/sglang/srt/entrypoints/api_contract.py b/python/sglang/srt/entrypoints/api_contract.py new file mode 100644 index 0000000000..5ea4fbf65f --- /dev/null +++ b/python/sglang/srt/entrypoints/api_contract.py @@ -0,0 +1,18 @@ +"""The HTTP JSON contract gate: `proto/sglang/api/v1` is the schema for both +the Rust and the Python server, so a `/generate` body the Python dataclass +parse admitted but the schema rejects is refused here with the schema's own +error text. The generated decoder is the single source of that judgment. +""" + +from typing import Any, Optional + +from sglang.api.v1.api_types import GenerateRequest, JsonContractError + + +def generate_contract_error(body: Any) -> Optional[str]: + """The contract violation in a parsed `/generate` JSON body, or None.""" + try: + GenerateRequest.from_json_value(body) + except JsonContractError as e: + return str(e) + return None diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py index 0fb6f4813a..9f621f06be 100644 --- a/python/sglang/srt/entrypoints/http_server.py +++ b/python/sglang/srt/entrypoints/http_server.py @@ -73,6 +73,7 @@ from sglang.srt.entrypoints.anthropic.protocol import ( AnthropicMessagesRequest, ) from sglang.srt.entrypoints.anthropic.serving import AnthropicServing +from sglang.srt.entrypoints.api_contract import generate_contract_error from sglang.srt.entrypoints.engine import ( Engine, init_tokenizer_manager, @@ -908,6 +909,11 @@ if os.environ.get("DUMPER_SERVER_PORT") == "reuse": ) async def generate_request(obj: GenerateReqInput, request: Request): """Handle a generate request.""" + # Starlette caches the parsed body, so this is the schema check, not a + # second decode. + contract_error = generate_contract_error(await request.json()) + if contract_error is not None: + return ORJSONResponse(status_code=400, content={"error": contract_error}) if envs.SGLANG_ENABLE_REQUEST_HEADER_OVERRIDES.get(): apply_header_overrides(obj, request.headers) if obj.stream: diff --git a/rust/.cargo/config.toml b/rust/.cargo/config.toml new file mode 100644 index 0000000000..03340c536b --- /dev/null +++ b/rust/.cargo/config.toml @@ -0,0 +1,5 @@ +# Workspace-local cargo aliases. +[alias] +# Regenerate the API types (Rust + Python) from proto/sglang; see +# sglang-api-codegen/src/main.rs. +gen-api = "run -p sglang-api-codegen" diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 2e2b522046..a5a51104f8 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -292,40 +292,13 @@ dependencies = [ "arrayvec", ] -[[package]] -name = "axum" -version = "0.7.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" -dependencies = [ - "async-trait", - "axum-core 0.4.5", - "bytes", - "futures-util", - "http", - "http-body", - "http-body-util", - "itoa", - "matchit 0.7.3", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "sync_wrapper", - "tower 0.5.3", - "tower-layer", - "tower-service", -] - [[package]] name = "axum" version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ - "axum-core 0.5.6", + "axum-core", "bytes", "form_urlencoded", "futures-util", @@ -335,7 +308,7 @@ dependencies = [ "hyper", "hyper-util", "itoa", - "matchit 0.8.4", + "matchit", "memchr", "mime", "percent-encoding", @@ -346,32 +319,12 @@ dependencies = [ "serde_urlencoded", "sync_wrapper", "tokio", - "tower 0.5.3", + "tower", "tower-layer", "tower-service", "tracing", ] -[[package]] -name = "axum-core" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http", - "http-body", - "http-body-util", - "mime", - "pin-project-lite", - "rustversion", - "sync_wrapper", - "tower-layer", - "tower-service", -] - [[package]] name = "axum-core" version = "0.5.6" @@ -1265,6 +1218,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -1492,6 +1451,15 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + [[package]] name = "hashbrown" version = "0.16.1" @@ -1658,7 +1626,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2", "system-configuration", "tokio", "tower-service", @@ -2088,12 +2056,6 @@ dependencies = [ "regex-automata", ] -[[package]] -name = "matchit" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" - [[package]] name = "matchit" version = "0.8.4" @@ -2558,11 +2520,12 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "petgraph" -version = "0.7.1" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset", + "hashbrown 0.15.5", "indexmap 2.14.0", ] @@ -2739,9 +2702,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.13.5" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" dependencies = [ "bytes", "prost-derive", @@ -2749,19 +2712,20 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.13.5" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck", "itertools", "log", "multimap", - "once_cell", "petgraph", "prettyplease", "prost", "prost-types", + "pulldown-cmark", + "pulldown-cmark-to-cmark", "regex", "syn 2.0.119", "tempfile", @@ -2769,9 +2733,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.13.5" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", "itertools", @@ -2781,10 +2745,20 @@ dependencies = [ ] [[package]] -name = "prost-types" -version = "0.13.5" +name = "prost-reflect" +version = "0.16.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +checksum = "01b80ea363c31af2de2b92e3c07ed1156628f7838c4afb4df75ee78a37fedbd1" +dependencies = [ + "prost", + "prost-types", +] + +[[package]] +name = "prost-types" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" dependencies = [ "prost", ] @@ -2853,6 +2827,26 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95067976aca6421a523e491fce939a3e65249bac4b977adee0ee9771568e8aa3" +[[package]] +name = "pulldown-cmark" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e" +dependencies = [ + "bitflags", + "memchr", + "unicase", +] + +[[package]] +name = "pulldown-cmark-to-cmark" +version = "22.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84bbb29c624230c4bd1047bbdb2aa47e41c860e9665ce62ba9504eebe91bf867" +dependencies = [ + "pulldown-cmark", +] + [[package]] name = "pulp" version = "0.22.3" @@ -2967,7 +2961,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.3", "rustls", - "socket2 0.5.10", + "socket2", "thiserror", "tokio", "tracing", @@ -3005,7 +2999,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2", "tracing", "windows-sys 0.61.2", ] @@ -3344,7 +3338,7 @@ dependencies = [ "tokio", "tokio-rustls", "tokio-util", - "tower 0.5.3", + "tower", "tower-http", "tower-service", "url", @@ -3688,6 +3682,30 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "sglang-api-codegen" +version = "0.1.0" +dependencies = [ + "prost", + "prost-build", + "prost-reflect", + "protoc-bin-vendored", + "tonic-prost-build", +] + +[[package]] +name = "sglang-api-types" +version = "0.1.0" +dependencies = [ + "bytes", + "prost", + "prost-types", + "serde", + "serde_json", + "tonic", + "tonic-prost", +] + [[package]] name = "sglang-grpc" version = "0.1.0" @@ -3696,28 +3714,17 @@ dependencies = [ "prost-types", "pyo3", "serde_json", - "sglang-grpc-types", + "sglang-api-types", "tokenizers", "tokio", "tokio-stream", "tonic", + "tonic-prost", "tracing", "tracing-subscriber", "uuid", ] -[[package]] -name = "sglang-grpc-types" -version = "0.1.0" -dependencies = [ - "prost", - "prost-build", - "prost-types", - "protoc-bin-vendored", - "tonic", - "tonic-build", -] - [[package]] name = "sglang-mm" version = "0.1.0" @@ -3757,7 +3764,7 @@ name = "sglang-renderer" version = "0.1.0" dependencies = [ "async-stream", - "axum 0.8.9", + "axum", "clap", "dynamo-parsers", "dynamo-protocols", @@ -3776,7 +3783,7 @@ dependencies = [ "sglang-processor", "thiserror", "tokio", - "tower 0.5.3", + "tower", "tracing", "tracing-subscriber", "uuid", @@ -3788,7 +3795,7 @@ version = "0.1.0" dependencies = [ "arc-swap", "async-stream", - "axum 0.8.9", + "axum", "bytemuck", "bytes", "core_affinity", @@ -3799,6 +3806,7 @@ dependencies = [ "flume", "futures", "hf-hub", + "http", "itertools", "minijinja", "numpy", @@ -3809,14 +3817,14 @@ dependencies = [ "rustix", "serde", "serde_json", - "sglang-grpc-types", + "sglang-api-types", "sglang-mm", - "socket2 0.6.5", + "socket2", "thiserror", "tokio", "tokio-stream", "tonic", - "tower 0.5.3", + "tower", "tracing", "tracing-appender", "tracing-subscriber", @@ -3903,16 +3911,6 @@ version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" -[[package]] -name = "socket2" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - [[package]] name = "socket2" version = "0.6.5" @@ -4253,7 +4251,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.5", + "socket2", "tokio-macros", "windows-sys 0.61.2", ] @@ -4306,16 +4304,14 @@ dependencies = [ [[package]] name = "tonic" -version = "0.12.3" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" +checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ - "async-stream", "async-trait", - "axum 0.7.9", + "axum", "base64 0.22.1", "bytes", - "flate2", "h2", "http", "http-body", @@ -4325,11 +4321,11 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "prost", - "socket2 0.5.10", + "socket2", + "sync_wrapper", "tokio", "tokio-stream", - "tower 0.4.13", + "tower", "tower-layer", "tower-service", "tracing", @@ -4337,9 +4333,32 @@ dependencies = [ [[package]] name = "tonic-build" -version = "0.12.3" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9557ce109ea773b399c9b9e5dca39294110b74f1f342cb347a80d1fce8c26a11" +checksum = "c68f61875ac5293cf72e6c8cf0158086428c82c37229e98c840878f1706b0322" +dependencies = [ + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tonic-prost" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" +dependencies = [ + "bytes", + "prost", + "tonic", +] + +[[package]] +name = "tonic-prost-build" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "654e5643eff75d7f8c99197ce1440ed19a3474eada74c12bbac488b2cafdae27" dependencies = [ "prettyplease", "proc-macro2", @@ -4347,26 +4366,8 @@ dependencies = [ "prost-types", "quote", "syn 2.0.119", -] - -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "indexmap 1.9.3", - "pin-project", - "pin-project-lite", - "rand 0.8.7", - "slab", - "tokio", - "tokio-util", - "tower-layer", - "tower-service", - "tracing", + "tempfile", + "tonic-build", ] [[package]] @@ -4377,9 +4378,12 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", + "indexmap 2.14.0", "pin-project-lite", + "slab", "sync_wrapper", "tokio", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -4397,7 +4401,7 @@ dependencies = [ "http", "http-body", "pin-project-lite", - "tower 0.5.3", + "tower", "tower-layer", "tower-service", "url", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 6e7d656f13..587f3388cc 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,7 +1,8 @@ [workspace] resolver = "3" members = [ - "../proto", + "sglang-api-codegen", + "sglang-api-types", "sglang-grpc", "sglang-mm", "sglang-processor", @@ -30,9 +31,9 @@ futures = "0.3" pyo3 = { version = "0.29.0", features = ["extension-module"] } # These versions define the generated public gRPC API. Update them together # and deliberately rather than changing bindings during an unrelated build. -prost = "=0.13.5" -prost-types = "=0.13.5" -prost-build = "=0.13.5" +prost = "0.14" +prost-types = "0.14" +prost-build = "0.14" protoc-bin-vendored = "=3.2.0" serde = { version = "1", features = ["derive"] } serde_json = "1" @@ -40,11 +41,13 @@ socket2 = "0.6" thiserror = "2" tokio = { version = "1", features = ["full"] } tokio-stream = { version = "0.1", features = ["net"] } +tonic = "0.14" +tonic-build = "0.14" +tonic-prost = "0.14" +tower = { version = "0.5", features = ["util"] } tracing = "0.1" tracing-appender = "0.2" tracing-subscriber = { version = "0.3", features = ["env-filter"] } -tonic = { version = "=0.12.3", features = ["gzip", "transport"] } -tonic-build = "=0.12.3" uuid = { version = "1", features = ["v4"] } # Profiles only take effect at the workspace root; per-member profile sections diff --git a/rust/sglang-api-codegen/Cargo.toml b/rust/sglang-api-codegen/Cargo.toml new file mode 100644 index 0000000000..32f142167e --- /dev/null +++ b/rust/sglang-api-codegen/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "sglang-api-codegen" +description = "Generator: proto/sglang -> checked-in bindings (prost structs, tonic services, schema-driven serde, Python types)" +version.workspace = true +edition.workspace = true +license.workspace = true + +# No [package.metadata.sglang]: not a Python extension; wheel builds skip it. + +[[bin]] +name = "sglang-api-codegen" +path = "src/main.rs" + +[dependencies] +prost = { workspace = true } +prost-build = "0.14" +prost-reflect = "0.16" +protoc-bin-vendored = "3" +tonic-prost-build = "0.14" diff --git a/rust/sglang-api-codegen/src/emit.rs b/rust/sglang-api-codegen/src/emit.rs new file mode 100644 index 0000000000..7c94eea85d --- /dev/null +++ b/rust/sglang-api-codegen/src/emit.rs @@ -0,0 +1,1081 @@ +//! Pass 2: render each message's serde implementation from its +//! [`model::MessageModel`]. String-template emission; the output is checked +//! in and rustfmt'd, so it is reviewed like hand-written code. +//! +//! JSON contracts implemented here (see proto/sglang/json/v1/options.proto): +//! bare_list, tuple, one_or_many, tagged unions with passthrough, untagged +//! unions, per-field null/default/presence rules, raw_json passthrough, and +//! the IGNORE/DENY unknown-key policies with serde's own error texts. + +use std::collections::HashMap; + +use crate::model::{self, FieldKind, FieldModel, MessageKind, MessageModel, ScalarTy}; +use prost_reflect::DescriptorPool; + +pub fn emit_serde(descriptor_bytes: &[u8], package: &str) -> String { + let pool = DescriptorPool::decode(descriptor_bytes).expect("decode descriptor set"); + let models = model::build_models(&pool, package); + let by_name: HashMap<&str, &MessageModel> = + models.iter().map(|m| (m.rust_name.as_str(), m)).collect(); + + let mut out = String::new(); + out.push_str( + "// Generated by sglang-api-codegen from proto/sglang — DO NOT EDIT.\n\ + // The sglang.json.v1 options in the schema are the JSON contract; edit the\n\ + // .proto files and `cargo gen-api`.\n\n", + ); + for m in &models { + match &m.kind { + MessageKind::Skip => {} + MessageKind::BareList { item } => emit_bare_list(&mut out, m, item), + MessageKind::Tuple { fields } => emit_tuple(&mut out, m, fields), + MessageKind::Transparent { field } => emit_transparent(&mut out, m, field), + MessageKind::OneOrMany { + oneof_mod, + one, + many, + } => emit_one_or_many(&mut out, m, oneof_mod, one, many, &by_name), + MessageKind::Untagged { + oneof_mod, + oneof_ident, + variants, + } => emit_untagged(&mut out, m, oneof_mod, oneof_ident, variants, &by_name), + MessageKind::TaggedUnion { + oneof_mod, + oneof_ident, + tag, + variants, + passthrough, + } => emit_tagged( + &mut out, + m, + oneof_mod, + oneof_ident, + tag, + variants, + passthrough.as_ref(), + ), + MessageKind::Struct { + deny_unknown, + fields, + } => emit_struct(&mut out, m, *deny_unknown, fields), + } + } + out +} + +fn item_rust_ty(kind: &FieldKind) -> String { + match kind { + FieldKind::RepeatedScalar { ty } => ty.rust().to_string(), + FieldKind::RepeatedMessage { rust_path } => rust_path.clone(), + other => panic!("bare_list item must be repeated, got {other:?}"), + } +} + +fn emit_bare_list(out: &mut String, m: &MessageModel, item: &FieldModel) { + let name = &m.rust_name; + let field = &item.rust_ident; + let item_ty = item_rust_ty(&item.kind); + out.push_str(&format!( + r#"impl ::serde::Serialize for {name} {{ + fn serialize(&self, serializer: S) -> Result {{ + serializer.collect_seq(self.{field}.iter()) + }} +}} + +impl<'de> ::serde::Deserialize<'de> for {name} {{ + fn deserialize>(deserializer: D) -> Result {{ + Ok({name} {{ + {field}: <::prost::alloc::vec::Vec<{item_ty}> as ::serde::Deserialize>::deserialize(deserializer)?, + }}) + }} +}} + +"# + )); +} + +fn emit_transparent(out: &mut String, m: &MessageModel, field: &FieldModel) { + let name = &m.rust_name; + let ident = &field.rust_ident; + let ty = match &field.kind { + FieldKind::OptScalar { ty, .. } => ty.rust().to_string(), + FieldKind::Message { rust_path, .. } => rust_path.clone(), + other => panic!( + "{name}: transparent field must be an optional scalar or a message, got {other:?}" + ), + }; + out.push_str(&format!( + r#"impl ::serde::Serialize for {name} {{ + fn serialize(&self, serializer: S) -> Result {{ + self.{ident}.serialize(serializer) + }} +}} + +impl<'de> ::serde::Deserialize<'de> for {name} {{ + fn deserialize>(deserializer: D) -> Result {{ + Ok({name} {{ + {ident}: as ::serde::Deserialize>::deserialize(deserializer)?, + }}) + }} +}} + +"# + )); +} + +fn emit_tuple(out: &mut String, m: &MessageModel, fields: &[FieldModel]) { + let name = &m.rust_name; + let len = fields.len(); + let mut ser_elems = String::new(); + let mut de_elems = String::new(); + for (i, f) in fields.iter().enumerate() { + let ident = &f.rust_ident; + ser_elems.push_str(&format!( + " ::serde::ser::SerializeSeq::serialize_element(&mut seq, &self.{ident})?;\n" + )); + let ty = match &f.kind { + FieldKind::Scalar { ty, .. } => ty.rust().to_string(), + FieldKind::OptScalar { ty, .. } => format!("Option<{}>", ty.rust()), + other => panic!("tuple member kind {other:?} unsupported"), + }; + de_elems.push_str(&format!( + " let {ident}: {ty} = seq + .next_element()? + .ok_or_else(|| ::serde::de::Error::invalid_length({i}, &\"a {len}-element tuple\"))?;\n" + )); + } + let build: Vec = fields.iter().map(|f| f.rust_ident.clone()).collect(); + let build = build.join(", "); + out.push_str(&format!( + r#"impl ::serde::Serialize for {name} {{ + fn serialize(&self, serializer: S) -> Result {{ + let mut seq = serializer.serialize_seq(Some({len}))?; +{ser_elems} ::serde::ser::SerializeSeq::end(seq) + }} +}} + +impl<'de> ::serde::Deserialize<'de> for {name} {{ + fn deserialize>(deserializer: D) -> Result {{ + struct V; + impl<'de> ::serde::de::Visitor<'de> for V {{ + type Value = {name}; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {{ + f.write_str("a {len}-element tuple") + }} + fn visit_seq>(self, mut seq: A) -> Result {{ +{de_elems} Ok({name} {{ {build} }}) + }} + }} + deserializer.deserialize_seq(V) + }} +}} + +"# + )); +} + +fn emit_one_or_many( + out: &mut String, + m: &MessageModel, + oneof_mod: &str, + one: &model::VariantModel, + many: &model::VariantModel, + models: &HashMap<&str, &MessageModel>, +) { + let name = &m.rust_name; + let one_v = &one.rust_variant; + let many_v = &many.rust_variant; + let many_ty = match &many.kind { + FieldKind::Message { rust_path, .. } => rust_path.clone(), + other => panic!("{name}: many arm must be a list message, got {other:?}"), + }; + + // The scalar-shape visitors the `one` arm answers to. + let one_visits = match &one.kind { + FieldKind::Scalar { + ty: ScalarTy::Str, .. + } + | FieldKind::OptScalar { + ty: ScalarTy::Str, .. + } => format!( + r#" fn visit_str(self, v: &str) -> Result {{ + Ok({name} {{ value: Some({oneof_mod}::Value::{one_v}(v.to_owned())) }}) + }} + fn visit_string(self, v: String) -> Result {{ + Ok({name} {{ value: Some({oneof_mod}::Value::{one_v}(v)) }}) + }} +"# + ), + FieldKind::Scalar { + ty: ScalarTy::Bool, .. + } + | FieldKind::OptScalar { + ty: ScalarTy::Bool, .. + } => format!( + r#" fn visit_bool(self, v: bool) -> Result {{ + Ok({name} {{ value: Some({oneof_mod}::Value::{one_v}(v)) }}) + }} +"# + ), + FieldKind::Scalar { + ty: ScalarTy::I64, .. + } + | FieldKind::OptScalar { + ty: ScalarTy::I64, .. + } => format!( + r#" fn visit_i64(self, v: i64) -> Result {{ + Ok({name} {{ value: Some({oneof_mod}::Value::{one_v}(v)) }}) + }} + fn visit_u64(self, v: u64) -> Result {{ + let v = i64::try_from(v).map_err(|_| E::custom("integer out of range"))?; + Ok({name} {{ value: Some({oneof_mod}::Value::{one_v}(v)) }}) + }} +"# + ), + // A message-typed `one` (SamplingParams, OptionalString wrappers): + // maps go to `one`; bare scalars for the Optional* wrappers arrive as + // visit_str/visit_i64 handled below in the wrapper-specific arm. + FieldKind::Message { rust_path, .. } => match rust_path.as_str() { + "OptionalString" => format!( + r#" fn visit_str(self, v: &str) -> Result {{ + Ok({name} {{ value: Some({oneof_mod}::Value::{one_v}(OptionalString {{ value: Some(v.to_owned()) }})) }}) + }} + fn visit_string(self, v: String) -> Result {{ + Ok({name} {{ value: Some({oneof_mod}::Value::{one_v}(OptionalString {{ value: Some(v) }})) }}) + }} +"# + ), + "OptionalInt64" => format!( + r#" fn visit_i64(self, v: i64) -> Result {{ + Ok({name} {{ value: Some({oneof_mod}::Value::{one_v}(OptionalInt64 {{ value: Some(v) }})) }}) + }} + fn visit_u64(self, v: u64) -> Result {{ + let v = i64::try_from(v).map_err(|_| E::custom("integer out of range"))?; + Ok({name} {{ value: Some({oneof_mod}::Value::{one_v}(OptionalInt64 {{ value: Some(v) }})) }}) + }} +"# + ), + _ => format!( + r#" fn visit_map>(self, map: A) -> Result {{ + let one = <{path} as ::serde::Deserialize>::deserialize( + ::serde::de::value::MapAccessDeserializer::new(map), + )?; + Ok({name} {{ value: Some({oneof_mod}::Value::{one_v}(one)) }}) + }} +"#, + path = rust_path + ), + }, + other => panic!("{name}: unsupported one arm {other:?}"), + }; + + // Sequence handling. For a message-typed `one` whose JSON is ITSELF an + // array (TokenIds), the outer array is ambiguous: dispatch on the first + // element's shape (number => one flat id list, array/other => many). + let seq_visit = if let Some((one_ty, first_check)) = bare_scalar_list(&one.kind, models) { + format!( + r#" fn visit_seq>(self, mut seq: A) -> Result {{ + let Some(first) = seq.next_element::<::serde_json::Value>()? else {{ + // [] is one empty list (the untagged first-arm rule). + return Ok({name} {{ value: Some({oneof_mod}::Value::{one_v}({one_ty}::default())) }}); + }}; + let is_number = {first_check}; + let mut rest = vec![first]; + while let Some(v) = seq.next_element::<::serde_json::Value>()? {{ + rest.push(v); + }} + let reparse = ::serde_json::Value::Array(rest); + if is_number {{ + let one: {one_ty} = ::serde_json::from_value(reparse) + .map_err(::serde::de::Error::custom)?; + Ok({name} {{ value: Some({oneof_mod}::Value::{one_v}(one)) }}) + }} else {{ + let many: {many_ty} = ::serde_json::from_value(reparse) + .map_err(::serde::de::Error::custom)?; + Ok({name} {{ value: Some({oneof_mod}::Value::{many_v}(many)) }}) + }} + }} +"# + ) + } else { + format!( + r#" fn visit_seq>(self, seq: A) -> Result {{ + let many = <{many_ty} as ::serde::Deserialize>::deserialize( + ::serde::de::value::SeqAccessDeserializer::new(seq), + )?; + Ok({name} {{ value: Some({oneof_mod}::Value::{many_v}(many)) }}) + }} +"# + ) + }; + + let companion = if matches!(&one.kind, FieldKind::Message { rust_path, .. } if rust_path == "SamplingParams") + { + format!( + "impl {name} {{\n /// Protobuf arrival: recurse the absent-defaults rule into the params.\n pub fn apply_absent_defaults(&mut self) {{\n match self.value.as_mut() {{\n Some({oneof_mod}::Value::{one_v}(p)) => p.apply_absent_defaults(),\n Some({oneof_mod}::Value::{many_v}(l)) => {{\n for p in &mut l.items {{\n p.apply_absent_defaults();\n }}\n }}\n None => {{}}\n }}\n }}\n}}\n\n" + ) + } else { + String::new() + }; + out.push_str(&companion); + let expecting = format!("a value or an array of values for {name}"); + out.push_str(&format!( + r#"impl ::serde::Serialize for {name} {{ + fn serialize(&self, serializer: S) -> Result {{ + match &self.value {{ + Some({oneof_mod}::Value::{one_v}(v)) => v.serialize(serializer), + Some({oneof_mod}::Value::{many_v}(v)) => v.serialize(serializer), + None => serializer.serialize_none(), + }} + }} +}} + +impl<'de> ::serde::Deserialize<'de> for {name} {{ + fn deserialize>(deserializer: D) -> Result {{ + struct V; + impl<'de> ::serde::de::Visitor<'de> for V {{ + type Value = {name}; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {{ + f.write_str({expecting:?}) + }} +{one_visits}{seq_visit} }} + deserializer.deserialize_any(V) + }} +}} + +"# + )); +} + +/// The JSON value shapes a field or message can take; the untagged dispatch +/// keys on these. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +enum JsonShape { + Str, + Int, + Float, + Bool, + Map, + Seq, + Null, +} + +fn scalar_shapes(ty: &ScalarTy) -> Vec { + match ty { + ScalarTy::Str => vec![JsonShape::Str], + ScalarTy::I32 | ScalarTy::I64 | ScalarTy::U32 | ScalarTy::U64 => vec![JsonShape::Int], + ScalarTy::F64 => vec![JsonShape::Int, JsonShape::Float], + ScalarTy::Bool => vec![JsonShape::Bool], + } +} + +fn kind_shapes(kind: &FieldKind, models: &HashMap<&str, &MessageModel>) -> Vec { + match kind { + FieldKind::Scalar { ty, .. } | FieldKind::OptScalar { ty, .. } => scalar_shapes(ty), + FieldKind::Message { rust_path, .. } => message_shapes(rust_path, models), + FieldKind::RepeatedScalar { .. } + | FieldKind::RepeatedMessage { .. } + | FieldKind::RepeatedRawJson => vec![JsonShape::Seq], + FieldKind::Map { .. } => vec![JsonShape::Map], + FieldKind::RawJson { .. } => vec![ + JsonShape::Str, + JsonShape::Int, + JsonShape::Float, + JsonShape::Bool, + JsonShape::Map, + JsonShape::Seq, + JsonShape::Null, + ], + } +} + +fn message_shapes(name: &str, models: &HashMap<&str, &MessageModel>) -> Vec { + let m = models + .get(name) + .unwrap_or_else(|| panic!("unknown message {name}")); + match &m.kind { + MessageKind::Struct { .. } | MessageKind::TaggedUnion { .. } => vec![JsonShape::Map], + MessageKind::Tuple { .. } | MessageKind::BareList { .. } => vec![JsonShape::Seq], + MessageKind::Transparent { field } => { + let mut s = kind_shapes(&field.kind, models); + s.push(JsonShape::Null); + s + } + MessageKind::Untagged { variants, .. } => { + let mut s = Vec::new(); + for v in variants { + for shape in kind_shapes(&v.kind, models) { + if !s.contains(&shape) { + s.push(shape); + } + } + } + s + } + MessageKind::OneOrMany { one, .. } => { + let mut s = kind_shapes(&one.kind, models); + if !s.contains(&JsonShape::Seq) { + s.push(JsonShape::Seq); + } + s + } + MessageKind::Skip => panic!("{name}: a gRPC-only message has no JSON shape"), + } +} + +/// For a bare-list message: the shapes of one ELEMENT. +fn element_shapes(name: &str, models: &HashMap<&str, &MessageModel>) -> Vec { + let m = models + .get(name) + .unwrap_or_else(|| panic!("unknown message {name}")); + match &m.kind { + MessageKind::BareList { item } => match &item.kind { + FieldKind::RepeatedScalar { ty } => scalar_shapes(ty), + FieldKind::RepeatedMessage { rust_path } => message_shapes(rust_path, models), + other => panic!("{name}: bare_list item must be repeated, got {other:?}"), + }, + other => panic!("{name}: not a bare list ({other:?})"), + } +} + +/// A `one` arm that is a bare list of scalars: its Rust type and the +/// `serde_json::Value` test that recognizes one of its elements. +fn bare_scalar_list( + kind: &FieldKind, + models: &HashMap<&str, &MessageModel>, +) -> Option<(String, &'static str)> { + let FieldKind::Message { rust_path, .. } = kind else { + return None; + }; + let m = models.get(rust_path.as_str())?; + let MessageKind::BareList { item } = &m.kind else { + return None; + }; + let FieldKind::RepeatedScalar { ty } = &item.kind else { + return None; + }; + let check = match ty { + ScalarTy::Str => "first.is_string()", + ScalarTy::Bool => "first.is_boolean()", + _ => "first.is_number()", + }; + Some((rust_path.clone(), check)) +} + +/// A message-typed arm built from a visited scalar `v`: the message's own +/// `Deserialize` runs over a value deserializer, so its rules (unknown-key +/// policy, nested unions) apply unchanged. +fn via_value_deserializer(ty: &str, de: &str) -> String { + format!("<{ty} as ::serde::Deserialize>::deserialize(::serde::de::value::{de}::::new(v))?") +} + +fn emit_untagged( + out: &mut String, + m: &MessageModel, + oneof_mod: &str, + oneof_ident: &str, + variants: &[model::VariantModel], + models: &HashMap<&str, &MessageModel>, +) { + let name = &m.rust_name; + let wrap = |var: &str, expr: &str| { + format!("Ok({name} {{ value: Some({oneof_mod}::{oneof_ident}::{var}({expr})) }})") + }; + let mut ser_arms = String::new(); + // shape -> the arms claiming it, in declaration order. + let mut claims: Vec<(JsonShape, Vec<&model::VariantModel>)> = Vec::new(); + for v in variants { + ser_arms.push_str(&format!( + " Some({oneof_mod}::{oneof_ident}::{}(v)) => v.serialize(serializer),\n", + v.rust_variant + )); + for shape in kind_shapes(&v.kind, models) { + match claims.iter_mut().find(|(s, _)| *s == shape) { + Some((_, arms)) => arms.push(v), + None => claims.push((shape, vec![v])), + } + } + } + + let mut visits = String::new(); + for (shape, arms) in &claims { + if *shape != JsonShape::Seq && arms.len() > 1 { + panic!( + "{name}: untagged arms {:?} all accept a JSON {shape:?}; the value's shape cannot pick one", + arms.iter().map(|a| &a.json_name).collect::>() + ); + } + let arm = arms[0]; + let var = &arm.rust_variant; + // A scalar arm takes the value as-is; a message arm runs the message's + // own deserializer over it. + let scalar_arm = matches!( + arm.kind, + FieldKind::Scalar { .. } | FieldKind::OptScalar { .. } + ); + let msg_ty = match &arm.kind { + FieldKind::Message { rust_path, .. } => Some(rust_path.as_str()), + _ => None, + }; + let build = |de: &str, direct: &str| -> String { + if scalar_arm { + wrap(var, direct) + } else { + wrap( + var, + &via_value_deserializer(msg_ty.expect("message arm"), de), + ) + } + }; + let unit = |var: &str| { + wrap( + var, + &format!( + "<{} as ::serde::Deserialize>::deserialize(::serde::de::value::UnitDeserializer::::new())?", + msg_ty.expect("null arm is a message") + ), + ) + }; + match shape { + JsonShape::Str => visits.push_str(&format!( + r#" fn visit_str(self, v: &str) -> Result {{ + {} + }} + fn visit_string(self, v: String) -> Result {{ + {} + }} +"#, + build("StrDeserializer", "v.to_owned()"), + build("StringDeserializer", "v") + )), + JsonShape::Int => visits.push_str(&format!( + r#" fn visit_i64(self, v: i64) -> Result {{ + {} + }} + fn visit_u64(self, v: u64) -> Result {{ + let v = i64::try_from(v).map_err(|_| E::custom("integer out of range"))?; + {} + }} +"#, + build("I64Deserializer", "v"), + build("I64Deserializer", "v") + )), + JsonShape::Float => visits.push_str(&format!( + r#" fn visit_f64(self, v: f64) -> Result {{ + {} + }} +"#, + build("F64Deserializer", "v") + )), + JsonShape::Bool => visits.push_str(&format!( + r#" fn visit_bool(self, v: bool) -> Result {{ + {} + }} +"#, + build("BoolDeserializer", "v") + )), + JsonShape::Null => visits.push_str(&format!( + r#" fn visit_unit(self) -> Result {{ + {} + }} + fn visit_none(self) -> Result {{ + {} + }} +"#, + unit(var), + unit(var) + )), + JsonShape::Map => visits.push_str(&format!( + r#" fn visit_map>(self, map: A) -> Result {{ + let v = <{ty} as ::serde::Deserialize>::deserialize( + ::serde::de::value::MapAccessDeserializer::new(map), + )?; + {} + }} +"#, + wrap(var, "v"), + ty = msg_ty.expect("map arm is a message") + )), + JsonShape::Seq => { + if arms.len() == 1 { + visits.push_str(&format!( + r#" fn visit_seq>(self, seq: A) -> Result {{ + let v = <{ty} as ::serde::Deserialize>::deserialize( + ::serde::de::value::SeqAccessDeserializer::new(seq), + )?; + {} + }} +"#, + wrap(var, "v"), + ty = msg_ty.expect("seq arm is a message") + )); + } else { + // Two array-shaped arms: the one whose ELEMENTS are arrays + // is the nested form; the first element decides, and `[]` + // is the flat form. + assert_eq!(arms.len(), 2, "{name}: more than two array-shaped untagged arms"); + let elem_is_seq = |a: &model::VariantModel| match &a.kind { + FieldKind::Message { rust_path, .. } => { + element_shapes(rust_path, models).contains(&JsonShape::Seq) + } + _ => false, + }; + let (nested, flat) = match (elem_is_seq(arms[0]), elem_is_seq(arms[1])) { + (true, false) => (arms[0], arms[1]), + (false, true) => (arms[1], arms[0]), + _ => panic!( + "{name}: array-shaped arms {} and {} cannot be told apart by their first element", + arms[0].json_name, arms[1].json_name + ), + }; + let ty_of = |a: &model::VariantModel| match &a.kind { + FieldKind::Message { rust_path, .. } => rust_path.clone(), + _ => unreachable!(), + }; + visits.push_str(&format!( + r#" fn visit_seq>(self, mut seq: A) -> Result {{ + let mut items: Vec<::serde_json::Value> = Vec::new(); + while let Some(v) = seq.next_element::<::serde_json::Value>()? {{ + items.push(v); + }} + let nested = items.first().is_some_and(|f| f.is_array()); + let reparse = ::serde_json::Value::Array(items); + if nested {{ + let v: {nested_ty} = ::serde_json::from_value(reparse) + .map_err(::serde::de::Error::custom)?; + {nested_ok} + }} else {{ + let v: {flat_ty} = ::serde_json::from_value(reparse) + .map_err(::serde::de::Error::custom)?; + {flat_ok} + }} + }} +"#, + nested_ty = ty_of(nested), + flat_ty = ty_of(flat), + nested_ok = wrap(&nested.rust_variant, "v"), + flat_ok = wrap(&flat.rust_variant, "v"), + )); + } + } + } + } + out.push_str(&format!( + r#"impl ::serde::Serialize for {name} {{ + fn serialize(&self, serializer: S) -> Result {{ + match &self.value {{ +{ser_arms} None => serializer.serialize_none(), + }} + }} +}} + +impl<'de> ::serde::Deserialize<'de> for {name} {{ + fn deserialize>(deserializer: D) -> Result {{ + struct V; + impl<'de> ::serde::de::Visitor<'de> for V {{ + type Value = {name}; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {{ + f.write_str("one of the {name} shapes") + }} +{visits} }} + deserializer.deserialize_any(V) + }} +}} + +"# + )); +} + +/// Serialize one struct/payload field into `map` (a SerializeMap), honoring +/// the field's presence/null rules. `always_emit_null` = tagged-union payload +/// semantics: optional members emit null instead of being skipped. +fn ser_field(f: &FieldModel, receiver: &str, always_emit_null: bool) -> String { + let key = &f.json_name; + let ident = &f.rust_ident; + let get = format!("{receiver}{ident}"); + match &f.kind { + FieldKind::Scalar { .. } => format!( + " ::serde::ser::SerializeMap::serialize_entry(&mut map, {key:?}, &{get})?;\n" + ), + FieldKind::OptScalar { + emit_null, + null_is_none_default, + .. + } => { + // null_is_none: None means "explicit null", distinct from absent + // (the default), so it must serialize as null to round-trip. + if *emit_null || always_emit_null || null_is_none_default.is_some() { + format!( + " ::serde::ser::SerializeMap::serialize_entry(&mut map, {key:?}, &{get})?;\n" + ) + } else { + format!( + " if let Some(v) = &{get} {{\n ::serde::ser::SerializeMap::serialize_entry(&mut map, {key:?}, v)?;\n }}\n" + ) + } + } + FieldKind::Message { emit_null, .. } => { + if *emit_null || always_emit_null { + format!( + " ::serde::ser::SerializeMap::serialize_entry(&mut map, {key:?}, &{get})?;\n" + ) + } else { + format!( + " if let Some(v) = &{get} {{\n ::serde::ser::SerializeMap::serialize_entry(&mut map, {key:?}, v)?;\n }}\n" + ) + } + } + FieldKind::RepeatedScalar { .. } | FieldKind::RepeatedMessage { .. } => format!( + " if !{get}.is_empty() {{\n ::serde::ser::SerializeMap::serialize_entry(&mut map, {key:?}, &{get})?;\n }}\n" + ), + FieldKind::RepeatedRawJson => format!( + " if !{get}.is_empty() {{\n let rows: Vec<::serde_json::Value> = {get}\n .iter()\n .map(|s| ::serde_json::from_str(s).unwrap_or(::serde_json::Value::Null))\n .collect();\n ::serde::ser::SerializeMap::serialize_entry(&mut map, {key:?}, &rows)?;\n }}\n" + ), + FieldKind::Map { .. } => format!( + " if !{get}.is_empty() {{\n ::serde::ser::SerializeMap::serialize_entry(&mut map, {key:?}, &{get})?;\n }}\n" + ), + FieldKind::RawJson { emit_null } => { + let body = format!( + "::serde::ser::SerializeMap::serialize_entry(&mut map, {key:?}, &::serde_json::from_str::<::serde_json::Value>(v).unwrap_or(::serde_json::Value::Null))?;" + ); + if *emit_null || always_emit_null { + format!( + " match &{get} {{\n Some(v) => {{ {body} }}\n None => {{ ::serde::ser::SerializeMap::serialize_entry(&mut map, {key:?}, &::serde_json::Value::Null)?; }}\n }}\n" + ) + } else { + format!(" if let Some(v) = &{get} {{\n {body}\n }}\n") + } + } + } +} + +/// The visitor-arm statements deserializing one field's value into its local +/// `Option<...>` slot named `f_` (positioned errors preserved via +/// `map.next_value`). +fn de_field_arm(f: &FieldModel) -> String { + let key = &f.json_name; + let ident = &f.rust_ident; + let slot = format!("f_{}", ident.trim_start_matches("r#")); + let dup = format!( + " if {slot}.is_some() {{\n return Err(::serde::de::Error::duplicate_field({key:?}));\n }}\n" + ); + let read = match &f.kind { + FieldKind::Scalar { ty, null_default } => match null_default { + // Explicit null resets to the default (Python __post_init__ + // parity); the value type still errors on shape mismatch. + Some(default) => format!( + " {slot} = Some(map.next_value::>()?.unwrap_or_else(|| {default}));\n", + ty = ty.rust(), + default = default.rust() + ), + None => format!( + " {slot} = Some(map.next_value::<{ty}>()?);\n", + ty = ty.rust() + ), + }, + FieldKind::OptScalar { + ty, null_default, .. + } => match null_default { + // null_resets_default: explicit null is the schema default. + Some(default) => format!( + " {slot} = Some(Some(map.next_value::>()?.unwrap_or_else(|| {default})));\n", + ty = ty.rust(), + default = default.rust() + ), + None => format!( + " {slot} = Some(map.next_value::>()?);\n", + ty = ty.rust() + ), + }, + FieldKind::Message { rust_path, .. } => format!( + " {slot} = Some(map.next_value::>()?);\n" + ), + FieldKind::RepeatedScalar { ty } => format!( + " {slot} = Some(map.next_value::>>()?.unwrap_or_default());\n", + ty = ty.rust() + ), + FieldKind::RepeatedMessage { rust_path } => format!( + " {slot} = Some(map.next_value::>>()?.unwrap_or_default());\n" + ), + FieldKind::RepeatedRawJson => format!( + " {slot} = Some(map.next_value::>>()?.unwrap_or_default().into_iter().map(|v| v.to_string()).collect());\n" + ), + FieldKind::Map { value } => format!( + " {slot} = Some(map.next_value::>>()?.unwrap_or_default());\n", + ty = value.rust() + ), + FieldKind::RawJson { .. } => format!( + " {slot} = Some(map.next_value::>()?.map(|v| v.to_string()));\n" + ), + }; + format!(" {key:?} => {{\n{dup}{read} }}\n") +} + +/// The absent-key fallback expression building the struct field from its +/// visitor slot. +fn de_field_finish(f: &FieldModel) -> String { + let ident = &f.rust_ident; + let slot = format!("f_{}", ident.trim_start_matches("r#")); + let expr = match &f.kind { + FieldKind::Scalar { null_default, .. } => match null_default { + Some(default) => format!( + "{slot}.unwrap_or_else(|| {default})", + default = default.rust() + ), + None => format!("{slot}.unwrap_or_default()"), + }, + FieldKind::OptScalar { + null_is_none_default, + null_default, + .. + } => match (null_is_none_default, null_default) { + // Absent => the default; explicit null => None (already in the + // slot as Some(None)). + (Some(default), _) | (_, Some(default)) => format!( + "{slot}.unwrap_or_else(|| Some({default}))", + default = default.rust() + ), + (None, None) => format!("{slot}.flatten()"), + }, + FieldKind::Message { .. } => format!("{slot}.flatten()"), + FieldKind::RepeatedScalar { .. } + | FieldKind::RepeatedMessage { .. } + | FieldKind::RepeatedRawJson + | FieldKind::Map { .. } => format!("{slot}.unwrap_or_default()"), + FieldKind::RawJson { .. } => format!("{slot}.flatten()"), + }; + format!(" {ident}: {expr},\n") +} + +/// Companion impl for a struct message: `*_or_default` accessors for every +/// schema-defaulted field (protobuf arrival has presence but no baked +/// defaults), and `apply_absent_defaults` filling every unset defaulted slot — +/// the gRPC entry's "proto unset = JSON absent" rule, recursing into +/// defaulted children. +fn emit_struct_companion(out: &mut String, m: &MessageModel, fields: &[FieldModel]) { + let name = &m.rust_name; + let mut accessors = String::new(); + let mut fills = String::new(); + for f in fields { + let ident = &f.rust_ident; + let plain = ident.trim_start_matches("r#"); + match &f.kind { + FieldKind::OptScalar { + ty, + null_default: Some(default), + .. + } + | FieldKind::OptScalar { + ty, + null_is_none_default: Some(default), + .. + } => { + let ty = ty.rust(); + let default = default.rust(); + accessors.push_str(&format!( + " /// The schema default when unset (protobuf arrival).\n pub fn {plain}_or_default(&self) -> {ty} {{\n self.{ident}.clone().unwrap_or_else(|| {default})\n }}\n" + )); + fills.push_str(&format!( + " if self.{ident}.is_none() {{\n self.{ident} = Some({default});\n }}\n" + )); + } + FieldKind::Message { rust_path, .. } + if matches!(rust_path.as_str(), "SamplingParamsOrList") => + { + fills.push_str(&format!( + " if let Some(v) = self.{ident}.as_mut() {{\n v.apply_absent_defaults();\n }}\n" + )); + } + _ => {} + } + } + if accessors.is_empty() && fills.is_empty() { + return; + } + out.push_str(&format!( + "impl {name} {{\n{accessors} /// Protobuf arrival: unset defaulted fields become their schema\n /// defaults (JSON arrival already baked them at deserialization).\n pub fn apply_absent_defaults(&mut self) {{\n{fills} }}\n}}\n\n" + )); +} + +fn emit_struct(out: &mut String, m: &MessageModel, deny_unknown: bool, fields: &[FieldModel]) { + emit_struct_companion(out, m, fields); + let name = &m.rust_name; + let map_binding = if fields.is_empty() { "map" } else { "mut map" }; + let mut ser_body = String::new(); + for f in fields { + ser_body.push_str(&ser_field(f, "self.", false)); + } + let mut arms = String::new(); + let mut slots = String::new(); + let mut finish = String::new(); + let mut field_names = String::new(); + for f in fields { + arms.push_str(&de_field_arm(f)); + let slot = format!("f_{}", f.rust_ident.trim_start_matches("r#")); + slots.push_str(&format!(" let mut {slot} = None;\n")); + finish.push_str(&de_field_finish(f)); + field_names.push_str(&format!("{:?}, ", f.json_name)); + } + let unknown_arm = if deny_unknown { + " other => {\n return Err(::serde::de::Error::unknown_field(other, FIELDS));\n }\n" + .to_string() + } else { + " _ => {\n map.next_value::<::serde::de::IgnoredAny>()?;\n }\n" + .to_string() + }; + let fields_const = if deny_unknown { + format!(" const FIELDS: &[&str] = &[{field_names}];\n") + } else { + String::new() + }; + out.push_str(&format!( + r#"impl ::serde::Serialize for {name} {{ + fn serialize(&self, serializer: S) -> Result {{ + let {map_binding} = serializer.serialize_map(None)?; +{ser_body} ::serde::ser::SerializeMap::end(map) + }} +}} + +impl<'de> ::serde::Deserialize<'de> for {name} {{ + fn deserialize>(deserializer: D) -> Result {{ +{fields_const} struct V; + impl<'de> ::serde::de::Visitor<'de> for V {{ + type Value = {name}; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {{ + f.write_str("struct {name}") + }} + fn visit_map>(self, mut map: A) -> Result {{ +{slots} while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? {{ + match key.as_ref() {{ +{arms}{unknown_arm} }} + }} + Ok({name} {{ +{finish} }}) + }} + }} + deserializer.deserialize_map(V) + }} +}} + +"# + )); +} + +fn emit_tagged( + out: &mut String, + m: &MessageModel, + oneof_mod: &str, + oneof_ident: &str, + tag: &str, + variants: &[model::VariantModel], + passthrough: Option<&model::VariantModel>, +) { + let name = &m.rust_name; + + // Serialize: {tag: variant_name, ...payload fields inlined (optional + // members as explicit nulls, matching serde's tagged-enum output)}. + let mut ser_arms = String::new(); + for v in variants { + let var = &v.rust_variant; + let tag_value = &v.json_name; + let mut payload = String::new(); + for f in &v.payload_fields { + payload.push_str(&ser_field(f, "v.", true)); + } + ser_arms.push_str(&format!( + r#" Some({oneof_mod}::{oneof_ident}::{var}(v)) => {{ + let mut map = serializer.serialize_map(None)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, {tag:?}, {tag_value:?})?; +{payload} ::serde::ser::SerializeMap::end(map) + }} +"# + )); + } + if let Some(p) = passthrough { + let var = &p.rust_variant; + ser_arms.push_str(&format!( + r#" Some({oneof_mod}::{oneof_ident}::{var}(v)) => {{ + ::serde::Serialize::serialize(&::serde_json::from_str::<::serde_json::Value>(v).unwrap_or(::serde_json::Value::Null), serializer) + }} +"# + )); + } + + // Deserialize: buffer the map (the tag can appear anywhere; a known tag + // whose payload fails to parse falls back to the passthrough arm, the + // hand-written FinishReason contract). + let mut de_arms = String::new(); + for v in variants { + let var = &v.rust_variant; + let tag_value = &v.json_name; + let mut fields = String::new(); + for f in &v.payload_fields { + let ident = &f.rust_ident; + let key = &f.json_name; + let read = match &f.kind { + FieldKind::OptScalar { ty, .. } => format!( + "match raw.get({key:?}) {{ None | Some(::serde_json::Value::Null) => None, Some(v) => match ::serde_json::from_value::<{ty}>(v.clone()) {{ Ok(v) => Some(v), Err(_) => return fallback(raw) }} }}", + ty = ty.rust() + ), + FieldKind::Message { rust_path, .. } => format!( + "match raw.get({key:?}) {{ None | Some(::serde_json::Value::Null) => None, Some(v) => match ::serde_json::from_value::<{rust_path}>(v.clone()) {{ Ok(v) => Some(v), Err(_) => return fallback(raw) }} }}" + ), + other => panic!("{name}: tagged payload field kind {other:?} unsupported"), + }; + fields.push_str(&format!(" {ident}: {read},\n")); + } + de_arms.push_str(&format!( + r#" Some({tag_value:?}) => {{ + let payload = {payload_ty} {{ +{fields} }}; + Ok({name} {{ kind: Some({oneof_mod}::{oneof_ident}::{var}(payload.into())) }}) + }} +"#, + payload_ty = match &v.kind { + FieldKind::Message { rust_path, .. } => rust_path.clone(), + other => panic!("{name}: tagged variant must be message-typed, got {other:?}"), + } + )); + } + let fallback = match passthrough { + Some(p) => { + let var = &p.rust_variant; + format!( + r#" fn fallback(raw: ::serde_json::Map) -> Result<{name}, E> {{ + Ok({name} {{ kind: Some({oneof_mod}::{oneof_ident}::{var}(::serde_json::Value::Object(raw).to_string())) }}) + }} +"# + ) + } + None => format!( + r#" fn fallback(_raw: ::serde_json::Map) -> Result<{name}, E> {{ + Err(E::custom("unknown {tag} for {name}")) + }} +"# + ), + }; + + out.push_str(&format!( + r#"impl ::serde::Serialize for {name} {{ + fn serialize(&self, serializer: S) -> Result {{ + match &self.kind {{ +{ser_arms} None => serializer.serialize_none(), + }} + }} +}} + +impl<'de> ::serde::Deserialize<'de> for {name} {{ + fn deserialize>(deserializer: D) -> Result {{ +{fallback} let raw = ::serde_json::Map::::deserialize(deserializer)?; + let tag = raw.get({tag:?}).and_then(::serde_json::Value::as_str); + #[allow(clippy::redundant_closure_call)] + (|raw: ::serde_json::Map| match tag {{ +{de_arms} _ => fallback(raw), + }})(raw.clone()) + }} +}} + +"# + )); +} diff --git a/rust/sglang-api-codegen/src/emit_py.rs b/rust/sglang-api-codegen/src/emit_py.rs new file mode 100644 index 0000000000..a9dd76d8c6 --- /dev/null +++ b/rust/sglang-api-codegen/src/emit_py.rs @@ -0,0 +1,1172 @@ +//! Pass 2 (Python): render the Python interface definition from the same +//! [`model::MessageModel`]s the Rust serde emitter consumes, so both languages +//! implement one JSON contract from one schema. +//! +//! The output is a single module over `typing` + `msgspec`. Each message maps +//! to the Python shape its JSON contract implies rather than to a 1:1 mirror +//! of the proto struct: plain objects and positional tuples are frozen +//! `msgspec.Struct`s; bare lists are `List[T]`; one-or-many carriers are +//! `Union[T, List[T]]`; transparent wrappers are `Optional[T]`; untagged +//! unions are a `Union` of their arms; tagged unions are a `Union` of their +//! payload Structs plus `Dict[str, Any]` for the passthrough arm. Every +//! message gets `decode_` / `encode_` functions over parsed JSON +//! values; Structs expose them as `from_json_value` / `to_json_value`. +//! Contract violations raise `JsonContractError` carrying serde's error texts, +//! so a 400 body reads the same from either server. + +use std::collections::{HashMap, HashSet}; + +use crate::model::{ + self, DefaultLit, FieldKind, FieldModel, MessageKind, MessageModel, ScalarTy, VariantModel, +}; +use prost_reflect::DescriptorPool; + +const HEADER: &str = "# Generated by sglang-api-codegen from proto/sglang -- DO NOT EDIT. +# The sglang.json.v1 options in the schema are the JSON contract; edit the +# .proto files and `cargo gen-api`. +\"\"\"Generated SGLang API types: the HTTP JSON interface of package {package}. + +Frozen msgspec Structs for JSON objects and tuples, type aliases for the other +JSON shapes, and `decode_*` / `encode_*` codecs implementing the sglang.json.v1 +options exactly as the Rust server does. Decode with +`X.from_json_value(json.loads(s))` and encode with `json.dumps(x.to_json_value())`. +\"\"\" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Tuple, Union + +import msgspec + +"; + +const PRELUDE: &str = r#"class JsonContractError(ValueError): + """The JSON value violates the schema contract (an HTTP 400 in the server).""" + + +def _is_number(v: Any) -> bool: + return isinstance(v, (int, float)) and not isinstance(v, bool) + + +def _is_int(v: Any) -> bool: + return isinstance(v, int) and not isinstance(v, bool) + + +def _unexpected(v: Any) -> str: + """serde's `Unexpected` display, so both languages report one text.""" + if v is None: + return "null" + if isinstance(v, bool): + return f"boolean `{'true' if v else 'false'}`" + if isinstance(v, int): + return f"integer `{v}`" + if isinstance(v, float): + return f"floating point `{v}`" + if isinstance(v, str): + return f'string "{v}"' + if isinstance(v, list): + return "sequence" + if isinstance(v, dict): + return "map" + return type(v).__name__ + + +def _invalid_type(v: Any, expected: str) -> JsonContractError: + return JsonContractError(f"invalid type: {_unexpected(v)}, expected {expected}") + + +def _expect_bool(v: Any, expected: str) -> bool: + if isinstance(v, bool): + return v + raise _invalid_type(v, expected=expected) + + +def _expect_int(v: Any, expected: str) -> int: + if _is_int(v): + return v + raise _invalid_type(v, expected=expected) + + +def _expect_uint(v: Any, expected: str) -> int: + if _is_int(v): + if v < 0: + raise JsonContractError(f"invalid value: integer `{v}`, expected {expected}") + return v + raise _invalid_type(v, expected=expected) + + +def _expect_float(v: Any, expected: str) -> float: + if _is_number(v): + return float(v) + raise _invalid_type(v, expected=expected) + + +def _expect_str(v: Any, expected: str) -> str: + if isinstance(v, str): + return v + raise _invalid_type(v, expected=expected) + + +def _expect_seq(v: Any, expected: str) -> List[Any]: + if isinstance(v, list): + return v + raise _invalid_type(v, expected=expected) + + +def _expect_map(v: Any, expected: str) -> Dict[str, Any]: + if isinstance(v, dict): + return v + raise _invalid_type(v, expected=expected) + + +def _unknown_field(key: str, fields: Tuple[str, ...]) -> JsonContractError: + """serde's `unknown_field` text: the DENY policy's 400 body.""" + if not fields: + return JsonContractError(f"unknown field `{key}`, there are no fields") + if len(fields) == 1: + return JsonContractError(f"unknown field `{key}`, expected `{fields[0]}`") + if len(fields) == 2: + return JsonContractError( + f"unknown field `{key}`, expected `{fields[0]}` or `{fields[1]}`" + ) + listed = ", ".join(f"`{f}`" for f in fields) + return JsonContractError(f"unknown field `{key}`, expected one of {listed}") + + +"#; + +/// The Python runtime shape a message name resolves to, for `isinstance` +/// dispatch in union codecs and for ordering alias definitions. +#[derive(Clone, Copy, PartialEq, Debug)] +enum Shape { + /// A msgspec Struct class (Struct, Tuple). + Class, + /// A `List[...]` alias (BareList). + List, + /// Any other alias (Transparent, OneOrMany, unions): no single runtime type. + Value, +} + +struct Ctx<'a> { + package: &'a str, + shapes: HashMap<&'a str, Shape>, + models: HashMap<&'a str, &'a MessageModel>, +} + +impl<'a> Ctx<'a> { + fn new(models: &'a [MessageModel], package: &'a str) -> Self { + let mut shapes = HashMap::new(); + let mut by_name = HashMap::new(); + for m in models { + let shape = match &m.kind { + MessageKind::Struct { .. } | MessageKind::Tuple { .. } => Shape::Class, + MessageKind::BareList { .. } => Shape::List, + _ => Shape::Value, + }; + shapes.insert(m.rust_name.as_str(), shape); + by_name.insert(m.rust_name.as_str(), m); + } + Ctx { + package, + shapes, + models: by_name, + } + } + + fn shape(&self, name: &str) -> Shape { + *self + .shapes + .get(name) + .unwrap_or_else(|| panic!("unknown message {name}")) + } + + fn model(&self, name: &str) -> &MessageModel { + self.models + .get(name) + .unwrap_or_else(|| panic!("unknown message {name}")) + } + + fn full_name(&self, name: &str) -> String { + format!("{}.{name}", self.package) + } +} + +pub fn emit_python(descriptor_bytes: &[u8], package: &str) -> String { + let pool = DescriptorPool::decode(descriptor_bytes).expect("decode descriptor set"); + let models = model::build_models(&pool, package); + let ctx = Ctx::new(&models, package); + + let mut out = String::new(); + out.push_str(&HEADER.replace("{package}", package)); + out.push_str(PRELUDE); + + out.push_str("# ---- Structs: JSON objects and positional tuples ----\n\n\n"); + for m in &models { + match &m.kind { + MessageKind::Struct { + deny_unknown, + fields, + } => emit_struct_class(&mut out, &ctx, m, *deny_unknown, fields), + MessageKind::Tuple { fields } => emit_tuple_class(&mut out, &ctx, m, fields), + _ => {} + } + } + + out.push_str("# ---- type aliases: JSON shapes without an object of their own ----\n\n"); + emit_aliases(&mut out, &ctx, &models); + + out.push_str("\n# ---- codecs: parsed JSON value <-> Python value ----\n\n\n"); + for m in &models { + match &m.kind { + MessageKind::Skip => {} + MessageKind::Struct { + deny_unknown, + fields, + } => emit_struct_codec(&mut out, m, *deny_unknown, fields), + MessageKind::Tuple { fields } => emit_tuple_codec(&mut out, m, fields), + MessageKind::BareList { item } => emit_bare_list_codec(&mut out, m, item), + MessageKind::Transparent { field } => emit_transparent_codec(&mut out, m, field), + MessageKind::OneOrMany { one, many, .. } => { + emit_one_or_many_codec(&mut out, &ctx, m, one, many) + } + MessageKind::Untagged { variants, .. } => { + emit_untagged_codec(&mut out, &ctx, m, variants) + } + MessageKind::TaggedUnion { + tag, + variants, + passthrough, + .. + } => emit_tagged_codec(&mut out, m, tag, variants, passthrough.as_ref()), + } + } + // One trailing newline: the end-of-file hook must not rewrite the output. + out.trim_end().to_string() + "\n" +} + +// ---- naming and typing ------------------------------------------------- + +/// Python keyword escaping for attribute names (the JSON key is unaffected). +fn py_ident(name: &str) -> String { + const KEYWORDS: &[&str] = &[ + "False", "None", "True", "and", "as", "assert", "async", "await", "break", "class", + "continue", "def", "del", "elif", "else", "except", "finally", "for", "from", "global", + "if", "import", "in", "is", "lambda", "nonlocal", "not", "or", "pass", "raise", "return", + "try", "while", "with", "yield", + ]; + if KEYWORDS.contains(&name) { + format!("{name}_") + } else { + name.to_string() + } +} + +fn py_scalar(ty: &ScalarTy) -> &'static str { + match ty { + ScalarTy::F64 => "float", + ScalarTy::I32 | ScalarTy::I64 | ScalarTy::U32 | ScalarTy::U64 => "int", + ScalarTy::Bool => "bool", + ScalarTy::Str => "str", + } +} + +fn py_lit(d: &DefaultLit) -> String { + match d { + DefaultLit::F64(v) => format!("{v:?}"), + DefaultLit::I64(v) => v.to_string(), + DefaultLit::Bool(v) => if *v { "True" } else { "False" }.to_string(), + DefaultLit::Str(v) => format!("{v:?}"), + } +} + +fn type_default(ty: &ScalarTy) -> &'static str { + match ty { + ScalarTy::F64 => "0.0", + ScalarTy::I32 | ScalarTy::I64 | ScalarTy::U32 | ScalarTy::U64 => "0", + ScalarTy::Bool => "False", + ScalarTy::Str => "\"\"", + } +} + +/// The element type of a field: what a list holds, what an Optional wraps. +fn inner_ty(kind: &FieldKind) -> String { + match kind { + FieldKind::Scalar { ty, .. } + | FieldKind::OptScalar { ty, .. } + | FieldKind::RepeatedScalar { ty } => py_scalar(ty).to_string(), + FieldKind::Message { rust_path, .. } | FieldKind::RepeatedMessage { rust_path } => { + rust_path.clone() + } + FieldKind::RepeatedRawJson | FieldKind::RawJson { .. } => "Any".to_string(), + FieldKind::Map { value } => format!("Dict[str, {}]", py_scalar(value)), + } +} + +/// The Struct attribute annotation. +fn field_ty(kind: &FieldKind) -> String { + match kind { + FieldKind::Scalar { .. } => inner_ty(kind), + FieldKind::OptScalar { .. } | FieldKind::Message { .. } | FieldKind::RawJson { .. } => { + format!("Optional[{}]", inner_ty(kind)) + } + FieldKind::RepeatedScalar { .. } + | FieldKind::RepeatedMessage { .. } + | FieldKind::RepeatedRawJson => format!("List[{}]", inner_ty(kind)), + FieldKind::Map { .. } => inner_ty(kind), + } +} + +/// The Struct default: the value an ABSENT JSON key yields. +fn field_default(kind: &FieldKind) -> String { + match kind { + FieldKind::Scalar { ty, null_default } => null_default + .as_ref() + .map(py_lit) + .unwrap_or_else(|| type_default(ty).to_string()), + FieldKind::OptScalar { + null_is_none_default, + null_default, + .. + } => null_is_none_default + .as_ref() + .or(null_default.as_ref()) + .map(py_lit) + .unwrap_or_else(|| "None".to_string()), + FieldKind::Message { .. } | FieldKind::RawJson { .. } => "None".to_string(), + FieldKind::RepeatedScalar { .. } + | FieldKind::RepeatedMessage { .. } + | FieldKind::RepeatedRawJson => "[]".to_string(), + FieldKind::Map { .. } => "{}".to_string(), + } +} + +// ---- scalar codecs ----------------------------------------------------- + +/// `_expect_*(var, "expected")` validating one scalar; `expected` defaults to +/// serde's own type name for that scalar. +fn expect_call(ty: &ScalarTy, var: &str, expected: Option<&str>) -> String { + let (func, default_expected) = match ty { + ScalarTy::F64 => ("_expect_float", "f64"), + ScalarTy::I32 => ("_expect_int", "i32"), + ScalarTy::I64 => ("_expect_int", "i64"), + ScalarTy::U32 => ("_expect_uint", "u32"), + ScalarTy::U64 => ("_expect_uint", "u64"), + ScalarTy::Bool => ("_expect_bool", "a boolean"), + ScalarTy::Str => ("_expect_str", "a string"), + }; + format!( + "{func}({var}, expected={:?})", + expected.unwrap_or(default_expected) + ) +} + +/// Encoding a scalar: identity, except doubles are normalized to float so an +/// int-valued temperature serializes as `1.0` like the Rust side. +fn enc_scalar(ty: &ScalarTy, var: &str) -> String { + match ty { + ScalarTy::F64 => format!("float({var})"), + _ => var.to_string(), + } +} + +/// The decode expression for one present key's value `x` (null handling per +/// the field's presence/default rules; absent keys never reach here). +fn de_expr(kind: &FieldKind, x: &str) -> String { + match kind { + FieldKind::Scalar { ty, null_default } => match null_default { + // null_resets_default: explicit null is the schema default. + Some(d) => format!( + "{} if {x} is None else {}", + py_lit(d), + expect_call(ty, x, None) + ), + None => expect_call(ty, x, None), + }, + FieldKind::OptScalar { + ty, null_default, .. + } => match null_default { + Some(d) => format!( + "{} if {x} is None else {}", + py_lit(d), + expect_call(ty, x, None) + ), + None => format!("None if {x} is None else {}", expect_call(ty, x, None)), + }, + FieldKind::Message { rust_path, .. } => { + format!("None if {x} is None else decode_{rust_path}({x})") + } + FieldKind::RepeatedScalar { ty } => format!( + "[] if {x} is None else [{} for e in _expect_seq({x}, expected=\"a sequence\")]", + expect_call(ty, "e", None) + ), + FieldKind::RepeatedMessage { rust_path } => format!( + "[] if {x} is None else [decode_{rust_path}(e) for e in _expect_seq({x}, expected=\"a sequence\")]" + ), + FieldKind::RepeatedRawJson => { + format!("[] if {x} is None else list(_expect_seq({x}, expected=\"a sequence\"))") + } + FieldKind::Map { value } => format!( + "{{}} if {x} is None else {{k: {} for k, e in _expect_map({x}, expected=\"a map\").items()}}", + expect_call(value, "e", None) + ), + // raw_json: the value passes through verbatim. + FieldKind::RawJson { .. } => x.to_string(), + } +} + +/// The encode expression for a non-None value `x` of this field. +fn enc_expr(kind: &FieldKind, x: &str) -> String { + match kind { + FieldKind::Scalar { ty, .. } | FieldKind::OptScalar { ty, .. } => enc_scalar(ty, x), + FieldKind::Message { rust_path, .. } => format!("encode_{rust_path}({x})"), + FieldKind::RepeatedScalar { ty } => match ty { + ScalarTy::F64 => format!("[float(e) for e in {x}]"), + _ => format!("list({x})"), + }, + FieldKind::RepeatedMessage { rust_path } => { + format!("[encode_{rust_path}(e) for e in {x}]") + } + FieldKind::RepeatedRawJson => format!("list({x})"), + FieldKind::Map { value } => match value { + ScalarTy::F64 => format!("{{k: float(e) for k, e in {x}.items()}}"), + _ => format!("dict({x})"), + }, + FieldKind::RawJson { .. } => x.to_string(), + } +} + +/// Statements storing one field into the dict `d`, honoring its presence and +/// null rules. `always_emit_null` = tagged-union payload semantics. +fn ser_stmts(f: &FieldModel, receiver: &str, always_emit_null: bool, indent: &str) -> String { + let key = &f.json_name; + let get = format!("{receiver}{}", py_ident(&f.json_name)); + let enc = enc_expr(&f.kind, &get); + let nullable = |emit_null: bool| -> String { + if emit_null || always_emit_null { + if enc == get { + format!("{indent}d[{key:?}] = {get}\n") + } else { + format!("{indent}d[{key:?}] = None if {get} is None else {enc}\n") + } + } else { + format!("{indent}if {get} is not None:\n{indent} d[{key:?}] = {enc}\n") + } + }; + match &f.kind { + FieldKind::Scalar { .. } => format!("{indent}d[{key:?}] = {enc}\n"), + // null_is_none: None means "explicit null", distinct from absent + // (the default), so it must encode as null to round-trip. + FieldKind::OptScalar { + emit_null, + null_is_none_default, + .. + } => nullable(*emit_null || null_is_none_default.is_some()), + FieldKind::Message { emit_null, .. } | FieldKind::RawJson { emit_null } => { + nullable(*emit_null) + } + FieldKind::RepeatedScalar { .. } + | FieldKind::RepeatedMessage { .. } + | FieldKind::RepeatedRawJson + | FieldKind::Map { .. } => { + format!("{indent}if {get}:\n{indent} d[{key:?}] = {enc}\n") + } + } +} + +// ---- Structs --------------------------------------------------------- + +fn emit_methods(out: &mut String, name: &str, ret: &str) { + out.push_str(&format!( + r#" @classmethod + def from_json_value(cls, value: Any) -> "{name}": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_{name}(value) + + def to_json_value(self) -> {ret}: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_{name}(self) + + +"# + )); +} + +fn emit_struct_class( + out: &mut String, + ctx: &Ctx, + m: &MessageModel, + deny_unknown: bool, + fields: &[FieldModel], +) { + let name = &m.rust_name; + let policy = if deny_unknown { "rejected" } else { "ignored" }; + out.push_str(&format!( + "class {name}(msgspec.Struct, kw_only=True, frozen=True):\n \"\"\"{}: JSON object (unknown keys {policy}).\"\"\"\n\n", + ctx.full_name(name) + )); + for f in fields { + out.push_str(&format!( + " {}: {} = {}\n", + py_ident(&f.json_name), + field_ty(&f.kind), + field_default(&f.kind) + )); + } + if !fields.is_empty() { + out.push('\n'); + } + emit_methods(out, name, "Dict[str, Any]"); +} + +fn emit_tuple_class(out: &mut String, ctx: &Ctx, m: &MessageModel, fields: &[FieldModel]) { + let name = &m.rust_name; + let members: Vec<&str> = fields.iter().map(|f| f.json_name.as_str()).collect(); + out.push_str(&format!( + "class {name}(msgspec.Struct, kw_only=True, frozen=True):\n \"\"\"{}: JSON positional array [{}].\"\"\"\n\n", + ctx.full_name(name), + members.join(", ") + )); + for f in fields { + out.push_str(&format!( + " {}: {} = {}\n", + py_ident(&f.json_name), + field_ty(&f.kind), + field_default(&f.kind) + )); + } + if !fields.is_empty() { + out.push('\n'); + } + emit_methods(out, name, "List[Any]"); +} + +// ---- aliases ----------------------------------------------------------- + +/// The alias definition (Python type expression, JSON note, referenced +/// message names) for a non-Struct message. +fn alias_def(m: &MessageModel) -> Option<(String, &'static str, Vec)> { + let refs = |kinds: &[&FieldKind]| -> Vec { + kinds + .iter() + .filter_map(|k| match k { + FieldKind::Message { rust_path, .. } | FieldKind::RepeatedMessage { rust_path } => { + Some(rust_path.clone()) + } + _ => None, + }) + .collect() + }; + match &m.kind { + MessageKind::BareList { item } => Some(( + format!("List[{}]", inner_ty(&item.kind)), + "JSON bare array", + refs(&[&item.kind]), + )), + MessageKind::Transparent { field } => Some(( + format!("Optional[{}]", inner_ty(&field.kind)), + "JSON: the wrapped value itself, null when unset", + refs(&[&field.kind]), + )), + MessageKind::OneOrMany { one, many, .. } => Some(( + format!("Union[{}, {}]", inner_ty(&one.kind), inner_ty(&many.kind)), + "JSON: one value or an array of values", + refs(&[&one.kind, &many.kind]), + )), + MessageKind::Untagged { variants, .. } => { + let arms: Vec = variants.iter().map(|v| inner_ty(&v.kind)).collect(); + let kinds: Vec<&FieldKind> = variants.iter().map(|v| &v.kind).collect(); + Some(( + format!("Union[{}]", arms.join(", ")), + "JSON: the value's shape picks the arm", + refs(&kinds), + )) + } + MessageKind::TaggedUnion { + variants, + passthrough, + .. + } => { + let mut arms: Vec = variants.iter().map(|v| inner_ty(&v.kind)).collect(); + let note = if passthrough.is_some() { + arms.push("Dict[str, Any]".to_string()); + "JSON object tagged by its type key; unknown tags pass through as a dict" + } else { + "JSON object tagged by its type key" + }; + let kinds: Vec<&FieldKind> = variants.iter().map(|v| &v.kind).collect(); + Some((format!("Union[{}]", arms.join(", ")), note, refs(&kinds))) + } + MessageKind::Struct { .. } | MessageKind::Tuple { .. } | MessageKind::Skip => None, + } +} + +/// Aliases are evaluated at import time, so each is emitted after the aliases +/// it references (Struct classes are already defined above them). +fn emit_aliases(out: &mut String, ctx: &Ctx, models: &[MessageModel]) { + let mut pending: Vec<&MessageModel> = + models.iter().filter(|m| alias_def(m).is_some()).collect(); + let mut done: HashSet = HashSet::new(); + while !pending.is_empty() { + let before = pending.len(); + let mut i = 0; + while i < pending.len() { + let m = pending[i]; + let (ty, note, refs) = alias_def(m).expect("alias"); + let ready = refs + .iter() + .all(|r| ctx.shape(r) == Shape::Class || done.contains(r)); + if ready { + out.push_str(&format!( + "# {}: {note}.\n{} = {ty}\n\n", + ctx.full_name(&m.rust_name), + m.rust_name + )); + done.insert(m.rust_name.clone()); + pending.remove(i); + } else { + i += 1; + } + } + assert!( + pending.len() < before, + "type alias cycle among {:?}", + pending.iter().map(|m| &m.rust_name).collect::>() + ); + } +} + +// ---- codecs ------------------------------------------------------------ + +fn emit_struct_codec( + out: &mut String, + m: &MessageModel, + deny_unknown: bool, + fields: &[FieldModel], +) { + let name = &m.rust_name; + if deny_unknown { + let names: Vec = fields + .iter() + .map(|f| format!("{:?},", f.json_name)) + .collect(); + out.push_str(&format!("_FIELDS_{name} = ({})\n\n\n", names.join(" "))); + } + + out.push_str(&format!( + "def decode_{name}(value: Any) -> {name}:\n m = _expect_map(value, expected=\"struct {name}\")\n" + )); + if fields.is_empty() { + if deny_unknown { + out.push_str(" for key in m:\n raise _unknown_field(key, fields=())\n"); + } + out.push_str(&format!(" return {name}()\n\n\n")); + } else { + out.push_str(" kw: Dict[str, Any] = {}\n for key, x in m.items():\n"); + for (i, f) in fields.iter().enumerate() { + let kw = if i == 0 { "if" } else { "elif" }; + out.push_str(&format!( + " {kw} key == {:?}:\n kw[{:?}] = {}\n", + f.json_name, + py_ident(&f.json_name), + de_expr(&f.kind, "x") + )); + } + if deny_unknown { + out.push_str(&format!( + " else:\n raise _unknown_field(key, fields=_FIELDS_{name})\n" + )); + } + out.push_str(&format!(" return {name}(**kw)\n\n\n")); + } + + out.push_str(&format!( + "def encode_{name}(v: {name}) -> Dict[str, Any]:\n d: Dict[str, Any] = {{}}\n" + )); + for f in fields { + out.push_str(&ser_stmts(f, "v.", false, " ")); + } + out.push_str(" return d\n\n\n"); +} + +fn emit_tuple_codec(out: &mut String, m: &MessageModel, fields: &[FieldModel]) { + let name = &m.rust_name; + let len = fields.len(); + let expected = format!("a {len}-element tuple"); + out.push_str(&format!( + "def decode_{name}(value: Any) -> {name}:\n s = _expect_seq(value, expected={expected:?})\n if len(s) != {len}:\n raise JsonContractError(f\"invalid length {{len(s)}}, expected {expected}\")\n return {name}(\n" + )); + for (i, f) in fields.iter().enumerate() { + let x = format!("s[{i}]"); + let expr = match &f.kind { + FieldKind::Scalar { .. } | FieldKind::OptScalar { .. } => de_expr(&f.kind, &x), + other => panic!("{name}: tuple member kind {other:?} unsupported"), + }; + out.push_str(&format!(" {}={expr},\n", py_ident(&f.json_name))); + } + out.push_str(" )\n\n\n"); + + out.push_str(&format!( + "def encode_{name}(v: {name}) -> List[Any]:\n return [\n" + )); + for f in fields { + let get = format!("v.{}", py_ident(&f.json_name)); + let expr = match &f.kind { + FieldKind::Scalar { .. } => enc_expr(&f.kind, &get), + FieldKind::OptScalar { .. } => { + let enc = enc_expr(&f.kind, &get); + if enc == get { + get + } else { + format!("None if {get} is None else {enc}") + } + } + other => panic!("{name}: tuple member kind {other:?} unsupported"), + }; + out.push_str(&format!(" {expr},\n")); + } + out.push_str(" ]\n\n\n"); +} + +fn emit_bare_list_codec(out: &mut String, m: &MessageModel, item: &FieldModel) { + let name = &m.rust_name; + let elem = match &item.kind { + FieldKind::RepeatedScalar { ty } => expect_call(ty, "e", None), + FieldKind::RepeatedMessage { rust_path } => format!("decode_{rust_path}(e)"), + other => panic!("{name}: bare_list item must be repeated, got {other:?}"), + }; + out.push_str(&format!( + "def decode_{name}(value: Any) -> {name}:\n return [{elem} for e in _expect_seq(value, expected=\"a sequence\")]\n\n\n" + )); + out.push_str(&format!( + "def encode_{name}(v: {name}) -> List[Any]:\n return {}\n\n\n", + enc_expr(&item.kind, "v") + )); +} + +fn emit_transparent_codec(out: &mut String, m: &MessageModel, field: &FieldModel) { + let name = &m.rust_name; + let (de, en) = match &field.kind { + FieldKind::OptScalar { ty, .. } => (expect_call(ty, "value", None), enc_scalar(ty, "v")), + FieldKind::Message { rust_path, .. } => ( + format!("decode_{rust_path}(value)"), + format!("encode_{rust_path}(v)"), + ), + other => panic!( + "{name}: transparent field must be an optional scalar or a message, got {other:?}" + ), + }; + out.push_str(&format!( + "def decode_{name}(value: Any) -> {name}:\n return None if value is None else {de}\n\n\n" + )); + out.push_str(&format!( + "def encode_{name}(v: {name}) -> Any:\n return None if v is None else {en}\n\n\n" + )); +} + +/// How the `one` arm of a one-or-many carrier is recognized and decoded. +enum OneArm { + Scalar(ScalarTy), + /// The `one` is itself a bare list of scalars (TokenIds, StringList): the + /// outer array is ambiguous, so dispatch on the first element's shape. + ListOfScalar(String, ScalarTy), + Class(String), +} + +/// The Python test recognizing one element of a bare scalar list. +fn first_elem_check(ty: &ScalarTy) -> &'static str { + match ty { + ScalarTy::Str => "isinstance(value[0], str)", + ScalarTy::Bool => "isinstance(value[0], bool)", + _ => "_is_number(value[0])", + } +} + +fn one_arm(ctx: &Ctx, name: &str, one: &VariantModel) -> OneArm { + match &one.kind { + FieldKind::Scalar { ty, .. } | FieldKind::OptScalar { ty, .. } => { + OneArm::Scalar(ty.clone()) + } + FieldKind::Message { rust_path, .. } => match &ctx.model(rust_path).kind { + // A presence wrapper's JSON is its bare scalar. + MessageKind::Transparent { + field: + FieldModel { + kind: FieldKind::OptScalar { ty, .. }, + .. + }, + } => OneArm::Scalar(ty.clone()), + MessageKind::BareList { + item: + FieldModel { + kind: FieldKind::RepeatedScalar { ty }, + .. + }, + } => OneArm::ListOfScalar(rust_path.clone(), ty.clone()), + MessageKind::Struct { .. } => OneArm::Class(rust_path.clone()), + other => panic!("{name}: unsupported one arm {rust_path} ({other:?})"), + }, + other => panic!("{name}: unsupported one arm {other:?}"), + } +} + +fn emit_one_or_many_codec( + out: &mut String, + ctx: &Ctx, + m: &MessageModel, + one: &VariantModel, + many: &VariantModel, +) { + let name = &m.rust_name; + let many_p = match &many.kind { + FieldKind::Message { rust_path, .. } => rust_path.clone(), + other => panic!("{name}: many arm must be a list message, got {other:?}"), + }; + let expected = format!("a value or an array of values for {name}"); + let (de_body, en_body) = match one_arm(ctx, name, one) { + OneArm::ListOfScalar(one_p, ty) => ( + format!( + " if isinstance(value, list):\n # [] and [scalar, ...] are one flat list; anything else is a batch.\n if not value or {check}:\n return decode_{one_p}(value)\n return decode_{many_p}(value)\n raise _invalid_type(value, expected={expected:?})\n", + check = first_elem_check(&ty) + ), + format!( + " if v and isinstance(v[0], list):\n return encode_{many_p}(v)\n return encode_{one_p}(v)\n" + ), + ), + OneArm::Scalar(ty) => ( + format!( + " if isinstance(value, list):\n return decode_{many_p}(value)\n return {}\n", + expect_call(&ty, "value", Some(&expected)) + ), + format!( + " if isinstance(v, list):\n return encode_{many_p}(v)\n return {}\n", + enc_scalar(&ty, "v") + ), + ), + OneArm::Class(one_p) => ( + format!( + " if isinstance(value, list):\n return decode_{many_p}(value)\n return decode_{one_p}(_expect_map(value, expected={expected:?}))\n" + ), + format!( + " if isinstance(v, list):\n return encode_{many_p}(v)\n return encode_{one_p}(v)\n" + ), + ), + }; + out.push_str(&format!( + "def decode_{name}(value: Any) -> {name}:\n{de_body}\n\ndef encode_{name}(v: {name}) -> Any:\n{en_body}\n\n" + )); +} + +/// The Python runtime shapes a field or message's JSON can take. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum PyShape { + Str, + Int, + Float, + Bool, + Dict, + List, + NoneT, +} + +fn py_scalar_shapes(ty: &ScalarTy) -> Vec { + match ty { + ScalarTy::Str => vec![PyShape::Str], + ScalarTy::I32 | ScalarTy::I64 | ScalarTy::U32 | ScalarTy::U64 => vec![PyShape::Int], + ScalarTy::F64 => vec![PyShape::Int, PyShape::Float], + ScalarTy::Bool => vec![PyShape::Bool], + } +} + +/// `(JSON shapes, runtime class names)` of a kind: the shapes drive the decode +/// dispatch, the classes the encode dispatch (a decoded dict is a Struct). +fn py_shapes(ctx: &Ctx, kind: &FieldKind) -> (Vec, Vec) { + match kind { + FieldKind::Scalar { ty, .. } | FieldKind::OptScalar { ty, .. } => { + (py_scalar_shapes(ty), vec![]) + } + FieldKind::Message { rust_path, .. } => py_message_shapes(ctx, rust_path), + FieldKind::RepeatedScalar { .. } + | FieldKind::RepeatedMessage { .. } + | FieldKind::RepeatedRawJson => (vec![PyShape::List], vec![]), + FieldKind::Map { .. } => (vec![PyShape::Dict], vec![]), + FieldKind::RawJson { .. } => panic!("raw_json has no fixed shape"), + } +} + +fn py_message_shapes(ctx: &Ctx, name: &str) -> (Vec, Vec) { + let mut shapes: Vec = Vec::new(); + let mut classes: Vec = Vec::new(); + let mut add = |(s, c): (Vec, Vec)| { + for x in s { + if !shapes.contains(&x) { + shapes.push(x); + } + } + for x in c { + if !classes.contains(&x) { + classes.push(x); + } + } + }; + match &ctx.model(name).kind { + MessageKind::Struct { .. } | MessageKind::TaggedUnion { .. } => { + add((vec![PyShape::Dict], vec![name.to_string()])) + } + MessageKind::Tuple { .. } | MessageKind::BareList { .. } => { + add((vec![PyShape::List], vec![])) + } + MessageKind::Transparent { field } => { + add(py_shapes(ctx, &field.kind)); + add((vec![PyShape::NoneT], vec![])); + } + MessageKind::Untagged { variants, .. } => { + for v in variants { + add(py_shapes(ctx, &v.kind)); + } + } + MessageKind::OneOrMany { one, .. } => { + add(py_shapes(ctx, &one.kind)); + add((vec![PyShape::List], vec![])); + } + MessageKind::Skip => panic!("{name}: a gRPC-only message has no JSON shape"), + } + (shapes, classes) +} + +/// Whether a bare-list message's elements are themselves arrays. +fn py_elements_are_lists(ctx: &Ctx, name: &str) -> bool { + match &ctx.model(name).kind { + MessageKind::BareList { item } => match &item.kind { + FieldKind::RepeatedMessage { rust_path } => { + py_message_shapes(ctx, rust_path).0.contains(&PyShape::List) + } + _ => false, + }, + _ => false, + } +} + +fn emit_untagged_codec(out: &mut String, ctx: &Ctx, m: &MessageModel, variants: &[VariantModel]) { + let name = &m.rust_name; + let mut de = format!("def decode_{name}(value: Any) -> {name}:\n"); + let mut en = format!("def encode_{name}(v: {name}) -> Any:\n"); + + struct Arm<'a> { + v: &'a VariantModel, + shapes: Vec, + classes: Vec, + } + let arms: Vec = variants + .iter() + .map(|v| { + let (shapes, classes) = py_shapes(ctx, &v.kind); + Arm { v, shapes, classes } + }) + .collect(); + let decode_expr = |a: &Arm| match &a.v.kind { + FieldKind::Scalar { .. } | FieldKind::OptScalar { .. } => "value".to_string(), + FieldKind::Message { rust_path, .. } => format!("decode_{rust_path}(value)"), + other => panic!("{name}: untagged variant kind {other:?} unsupported"), + }; + let encode_expr = |a: &Arm| match &a.v.kind { + FieldKind::Scalar { .. } | FieldKind::OptScalar { .. } => "v".to_string(), + FieldKind::Message { rust_path, .. } => format!("encode_{rust_path}(v)"), + _ => unreachable!(), + }; + // Two list-shaped arms: the one whose elements are lists is the nested + // form, told apart by the first element (`[]` is the flat form). + let list_arms: Vec<&Arm> = arms + .iter() + .filter(|a| a.shapes.contains(&PyShape::List)) + .collect(); + let nested_arm: Option = match list_arms.len() { + 0 | 1 => None, + 2 => { + let is_nested = |a: &Arm| match &a.v.kind { + FieldKind::Message { rust_path, .. } => py_elements_are_lists(ctx, rust_path), + _ => false, + }; + match (is_nested(list_arms[0]), is_nested(list_arms[1])) { + (true, false) => Some(list_arms[0].v.json_name.clone()), + (false, true) => Some(list_arms[1].v.json_name.clone()), + _ => panic!("{name}: two list-shaped arms cannot be told apart"), + } + } + _ => panic!("{name}: more than two list-shaped untagged arms"), + }; + + // Decode: one test per shape, bool before int (a Python bool is an int). + for shape in [ + PyShape::NoneT, + PyShape::Bool, + PyShape::Str, + PyShape::Int, + PyShape::Float, + PyShape::Dict, + PyShape::List, + ] { + let claimants: Vec<&Arm> = arms.iter().filter(|a| a.shapes.contains(&shape)).collect(); + if claimants.is_empty() { + continue; + } + if shape != PyShape::List && claimants.len() > 1 { + panic!( + "{name}: untagged arms {:?} all accept a JSON {shape:?}", + claimants.iter().map(|a| &a.v.json_name).collect::>() + ); + } + if shape == PyShape::List && claimants.len() == 2 { + let nested = nested_arm.as_deref().expect("nested arm"); + let (n, f) = if claimants[0].v.json_name == nested { + (claimants[0], claimants[1]) + } else { + (claimants[1], claimants[0]) + }; + de.push_str(&format!( + " if isinstance(value, list):\n if value and isinstance(value[0], list):\n return {}\n return {}\n", + decode_expr(n), + decode_expr(f) + )); + continue; + } + let check = match shape { + PyShape::NoneT => "value is None", + PyShape::Bool => "isinstance(value, bool)", + PyShape::Str => "isinstance(value, str)", + PyShape::Int => "_is_int(value)", + PyShape::Float => "_is_number(value)", + PyShape::Dict => "isinstance(value, dict)", + PyShape::List => "isinstance(value, list)", + }; + de.push_str(&format!( + " if {check}:\n return {}\n", + decode_expr(claimants[0]) + )); + } + de.push_str(&format!( + " raise _invalid_type(value, expected=\"one of the {name} shapes\")\n\n\n" + )); + + // Encode: the runtime value's type picks the arm (a decoded object is its + // Struct class); list arms by first element. + for a in &arms { + let mut types: Vec = Vec::new(); + for s in &a.shapes { + let t = match s { + PyShape::Str => "str", + PyShape::Bool => "bool", + PyShape::Int => "int", + PyShape::Float => "float", + PyShape::List => "list", + PyShape::Dict | PyShape::NoneT => continue, + }; + if !types.iter().any(|x| x == t) { + types.push(t.to_string()); + } + } + types.extend(a.classes.iter().cloned()); + let mut check = if types.is_empty() { + "False".to_string() + } else { + format!("isinstance(v, ({},))", types.join(", ")) + }; + if a.shapes.contains(&PyShape::NoneT) { + check = format!("v is None or {check}"); + } + if a.shapes.contains(&PyShape::List) && list_arms.len() == 2 { + let elem = if nested_arm.as_deref() == Some(a.v.json_name.as_str()) { + "(v and isinstance(v[0], list))" + } else { + "not (v and isinstance(v[0], list))" + }; + check = format!("isinstance(v, list) and {elem}"); + } + en.push_str(&format!( + " if {check}:\n return {}\n", + encode_expr(a) + )); + } + en.push_str(" return v\n\n\n"); + out.push_str(&de); + out.push_str(&en); +} + +fn emit_tagged_codec( + out: &mut String, + m: &MessageModel, + tag: &str, + variants: &[VariantModel], + passthrough: Option<&VariantModel>, +) { + let name = &m.rust_name; + // Decode: a known tag builds its payload from the declared keys; a payload + // that fails to parse, or an unknown tag, falls back to the passthrough + // (or errors when the schema declares none). + let fallback = if passthrough.is_some() { + "return dict(raw)".to_string() + } else { + format!("raise JsonContractError(\"unknown {tag} for {name}\")") + }; + let mut de = format!( + "def decode_{name}(value: Any) -> {name}:\n raw = _expect_map(value, expected=\"map\")\n tag = raw.get({tag:?})\n" + ); + for (i, v) in variants.iter().enumerate() { + let payload = match &v.kind { + FieldKind::Message { rust_path, .. } => rust_path.clone(), + other => panic!("{name}: tagged variant must be message-typed, got {other:?}"), + }; + let kw = if i == 0 { "if" } else { "elif" }; + de.push_str(&format!( + " {kw} tag == {:?}:\n try:\n return {payload}(\n", + v.json_name + )); + for f in &v.payload_fields { + let key = &f.json_name; + let read = match &f.kind { + FieldKind::OptScalar { ty, .. } => expect_call(ty, &format!("raw[{key:?}]"), None), + FieldKind::Message { rust_path, .. } => { + format!("decode_{rust_path}(raw[{key:?}])") + } + other => panic!("{name}: tagged payload field kind {other:?} unsupported"), + }; + de.push_str(&format!( + " {}=None if raw.get({key:?}) is None else {read},\n", + py_ident(key) + )); + } + let on_error = if passthrough.is_some() { + "return dict(raw)" + } else { + "raise" + }; + de.push_str(&format!( + " )\n except JsonContractError:\n {on_error}\n" + )); + } + de.push_str(&format!(" {fallback}\n\n\n")); + + // Encode: {tag: variant name, ...payload fields with explicit nulls}. + let mut en = format!("def encode_{name}(v: {name}) -> Any:\n"); + for v in variants { + let payload = match &v.kind { + FieldKind::Message { rust_path, .. } => rust_path.clone(), + other => panic!("{name}: tagged variant must be message-typed, got {other:?}"), + }; + en.push_str(&format!( + " if isinstance(v, {payload}):\n d: Dict[str, Any] = {{{tag:?}: {:?}}}\n", + v.json_name + )); + for f in &v.payload_fields { + en.push_str(&ser_stmts(f, "v.", true, " ")); + } + en.push_str(" return d\n"); + } + if passthrough.is_some() { + en.push_str(" if isinstance(v, dict):\n return dict(v)\n"); + } + en.push_str(&format!( + " raise TypeError(f\"{name}: unsupported value {{type(v).__name__}}\")\n\n\n" + )); + out.push_str(&de); + out.push_str(&en); +} diff --git a/rust/sglang-api-codegen/src/main.rs b/rust/sglang-api-codegen/src/main.rs new file mode 100644 index 0000000000..89ad2070d5 --- /dev/null +++ b/rust/sglang-api-codegen/src/main.rs @@ -0,0 +1,179 @@ +//! The generator: every checked-in binding of `proto/sglang/`, from one protoc. +//! +//! Targets (see [`TARGETS`]): +//! Both targets land in `rust/sglang-api-types/src/generated/`: +//! - `sglang.api.v1`: prost structs, tonic services, schema-driven serde, plus +//! `python/sglang/api/v1/types.py`. +//! - `sglang.runtime.v1`: prost structs and tonic services only. +//! +//! Pass 1 (prost-build + tonic-prost-build) emits the prost structs and tonic +//! service code, plus a `FileDescriptorSet`. For targets with a JSON contract +//! pass 2 reads the `sglang.json.v1` options from that set through +//! prost-reflect and emits, from one model (`model`), the Rust serde +//! implementations (`emit`) and the Python interface definition (`emit_py`); +//! proto3's canonical JSON mapping is NOT used, the options are the contract. +//! +//! Regeneration is manual. From anywhere in the `rust/` workspace: +//! +//! ```text +//! cargo gen-api +//! ``` +//! +//! Each target has a single root .proto: every API must be defined (or +//! imported) there, and protoc's import closure supplies the rest of the +//! package; a file the root does not reach gets no generated code. All output +//! is checked in, so commit it together with the .proto change. The vendored +//! protoc (pinned by Cargo.lock) is used unless `PROTOC` is set, so the output +//! is byte-stable across machines. + +mod emit; +mod emit_py; +mod model; + +use std::path::{Path, PathBuf}; + +/// One generated binding set. Paths are relative to the repository root. +struct Target { + /// The root .proto under `proto/`; its import closure is the API surface. + root: &'static str, + package: &'static str, + rust_out: &'static str, + /// Set when the schema carries `sglang.json.v1` options: emits the serde + /// implementations next to the structs and the Python module here. + python_out: Option<&'static str>, + /// prost `boxed` paths (message fields stored as `Box`). + boxed: &'static [&'static str], + /// Emit every service method with an UNIMPLEMENTED default body, so a + /// server may implement a subset (the runtime.v1 follower metadata + /// service). Off for a contract every server implements in full. + default_stubs: bool, +} + +const TARGETS: &[Target] = &[ + Target { + root: "sglang/api/v1/service.proto", + package: "sglang.api.v1", + rust_out: "rust/sglang-api-types/src/generated", + python_out: Some("python/sglang/api/v1/api_types.py"), + // Boxed abort payload won't set the size of every ChunkEvent. + boxed: &[".sglang.api.v1.FinishReason.kind.abort"], + default_stubs: false, + }, + Target { + root: "sglang/runtime/v1/sglang.proto", + package: "sglang.runtime.v1", + rust_out: "rust/sglang-api-types/src/generated", + python_out: None, + boxed: &[], + default_stubs: true, + }, +]; + +fn main() { + let repo_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../.."); + let proto_root = repo_root.join("proto"); + + // Vendored protoc first: the checked-in output must not depend on + // whichever protoc a machine has. `PROTOC` still overrides for platforms + // the vendored crate lacks. + if std::env::var_os("PROTOC").is_none() + && let Ok(vendored) = protoc_bin_vendored::protoc_bin_path() + { + // SAFETY: nothing else is running yet; the generator is single-threaded. + unsafe { std::env::set_var("PROTOC", vendored) }; + } + + // The descriptor set is only pass 2's input, not part of any crate; it is + // a protoc-version-dependent binary, so keep it out of the tree. + let scratch_dir = std::env::temp_dir().join("sglang-api-codegen"); + std::fs::create_dir_all(&scratch_dir).expect("create scratch dir"); + + for target in TARGETS { + generate(target, &repo_root, &proto_root, &scratch_dir); + } +} + +fn generate(target: &Target, repo_root: &Path, proto_root: &Path, scratch_dir: &Path) { + let package = target.package; + let out_dir = repo_root.join(target.rust_out); + std::fs::create_dir_all(&out_dir).expect("create generated dir"); + let descriptor_path = scratch_dir.join(format!("{package}.descriptor.bin")); + + // Pass 1: structs + tonic services (+ the descriptor set pass 2 reads). + // Everything the root imports comes in transitively via the include path, + // in topological order, so the output is stable. + let mut config = prost_build::Config::new(); + config + .out_dir(&out_dir) + .file_descriptor_set_path(&descriptor_path) + .service_generator( + tonic_prost_build::configure() + .generate_default_stubs(target.default_stubs) + .service_generator(), + ) + .protoc_arg("--experimental_allow_proto3_optional"); + for path in target.boxed { + config.boxed(path); + } + config + .compile_protos(&[proto_root.join(target.root)], &[proto_root]) + .unwrap_or_else(|e| panic!("pass 1 ({package}): {e}")); + + // prost writes structs + services into one file; split the tonic modules + // out so the include! layout stays stable. + split_tonic(&out_dir, package); + + if let Some(python_out) = target.python_out { + let bytes = std::fs::read(&descriptor_path).expect("descriptor set"); + + // Pass 2: schema-driven serde from the options in the descriptor set. + let serde_src = emit::emit_serde(&bytes, package); + std::fs::write(out_dir.join(format!("{package}.serde.rs")), serde_src) + .expect("write serde"); + + // Pass 2 (Python): the same model rendered as Structs + codecs. + let py_out = repo_root.join(python_out); + std::fs::create_dir_all(py_out.parent().expect("py out dir")).expect("create py dir"); + std::fs::write(&py_out, emit_py::emit_python(&bytes, package)).expect("write python types"); + println!("generated {}", py_out.display()); + + // The options package itself needs no Rust types (it exists for the + // generator); drop pass-1 output for it if prost emitted one. + let _ = std::fs::remove_file(out_dir.join("sglang.json.v1.rs")); + } + + rustfmt(&out_dir); + println!("generated into {}", out_dir.display()); +} + +/// Move the tonic `pub mod *_client/_server` blocks from the prost output +/// `.rs` into `.tonic.rs`. +fn split_tonic(out_dir: &Path, package: &str) { + let prost_file = out_dir.join(format!("{package}.rs")); + let src = std::fs::read_to_string(&prost_file).expect("read pass-1 output"); + let marker = "/// Generated client implementations."; + let (structs, tonic) = match src.find(marker) { + Some(pos) => src.split_at(pos), + None => (src.as_str(), ""), + }; + std::fs::write(&prost_file, structs.trim_end().to_string() + "\n").expect("write structs"); + std::fs::write( + out_dir.join(format!("{package}.tonic.rs")), + tonic.trim_start(), + ) + .expect("write tonic"); +} + +fn rustfmt(out_dir: &Path) { + for entry in std::fs::read_dir(out_dir).expect("read generated dir") { + let path = entry.expect("dir entry").path(); + if path.extension().is_some_and(|e| e == "rs") { + // Best effort: the output is already well-formed; rustfmt only + // normalizes. + let _ = std::process::Command::new("rustfmt") + .arg("--edition=2024") + .arg(&path) + .status(); + } + } +} diff --git a/rust/sglang-api-codegen/src/model.rs b/rust/sglang-api-codegen/src/model.rs new file mode 100644 index 0000000000..ee94e770ba --- /dev/null +++ b/rust/sglang-api-codegen/src/model.rs @@ -0,0 +1,550 @@ +//! Pass 2's input model: each message classified by the `sglang.json.v1` +//! options into the JSON shape the emitter renders. The options are read from +//! the FileDescriptorSet through prost-reflect, so the schema file is the +//! single source of truth. + +use prost_reflect::{ + DescriptorPool, ExtensionDescriptor, FieldDescriptor, Kind, MessageDescriptor, +}; + +pub struct Extensions { + message_json: ExtensionDescriptor, + field_json: ExtensionDescriptor, + oneof_json: ExtensionDescriptor, +} + +impl Extensions { + pub fn resolve(pool: &DescriptorPool) -> Extensions { + let get = |name: &str| { + pool.get_extension_by_name(name) + .unwrap_or_else(|| panic!("options.proto extension missing: {name}")) + }; + Extensions { + message_json: get("sglang.json.v1.message_json"), + field_json: get("sglang.json.v1.field_json"), + oneof_json: get("sglang.json.v1.oneof_json"), + } + } +} + +fn opt_bool(options: &prost_reflect::DynamicMessage, name: &str) -> bool { + options + .get_field_by_name(name) + .and_then(|v| v.as_bool()) + .unwrap_or(false) +} + +/// An enum-typed option as its number; 0 (the unset/default variant) when absent. +fn opt_enum(options: &prost_reflect::DynamicMessage, name: &str) -> i32 { + options + .get_field_by_name(name) + .and_then(|v| v.as_enum_number()) + .unwrap_or(0) +} + +fn opt_str(options: &prost_reflect::DynamicMessage, name: &str) -> Option { + options + .get_field_by_name(name) + .and_then(|v| v.as_str().map(str::to_owned)) + .filter(|s| !s.is_empty()) +} + +#[derive(Debug, Clone, PartialEq)] +pub enum ScalarTy { + F64, + I32, + I64, + U32, + U64, + Bool, + Str, +} + +impl ScalarTy { + pub fn rust(&self) -> &'static str { + match self { + ScalarTy::F64 => "f64", + ScalarTy::I32 => "i32", + ScalarTy::I64 => "i64", + ScalarTy::U32 => "u32", + ScalarTy::U64 => "u64", + ScalarTy::Bool => "bool", + ScalarTy::Str => "::prost::alloc::string::String", + } + } +} + +#[derive(Debug, Clone)] +pub enum DefaultLit { + F64(f64), + I64(i64), + Bool(bool), + Str(String), +} + +impl DefaultLit { + pub fn rust(&self) -> String { + match self { + DefaultLit::F64(v) => format!("{v:?}f64"), + DefaultLit::I64(v) => format!("{v}i64"), + DefaultLit::Bool(v) => format!("{v}"), + DefaultLit::Str(v) => format!("{v:?}.to_string()"), + } + } +} + +#[derive(Debug, Clone)] +pub enum FieldKind { + /// A plain proto3 scalar (no presence). `null_default` set = + /// null_resets_default: absent AND explicit null yield the default. + Scalar { + ty: ScalarTy, + null_default: Option, + }, + /// proto3 `optional` scalar. `null_is_none_default` set: absent yields the + /// default, explicit null clears to None. `null_default` set + /// (null_resets_default): absent AND null yield Some(default), and an + /// `*_or_default` accessor covers protobuf arrival (unset = None). + OptScalar { + ty: ScalarTy, + emit_null: bool, + null_is_none_default: Option, + null_default: Option, + }, + /// A message-typed field: `Option` presence; the value's JSON shape is + /// the message's own impl. + Message { + rust_path: String, + emit_null: bool, + }, + RepeatedScalar { + ty: ScalarTy, + }, + RepeatedMessage { + rust_path: String, + }, + /// repeated string with raw_json: Vec of JSON texts passing + /// through as an array of raw values. + RepeatedRawJson, + /// map; int_key means numeric-string keys (no transform + /// needed on the JSON side, documented parity). + Map { + value: ScalarTy, + }, + /// A string field holding JSON text (raw_json): the value passes through + /// verbatim on the HTTP side. + RawJson { + emit_null: bool, + }, +} + +#[derive(Debug, Clone)] +pub struct FieldModel { + /// The JSON key (= the proto field name). + pub json_name: String, + /// The prost struct field ident (keyword-escaped). + pub rust_ident: String, + pub kind: FieldKind, +} + +#[derive(Debug, Clone)] +pub struct VariantModel { + /// The JSON discriminator value / shape source (= the proto field name). + pub json_name: String, + /// The prost oneof enum variant (PascalCase). + pub rust_variant: String, + pub kind: FieldKind, + /// For tagged unions: the payload message's fields, inlined into the + /// tagged object. + pub payload_fields: Vec, +} + +#[derive(Debug)] +pub enum MessageKind { + /// Single-repeated-field carrier: JSON is the bare array. + BareList { item: FieldModel }, + /// Positional tuple: fields in declaration order form a bare array. + Tuple { fields: Vec }, + /// Single-field wrapper serializing as its sole field's value. + Transparent { field: FieldModel }, + /// Single oneof { one, many }: scalar-or-list flattening. + OneOrMany { + oneof_mod: String, + one: VariantModel, + many: VariantModel, + }, + /// Single oneof, internally tagged by `tag`; `passthrough` catches + /// unknown tags as a raw map (and any known-tag payload that fails to + /// parse, matching the hand-written FinishReason). + TaggedUnion { + oneof_mod: String, + oneof_ident: String, + tag: String, + variants: Vec, + passthrough: Option, + }, + /// Single oneof, untagged: the value's shape picks the variant. + Untagged { + oneof_mod: String, + oneof_ident: String, + variants: Vec, + }, + /// A plain object. + Struct { + deny_unknown: bool, + fields: Vec, + }, + /// gRPC-only (a plain oneof container like GenerateStreamItem): no HTTP + /// JSON contract, no serde emitted. + Skip, +} + +#[derive(Debug)] +pub struct MessageModel { + /// Rust type name (= proto message name). + pub rust_name: String, + pub kind: MessageKind, +} + +pub fn build_models(pool: &DescriptorPool, package: &str) -> Vec { + let ext = Extensions::resolve(pool); + pool.all_messages() + .filter(|m| m.full_name().starts_with(package)) + // Map-entry synthetic messages have no JSON identity of their own. + .filter(|m| !m.is_map_entry()) + .map(|m| message_model(&m, &ext)) + .collect() +} + +fn message_model(msg: &MessageDescriptor, ext: &Extensions) -> MessageModel { + let rust_name = msg.name().to_string(); + let options = msg.options(); + let mopts = options.get_extension(&ext.message_json); + let mopts = mopts.as_message(); + + // `MessageShape`: one shape per message, by construction of the enum. + let shape = mopts.map(|m| opt_enum(m, "shape")).unwrap_or(0); + let (bare_list, tuple, transparent) = (shape == 1, shape == 2, shape == 3); + let deny_unknown = mopts + .and_then(|m| m.get_field_by_name("unknown_fields")) + .and_then(|v| v.as_enum_number()) + .unwrap_or(0) + == 1; + + // proto3 `optional` is implemented as a synthetic oneof: those fields are + // plain fields with presence, not union members. + let oneofs: Vec<_> = msg.oneofs().filter(|o| !o.is_synthetic()).collect(); + let plain_fields: Vec = msg + .fields() + .filter(|f| f.containing_oneof().is_none_or(|o| o.is_synthetic())) + .collect(); + + if bare_list { + assert_eq!( + plain_fields.len(), + 1, + "{}: bare_list needs exactly one repeated field", + msg.full_name() + ); + return MessageModel { + rust_name, + kind: MessageKind::BareList { + item: field_model(&plain_fields[0], ext), + }, + }; + } + if transparent { + assert_eq!( + plain_fields.len(), + 1, + "{}: transparent needs exactly one field", + msg.full_name() + ); + return MessageModel { + rust_name, + kind: MessageKind::Transparent { + field: field_model(&plain_fields[0], ext), + }, + }; + } + if tuple { + return MessageModel { + rust_name, + kind: MessageKind::Tuple { + fields: plain_fields.iter().map(|f| field_model(f, ext)).collect(), + }, + }; + } + if oneofs.len() == 1 && plain_fields.is_empty() { + let oneof = &oneofs[0]; + let oopts = oneof.options(); + let oopts = oopts.get_extension(&ext.oneof_json); + let oopts = oopts.as_message(); + // `OneofShape`: one spelling per union, by construction of the enum. + let shape = oopts.map(|m| opt_enum(m, "shape")).unwrap_or(0); + let (untagged, one_or_many, tagged) = (shape == 1, shape == 2, shape == 3); + let tag = oopts.and_then(|m| opt_str(m, "tag")); + assert_eq!( + tagged, + tag.is_some(), + "{}: ONEOF_SHAPE_TAGGED and `tag` go together", + msg.full_name() + ); + let passthrough_on = oopts + .map(|m| opt_bool(m, "unknown_variant_passthrough")) + .unwrap_or(false); + let oneof_mod = snake(&rust_name); + let oneof_ident = pascal(oneof.name()); + + let mut variants: Vec = oneof + .fields() + .map(|f| VariantModel { + json_name: f.name().to_string(), + rust_variant: pascal(f.name()), + kind: field_model(&f, ext).kind, + payload_fields: match f.kind() { + // The raw_json passthrough arm carries a well-known type; + // its innards are never inlined. + Kind::Message(payload) + if tag.is_some() && !payload.full_name().starts_with("google.protobuf") => + { + payload + .fields() + .map(|pf| field_model(&pf, ext)) + .collect::>() + } + _ => vec![], + }, + }) + .collect(); + + if one_or_many { + assert_eq!(variants.len(), 2, "{}: one_or_many", msg.full_name()); + let many = variants.pop().expect("many variant"); + let one = variants.pop().expect("one variant"); + return MessageModel { + rust_name, + kind: MessageKind::OneOrMany { + oneof_mod, + one, + many, + }, + }; + } + if let Some(tag) = tag { + let passthrough = passthrough_on.then(|| { + let idx = variants + .iter() + .position(|v| matches!(v.kind, FieldKind::RawJson { .. })) + .expect("passthrough needs a raw_json string variant"); + variants.remove(idx) + }); + return MessageModel { + rust_name, + kind: MessageKind::TaggedUnion { + oneof_mod, + oneof_ident, + tag, + variants, + passthrough, + }, + }; + } + if untagged { + return MessageModel { + rust_name, + kind: MessageKind::Untagged { + oneof_mod, + oneof_ident, + variants, + }, + }; + } + // A plain oneof container (e.g. GenerateStreamItem) has no HTTP JSON + // contract of its own — it is gRPC-only. Emit nothing. + return MessageModel { + rust_name, + kind: MessageKind::Skip, + }; + } + + MessageModel { + rust_name, + kind: MessageKind::Struct { + deny_unknown, + fields: plain_fields.iter().map(|f| field_model(f, ext)).collect(), + }, + } +} + +fn field_model(f: &FieldDescriptor, ext: &Extensions) -> FieldModel { + let options = f.options(); + let fopts = options.get_extension(&ext.field_json); + let fopts = fopts.as_message(); + let get_bool = |name: &str| fopts.map(|m| opt_bool(m, name)).unwrap_or(false); + let default_lit = |m: &prost_reflect::DynamicMessage| -> Option { + // has_field_by_name: the default_* members are proto3 optional, so + // presence is meaningful. + if m.has_field_by_name("default_f64") { + return m + .get_field_by_name("default_f64") + .and_then(|v| v.as_f64()) + .map(DefaultLit::F64); + } + if m.has_field_by_name("default_i64") { + return m + .get_field_by_name("default_i64") + .and_then(|v| v.as_i64()) + .map(DefaultLit::I64); + } + if m.has_field_by_name("default_bool") { + return m + .get_field_by_name("default_bool") + .and_then(|v| v.as_bool()) + .map(DefaultLit::Bool); + } + if m.has_field_by_name("default_string") { + return m + .get_field_by_name("default_string") + .and_then(|v| v.as_str().map(str::to_owned)) + .map(DefaultLit::Str); + } + None + }; + let raw_json = get_bool("raw_json"); + let emit_null = get_bool("emit_null_when_absent"); + // `NullPolicy`: one policy per field, by construction of the enum. + let null_policy = fopts.map(|m| opt_enum(m, "null_policy")).unwrap_or(0); + let (null_resets, null_is_none) = (null_policy == 1, null_policy == 2); + let default = fopts.and_then(default_lit); + + let kind = if raw_json { + if f.is_list() { + FieldKind::RepeatedRawJson + } else { + assert!( + matches!(f.kind(), Kind::String), + "{}: raw_json requires a string field", + f.full_name() + ); + FieldKind::RawJson { emit_null } + } + } else if f.is_map() { + let value_ty = match f.kind() { + Kind::Message(entry) => scalar_ty( + &entry + .get_field_by_name("value") + .expect("map value field") + .kind(), + ), + _ => unreachable!("map fields are message-kinded"), + }; + FieldKind::Map { value: value_ty } + } else if f.is_list() { + match f.kind() { + Kind::Message(m) => FieldKind::RepeatedMessage { + rust_path: m.name().to_string(), + }, + other => FieldKind::RepeatedScalar { + ty: scalar_ty(&other), + }, + } + } else { + match f.kind() { + Kind::Message(m) => FieldKind::Message { + rust_path: m.name().to_string(), + emit_null, + }, + other => { + let ty = scalar_ty(&other); + if f.supports_presence() { + FieldKind::OptScalar { + ty, + emit_null, + null_is_none_default: null_is_none.then(|| { + default + .clone() + .expect("NULL_POLICY_IS_NONE requires a default_*") + }), + null_default: null_resets.then(|| { + default + .clone() + .expect("NULL_POLICY_RESETS_DEFAULT requires a default_*") + }), + } + } else { + FieldKind::Scalar { + ty, + null_default: null_resets.then(|| { + default + .clone() + .expect("NULL_POLICY_RESETS_DEFAULT requires a default_*") + }), + } + } + } + } + }; + + FieldModel { + json_name: f.name().to_string(), + rust_ident: escape_ident(f.name()), + kind, + } +} + +fn scalar_ty(kind: &Kind) -> ScalarTy { + match kind { + Kind::Double | Kind::Float => ScalarTy::F64, + Kind::Int64 | Kind::Sint64 | Kind::Sfixed64 => ScalarTy::I64, + Kind::Int32 | Kind::Sint32 | Kind::Sfixed32 => ScalarTy::I32, + Kind::Uint32 | Kind::Fixed32 => ScalarTy::U32, + Kind::Uint64 | Kind::Fixed64 => ScalarTy::U64, + Kind::Bool => ScalarTy::Bool, + Kind::String => ScalarTy::Str, + other => panic!("unsupported scalar kind {other:?}"), + } +} + +/// prost's keyword escaping for field idents. +fn escape_ident(name: &str) -> String { + const KEYWORDS: &[&str] = &[ + "as", "break", "const", "continue", "crate", "dyn", "else", "enum", "extern", "false", + "fn", "for", "if", "impl", "in", "let", "loop", "match", "mod", "move", "mut", "pub", + "ref", "return", "self", "static", "struct", "super", "trait", "true", "type", "unsafe", + "use", "where", "while", + ]; + if KEYWORDS.contains(&name) { + format!("r#{name}") + } else { + name.to_string() + } +} + +pub fn pascal(name: &str) -> String { + name.split('_') + .filter(|s| !s.is_empty()) + .map(|s| { + let mut chars = s.chars(); + match chars.next() { + Some(first) => first.to_ascii_uppercase().to_string() + chars.as_str(), + None => String::new(), + } + }) + .collect() +} + +pub fn snake(name: &str) -> String { + let mut out = String::new(); + for (i, ch) in name.chars().enumerate() { + if ch.is_ascii_uppercase() { + if i > 0 { + out.push('_'); + } + out.push(ch.to_ascii_lowercase()); + } else { + out.push(ch); + } + } + out +} diff --git a/rust/sglang-api-types/Cargo.toml b/rust/sglang-api-types/Cargo.toml new file mode 100644 index 0000000000..ea7105d315 --- /dev/null +++ b/rust/sglang-api-types/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "sglang-api-types" +description = "Generated SGLang API types: prost structs, tonic services, schema-driven serde" +version.workspace = true +edition.workspace = true +license.workspace = true + +# No [package.metadata.sglang]: not a Python extension; wheel builds skip it. + +[lib] +name = "sglang_api_types" +crate-type = ["rlib"] + +[dependencies] +bytes = { workspace = true } +prost = { workspace = true } +prost-types = { workspace = true } +serde = { workspace = true } +# preserve_order: raw_json passthrough (custom_params, internal_states, an +# unknown finish reason) re-serializes through serde_json::Value, and the +# contract is "verbatim": key order must not depend on what else links the +# crate. +serde_json = { workspace = true, features = ["preserve_order"] } +tonic = { workspace = true } +tonic-prost = { workspace = true } diff --git a/rust/sglang-api-types/src/ext.rs b/rust/sglang-api-types/src/ext.rs new file mode 100644 index 0000000000..e89a72c519 --- /dev/null +++ b/rust/sglang-api-types/src/ext.rs @@ -0,0 +1,63 @@ +//! Hand-written extensions on the generated types: the accessors and +//! constructors server logic needs beyond the wire shapes. Grows with each +//! migration stage. + +use std::borrow::Cow; + +use crate::api::v1::{FinishReason, FinishStop, Matched, MatchedTokens, finish_reason, matched}; + +impl FinishReason { + /// A stop reason (`type: "stop"`), optionally carrying what matched. + pub fn stop(matched: Option) -> Self { + FinishReason { + kind: Some(finish_reason::Kind::Stop(FinishStop { + matched: matched.map(|value| Matched { value: Some(value) }), + })), + } + } + + /// Returns the wire `type` without reserializing the finish reason (an + /// unknown reason's is read out of its raw JSON, hence the `Cow`). + pub fn kind_name(&self) -> Option> { + match self.kind.as_ref()? { + finish_reason::Kind::Stop(_) => Some(Cow::Borrowed("stop")), + finish_reason::Kind::Length(_) => Some(Cow::Borrowed("length")), + finish_reason::Kind::Abort(_) => Some(Cow::Borrowed("abort")), + finish_reason::Kind::Unknown(raw) => { + let parsed: serde_json::Value = serde_json::from_str(raw).ok()?; + parsed + .get("type") + .and_then(serde_json::Value::as_str) + .map(|s| Cow::Owned(s.to_owned())) + } + } + } + + /// The stop this request matched, if it stopped on one. `None` for + /// length/abort and for an unknown type. + pub fn matched(&self) -> Option<&matched::Value> { + match self.kind.as_ref()? { + finish_reason::Kind::Stop(stop) => stop.matched.as_ref()?.value.as_ref(), + _ => None, + } + } + + /// `Some((status, message))` when this is an abort carrying a `status_code` — + /// a scheduler-side request error the API surfaces as that HTTP status instead + /// of as a normal completion. A plain abort (no code) reads as `None`. + pub fn abort_status(&self) -> Option<(u16, &str)> { + match self.kind.as_ref()? { + finish_reason::Kind::Abort(abort) => Some(( + u16::try_from(abort.status_code?).ok()?, + abort.message.as_deref().unwrap_or("request aborted"), + )), + _ => None, + } + } +} + +impl From> for MatchedTokens { + fn from(ids: Vec) -> Self { + MatchedTokens { ids } + } +} diff --git a/rust/sglang-api-types/src/generated/sglang.api.v1.rs b/rust/sglang-api-types/src/generated/sglang.api.v1.rs new file mode 100644 index 0000000000..8c26c921bb --- /dev/null +++ b/rust/sglang-api-types/src/generated/sglang.api.v1.rs @@ -0,0 +1,657 @@ +// This file is @generated by prost-build. +/// A token-id sequence. int64 matches the servers' token id width (Rust +/// `TokenIds = Vec`, Python `array("q")`). +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct TokenIds { + #[prost(int64, repeated, tag = "1")] + pub ids: ::prost::alloc::vec::Vec, +} +/// Scalar-or-list carriers: `/generate` accepts every per-request field as one +/// value (single request) or a list (batch). JSON flattens via the one_or_many +/// option; gRPC clients pick the variant explicitly. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct StringOrList { + #[prost(oneof = "string_or_list::Value", tags = "1, 2")] + pub value: ::core::option::Option, +} +/// Nested message and enum types in `StringOrList`. +pub mod string_or_list { + #[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)] + pub enum Value { + #[prost(string, tag = "1")] + One(::prost::alloc::string::String), + #[prost(message, tag = "2")] + Many(super::StringList), + } +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct StringList { + #[prost(string, repeated, tag = "1")] + pub items: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, +} +/// A flat string list for one request, or one list per batch entry. JSON: +/// `\["a", "b"\]` is `one`; `\[["a"\], \["b", "c"]\]` is `many` (first element +/// decides, `\[\]` is an empty `one`). +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct StringListOrList { + #[prost(oneof = "string_list_or_list::Value", tags = "1, 2")] + pub value: ::core::option::Option, +} +/// Nested message and enum types in `StringListOrList`. +pub mod string_list_or_list { + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Value { + #[prost(message, tag = "1")] + One(super::StringList), + #[prost(message, tag = "2")] + Many(super::StringListList), + } +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct StringListList { + #[prost(message, repeated, tag = "1")] + pub items: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct BoolOrList { + #[prost(oneof = "bool_or_list::Value", tags = "1, 2")] + pub value: ::core::option::Option, +} +/// Nested message and enum types in `BoolOrList`. +pub mod bool_or_list { + #[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)] + pub enum Value { + #[prost(bool, tag = "1")] + One(bool), + #[prost(message, tag = "2")] + Many(super::BoolList), + } +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct BoolList { + #[prost(bool, repeated, tag = "1")] + pub items: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct Int64OrList { + #[prost(oneof = "int64_or_list::Value", tags = "1, 2")] + pub value: ::core::option::Option, +} +/// Nested message and enum types in `Int64OrList`. +pub mod int64_or_list { + #[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)] + pub enum Value { + #[prost(int64, tag = "1")] + One(i64), + #[prost(message, tag = "2")] + Many(super::Int64List), + } +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct Int64List { + #[prost(int64, repeated, tag = "1")] + pub items: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TokenIdsOrList { + #[prost(oneof = "token_ids_or_list::Value", tags = "1, 2")] + pub value: ::core::option::Option, +} +/// Nested message and enum types in `TokenIdsOrList`. +pub mod token_ids_or_list { + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Value { + #[prost(message, tag = "1")] + One(super::TokenIds), + #[prost(message, tag = "2")] + Many(super::TokenIdsList), + } +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TokenIdsList { + #[prost(message, repeated, tag = "1")] + pub items: ::prost::alloc::vec::Vec, +} +/// List variants whose ELEMENTS may be JSON null (PD bootstrap columns: a +/// batch mixes bootstrapped and plain requests). Proto carries element +/// presence as a per-element oneof-free wrapper. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct OptionalString { + #[prost(string, optional, tag = "1")] + pub value: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct OptionalStringOrList { + #[prost(oneof = "optional_string_or_list::Value", tags = "1, 2")] + pub value: ::core::option::Option, +} +/// Nested message and enum types in `OptionalStringOrList`. +pub mod optional_string_or_list { + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Value { + #[prost(message, tag = "1")] + One(super::OptionalString), + #[prost(message, tag = "2")] + Many(super::OptionalStringList), + } +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct OptionalStringList { + #[prost(message, repeated, tag = "1")] + pub items: ::prost::alloc::vec::Vec, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct OptionalInt64 { + #[prost(int64, optional, tag = "1")] + pub value: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct OptionalInt64OrList { + #[prost(oneof = "optional_int64_or_list::Value", tags = "1, 2")] + pub value: ::core::option::Option, +} +/// Nested message and enum types in `OptionalInt64OrList`. +pub mod optional_int64_or_list { + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Value { + #[prost(message, tag = "1")] + One(super::OptionalInt64), + #[prost(message, tag = "2")] + Many(super::OptionalInt64List), + } +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct OptionalInt64List { + #[prost(message, repeated, tag = "1")] + pub items: ::prost::alloc::vec::Vec, +} +/// The native error body: `{"error": {"message", "code"}}` (Python +/// `http_server.generate_request` parity). Transports frame it themselves +/// (unary status + body, SSE error frame, gRPC status detail). +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ErrorBody { + #[prost(string, tag = "1")] + pub message: ::prost::alloc::string::String, + #[prost(uint32, tag = "2")] + pub code: u32, +} +/// Python's `matched` key, `Union\[str, int, List[int]\]`: the three wire shapes +/// are disjoint, so shape alone picks the arm (untagged). +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct Matched { + #[prost(oneof = "matched::Value", tags = "1, 2, 3")] + pub value: ::core::option::Option, +} +/// Nested message and enum types in `Matched`. +pub mod matched { + #[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)] + pub enum Value { + #[prost(int64, tag = "1")] + Token(i64), + #[prost(string, tag = "2")] + Str(::prost::alloc::string::String), + #[prost(message, tag = "3")] + Tokens(super::MatchedTokens), + } +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct MatchedTokens { + #[prost(int64, repeated, tag = "1")] + pub ids: ::prost::alloc::vec::Vec, +} +/// `FINISH_MATCHED_TOKEN` / `FINISH_MATCHED_STR` / `FINISHED_MATCHED_REGEX` — +/// all three report `type: "stop"`, matching the OpenAI API's value. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct FinishStop { + #[prost(message, optional, tag = "1")] + pub matched: ::core::option::Option, +} +/// `FINISH_LENGTH` — hit `max_new_tokens` (or the context limit). +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct FinishLength { + #[prost(uint64, optional, tag = "1")] + pub length: ::core::option::Option, +} +/// The `FINISH_ABORT` payload. Python emits all three keys, nulls included. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct FinishAbort { + #[prost(string, optional, tag = "1")] + pub message: ::core::option::Option<::prost::alloc::string::String>, + #[prost(uint32, optional, tag = "2")] + pub status_code: ::core::option::Option, + #[prost(string, optional, tag = "3")] + pub err_type: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct FinishReason { + #[prost(oneof = "finish_reason::Kind", tags = "1, 2, 3, 4")] + pub kind: ::core::option::Option, +} +/// Nested message and enum types in `FinishReason`. +pub mod finish_reason { + #[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)] + pub enum Kind { + #[prost(message, tag = "1")] + Stop(super::FinishStop), + #[prost(message, tag = "2")] + Length(super::FinishLength), + #[prost(message, tag = "3")] + Abort(::prost::alloc::boxed::Box), + /// The passthrough arm: an unrecognized `type`, kept as its raw map + /// (JSON text — Struct would coerce integer values to doubles). + #[prost(string, tag = "4")] + Unknown(::prost::alloc::string::String), + } +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SamplingParams { + /// JSON: absent = 128, explicit null = no limit (Python `Optional\[int\] = 128`). Protobuf has no null: unset = 128 (apply_absent_defaults at the + /// gRPC entry); no limit needs an explicit large value. + #[prost(int64, optional, tag = "1")] + pub max_new_tokens: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub stop: ::core::option::Option, + #[prost(message, optional, tag = "3")] + pub stop_token_ids: ::core::option::Option, + #[prost(message, optional, tag = "4")] + pub stop_regex: ::core::option::Option, + #[prost(double, optional, tag = "5")] + pub temperature: ::core::option::Option, + #[prost(double, optional, tag = "6")] + pub top_p: ::core::option::Option, + /// Default 1 \<\< 30 (TOP_K_ALL): "consider the whole vocabulary". + #[prost(int64, optional, tag = "7")] + pub top_k: ::core::option::Option, + #[prost(double, optional, tag = "8")] + pub min_p: ::core::option::Option, + #[prost(double, optional, tag = "9")] + pub frequency_penalty: ::core::option::Option, + #[prost(double, optional, tag = "10")] + pub presence_penalty: ::core::option::Option, + #[prost(double, optional, tag = "11")] + pub repetition_penalty: ::core::option::Option, + #[prost(int64, optional, tag = "12")] + pub min_new_tokens: ::core::option::Option, + #[prost(int64, optional, tag = "13")] + pub n: ::core::option::Option, + #[prost(string, optional, tag = "14")] + pub json_schema: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "15")] + pub regex: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "16")] + pub ebnf: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "17")] + pub structural_tag: ::core::option::Option<::prost::alloc::string::String>, + #[prost(bool, optional, tag = "18")] + pub ignore_eos: ::core::option::Option, + #[prost(bool, optional, tag = "19")] + pub skip_special_tokens: ::core::option::Option, + #[prost(bool, optional, tag = "20")] + pub spaces_between_special_tokens: ::core::option::Option, + #[prost(bool, optional, tag = "21")] + pub no_stop_trim: ::core::option::Option, + #[prost(int64, optional, tag = "22")] + pub stream_interval: ::core::option::Option, + /// Token id (numeric-string key, Python parity) -> bias. + #[prost(map = "string, double", tag = "23")] + pub logit_bias: ::std::collections::HashMap<::prost::alloc::string::String, f64>, + #[prost(int64, optional, tag = "24")] + pub sampling_seed: ::core::option::Option, + /// Opaque JSON forwarded to a custom logit processor; never inspected here. + #[prost(string, optional, tag = "25")] + pub custom_params: ::core::option::Option<::prost::alloc::string::String>, + /// beam_width > 1 turns the request into a beam search request; `n` is then + /// the number of returned sequences (n \<= beam_width). + #[prost(int64, optional, tag = "26")] + pub beam_width: ::core::option::Option, +} +/// (message_json).unknown_fields defaults to IGNORE — stated here as schema +/// intent, load-bearing for client compat. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GenerateRequest { + #[prost(message, optional, tag = "1")] + pub rid: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub text: ::core::option::Option, + #[prost(message, optional, tag = "3")] + pub input_ids: ::core::option::Option, + #[prost(bool, optional, tag = "4")] + pub stream: ::core::option::Option, + /// One set of params, or one per batch item. JSON errors keep serde's + /// field-level texts (the hand-written SamplingParamsInput contract). + #[prost(message, optional, tag = "5")] + pub sampling_params: ::core::option::Option, + #[prost(message, optional, tag = "6")] + pub return_logprob: ::core::option::Option, + #[prost(message, optional, tag = "7")] + pub logprob_start_len: ::core::option::Option, + #[prost(message, optional, tag = "8")] + pub top_logprobs_num: ::core::option::Option, + #[prost(message, optional, tag = "9")] + pub token_ids_logprob: ::core::option::Option, + #[prost(message, optional, tag = "10")] + pub return_hidden_states: ::core::option::Option, + #[prost(bool, optional, tag = "11")] + pub return_text_in_logprobs: ::core::option::Option, + /// PD bootstrap columns: a batch mixes bootstrapped and plain requests, so + /// elements are individually nullable. + #[prost(message, optional, tag = "12")] + pub bootstrap_host: ::core::option::Option, + #[prost(message, optional, tag = "13")] + pub bootstrap_port: ::core::option::Option, + #[prost(message, optional, tag = "14")] + pub bootstrap_room: ::core::option::Option, + #[prost(message, optional, tag = "15")] + pub bootstrap_pair_key: ::core::option::Option, + #[prost(message, optional, tag = "16")] + pub decode_tp_size: ::core::option::Option, + #[prost(int64, optional, tag = "17")] + pub routed_dp_rank: ::core::option::Option, + #[prost(int64, optional, tag = "18")] + pub disagg_prefill_dp_rank: ::core::option::Option, + /// Multimodal payloads pass through untyped (data URIs, URLs, file paths, or + /// nested batch lists) to the MM pipeline, which owns their validation. + /// Media inputs, one container per modality (see MediaInput). The item + /// hints a modality does not support are rejected at the processor. + #[prost(message, optional, tag = "19")] + pub image_data: ::core::option::Option, + /// Per-item content hashes overriding the computed ones, in item order; a + /// flat list for one request (the batch form is rejected downstream). + #[prost(message, optional, tag = "20")] + pub mm_hashes: ::core::option::Option, + #[prost(message, optional, tag = "21")] + pub video_data: ::core::option::Option, + #[prost(message, optional, tag = "22")] + pub audio_data: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SamplingParamsOrList { + #[prost(oneof = "sampling_params_or_list::Value", tags = "1, 2")] + pub value: ::core::option::Option, +} +/// Nested message and enum types in `SamplingParamsOrList`. +pub mod sampling_params_or_list { + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Value { + #[prost(message, tag = "1")] + One(super::SamplingParams), + #[prost(message, tag = "2")] + Many(super::SamplingParamsList), + } +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SamplingParamsList { + #[prost(message, repeated, tag = "1")] + pub items: ::prost::alloc::vec::Vec, +} +/// One `\[logprob, token_id, text|null\]` entry (text present when +/// `return_text_in_logprobs`, decoded on the detok shard). The logprob slot is +/// optional: Python emits null for a position with no logprob (the first +/// prefill token; Rust maps the scheduler's NaN to the same null). +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct LogprobEntry { + #[prost(double, optional, tag = "1")] + pub logprob: ::core::option::Option, + #[prost(int64, tag = "2")] + pub token_id: i64, + #[prost(string, optional, tag = "3")] + pub text: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct LogprobEntries { + #[prost(message, repeated, tag = "1")] + pub entries: ::prost::alloc::vec::Vec, +} +/// One decode position's top-k alternatives. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TopLogprobRow { + #[prost(message, repeated, tag = "1")] + pub entries: ::prost::alloc::vec::Vec, +} +/// A position's row, or null: a position that recorded no alternatives is +/// null on the wire (Python parity), never \[\]. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct NullableTopLogprobs { + #[prost(message, optional, tag = "1")] + pub row: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct HiddenStateRow { + #[prost(double, repeated, tag = "1")] + pub values: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GenerateMetaInfo { + /// The client's rid, byte-for-byte (`Rid::client_facing`). + #[prost(string, tag = "1")] + pub id: ::prost::alloc::string::String, + #[prost(uint32, tag = "2")] + pub prompt_tokens: u32, + #[prost(uint64, tag = "3")] + pub completion_tokens: u64, + /// Always present in JSON, null until a terminal frame. + #[prost(message, optional, tag = "4")] + pub finish_reason: ::core::option::Option, + /// input+output token logprobs are set together, empty lists included: a PD + /// decode node never receives input logprobs, but the PD router keys its + /// merge of prefill's on the key's presence. + #[prost(message, optional, tag = "5")] + pub output_token_logprobs: ::core::option::Option, + #[prost(message, optional, tag = "6")] + pub input_token_logprobs: ::core::option::Option, + #[prost(message, repeated, tag = "7")] + pub output_top_logprobs: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "8")] + pub input_top_logprobs: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "9")] + pub output_token_ids_logprobs: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "10")] + pub input_token_ids_logprobs: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "11")] + pub hidden_states: ::prost::alloc::vec::Vec, + /// Seconds from normalized-request acceptance to the terminal output; only + /// on the terminal frame. + #[prost(double, optional, tag = "12")] + pub e2e_latency: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GenerateResponse { + /// JSON key order is FIELD-NUMBER order: text, meta_info, then the + /// conditionally-present output_ids and index — matching frame_value's + /// insertion order byte-for-byte. + #[prost(string, tag = "1")] + pub text: ::prost::alloc::string::String, + #[prost(message, optional, tag = "2")] + pub meta_info: ::core::option::Option, + /// Omitted while empty (a text-only frame). + #[prost(message, optional, tag = "3")] + pub output_ids: ::core::option::Option, + /// Batch position; only on batch responses/frames. + #[prost(uint32, optional, tag = "4")] + pub index: ::core::option::Option, +} +/// One request's failure inside a Generate stream; carries its batch position +/// (the HTTP SSE shape puts `index` beside the error object the same way). +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GenerateStreamError { + #[prost(message, optional, tag = "1")] + pub error: ::core::option::Option, + #[prost(uint32, optional, tag = "2")] + pub index: ::core::option::Option, +} +/// One item of the gRPC Generate stream: a frame, or one request's failure +/// (the stream continues — other batch items may still be running). On HTTP +/// the same failure is an in-stream SSE error frame. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GenerateStreamItem { + #[prost(oneof = "generate_stream_item::Item", tags = "1, 2")] + pub item: ::core::option::Option, +} +/// Nested message and enum types in `GenerateStreamItem`. +pub mod generate_stream_item { + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Item { + #[prost(message, tag = "1")] + Frame(super::GenerateResponse), + #[prost(message, tag = "2")] + Error(super::GenerateStreamError), + } +} +/// One media item's source and per-item hints. Unknown keys are an error: a +/// hint this schema does not name would otherwise be silently dropped. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct MediaRef { + /// URL, `file://` / absolute path, `data:` URI, or bare base64. + #[prost(string, tag = "1")] + pub url: ::prost::alloc::string::String, + /// Image: "auto" | "low" | "high" (the OpenAI detail hint). + #[prost(string, optional, tag = "2")] + pub detail: ::core::option::Option<::prost::alloc::string::String>, + /// Image/video: dynamic-patch budget (InternVL-style processors). + #[prost(int64, optional, tag = "3")] + pub max_dynamic_patch: ::core::option::Option, + /// Image: a caller-supplied content hash that overrides the computed one. + #[prost(string, optional, tag = "4")] + pub content_hash: ::core::option::Option<::prost::alloc::string::String>, + /// Video/image budgets read by the GLM and MiMo processors. + #[prost(int64, optional, tag = "5")] + pub min_pixels: ::core::option::Option, + #[prost(int64, optional, tag = "6")] + pub max_pixels: ::core::option::Option, + #[prost(int64, optional, tag = "7")] + pub total_max_pixels: ::core::option::Option, + #[prost(double, optional, tag = "8")] + pub fps: ::core::option::Option, + #[prost(int64, optional, tag = "9")] + pub num_frames: ::core::option::Option, + #[prost(int64, optional, tag = "10")] + pub min_frames: ::core::option::Option, + #[prost(int64, optional, tag = "11")] + pub max_frames: ::core::option::Option, + #[prost(int64, optional, tag = "12")] + pub max_tokens_per_frame: ::core::option::Option, + #[prost(int64, optional, tag = "13")] + pub max_image_tokens: ::core::option::Option, + /// Video: also decode the audio track. + #[prost(bool, optional, tag = "14")] + pub use_audio: ::core::option::Option, +} +/// One media item. JSON: a bare string is the source; an object is a MediaRef. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct MediaItem { + #[prost(oneof = "media_item::Value", tags = "1, 2")] + pub value: ::core::option::Option, +} +/// Nested message and enum types in `MediaItem`. +pub mod media_item { + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Value { + #[prost(string, tag = "1")] + Source(::prost::alloc::string::String), + #[prost(message, tag = "2")] + Ref(super::MediaRef), + } +} +/// A list element that may be JSON null (Python's `\[img, None\]`). +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct OptionalMediaItem { + #[prost(message, optional, tag = "1")] + pub item: ::core::option::Option, +} +/// One request's items (`\[a, null, b\]`), or one batch entry's items. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct MediaItems { + #[prost(message, repeated, tag = "1")] + pub items: ::prost::alloc::vec::Vec, +} +/// A batch entry whose item list may be JSON null. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct OptionalMediaItems { + #[prost(message, optional, tag = "1")] + pub items: ::core::option::Option, +} +/// One item list per batch entry (`\[[a, b\], null, \[]\]`). +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct MediaItemsList { + #[prost(message, repeated, tag = "1")] + pub items: ::prost::alloc::vec::Vec, +} +/// The three container forms Python's `_normalize_image_data` distinguishes. +/// JSON: a string or object is `item` (one item; a broadcast in a batch); an +/// array whose first element is not an array is `many` (a flat list; `\[\]` is +/// an empty `many`); an array whose first element is an array is `nested` +/// (one list per batch entry). Which forms a request may use depends on +/// whether it is a batch, and is checked downstream, not here. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct MediaInput { + #[prost(oneof = "media_input::Value", tags = "1, 2, 3")] + pub value: ::core::option::Option, +} +/// Nested message and enum types in `MediaInput`. +pub mod media_input { + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Value { + #[prost(message, tag = "1")] + Item(super::MediaItem), + #[prost(message, tag = "2")] + Many(super::MediaItems), + #[prost(message, tag = "3")] + Nested(super::MediaItemsList), + } +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct HealthCheckRequest {} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct HealthCheckResponse { + #[prost(bool, tag = "1")] + pub healthy: bool, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetModelInfoRequest {} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetModelInfoResponse { + #[prost(string, tag = "1")] + pub model_path: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub served_model_name: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub tokenizer_path: ::prost::alloc::string::String, + #[prost(bool, tag = "4")] + pub is_generation: bool, + /// The launch-time `--preferred-sampling-params` object, or null when unset. + /// JSON carries the object itself (the HTTP `/get_model_info` shape); the + /// protobuf field holds it as JSON text. + #[prost(string, optional, tag = "5")] + pub preferred_sampling_params: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "6")] + pub weight_version: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "7")] + pub load_format: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "8")] + pub reasoning_parser: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "9")] + pub tool_call_parser: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetServerInfoRequest {} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetServerInfoResponse { + #[prost(string, tag = "1")] + pub model_path: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub served_model_name: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub tokenizer_path: ::prost::alloc::string::String, + #[prost(uint32, tag = "4")] + pub max_context_length: u32, + #[prost(uint64, optional, tag = "5")] + pub max_total_num_tokens: ::core::option::Option, + #[prost(string, tag = "6")] + pub version: ::prost::alloc::string::String, + /// The allowlisted runtime metrics (last_gen_throughput, memory_usage, + /// effective_max_running_requests_per_dp, avg_spec_accept_length, + /// step_time_dict), untyped because the scheduler owns their shapes; JSON + /// passes them through verbatim. + #[prost(string, repeated, tag = "7")] + pub internal_states: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, +} diff --git a/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs b/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs new file mode 100644 index 0000000000..d84aa80ab3 --- /dev/null +++ b/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs @@ -0,0 +1,3115 @@ +// Generated by sglang-api-codegen from proto/sglang — DO NOT EDIT. +// The sglang.json.v1 options in the schema are the JSON contract; edit the +// .proto files and `cargo gen-api`. + +impl ::serde::Serialize for TokenIds { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_seq(self.ids.iter()) + } +} + +impl<'de> ::serde::Deserialize<'de> for TokenIds { + fn deserialize>(deserializer: D) -> Result { + Ok(TokenIds { + ids: <::prost::alloc::vec::Vec as ::serde::Deserialize>::deserialize( + deserializer, + )?, + }) + } +} + +impl ::serde::Serialize for StringOrList { + fn serialize(&self, serializer: S) -> Result { + match &self.value { + Some(string_or_list::Value::One(v)) => v.serialize(serializer), + Some(string_or_list::Value::Many(v)) => v.serialize(serializer), + None => serializer.serialize_none(), + } + } +} + +impl<'de> ::serde::Deserialize<'de> for StringOrList { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = StringOrList; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("a value or an array of values for StringOrList") + } + fn visit_str(self, v: &str) -> Result { + Ok(StringOrList { + value: Some(string_or_list::Value::One(v.to_owned())), + }) + } + fn visit_string(self, v: String) -> Result { + Ok(StringOrList { + value: Some(string_or_list::Value::One(v)), + }) + } + fn visit_seq>( + self, + seq: A, + ) -> Result { + let many = ::deserialize( + ::serde::de::value::SeqAccessDeserializer::new(seq), + )?; + Ok(StringOrList { + value: Some(string_or_list::Value::Many(many)), + }) + } + } + deserializer.deserialize_any(V) + } +} + +impl ::serde::Serialize for StringList { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_seq(self.items.iter()) + } +} + +impl<'de> ::serde::Deserialize<'de> for StringList { + fn deserialize>(deserializer: D) -> Result { + Ok(StringList { + items: <::prost::alloc::vec::Vec<::prost::alloc::string::String> as ::serde::Deserialize>::deserialize(deserializer)?, + }) + } +} + +impl ::serde::Serialize for StringListOrList { + fn serialize(&self, serializer: S) -> Result { + match &self.value { + Some(string_list_or_list::Value::One(v)) => v.serialize(serializer), + Some(string_list_or_list::Value::Many(v)) => v.serialize(serializer), + None => serializer.serialize_none(), + } + } +} + +impl<'de> ::serde::Deserialize<'de> for StringListOrList { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = StringListOrList; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("a value or an array of values for StringListOrList") + } + fn visit_map>( + self, + map: A, + ) -> Result { + let one = ::deserialize( + ::serde::de::value::MapAccessDeserializer::new(map), + )?; + Ok(StringListOrList { + value: Some(string_list_or_list::Value::One(one)), + }) + } + fn visit_seq>( + self, + mut seq: A, + ) -> Result { + let Some(first) = seq.next_element::<::serde_json::Value>()? else { + // [] is one empty list (the untagged first-arm rule). + return Ok(StringListOrList { + value: Some(string_list_or_list::Value::One(StringList::default())), + }); + }; + let is_number = first.is_string(); + let mut rest = vec![first]; + while let Some(v) = seq.next_element::<::serde_json::Value>()? { + rest.push(v); + } + let reparse = ::serde_json::Value::Array(rest); + if is_number { + let one: StringList = + ::serde_json::from_value(reparse).map_err(::serde::de::Error::custom)?; + Ok(StringListOrList { + value: Some(string_list_or_list::Value::One(one)), + }) + } else { + let many: StringListList = + ::serde_json::from_value(reparse).map_err(::serde::de::Error::custom)?; + Ok(StringListOrList { + value: Some(string_list_or_list::Value::Many(many)), + }) + } + } + } + deserializer.deserialize_any(V) + } +} + +impl ::serde::Serialize for StringListList { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_seq(self.items.iter()) + } +} + +impl<'de> ::serde::Deserialize<'de> for StringListList { + fn deserialize>(deserializer: D) -> Result { + Ok(StringListList { + items: <::prost::alloc::vec::Vec as ::serde::Deserialize>::deserialize( + deserializer, + )?, + }) + } +} + +impl ::serde::Serialize for BoolOrList { + fn serialize(&self, serializer: S) -> Result { + match &self.value { + Some(bool_or_list::Value::One(v)) => v.serialize(serializer), + Some(bool_or_list::Value::Many(v)) => v.serialize(serializer), + None => serializer.serialize_none(), + } + } +} + +impl<'de> ::serde::Deserialize<'de> for BoolOrList { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = BoolOrList; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("a value or an array of values for BoolOrList") + } + fn visit_bool(self, v: bool) -> Result { + Ok(BoolOrList { + value: Some(bool_or_list::Value::One(v)), + }) + } + fn visit_seq>( + self, + seq: A, + ) -> Result { + let many = ::deserialize( + ::serde::de::value::SeqAccessDeserializer::new(seq), + )?; + Ok(BoolOrList { + value: Some(bool_or_list::Value::Many(many)), + }) + } + } + deserializer.deserialize_any(V) + } +} + +impl ::serde::Serialize for BoolList { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_seq(self.items.iter()) + } +} + +impl<'de> ::serde::Deserialize<'de> for BoolList { + fn deserialize>(deserializer: D) -> Result { + Ok(BoolList { + items: <::prost::alloc::vec::Vec as ::serde::Deserialize>::deserialize( + deserializer, + )?, + }) + } +} + +impl ::serde::Serialize for Int64OrList { + fn serialize(&self, serializer: S) -> Result { + match &self.value { + Some(int64_or_list::Value::One(v)) => v.serialize(serializer), + Some(int64_or_list::Value::Many(v)) => v.serialize(serializer), + None => serializer.serialize_none(), + } + } +} + +impl<'de> ::serde::Deserialize<'de> for Int64OrList { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = Int64OrList; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("a value or an array of values for Int64OrList") + } + fn visit_i64(self, v: i64) -> Result { + Ok(Int64OrList { + value: Some(int64_or_list::Value::One(v)), + }) + } + fn visit_u64(self, v: u64) -> Result { + let v = i64::try_from(v).map_err(|_| E::custom("integer out of range"))?; + Ok(Int64OrList { + value: Some(int64_or_list::Value::One(v)), + }) + } + fn visit_seq>( + self, + seq: A, + ) -> Result { + let many = ::deserialize( + ::serde::de::value::SeqAccessDeserializer::new(seq), + )?; + Ok(Int64OrList { + value: Some(int64_or_list::Value::Many(many)), + }) + } + } + deserializer.deserialize_any(V) + } +} + +impl ::serde::Serialize for Int64List { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_seq(self.items.iter()) + } +} + +impl<'de> ::serde::Deserialize<'de> for Int64List { + fn deserialize>(deserializer: D) -> Result { + Ok(Int64List { + items: <::prost::alloc::vec::Vec as ::serde::Deserialize>::deserialize( + deserializer, + )?, + }) + } +} + +impl ::serde::Serialize for TokenIdsOrList { + fn serialize(&self, serializer: S) -> Result { + match &self.value { + Some(token_ids_or_list::Value::One(v)) => v.serialize(serializer), + Some(token_ids_or_list::Value::Many(v)) => v.serialize(serializer), + None => serializer.serialize_none(), + } + } +} + +impl<'de> ::serde::Deserialize<'de> for TokenIdsOrList { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = TokenIdsOrList; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("a value or an array of values for TokenIdsOrList") + } + fn visit_map>( + self, + map: A, + ) -> Result { + let one = ::deserialize( + ::serde::de::value::MapAccessDeserializer::new(map), + )?; + Ok(TokenIdsOrList { + value: Some(token_ids_or_list::Value::One(one)), + }) + } + fn visit_seq>( + self, + mut seq: A, + ) -> Result { + let Some(first) = seq.next_element::<::serde_json::Value>()? else { + // [] is one empty list (the untagged first-arm rule). + return Ok(TokenIdsOrList { + value: Some(token_ids_or_list::Value::One(TokenIds::default())), + }); + }; + let is_number = first.is_number(); + let mut rest = vec![first]; + while let Some(v) = seq.next_element::<::serde_json::Value>()? { + rest.push(v); + } + let reparse = ::serde_json::Value::Array(rest); + if is_number { + let one: TokenIds = + ::serde_json::from_value(reparse).map_err(::serde::de::Error::custom)?; + Ok(TokenIdsOrList { + value: Some(token_ids_or_list::Value::One(one)), + }) + } else { + let many: TokenIdsList = + ::serde_json::from_value(reparse).map_err(::serde::de::Error::custom)?; + Ok(TokenIdsOrList { + value: Some(token_ids_or_list::Value::Many(many)), + }) + } + } + } + deserializer.deserialize_any(V) + } +} + +impl ::serde::Serialize for TokenIdsList { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_seq(self.items.iter()) + } +} + +impl<'de> ::serde::Deserialize<'de> for TokenIdsList { + fn deserialize>(deserializer: D) -> Result { + Ok(TokenIdsList { + items: <::prost::alloc::vec::Vec as ::serde::Deserialize>::deserialize( + deserializer, + )?, + }) + } +} + +impl ::serde::Serialize for OptionalString { + fn serialize(&self, serializer: S) -> Result { + self.value.serialize(serializer) + } +} + +impl<'de> ::serde::Deserialize<'de> for OptionalString { + fn deserialize>(deserializer: D) -> Result { + Ok(OptionalString { + value: as ::serde::Deserialize>::deserialize( + deserializer, + )?, + }) + } +} + +impl ::serde::Serialize for OptionalStringOrList { + fn serialize(&self, serializer: S) -> Result { + match &self.value { + Some(optional_string_or_list::Value::One(v)) => v.serialize(serializer), + Some(optional_string_or_list::Value::Many(v)) => v.serialize(serializer), + None => serializer.serialize_none(), + } + } +} + +impl<'de> ::serde::Deserialize<'de> for OptionalStringOrList { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = OptionalStringOrList; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("a value or an array of values for OptionalStringOrList") + } + fn visit_str(self, v: &str) -> Result { + Ok(OptionalStringOrList { + value: Some(optional_string_or_list::Value::One(OptionalString { + value: Some(v.to_owned()), + })), + }) + } + fn visit_string(self, v: String) -> Result { + Ok(OptionalStringOrList { + value: Some(optional_string_or_list::Value::One(OptionalString { + value: Some(v), + })), + }) + } + fn visit_seq>( + self, + seq: A, + ) -> Result { + let many = ::deserialize( + ::serde::de::value::SeqAccessDeserializer::new(seq), + )?; + Ok(OptionalStringOrList { + value: Some(optional_string_or_list::Value::Many(many)), + }) + } + } + deserializer.deserialize_any(V) + } +} + +impl ::serde::Serialize for OptionalStringList { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_seq(self.items.iter()) + } +} + +impl<'de> ::serde::Deserialize<'de> for OptionalStringList { + fn deserialize>(deserializer: D) -> Result { + Ok(OptionalStringList { + items: <::prost::alloc::vec::Vec as ::serde::Deserialize>::deserialize( + deserializer, + )?, + }) + } +} + +impl ::serde::Serialize for OptionalInt64 { + fn serialize(&self, serializer: S) -> Result { + self.value.serialize(serializer) + } +} + +impl<'de> ::serde::Deserialize<'de> for OptionalInt64 { + fn deserialize>(deserializer: D) -> Result { + Ok(OptionalInt64 { + value: as ::serde::Deserialize>::deserialize(deserializer)?, + }) + } +} + +impl ::serde::Serialize for OptionalInt64OrList { + fn serialize(&self, serializer: S) -> Result { + match &self.value { + Some(optional_int64_or_list::Value::One(v)) => v.serialize(serializer), + Some(optional_int64_or_list::Value::Many(v)) => v.serialize(serializer), + None => serializer.serialize_none(), + } + } +} + +impl<'de> ::serde::Deserialize<'de> for OptionalInt64OrList { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = OptionalInt64OrList; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("a value or an array of values for OptionalInt64OrList") + } + fn visit_i64(self, v: i64) -> Result { + Ok(OptionalInt64OrList { + value: Some(optional_int64_or_list::Value::One(OptionalInt64 { + value: Some(v), + })), + }) + } + fn visit_u64(self, v: u64) -> Result { + let v = i64::try_from(v).map_err(|_| E::custom("integer out of range"))?; + Ok(OptionalInt64OrList { + value: Some(optional_int64_or_list::Value::One(OptionalInt64 { + value: Some(v), + })), + }) + } + fn visit_seq>( + self, + seq: A, + ) -> Result { + let many = ::deserialize( + ::serde::de::value::SeqAccessDeserializer::new(seq), + )?; + Ok(OptionalInt64OrList { + value: Some(optional_int64_or_list::Value::Many(many)), + }) + } + } + deserializer.deserialize_any(V) + } +} + +impl ::serde::Serialize for OptionalInt64List { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_seq(self.items.iter()) + } +} + +impl<'de> ::serde::Deserialize<'de> for OptionalInt64List { + fn deserialize>(deserializer: D) -> Result { + Ok(OptionalInt64List { + items: <::prost::alloc::vec::Vec as ::serde::Deserialize>::deserialize( + deserializer, + )?, + }) + } +} + +impl ::serde::Serialize for ErrorBody { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "message", &self.message)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "code", &self.code)?; + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for ErrorBody { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = ErrorBody; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct ErrorBody") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut f_message = None; + let mut f_code = None; + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + "message" => { + if f_message.is_some() { + return Err(::serde::de::Error::duplicate_field("message")); + } + f_message = Some(map.next_value::<::prost::alloc::string::String>()?); + } + "code" => { + if f_code.is_some() { + return Err(::serde::de::Error::duplicate_field("code")); + } + f_code = Some(map.next_value::()?); + } + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(ErrorBody { + message: f_message.unwrap_or_default(), + code: f_code.unwrap_or_default(), + }) + } + } + deserializer.deserialize_map(V) + } +} + +impl ::serde::Serialize for Matched { + fn serialize(&self, serializer: S) -> Result { + match &self.value { + Some(matched::Value::Token(v)) => v.serialize(serializer), + Some(matched::Value::Str(v)) => v.serialize(serializer), + Some(matched::Value::Tokens(v)) => v.serialize(serializer), + None => serializer.serialize_none(), + } + } +} + +impl<'de> ::serde::Deserialize<'de> for Matched { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = Matched; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("one of the Matched shapes") + } + fn visit_i64(self, v: i64) -> Result { + Ok(Matched { + value: Some(matched::Value::Token(v)), + }) + } + fn visit_u64(self, v: u64) -> Result { + let v = i64::try_from(v).map_err(|_| E::custom("integer out of range"))?; + Ok(Matched { + value: Some(matched::Value::Token(v)), + }) + } + fn visit_str(self, v: &str) -> Result { + Ok(Matched { + value: Some(matched::Value::Str(v.to_owned())), + }) + } + fn visit_string(self, v: String) -> Result { + Ok(Matched { + value: Some(matched::Value::Str(v)), + }) + } + fn visit_seq>( + self, + seq: A, + ) -> Result { + let v = ::deserialize( + ::serde::de::value::SeqAccessDeserializer::new(seq), + )?; + Ok(Matched { + value: Some(matched::Value::Tokens(v)), + }) + } + } + deserializer.deserialize_any(V) + } +} + +impl ::serde::Serialize for MatchedTokens { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_seq(self.ids.iter()) + } +} + +impl<'de> ::serde::Deserialize<'de> for MatchedTokens { + fn deserialize>(deserializer: D) -> Result { + Ok(MatchedTokens { + ids: <::prost::alloc::vec::Vec as ::serde::Deserialize>::deserialize( + deserializer, + )?, + }) + } +} + +impl ::serde::Serialize for FinishStop { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + if let Some(v) = &self.matched { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "matched", v)?; + } + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for FinishStop { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = FinishStop; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct FinishStop") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut f_matched = None; + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + "matched" => { + if f_matched.is_some() { + return Err(::serde::de::Error::duplicate_field("matched")); + } + f_matched = Some(map.next_value::>()?); + } + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(FinishStop { + matched: f_matched.flatten(), + }) + } + } + deserializer.deserialize_map(V) + } +} + +impl ::serde::Serialize for FinishLength { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + if let Some(v) = &self.length { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "length", v)?; + } + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for FinishLength { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = FinishLength; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct FinishLength") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut f_length = None; + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + "length" => { + if f_length.is_some() { + return Err(::serde::de::Error::duplicate_field("length")); + } + f_length = Some(map.next_value::>()?); + } + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(FinishLength { + length: f_length.flatten(), + }) + } + } + deserializer.deserialize_map(V) + } +} + +impl ::serde::Serialize for FinishAbort { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "message", &self.message)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "status_code", &self.status_code)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "err_type", &self.err_type)?; + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for FinishAbort { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = FinishAbort; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct FinishAbort") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut f_message = None; + let mut f_status_code = None; + let mut f_err_type = None; + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + "message" => { + if f_message.is_some() { + return Err(::serde::de::Error::duplicate_field("message")); + } + f_message = + Some(map.next_value::>()?); + } + "status_code" => { + if f_status_code.is_some() { + return Err(::serde::de::Error::duplicate_field("status_code")); + } + f_status_code = Some(map.next_value::>()?); + } + "err_type" => { + if f_err_type.is_some() { + return Err(::serde::de::Error::duplicate_field("err_type")); + } + f_err_type = + Some(map.next_value::>()?); + } + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(FinishAbort { + message: f_message.flatten(), + status_code: f_status_code.flatten(), + err_type: f_err_type.flatten(), + }) + } + } + deserializer.deserialize_map(V) + } +} + +impl ::serde::Serialize for FinishReason { + fn serialize(&self, serializer: S) -> Result { + match &self.kind { + Some(finish_reason::Kind::Stop(v)) => { + let mut map = serializer.serialize_map(None)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "type", "stop")?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "matched", &v.matched)?; + ::serde::ser::SerializeMap::end(map) + } + Some(finish_reason::Kind::Length(v)) => { + let mut map = serializer.serialize_map(None)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "type", "length")?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "length", &v.length)?; + ::serde::ser::SerializeMap::end(map) + } + Some(finish_reason::Kind::Abort(v)) => { + let mut map = serializer.serialize_map(None)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "type", "abort")?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "message", &v.message)?; + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "status_code", + &v.status_code, + )?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "err_type", &v.err_type)?; + ::serde::ser::SerializeMap::end(map) + } + Some(finish_reason::Kind::Unknown(v)) => ::serde::Serialize::serialize( + &::serde_json::from_str::<::serde_json::Value>(v) + .unwrap_or(::serde_json::Value::Null), + serializer, + ), + None => serializer.serialize_none(), + } + } +} + +impl<'de> ::serde::Deserialize<'de> for FinishReason { + fn deserialize>(deserializer: D) -> Result { + fn fallback( + raw: ::serde_json::Map, + ) -> Result { + Ok(FinishReason { + kind: Some(finish_reason::Kind::Unknown( + ::serde_json::Value::Object(raw).to_string(), + )), + }) + } + let raw = ::serde_json::Map::::deserialize(deserializer)?; + let tag = raw.get("type").and_then(::serde_json::Value::as_str); + #[allow(clippy::redundant_closure_call)] + (|raw: ::serde_json::Map| match tag { + Some("stop") => { + let payload = FinishStop { + matched: match raw.get("matched") { + None | Some(::serde_json::Value::Null) => None, + Some(v) => match ::serde_json::from_value::(v.clone()) { + Ok(v) => Some(v), + Err(_) => return fallback(raw), + }, + }, + }; + Ok(FinishReason { + kind: Some(finish_reason::Kind::Stop(payload.into())), + }) + } + Some("length") => { + let payload = FinishLength { + length: match raw.get("length") { + None | Some(::serde_json::Value::Null) => None, + Some(v) => match ::serde_json::from_value::(v.clone()) { + Ok(v) => Some(v), + Err(_) => return fallback(raw), + }, + }, + }; + Ok(FinishReason { + kind: Some(finish_reason::Kind::Length(payload.into())), + }) + } + Some("abort") => { + let payload = + FinishAbort { + message: match raw.get("message") { + None | Some(::serde_json::Value::Null) => None, + Some(v) => match ::serde_json::from_value::< + ::prost::alloc::string::String, + >(v.clone()) + { + Ok(v) => Some(v), + Err(_) => return fallback(raw), + }, + }, + status_code: match raw.get("status_code") { + None | Some(::serde_json::Value::Null) => None, + Some(v) => match ::serde_json::from_value::(v.clone()) { + Ok(v) => Some(v), + Err(_) => return fallback(raw), + }, + }, + err_type: match raw.get("err_type") { + None | Some(::serde_json::Value::Null) => None, + Some(v) => match ::serde_json::from_value::< + ::prost::alloc::string::String, + >(v.clone()) + { + Ok(v) => Some(v), + Err(_) => return fallback(raw), + }, + }, + }; + Ok(FinishReason { + kind: Some(finish_reason::Kind::Abort(payload.into())), + }) + } + _ => fallback(raw), + })(raw.clone()) + } +} + +impl SamplingParams { + /// The schema default when unset (protobuf arrival). + pub fn max_new_tokens_or_default(&self) -> i64 { + self.max_new_tokens.clone().unwrap_or_else(|| 128i64) + } + /// The schema default when unset (protobuf arrival). + pub fn temperature_or_default(&self) -> f64 { + self.temperature.clone().unwrap_or_else(|| 1.0f64) + } + /// The schema default when unset (protobuf arrival). + pub fn top_p_or_default(&self) -> f64 { + self.top_p.clone().unwrap_or_else(|| 1.0f64) + } + /// The schema default when unset (protobuf arrival). + pub fn top_k_or_default(&self) -> i64 { + self.top_k.clone().unwrap_or_else(|| 1073741824i64) + } + /// The schema default when unset (protobuf arrival). + pub fn min_p_or_default(&self) -> f64 { + self.min_p.clone().unwrap_or_else(|| 0.0f64) + } + /// The schema default when unset (protobuf arrival). + pub fn frequency_penalty_or_default(&self) -> f64 { + self.frequency_penalty.clone().unwrap_or_else(|| 0.0f64) + } + /// The schema default when unset (protobuf arrival). + pub fn presence_penalty_or_default(&self) -> f64 { + self.presence_penalty.clone().unwrap_or_else(|| 0.0f64) + } + /// The schema default when unset (protobuf arrival). + pub fn repetition_penalty_or_default(&self) -> f64 { + self.repetition_penalty.clone().unwrap_or_else(|| 1.0f64) + } + /// The schema default when unset (protobuf arrival). + pub fn min_new_tokens_or_default(&self) -> i64 { + self.min_new_tokens.clone().unwrap_or_else(|| 0i64) + } + /// The schema default when unset (protobuf arrival). + pub fn n_or_default(&self) -> i64 { + self.n.clone().unwrap_or_else(|| 1i64) + } + /// The schema default when unset (protobuf arrival). + pub fn ignore_eos_or_default(&self) -> bool { + self.ignore_eos.clone().unwrap_or_else(|| false) + } + /// The schema default when unset (protobuf arrival). + pub fn skip_special_tokens_or_default(&self) -> bool { + self.skip_special_tokens.clone().unwrap_or_else(|| true) + } + /// The schema default when unset (protobuf arrival). + pub fn spaces_between_special_tokens_or_default(&self) -> bool { + self.spaces_between_special_tokens + .clone() + .unwrap_or_else(|| true) + } + /// The schema default when unset (protobuf arrival). + pub fn no_stop_trim_or_default(&self) -> bool { + self.no_stop_trim.clone().unwrap_or_else(|| false) + } + /// Protobuf arrival: unset defaulted fields become their schema + /// defaults (JSON arrival already baked them at deserialization). + pub fn apply_absent_defaults(&mut self) { + if self.max_new_tokens.is_none() { + self.max_new_tokens = Some(128i64); + } + if self.temperature.is_none() { + self.temperature = Some(1.0f64); + } + if self.top_p.is_none() { + self.top_p = Some(1.0f64); + } + if self.top_k.is_none() { + self.top_k = Some(1073741824i64); + } + if self.min_p.is_none() { + self.min_p = Some(0.0f64); + } + if self.frequency_penalty.is_none() { + self.frequency_penalty = Some(0.0f64); + } + if self.presence_penalty.is_none() { + self.presence_penalty = Some(0.0f64); + } + if self.repetition_penalty.is_none() { + self.repetition_penalty = Some(1.0f64); + } + if self.min_new_tokens.is_none() { + self.min_new_tokens = Some(0i64); + } + if self.n.is_none() { + self.n = Some(1i64); + } + if self.ignore_eos.is_none() { + self.ignore_eos = Some(false); + } + if self.skip_special_tokens.is_none() { + self.skip_special_tokens = Some(true); + } + if self.spaces_between_special_tokens.is_none() { + self.spaces_between_special_tokens = Some(true); + } + if self.no_stop_trim.is_none() { + self.no_stop_trim = Some(false); + } + } +} + +impl ::serde::Serialize for SamplingParams { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "max_new_tokens", + &self.max_new_tokens, + )?; + if let Some(v) = &self.stop { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "stop", v)?; + } + if let Some(v) = &self.stop_token_ids { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "stop_token_ids", v)?; + } + if let Some(v) = &self.stop_regex { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "stop_regex", v)?; + } + if let Some(v) = &self.temperature { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "temperature", v)?; + } + if let Some(v) = &self.top_p { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "top_p", v)?; + } + if let Some(v) = &self.top_k { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "top_k", v)?; + } + if let Some(v) = &self.min_p { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "min_p", v)?; + } + if let Some(v) = &self.frequency_penalty { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "frequency_penalty", v)?; + } + if let Some(v) = &self.presence_penalty { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "presence_penalty", v)?; + } + if let Some(v) = &self.repetition_penalty { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "repetition_penalty", v)?; + } + if let Some(v) = &self.min_new_tokens { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "min_new_tokens", v)?; + } + if let Some(v) = &self.n { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "n", v)?; + } + if let Some(v) = &self.json_schema { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "json_schema", v)?; + } + if let Some(v) = &self.regex { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "regex", v)?; + } + if let Some(v) = &self.ebnf { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "ebnf", v)?; + } + if let Some(v) = &self.structural_tag { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "structural_tag", v)?; + } + if let Some(v) = &self.ignore_eos { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "ignore_eos", v)?; + } + if let Some(v) = &self.skip_special_tokens { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "skip_special_tokens", v)?; + } + if let Some(v) = &self.spaces_between_special_tokens { + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "spaces_between_special_tokens", + v, + )?; + } + if let Some(v) = &self.no_stop_trim { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "no_stop_trim", v)?; + } + if let Some(v) = &self.stream_interval { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "stream_interval", v)?; + } + if !self.logit_bias.is_empty() { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "logit_bias", &self.logit_bias)?; + } + if let Some(v) = &self.sampling_seed { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "sampling_seed", v)?; + } + if let Some(v) = &self.custom_params { + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "custom_params", + &::serde_json::from_str::<::serde_json::Value>(v) + .unwrap_or(::serde_json::Value::Null), + )?; + } + if let Some(v) = &self.beam_width { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "beam_width", v)?; + } + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for SamplingParams { + fn deserialize>(deserializer: D) -> Result { + const FIELDS: &[&str] = &[ + "max_new_tokens", + "stop", + "stop_token_ids", + "stop_regex", + "temperature", + "top_p", + "top_k", + "min_p", + "frequency_penalty", + "presence_penalty", + "repetition_penalty", + "min_new_tokens", + "n", + "json_schema", + "regex", + "ebnf", + "structural_tag", + "ignore_eos", + "skip_special_tokens", + "spaces_between_special_tokens", + "no_stop_trim", + "stream_interval", + "logit_bias", + "sampling_seed", + "custom_params", + "beam_width", + ]; + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = SamplingParams; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct SamplingParams") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut f_max_new_tokens = None; + let mut f_stop = None; + let mut f_stop_token_ids = None; + let mut f_stop_regex = None; + let mut f_temperature = None; + let mut f_top_p = None; + let mut f_top_k = None; + let mut f_min_p = None; + let mut f_frequency_penalty = None; + let mut f_presence_penalty = None; + let mut f_repetition_penalty = None; + let mut f_min_new_tokens = None; + let mut f_n = None; + let mut f_json_schema = None; + let mut f_regex = None; + let mut f_ebnf = None; + let mut f_structural_tag = None; + let mut f_ignore_eos = None; + let mut f_skip_special_tokens = None; + let mut f_spaces_between_special_tokens = None; + let mut f_no_stop_trim = None; + let mut f_stream_interval = None; + let mut f_logit_bias = None; + let mut f_sampling_seed = None; + let mut f_custom_params = None; + let mut f_beam_width = None; + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + "max_new_tokens" => { + if f_max_new_tokens.is_some() { + return Err(::serde::de::Error::duplicate_field("max_new_tokens")); + } + f_max_new_tokens = Some(map.next_value::>()?); + } + "stop" => { + if f_stop.is_some() { + return Err(::serde::de::Error::duplicate_field("stop")); + } + f_stop = Some(map.next_value::>()?); + } + "stop_token_ids" => { + if f_stop_token_ids.is_some() { + return Err(::serde::de::Error::duplicate_field("stop_token_ids")); + } + f_stop_token_ids = Some(map.next_value::>()?); + } + "stop_regex" => { + if f_stop_regex.is_some() { + return Err(::serde::de::Error::duplicate_field("stop_regex")); + } + f_stop_regex = Some(map.next_value::>()?); + } + "temperature" => { + if f_temperature.is_some() { + return Err(::serde::de::Error::duplicate_field("temperature")); + } + f_temperature = Some(Some( + map.next_value::>()?.unwrap_or_else(|| 1.0f64), + )); + } + "top_p" => { + if f_top_p.is_some() { + return Err(::serde::de::Error::duplicate_field("top_p")); + } + f_top_p = Some(Some( + map.next_value::>()?.unwrap_or_else(|| 1.0f64), + )); + } + "top_k" => { + if f_top_k.is_some() { + return Err(::serde::de::Error::duplicate_field("top_k")); + } + f_top_k = Some(Some( + map.next_value::>()? + .unwrap_or_else(|| 1073741824i64), + )); + } + "min_p" => { + if f_min_p.is_some() { + return Err(::serde::de::Error::duplicate_field("min_p")); + } + f_min_p = Some(Some( + map.next_value::>()?.unwrap_or_else(|| 0.0f64), + )); + } + "frequency_penalty" => { + if f_frequency_penalty.is_some() { + return Err(::serde::de::Error::duplicate_field( + "frequency_penalty", + )); + } + f_frequency_penalty = Some(Some( + map.next_value::>()?.unwrap_or_else(|| 0.0f64), + )); + } + "presence_penalty" => { + if f_presence_penalty.is_some() { + return Err(::serde::de::Error::duplicate_field( + "presence_penalty", + )); + } + f_presence_penalty = Some(Some( + map.next_value::>()?.unwrap_or_else(|| 0.0f64), + )); + } + "repetition_penalty" => { + if f_repetition_penalty.is_some() { + return Err(::serde::de::Error::duplicate_field( + "repetition_penalty", + )); + } + f_repetition_penalty = Some(Some( + map.next_value::>()?.unwrap_or_else(|| 1.0f64), + )); + } + "min_new_tokens" => { + if f_min_new_tokens.is_some() { + return Err(::serde::de::Error::duplicate_field("min_new_tokens")); + } + f_min_new_tokens = Some(Some( + map.next_value::>()?.unwrap_or_else(|| 0i64), + )); + } + "n" => { + if f_n.is_some() { + return Err(::serde::de::Error::duplicate_field("n")); + } + f_n = Some(Some( + map.next_value::>()?.unwrap_or_else(|| 1i64), + )); + } + "json_schema" => { + if f_json_schema.is_some() { + return Err(::serde::de::Error::duplicate_field("json_schema")); + } + f_json_schema = + Some(map.next_value::>()?); + } + "regex" => { + if f_regex.is_some() { + return Err(::serde::de::Error::duplicate_field("regex")); + } + f_regex = + Some(map.next_value::>()?); + } + "ebnf" => { + if f_ebnf.is_some() { + return Err(::serde::de::Error::duplicate_field("ebnf")); + } + f_ebnf = + Some(map.next_value::>()?); + } + "structural_tag" => { + if f_structural_tag.is_some() { + return Err(::serde::de::Error::duplicate_field("structural_tag")); + } + f_structural_tag = + Some(map.next_value::>()?); + } + "ignore_eos" => { + if f_ignore_eos.is_some() { + return Err(::serde::de::Error::duplicate_field("ignore_eos")); + } + f_ignore_eos = Some(Some( + map.next_value::>()?.unwrap_or_else(|| false), + )); + } + "skip_special_tokens" => { + if f_skip_special_tokens.is_some() { + return Err(::serde::de::Error::duplicate_field( + "skip_special_tokens", + )); + } + f_skip_special_tokens = Some(Some( + map.next_value::>()?.unwrap_or_else(|| true), + )); + } + "spaces_between_special_tokens" => { + if f_spaces_between_special_tokens.is_some() { + return Err(::serde::de::Error::duplicate_field( + "spaces_between_special_tokens", + )); + } + f_spaces_between_special_tokens = Some(Some( + map.next_value::>()?.unwrap_or_else(|| true), + )); + } + "no_stop_trim" => { + if f_no_stop_trim.is_some() { + return Err(::serde::de::Error::duplicate_field("no_stop_trim")); + } + f_no_stop_trim = Some(Some( + map.next_value::>()?.unwrap_or_else(|| false), + )); + } + "stream_interval" => { + if f_stream_interval.is_some() { + return Err(::serde::de::Error::duplicate_field("stream_interval")); + } + f_stream_interval = Some(map.next_value::>()?); + } + "logit_bias" => { + if f_logit_bias.is_some() { + return Err(::serde::de::Error::duplicate_field("logit_bias")); + } + f_logit_bias = Some( + map.next_value::>>( + )? + .unwrap_or_default(), + ); + } + "sampling_seed" => { + if f_sampling_seed.is_some() { + return Err(::serde::de::Error::duplicate_field("sampling_seed")); + } + f_sampling_seed = Some(map.next_value::>()?); + } + "custom_params" => { + if f_custom_params.is_some() { + return Err(::serde::de::Error::duplicate_field("custom_params")); + } + f_custom_params = Some( + map.next_value::>()? + .map(|v| v.to_string()), + ); + } + "beam_width" => { + if f_beam_width.is_some() { + return Err(::serde::de::Error::duplicate_field("beam_width")); + } + f_beam_width = Some(map.next_value::>()?); + } + other => { + return Err(::serde::de::Error::unknown_field(other, FIELDS)); + } + } + } + Ok(SamplingParams { + max_new_tokens: f_max_new_tokens.unwrap_or_else(|| Some(128i64)), + stop: f_stop.flatten(), + stop_token_ids: f_stop_token_ids.flatten(), + stop_regex: f_stop_regex.flatten(), + temperature: f_temperature.unwrap_or_else(|| Some(1.0f64)), + top_p: f_top_p.unwrap_or_else(|| Some(1.0f64)), + top_k: f_top_k.unwrap_or_else(|| Some(1073741824i64)), + min_p: f_min_p.unwrap_or_else(|| Some(0.0f64)), + frequency_penalty: f_frequency_penalty.unwrap_or_else(|| Some(0.0f64)), + presence_penalty: f_presence_penalty.unwrap_or_else(|| Some(0.0f64)), + repetition_penalty: f_repetition_penalty.unwrap_or_else(|| Some(1.0f64)), + min_new_tokens: f_min_new_tokens.unwrap_or_else(|| Some(0i64)), + n: f_n.unwrap_or_else(|| Some(1i64)), + json_schema: f_json_schema.flatten(), + regex: f_regex.flatten(), + ebnf: f_ebnf.flatten(), + structural_tag: f_structural_tag.flatten(), + ignore_eos: f_ignore_eos.unwrap_or_else(|| Some(false)), + skip_special_tokens: f_skip_special_tokens.unwrap_or_else(|| Some(true)), + spaces_between_special_tokens: f_spaces_between_special_tokens + .unwrap_or_else(|| Some(true)), + no_stop_trim: f_no_stop_trim.unwrap_or_else(|| Some(false)), + stream_interval: f_stream_interval.flatten(), + logit_bias: f_logit_bias.unwrap_or_default(), + sampling_seed: f_sampling_seed.flatten(), + custom_params: f_custom_params.flatten(), + beam_width: f_beam_width.flatten(), + }) + } + } + deserializer.deserialize_map(V) + } +} + +impl GenerateRequest { + /// The schema default when unset (protobuf arrival). + pub fn stream_or_default(&self) -> bool { + self.stream.clone().unwrap_or_else(|| false) + } + /// Protobuf arrival: unset defaulted fields become their schema + /// defaults (JSON arrival already baked them at deserialization). + pub fn apply_absent_defaults(&mut self) { + if self.stream.is_none() { + self.stream = Some(false); + } + if let Some(v) = self.sampling_params.as_mut() { + v.apply_absent_defaults(); + } + } +} + +impl ::serde::Serialize for GenerateRequest { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + if let Some(v) = &self.rid { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "rid", v)?; + } + if let Some(v) = &self.text { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "text", v)?; + } + if let Some(v) = &self.input_ids { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "input_ids", v)?; + } + if let Some(v) = &self.stream { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "stream", v)?; + } + if let Some(v) = &self.sampling_params { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "sampling_params", v)?; + } + if let Some(v) = &self.return_logprob { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "return_logprob", v)?; + } + if let Some(v) = &self.logprob_start_len { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "logprob_start_len", v)?; + } + if let Some(v) = &self.top_logprobs_num { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "top_logprobs_num", v)?; + } + if let Some(v) = &self.token_ids_logprob { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "token_ids_logprob", v)?; + } + if let Some(v) = &self.return_hidden_states { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "return_hidden_states", v)?; + } + if let Some(v) = &self.return_text_in_logprobs { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "return_text_in_logprobs", v)?; + } + if let Some(v) = &self.bootstrap_host { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "bootstrap_host", v)?; + } + if let Some(v) = &self.bootstrap_port { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "bootstrap_port", v)?; + } + if let Some(v) = &self.bootstrap_room { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "bootstrap_room", v)?; + } + if let Some(v) = &self.bootstrap_pair_key { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "bootstrap_pair_key", v)?; + } + if let Some(v) = &self.decode_tp_size { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "decode_tp_size", v)?; + } + if let Some(v) = &self.routed_dp_rank { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "routed_dp_rank", v)?; + } + if let Some(v) = &self.disagg_prefill_dp_rank { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "disagg_prefill_dp_rank", v)?; + } + if let Some(v) = &self.image_data { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "image_data", v)?; + } + if let Some(v) = &self.mm_hashes { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "mm_hashes", v)?; + } + if let Some(v) = &self.video_data { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "video_data", v)?; + } + if let Some(v) = &self.audio_data { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "audio_data", v)?; + } + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for GenerateRequest { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = GenerateRequest; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct GenerateRequest") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut f_rid = None; + let mut f_text = None; + let mut f_input_ids = None; + let mut f_stream = None; + let mut f_sampling_params = None; + let mut f_return_logprob = None; + let mut f_logprob_start_len = None; + let mut f_top_logprobs_num = None; + let mut f_token_ids_logprob = None; + let mut f_return_hidden_states = None; + let mut f_return_text_in_logprobs = None; + let mut f_bootstrap_host = None; + let mut f_bootstrap_port = None; + let mut f_bootstrap_room = None; + let mut f_bootstrap_pair_key = None; + let mut f_decode_tp_size = None; + let mut f_routed_dp_rank = None; + let mut f_disagg_prefill_dp_rank = None; + let mut f_image_data = None; + let mut f_mm_hashes = None; + let mut f_video_data = None; + let mut f_audio_data = None; + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + "rid" => { + if f_rid.is_some() { + return Err(::serde::de::Error::duplicate_field("rid")); + } + f_rid = Some(map.next_value::>()?); + } + "text" => { + if f_text.is_some() { + return Err(::serde::de::Error::duplicate_field("text")); + } + f_text = Some(map.next_value::>()?); + } + "input_ids" => { + if f_input_ids.is_some() { + return Err(::serde::de::Error::duplicate_field("input_ids")); + } + f_input_ids = Some(map.next_value::>()?); + } + "stream" => { + if f_stream.is_some() { + return Err(::serde::de::Error::duplicate_field("stream")); + } + f_stream = Some(Some( + map.next_value::>()?.unwrap_or_else(|| false), + )); + } + "sampling_params" => { + if f_sampling_params.is_some() { + return Err(::serde::de::Error::duplicate_field("sampling_params")); + } + f_sampling_params = + Some(map.next_value::>()?); + } + "return_logprob" => { + if f_return_logprob.is_some() { + return Err(::serde::de::Error::duplicate_field("return_logprob")); + } + f_return_logprob = Some(map.next_value::>()?); + } + "logprob_start_len" => { + if f_logprob_start_len.is_some() { + return Err(::serde::de::Error::duplicate_field( + "logprob_start_len", + )); + } + f_logprob_start_len = Some(map.next_value::>()?); + } + "top_logprobs_num" => { + if f_top_logprobs_num.is_some() { + return Err(::serde::de::Error::duplicate_field( + "top_logprobs_num", + )); + } + f_top_logprobs_num = Some(map.next_value::>()?); + } + "token_ids_logprob" => { + if f_token_ids_logprob.is_some() { + return Err(::serde::de::Error::duplicate_field( + "token_ids_logprob", + )); + } + f_token_ids_logprob = Some(map.next_value::>()?); + } + "return_hidden_states" => { + if f_return_hidden_states.is_some() { + return Err(::serde::de::Error::duplicate_field( + "return_hidden_states", + )); + } + f_return_hidden_states = Some(map.next_value::>()?); + } + "return_text_in_logprobs" => { + if f_return_text_in_logprobs.is_some() { + return Err(::serde::de::Error::duplicate_field( + "return_text_in_logprobs", + )); + } + f_return_text_in_logprobs = Some(map.next_value::>()?); + } + "bootstrap_host" => { + if f_bootstrap_host.is_some() { + return Err(::serde::de::Error::duplicate_field("bootstrap_host")); + } + f_bootstrap_host = + Some(map.next_value::>()?); + } + "bootstrap_port" => { + if f_bootstrap_port.is_some() { + return Err(::serde::de::Error::duplicate_field("bootstrap_port")); + } + f_bootstrap_port = + Some(map.next_value::>()?); + } + "bootstrap_room" => { + if f_bootstrap_room.is_some() { + return Err(::serde::de::Error::duplicate_field("bootstrap_room")); + } + f_bootstrap_room = + Some(map.next_value::>()?); + } + "bootstrap_pair_key" => { + if f_bootstrap_pair_key.is_some() { + return Err(::serde::de::Error::duplicate_field( + "bootstrap_pair_key", + )); + } + f_bootstrap_pair_key = + Some(map.next_value::>()?); + } + "decode_tp_size" => { + if f_decode_tp_size.is_some() { + return Err(::serde::de::Error::duplicate_field("decode_tp_size")); + } + f_decode_tp_size = + Some(map.next_value::>()?); + } + "routed_dp_rank" => { + if f_routed_dp_rank.is_some() { + return Err(::serde::de::Error::duplicate_field("routed_dp_rank")); + } + f_routed_dp_rank = Some(map.next_value::>()?); + } + "disagg_prefill_dp_rank" => { + if f_disagg_prefill_dp_rank.is_some() { + return Err(::serde::de::Error::duplicate_field( + "disagg_prefill_dp_rank", + )); + } + f_disagg_prefill_dp_rank = Some(map.next_value::>()?); + } + "image_data" => { + if f_image_data.is_some() { + return Err(::serde::de::Error::duplicate_field("image_data")); + } + f_image_data = Some(map.next_value::>()?); + } + "mm_hashes" => { + if f_mm_hashes.is_some() { + return Err(::serde::de::Error::duplicate_field("mm_hashes")); + } + f_mm_hashes = Some(map.next_value::>()?); + } + "video_data" => { + if f_video_data.is_some() { + return Err(::serde::de::Error::duplicate_field("video_data")); + } + f_video_data = Some(map.next_value::>()?); + } + "audio_data" => { + if f_audio_data.is_some() { + return Err(::serde::de::Error::duplicate_field("audio_data")); + } + f_audio_data = Some(map.next_value::>()?); + } + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(GenerateRequest { + rid: f_rid.flatten(), + text: f_text.flatten(), + input_ids: f_input_ids.flatten(), + stream: f_stream.unwrap_or_else(|| Some(false)), + sampling_params: f_sampling_params.flatten(), + return_logprob: f_return_logprob.flatten(), + logprob_start_len: f_logprob_start_len.flatten(), + top_logprobs_num: f_top_logprobs_num.flatten(), + token_ids_logprob: f_token_ids_logprob.flatten(), + return_hidden_states: f_return_hidden_states.flatten(), + return_text_in_logprobs: f_return_text_in_logprobs.flatten(), + bootstrap_host: f_bootstrap_host.flatten(), + bootstrap_port: f_bootstrap_port.flatten(), + bootstrap_room: f_bootstrap_room.flatten(), + bootstrap_pair_key: f_bootstrap_pair_key.flatten(), + decode_tp_size: f_decode_tp_size.flatten(), + routed_dp_rank: f_routed_dp_rank.flatten(), + disagg_prefill_dp_rank: f_disagg_prefill_dp_rank.flatten(), + image_data: f_image_data.flatten(), + mm_hashes: f_mm_hashes.flatten(), + video_data: f_video_data.flatten(), + audio_data: f_audio_data.flatten(), + }) + } + } + deserializer.deserialize_map(V) + } +} + +impl SamplingParamsOrList { + /// Protobuf arrival: recurse the absent-defaults rule into the params. + pub fn apply_absent_defaults(&mut self) { + match self.value.as_mut() { + Some(sampling_params_or_list::Value::One(p)) => p.apply_absent_defaults(), + Some(sampling_params_or_list::Value::Many(l)) => { + for p in &mut l.items { + p.apply_absent_defaults(); + } + } + None => {} + } + } +} + +impl ::serde::Serialize for SamplingParamsOrList { + fn serialize(&self, serializer: S) -> Result { + match &self.value { + Some(sampling_params_or_list::Value::One(v)) => v.serialize(serializer), + Some(sampling_params_or_list::Value::Many(v)) => v.serialize(serializer), + None => serializer.serialize_none(), + } + } +} + +impl<'de> ::serde::Deserialize<'de> for SamplingParamsOrList { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = SamplingParamsOrList; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("a value or an array of values for SamplingParamsOrList") + } + fn visit_map>( + self, + map: A, + ) -> Result { + let one = ::deserialize( + ::serde::de::value::MapAccessDeserializer::new(map), + )?; + Ok(SamplingParamsOrList { + value: Some(sampling_params_or_list::Value::One(one)), + }) + } + fn visit_seq>( + self, + seq: A, + ) -> Result { + let many = ::deserialize( + ::serde::de::value::SeqAccessDeserializer::new(seq), + )?; + Ok(SamplingParamsOrList { + value: Some(sampling_params_or_list::Value::Many(many)), + }) + } + } + deserializer.deserialize_any(V) + } +} + +impl ::serde::Serialize for SamplingParamsList { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_seq(self.items.iter()) + } +} + +impl<'de> ::serde::Deserialize<'de> for SamplingParamsList { + fn deserialize>(deserializer: D) -> Result { + Ok(SamplingParamsList { + items: <::prost::alloc::vec::Vec as ::serde::Deserialize>::deserialize( + deserializer, + )?, + }) + } +} + +impl ::serde::Serialize for LogprobEntry { + fn serialize(&self, serializer: S) -> Result { + let mut seq = serializer.serialize_seq(Some(3))?; + ::serde::ser::SerializeSeq::serialize_element(&mut seq, &self.logprob)?; + ::serde::ser::SerializeSeq::serialize_element(&mut seq, &self.token_id)?; + ::serde::ser::SerializeSeq::serialize_element(&mut seq, &self.text)?; + ::serde::ser::SerializeSeq::end(seq) + } +} + +impl<'de> ::serde::Deserialize<'de> for LogprobEntry { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = LogprobEntry; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("a 3-element tuple") + } + fn visit_seq>( + self, + mut seq: A, + ) -> Result { + let logprob: Option = seq + .next_element()? + .ok_or_else(|| ::serde::de::Error::invalid_length(0, &"a 3-element tuple"))?; + let token_id: i64 = seq + .next_element()? + .ok_or_else(|| ::serde::de::Error::invalid_length(1, &"a 3-element tuple"))?; + let text: Option<::prost::alloc::string::String> = seq + .next_element()? + .ok_or_else(|| ::serde::de::Error::invalid_length(2, &"a 3-element tuple"))?; + Ok(LogprobEntry { + logprob, + token_id, + text, + }) + } + } + deserializer.deserialize_seq(V) + } +} + +impl ::serde::Serialize for LogprobEntries { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_seq(self.entries.iter()) + } +} + +impl<'de> ::serde::Deserialize<'de> for LogprobEntries { + fn deserialize>(deserializer: D) -> Result { + Ok(LogprobEntries { + entries: <::prost::alloc::vec::Vec as ::serde::Deserialize>::deserialize( + deserializer, + )?, + }) + } +} + +impl ::serde::Serialize for TopLogprobRow { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_seq(self.entries.iter()) + } +} + +impl<'de> ::serde::Deserialize<'de> for TopLogprobRow { + fn deserialize>(deserializer: D) -> Result { + Ok(TopLogprobRow { + entries: <::prost::alloc::vec::Vec as ::serde::Deserialize>::deserialize( + deserializer, + )?, + }) + } +} + +impl ::serde::Serialize for NullableTopLogprobs { + fn serialize(&self, serializer: S) -> Result { + self.row.serialize(serializer) + } +} + +impl<'de> ::serde::Deserialize<'de> for NullableTopLogprobs { + fn deserialize>(deserializer: D) -> Result { + Ok(NullableTopLogprobs { + row: as ::serde::Deserialize>::deserialize(deserializer)?, + }) + } +} + +impl ::serde::Serialize for HiddenStateRow { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_seq(self.values.iter()) + } +} + +impl<'de> ::serde::Deserialize<'de> for HiddenStateRow { + fn deserialize>(deserializer: D) -> Result { + Ok(HiddenStateRow { + values: <::prost::alloc::vec::Vec as ::serde::Deserialize>::deserialize( + deserializer, + )?, + }) + } +} + +impl ::serde::Serialize for GenerateMetaInfo { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "id", &self.id)?; + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "prompt_tokens", + &self.prompt_tokens, + )?; + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "completion_tokens", + &self.completion_tokens, + )?; + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "finish_reason", + &self.finish_reason, + )?; + if let Some(v) = &self.output_token_logprobs { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "output_token_logprobs", v)?; + } + if let Some(v) = &self.input_token_logprobs { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "input_token_logprobs", v)?; + } + if !self.output_top_logprobs.is_empty() { + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "output_top_logprobs", + &self.output_top_logprobs, + )?; + } + if !self.input_top_logprobs.is_empty() { + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "input_top_logprobs", + &self.input_top_logprobs, + )?; + } + if !self.output_token_ids_logprobs.is_empty() { + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "output_token_ids_logprobs", + &self.output_token_ids_logprobs, + )?; + } + if !self.input_token_ids_logprobs.is_empty() { + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "input_token_ids_logprobs", + &self.input_token_ids_logprobs, + )?; + } + if !self.hidden_states.is_empty() { + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "hidden_states", + &self.hidden_states, + )?; + } + if let Some(v) = &self.e2e_latency { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "e2e_latency", v)?; + } + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for GenerateMetaInfo { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = GenerateMetaInfo; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct GenerateMetaInfo") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut f_id = None; + let mut f_prompt_tokens = None; + let mut f_completion_tokens = None; + let mut f_finish_reason = None; + let mut f_output_token_logprobs = None; + let mut f_input_token_logprobs = None; + let mut f_output_top_logprobs = None; + let mut f_input_top_logprobs = None; + let mut f_output_token_ids_logprobs = None; + let mut f_input_token_ids_logprobs = None; + let mut f_hidden_states = None; + let mut f_e2e_latency = None; + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + "id" => { + if f_id.is_some() { + return Err(::serde::de::Error::duplicate_field("id")); + } + f_id = Some(map.next_value::<::prost::alloc::string::String>()?); + } + "prompt_tokens" => { + if f_prompt_tokens.is_some() { + return Err(::serde::de::Error::duplicate_field("prompt_tokens")); + } + f_prompt_tokens = Some(map.next_value::()?); + } + "completion_tokens" => { + if f_completion_tokens.is_some() { + return Err(::serde::de::Error::duplicate_field( + "completion_tokens", + )); + } + f_completion_tokens = Some(map.next_value::()?); + } + "finish_reason" => { + if f_finish_reason.is_some() { + return Err(::serde::de::Error::duplicate_field("finish_reason")); + } + f_finish_reason = Some(map.next_value::>()?); + } + "output_token_logprobs" => { + if f_output_token_logprobs.is_some() { + return Err(::serde::de::Error::duplicate_field( + "output_token_logprobs", + )); + } + f_output_token_logprobs = + Some(map.next_value::>()?); + } + "input_token_logprobs" => { + if f_input_token_logprobs.is_some() { + return Err(::serde::de::Error::duplicate_field( + "input_token_logprobs", + )); + } + f_input_token_logprobs = + Some(map.next_value::>()?); + } + "output_top_logprobs" => { + if f_output_top_logprobs.is_some() { + return Err(::serde::de::Error::duplicate_field( + "output_top_logprobs", + )); + } + f_output_top_logprobs = Some( + map.next_value::>>()? + .unwrap_or_default(), + ); + } + "input_top_logprobs" => { + if f_input_top_logprobs.is_some() { + return Err(::serde::de::Error::duplicate_field( + "input_top_logprobs", + )); + } + f_input_top_logprobs = Some( + map.next_value::>>()? + .unwrap_or_default(), + ); + } + "output_token_ids_logprobs" => { + if f_output_token_ids_logprobs.is_some() { + return Err(::serde::de::Error::duplicate_field( + "output_token_ids_logprobs", + )); + } + f_output_token_ids_logprobs = Some( + map.next_value::>>()? + .unwrap_or_default(), + ); + } + "input_token_ids_logprobs" => { + if f_input_token_ids_logprobs.is_some() { + return Err(::serde::de::Error::duplicate_field( + "input_token_ids_logprobs", + )); + } + f_input_token_ids_logprobs = Some( + map.next_value::>>()? + .unwrap_or_default(), + ); + } + "hidden_states" => { + if f_hidden_states.is_some() { + return Err(::serde::de::Error::duplicate_field("hidden_states")); + } + f_hidden_states = Some( + map.next_value::>>()? + .unwrap_or_default(), + ); + } + "e2e_latency" => { + if f_e2e_latency.is_some() { + return Err(::serde::de::Error::duplicate_field("e2e_latency")); + } + f_e2e_latency = Some(map.next_value::>()?); + } + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(GenerateMetaInfo { + id: f_id.unwrap_or_default(), + prompt_tokens: f_prompt_tokens.unwrap_or_default(), + completion_tokens: f_completion_tokens.unwrap_or_default(), + finish_reason: f_finish_reason.flatten(), + output_token_logprobs: f_output_token_logprobs.flatten(), + input_token_logprobs: f_input_token_logprobs.flatten(), + output_top_logprobs: f_output_top_logprobs.unwrap_or_default(), + input_top_logprobs: f_input_top_logprobs.unwrap_or_default(), + output_token_ids_logprobs: f_output_token_ids_logprobs.unwrap_or_default(), + input_token_ids_logprobs: f_input_token_ids_logprobs.unwrap_or_default(), + hidden_states: f_hidden_states.unwrap_or_default(), + e2e_latency: f_e2e_latency.flatten(), + }) + } + } + deserializer.deserialize_map(V) + } +} + +impl ::serde::Serialize for GenerateResponse { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "text", &self.text)?; + if let Some(v) = &self.meta_info { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "meta_info", v)?; + } + if let Some(v) = &self.output_ids { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "output_ids", v)?; + } + if let Some(v) = &self.index { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "index", v)?; + } + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for GenerateResponse { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = GenerateResponse; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct GenerateResponse") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut f_text = None; + let mut f_meta_info = None; + let mut f_output_ids = None; + let mut f_index = None; + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + "text" => { + if f_text.is_some() { + return Err(::serde::de::Error::duplicate_field("text")); + } + f_text = Some(map.next_value::<::prost::alloc::string::String>()?); + } + "meta_info" => { + if f_meta_info.is_some() { + return Err(::serde::de::Error::duplicate_field("meta_info")); + } + f_meta_info = Some(map.next_value::>()?); + } + "output_ids" => { + if f_output_ids.is_some() { + return Err(::serde::de::Error::duplicate_field("output_ids")); + } + f_output_ids = Some(map.next_value::>()?); + } + "index" => { + if f_index.is_some() { + return Err(::serde::de::Error::duplicate_field("index")); + } + f_index = Some(map.next_value::>()?); + } + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(GenerateResponse { + text: f_text.unwrap_or_default(), + meta_info: f_meta_info.flatten(), + output_ids: f_output_ids.flatten(), + index: f_index.flatten(), + }) + } + } + deserializer.deserialize_map(V) + } +} + +impl ::serde::Serialize for GenerateStreamError { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + if let Some(v) = &self.error { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "error", v)?; + } + if let Some(v) = &self.index { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "index", v)?; + } + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for GenerateStreamError { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = GenerateStreamError; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct GenerateStreamError") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut f_error = None; + let mut f_index = None; + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + "error" => { + if f_error.is_some() { + return Err(::serde::de::Error::duplicate_field("error")); + } + f_error = Some(map.next_value::>()?); + } + "index" => { + if f_index.is_some() { + return Err(::serde::de::Error::duplicate_field("index")); + } + f_index = Some(map.next_value::>()?); + } + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(GenerateStreamError { + error: f_error.flatten(), + index: f_index.flatten(), + }) + } + } + deserializer.deserialize_map(V) + } +} + +impl ::serde::Serialize for MediaRef { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "url", &self.url)?; + if let Some(v) = &self.detail { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "detail", v)?; + } + if let Some(v) = &self.max_dynamic_patch { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "max_dynamic_patch", v)?; + } + if let Some(v) = &self.content_hash { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "content_hash", v)?; + } + if let Some(v) = &self.min_pixels { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "min_pixels", v)?; + } + if let Some(v) = &self.max_pixels { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "max_pixels", v)?; + } + if let Some(v) = &self.total_max_pixels { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "total_max_pixels", v)?; + } + if let Some(v) = &self.fps { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "fps", v)?; + } + if let Some(v) = &self.num_frames { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "num_frames", v)?; + } + if let Some(v) = &self.min_frames { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "min_frames", v)?; + } + if let Some(v) = &self.max_frames { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "max_frames", v)?; + } + if let Some(v) = &self.max_tokens_per_frame { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "max_tokens_per_frame", v)?; + } + if let Some(v) = &self.max_image_tokens { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "max_image_tokens", v)?; + } + if let Some(v) = &self.use_audio { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "use_audio", v)?; + } + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for MediaRef { + fn deserialize>(deserializer: D) -> Result { + const FIELDS: &[&str] = &[ + "url", + "detail", + "max_dynamic_patch", + "content_hash", + "min_pixels", + "max_pixels", + "total_max_pixels", + "fps", + "num_frames", + "min_frames", + "max_frames", + "max_tokens_per_frame", + "max_image_tokens", + "use_audio", + ]; + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = MediaRef; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct MediaRef") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut f_url = None; + let mut f_detail = None; + let mut f_max_dynamic_patch = None; + let mut f_content_hash = None; + let mut f_min_pixels = None; + let mut f_max_pixels = None; + let mut f_total_max_pixels = None; + let mut f_fps = None; + let mut f_num_frames = None; + let mut f_min_frames = None; + let mut f_max_frames = None; + let mut f_max_tokens_per_frame = None; + let mut f_max_image_tokens = None; + let mut f_use_audio = None; + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + "url" => { + if f_url.is_some() { + return Err(::serde::de::Error::duplicate_field("url")); + } + f_url = Some(map.next_value::<::prost::alloc::string::String>()?); + } + "detail" => { + if f_detail.is_some() { + return Err(::serde::de::Error::duplicate_field("detail")); + } + f_detail = + Some(map.next_value::>()?); + } + "max_dynamic_patch" => { + if f_max_dynamic_patch.is_some() { + return Err(::serde::de::Error::duplicate_field( + "max_dynamic_patch", + )); + } + f_max_dynamic_patch = Some(map.next_value::>()?); + } + "content_hash" => { + if f_content_hash.is_some() { + return Err(::serde::de::Error::duplicate_field("content_hash")); + } + f_content_hash = + Some(map.next_value::>()?); + } + "min_pixels" => { + if f_min_pixels.is_some() { + return Err(::serde::de::Error::duplicate_field("min_pixels")); + } + f_min_pixels = Some(map.next_value::>()?); + } + "max_pixels" => { + if f_max_pixels.is_some() { + return Err(::serde::de::Error::duplicate_field("max_pixels")); + } + f_max_pixels = Some(map.next_value::>()?); + } + "total_max_pixels" => { + if f_total_max_pixels.is_some() { + return Err(::serde::de::Error::duplicate_field( + "total_max_pixels", + )); + } + f_total_max_pixels = Some(map.next_value::>()?); + } + "fps" => { + if f_fps.is_some() { + return Err(::serde::de::Error::duplicate_field("fps")); + } + f_fps = Some(map.next_value::>()?); + } + "num_frames" => { + if f_num_frames.is_some() { + return Err(::serde::de::Error::duplicate_field("num_frames")); + } + f_num_frames = Some(map.next_value::>()?); + } + "min_frames" => { + if f_min_frames.is_some() { + return Err(::serde::de::Error::duplicate_field("min_frames")); + } + f_min_frames = Some(map.next_value::>()?); + } + "max_frames" => { + if f_max_frames.is_some() { + return Err(::serde::de::Error::duplicate_field("max_frames")); + } + f_max_frames = Some(map.next_value::>()?); + } + "max_tokens_per_frame" => { + if f_max_tokens_per_frame.is_some() { + return Err(::serde::de::Error::duplicate_field( + "max_tokens_per_frame", + )); + } + f_max_tokens_per_frame = Some(map.next_value::>()?); + } + "max_image_tokens" => { + if f_max_image_tokens.is_some() { + return Err(::serde::de::Error::duplicate_field( + "max_image_tokens", + )); + } + f_max_image_tokens = Some(map.next_value::>()?); + } + "use_audio" => { + if f_use_audio.is_some() { + return Err(::serde::de::Error::duplicate_field("use_audio")); + } + f_use_audio = Some(map.next_value::>()?); + } + other => { + return Err(::serde::de::Error::unknown_field(other, FIELDS)); + } + } + } + Ok(MediaRef { + url: f_url.unwrap_or_default(), + detail: f_detail.flatten(), + max_dynamic_patch: f_max_dynamic_patch.flatten(), + content_hash: f_content_hash.flatten(), + min_pixels: f_min_pixels.flatten(), + max_pixels: f_max_pixels.flatten(), + total_max_pixels: f_total_max_pixels.flatten(), + fps: f_fps.flatten(), + num_frames: f_num_frames.flatten(), + min_frames: f_min_frames.flatten(), + max_frames: f_max_frames.flatten(), + max_tokens_per_frame: f_max_tokens_per_frame.flatten(), + max_image_tokens: f_max_image_tokens.flatten(), + use_audio: f_use_audio.flatten(), + }) + } + } + deserializer.deserialize_map(V) + } +} + +impl ::serde::Serialize for MediaItem { + fn serialize(&self, serializer: S) -> Result { + match &self.value { + Some(media_item::Value::Source(v)) => v.serialize(serializer), + Some(media_item::Value::Ref(v)) => v.serialize(serializer), + None => serializer.serialize_none(), + } + } +} + +impl<'de> ::serde::Deserialize<'de> for MediaItem { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = MediaItem; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("one of the MediaItem shapes") + } + fn visit_str(self, v: &str) -> Result { + Ok(MediaItem { + value: Some(media_item::Value::Source(v.to_owned())), + }) + } + fn visit_string(self, v: String) -> Result { + Ok(MediaItem { + value: Some(media_item::Value::Source(v)), + }) + } + fn visit_map>( + self, + map: A, + ) -> Result { + let v = ::deserialize( + ::serde::de::value::MapAccessDeserializer::new(map), + )?; + Ok(MediaItem { + value: Some(media_item::Value::Ref(v)), + }) + } + } + deserializer.deserialize_any(V) + } +} + +impl ::serde::Serialize for OptionalMediaItem { + fn serialize(&self, serializer: S) -> Result { + self.item.serialize(serializer) + } +} + +impl<'de> ::serde::Deserialize<'de> for OptionalMediaItem { + fn deserialize>(deserializer: D) -> Result { + Ok(OptionalMediaItem { + item: as ::serde::Deserialize>::deserialize(deserializer)?, + }) + } +} + +impl ::serde::Serialize for MediaItems { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_seq(self.items.iter()) + } +} + +impl<'de> ::serde::Deserialize<'de> for MediaItems { + fn deserialize>(deserializer: D) -> Result { + Ok(MediaItems { + items: + <::prost::alloc::vec::Vec as ::serde::Deserialize>::deserialize( + deserializer, + )?, + }) + } +} + +impl ::serde::Serialize for OptionalMediaItems { + fn serialize(&self, serializer: S) -> Result { + self.items.serialize(serializer) + } +} + +impl<'de> ::serde::Deserialize<'de> for OptionalMediaItems { + fn deserialize>(deserializer: D) -> Result { + Ok(OptionalMediaItems { + items: as ::serde::Deserialize>::deserialize(deserializer)?, + }) + } +} + +impl ::serde::Serialize for MediaItemsList { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_seq(self.items.iter()) + } +} + +impl<'de> ::serde::Deserialize<'de> for MediaItemsList { + fn deserialize>(deserializer: D) -> Result { + Ok(MediaItemsList { + items: + <::prost::alloc::vec::Vec as ::serde::Deserialize>::deserialize( + deserializer, + )?, + }) + } +} + +impl ::serde::Serialize for MediaInput { + fn serialize(&self, serializer: S) -> Result { + match &self.value { + Some(media_input::Value::Item(v)) => v.serialize(serializer), + Some(media_input::Value::Many(v)) => v.serialize(serializer), + Some(media_input::Value::Nested(v)) => v.serialize(serializer), + None => serializer.serialize_none(), + } + } +} + +impl<'de> ::serde::Deserialize<'de> for MediaInput { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = MediaInput; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("one of the MediaInput shapes") + } + fn visit_str(self, v: &str) -> Result { + Ok(MediaInput { + value: Some(media_input::Value::Item( + ::deserialize( + ::serde::de::value::StrDeserializer::::new(v), + )?, + )), + }) + } + fn visit_string(self, v: String) -> Result { + Ok(MediaInput { + value: Some(media_input::Value::Item( + ::deserialize( + ::serde::de::value::StringDeserializer::::new(v), + )?, + )), + }) + } + fn visit_map>( + self, + map: A, + ) -> Result { + let v = ::deserialize( + ::serde::de::value::MapAccessDeserializer::new(map), + )?; + Ok(MediaInput { + value: Some(media_input::Value::Item(v)), + }) + } + fn visit_seq>( + self, + mut seq: A, + ) -> Result { + let mut items: Vec<::serde_json::Value> = Vec::new(); + while let Some(v) = seq.next_element::<::serde_json::Value>()? { + items.push(v); + } + let nested = items.first().is_some_and(|f| f.is_array()); + let reparse = ::serde_json::Value::Array(items); + if nested { + let v: MediaItemsList = + ::serde_json::from_value(reparse).map_err(::serde::de::Error::custom)?; + Ok(MediaInput { + value: Some(media_input::Value::Nested(v)), + }) + } else { + let v: MediaItems = + ::serde_json::from_value(reparse).map_err(::serde::de::Error::custom)?; + Ok(MediaInput { + value: Some(media_input::Value::Many(v)), + }) + } + } + } + deserializer.deserialize_any(V) + } +} + +impl ::serde::Serialize for HealthCheckRequest { + fn serialize(&self, serializer: S) -> Result { + let map = serializer.serialize_map(None)?; + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for HealthCheckRequest { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = HealthCheckRequest; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct HealthCheckRequest") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(HealthCheckRequest {}) + } + } + deserializer.deserialize_map(V) + } +} + +impl ::serde::Serialize for HealthCheckResponse { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "healthy", &self.healthy)?; + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for HealthCheckResponse { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = HealthCheckResponse; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct HealthCheckResponse") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut f_healthy = None; + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + "healthy" => { + if f_healthy.is_some() { + return Err(::serde::de::Error::duplicate_field("healthy")); + } + f_healthy = Some(map.next_value::()?); + } + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(HealthCheckResponse { + healthy: f_healthy.unwrap_or_default(), + }) + } + } + deserializer.deserialize_map(V) + } +} + +impl ::serde::Serialize for GetModelInfoRequest { + fn serialize(&self, serializer: S) -> Result { + let map = serializer.serialize_map(None)?; + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for GetModelInfoRequest { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = GetModelInfoRequest; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct GetModelInfoRequest") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(GetModelInfoRequest {}) + } + } + deserializer.deserialize_map(V) + } +} + +impl ::serde::Serialize for GetModelInfoResponse { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "model_path", &self.model_path)?; + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "served_model_name", + &self.served_model_name, + )?; + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "tokenizer_path", + &self.tokenizer_path, + )?; + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "is_generation", + &self.is_generation, + )?; + match &self.preferred_sampling_params { + Some(v) => { + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "preferred_sampling_params", + &::serde_json::from_str::<::serde_json::Value>(v) + .unwrap_or(::serde_json::Value::Null), + )?; + } + None => { + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "preferred_sampling_params", + &::serde_json::Value::Null, + )?; + } + } + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "weight_version", + &self.weight_version, + )?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "load_format", &self.load_format)?; + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "reasoning_parser", + &self.reasoning_parser, + )?; + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "tool_call_parser", + &self.tool_call_parser, + )?; + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for GetModelInfoResponse { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = GetModelInfoResponse; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct GetModelInfoResponse") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut f_model_path = None; + let mut f_served_model_name = None; + let mut f_tokenizer_path = None; + let mut f_is_generation = None; + let mut f_preferred_sampling_params = None; + let mut f_weight_version = None; + let mut f_load_format = None; + let mut f_reasoning_parser = None; + let mut f_tool_call_parser = None; + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + "model_path" => { + if f_model_path.is_some() { + return Err(::serde::de::Error::duplicate_field("model_path")); + } + f_model_path = + Some(map.next_value::<::prost::alloc::string::String>()?); + } + "served_model_name" => { + if f_served_model_name.is_some() { + return Err(::serde::de::Error::duplicate_field( + "served_model_name", + )); + } + f_served_model_name = + Some(map.next_value::<::prost::alloc::string::String>()?); + } + "tokenizer_path" => { + if f_tokenizer_path.is_some() { + return Err(::serde::de::Error::duplicate_field("tokenizer_path")); + } + f_tokenizer_path = + Some(map.next_value::<::prost::alloc::string::String>()?); + } + "is_generation" => { + if f_is_generation.is_some() { + return Err(::serde::de::Error::duplicate_field("is_generation")); + } + f_is_generation = Some(map.next_value::()?); + } + "preferred_sampling_params" => { + if f_preferred_sampling_params.is_some() { + return Err(::serde::de::Error::duplicate_field( + "preferred_sampling_params", + )); + } + f_preferred_sampling_params = Some( + map.next_value::>()? + .map(|v| v.to_string()), + ); + } + "weight_version" => { + if f_weight_version.is_some() { + return Err(::serde::de::Error::duplicate_field("weight_version")); + } + f_weight_version = + Some(map.next_value::>()?); + } + "load_format" => { + if f_load_format.is_some() { + return Err(::serde::de::Error::duplicate_field("load_format")); + } + f_load_format = + Some(map.next_value::>()?); + } + "reasoning_parser" => { + if f_reasoning_parser.is_some() { + return Err(::serde::de::Error::duplicate_field( + "reasoning_parser", + )); + } + f_reasoning_parser = + Some(map.next_value::>()?); + } + "tool_call_parser" => { + if f_tool_call_parser.is_some() { + return Err(::serde::de::Error::duplicate_field( + "tool_call_parser", + )); + } + f_tool_call_parser = + Some(map.next_value::>()?); + } + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(GetModelInfoResponse { + model_path: f_model_path.unwrap_or_default(), + served_model_name: f_served_model_name.unwrap_or_default(), + tokenizer_path: f_tokenizer_path.unwrap_or_default(), + is_generation: f_is_generation.unwrap_or_default(), + preferred_sampling_params: f_preferred_sampling_params.flatten(), + weight_version: f_weight_version.flatten(), + load_format: f_load_format.flatten(), + reasoning_parser: f_reasoning_parser.flatten(), + tool_call_parser: f_tool_call_parser.flatten(), + }) + } + } + deserializer.deserialize_map(V) + } +} + +impl ::serde::Serialize for GetServerInfoRequest { + fn serialize(&self, serializer: S) -> Result { + let map = serializer.serialize_map(None)?; + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for GetServerInfoRequest { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = GetServerInfoRequest; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct GetServerInfoRequest") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(GetServerInfoRequest {}) + } + } + deserializer.deserialize_map(V) + } +} + +impl ::serde::Serialize for GetServerInfoResponse { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "model_path", &self.model_path)?; + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "served_model_name", + &self.served_model_name, + )?; + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "tokenizer_path", + &self.tokenizer_path, + )?; + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "max_context_length", + &self.max_context_length, + )?; + if let Some(v) = &self.max_total_num_tokens { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "max_total_num_tokens", v)?; + } + ::serde::ser::SerializeMap::serialize_entry(&mut map, "version", &self.version)?; + if !self.internal_states.is_empty() { + let rows: Vec<::serde_json::Value> = self + .internal_states + .iter() + .map(|s| ::serde_json::from_str(s).unwrap_or(::serde_json::Value::Null)) + .collect(); + ::serde::ser::SerializeMap::serialize_entry(&mut map, "internal_states", &rows)?; + } + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for GetServerInfoResponse { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = GetServerInfoResponse; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct GetServerInfoResponse") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut f_model_path = None; + let mut f_served_model_name = None; + let mut f_tokenizer_path = None; + let mut f_max_context_length = None; + let mut f_max_total_num_tokens = None; + let mut f_version = None; + let mut f_internal_states = None; + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + "model_path" => { + if f_model_path.is_some() { + return Err(::serde::de::Error::duplicate_field("model_path")); + } + f_model_path = + Some(map.next_value::<::prost::alloc::string::String>()?); + } + "served_model_name" => { + if f_served_model_name.is_some() { + return Err(::serde::de::Error::duplicate_field( + "served_model_name", + )); + } + f_served_model_name = + Some(map.next_value::<::prost::alloc::string::String>()?); + } + "tokenizer_path" => { + if f_tokenizer_path.is_some() { + return Err(::serde::de::Error::duplicate_field("tokenizer_path")); + } + f_tokenizer_path = + Some(map.next_value::<::prost::alloc::string::String>()?); + } + "max_context_length" => { + if f_max_context_length.is_some() { + return Err(::serde::de::Error::duplicate_field( + "max_context_length", + )); + } + f_max_context_length = Some(map.next_value::()?); + } + "max_total_num_tokens" => { + if f_max_total_num_tokens.is_some() { + return Err(::serde::de::Error::duplicate_field( + "max_total_num_tokens", + )); + } + f_max_total_num_tokens = Some(map.next_value::>()?); + } + "version" => { + if f_version.is_some() { + return Err(::serde::de::Error::duplicate_field("version")); + } + f_version = Some(map.next_value::<::prost::alloc::string::String>()?); + } + "internal_states" => { + if f_internal_states.is_some() { + return Err(::serde::de::Error::duplicate_field("internal_states")); + } + f_internal_states = Some( + map.next_value::>>()? + .unwrap_or_default() + .into_iter() + .map(|v| v.to_string()) + .collect(), + ); + } + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(GetServerInfoResponse { + model_path: f_model_path.unwrap_or_default(), + served_model_name: f_served_model_name.unwrap_or_default(), + tokenizer_path: f_tokenizer_path.unwrap_or_default(), + max_context_length: f_max_context_length.unwrap_or_default(), + max_total_num_tokens: f_max_total_num_tokens.flatten(), + version: f_version.unwrap_or_default(), + internal_states: f_internal_states.unwrap_or_default(), + }) + } + } + deserializer.deserialize_map(V) + } +} diff --git a/rust/sglang-api-types/src/generated/sglang.api.v1.tonic.rs b/rust/sglang-api-types/src/generated/sglang.api.v1.tonic.rs new file mode 100644 index 0000000000..0c7a0e53eb --- /dev/null +++ b/rust/sglang-api-types/src/generated/sglang.api.v1.tonic.rs @@ -0,0 +1,489 @@ +/// Generated client implementations. +pub mod sglang_service_client { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value + )] + use tonic::codegen::http::Uri; + use tonic::codegen::*; + #[derive(Debug, Clone)] + pub struct SglangServiceClient { + inner: tonic::client::Grpc, + } + impl SglangServiceClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl SglangServiceClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> SglangServiceClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + >>::Error: + Into + std::marker::Send + std::marker::Sync, + { + SglangServiceClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + /// `/generate`: always server-streaming (a unary request is a one-frame + /// stream); per-batch-item failures ride the stream as GenerateStreamItem + /// errors, pre-submit failures end the RPC with a gRPC status. + pub async fn generate( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response>, + tonic::Status, + > { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = + http::uri::PathAndQuery::from_static("/sglang.api.v1.SglangService/Generate"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("sglang.api.v1.SglangService", "Generate")); + self.inner.server_streaming(req, path, codec).await + } + /// `/health_generate`: fires a 1-token probe and watches the response + /// heartbeat. UNAVAILABLE on a stalled pipeline. + pub async fn health_check( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = + http::uri::PathAndQuery::from_static("/sglang.api.v1.SglangService/HealthCheck"); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "sglang.api.v1.SglangService", + "HealthCheck", + )); + self.inner.unary(req, path, codec).await + } + /// `/get_model_info`: static model metadata, no scheduler round-trip. + pub async fn get_model_info( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = + http::uri::PathAndQuery::from_static("/sglang.api.v1.SglangService/GetModelInfo"); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "sglang.api.v1.SglangService", + "GetModelInfo", + )); + self.inner.unary(req, path, codec).await + } + /// `/server_info`: one control round-trip, shaped through the runtime-metric + /// allowlist (never the raw server-args dump, which embeds keys). + pub async fn get_server_info( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = + http::uri::PathAndQuery::from_static("/sglang.api.v1.SglangService/GetServerInfo"); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "sglang.api.v1.SglangService", + "GetServerInfo", + )); + self.inner.unary(req, path, codec).await + } + } +} +/// Generated server implementations. +pub mod sglang_service_server { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value + )] + use tonic::codegen::*; + /// Generated trait containing gRPC methods that should be implemented for use with SglangServiceServer. + #[async_trait] + pub trait SglangService: std::marker::Send + std::marker::Sync + 'static { + /// Server streaming response type for the Generate method. + type GenerateStream: tonic::codegen::tokio_stream::Stream< + Item = std::result::Result, + > + std::marker::Send + + 'static; + /// `/generate`: always server-streaming (a unary request is a one-frame + /// stream); per-batch-item failures ride the stream as GenerateStreamItem + /// errors, pre-submit failures end the RPC with a gRPC status. + async fn generate( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + /// `/health_generate`: fires a 1-token probe and watches the response + /// heartbeat. UNAVAILABLE on a stalled pipeline. + async fn health_check( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + /// `/get_model_info`: static model metadata, no scheduler round-trip. + async fn get_model_info( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + /// `/server_info`: one control round-trip, shaped through the runtime-metric + /// allowlist (never the raw server-args dump, which embeds keys). + async fn get_server_info( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + } + #[derive(Debug)] + pub struct SglangServiceServer { + inner: Arc, + accept_compression_encodings: EnabledCompressionEncodings, + send_compression_encodings: EnabledCompressionEncodings, + max_decoding_message_size: Option, + max_encoding_message_size: Option, + } + impl SglangServiceServer { + pub fn new(inner: T) -> Self { + Self::from_arc(Arc::new(inner)) + } + pub fn from_arc(inner: Arc) -> Self { + Self { + inner, + accept_compression_encodings: Default::default(), + send_compression_encodings: Default::default(), + max_decoding_message_size: None, + max_encoding_message_size: None, + } + } + pub fn with_interceptor(inner: T, interceptor: F) -> InterceptedService + where + F: tonic::service::Interceptor, + { + InterceptedService::new(Self::new(inner), interceptor) + } + /// Enable decompressing requests with the given encoding. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.accept_compression_encodings.enable(encoding); + self + } + /// Compress responses with the given encoding, if the client supports it. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.send_compression_encodings.enable(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.max_decoding_message_size = Some(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.max_encoding_message_size = Some(limit); + self + } + } + impl tonic::codegen::Service> for SglangServiceServer + where + T: SglangService, + B: Body + std::marker::Send + 'static, + B::Error: Into + std::marker::Send + 'static, + { + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + match req.uri().path() { + "/sglang.api.v1.SglangService/Generate" => { + #[allow(non_camel_case_types)] + struct GenerateSvc(pub Arc); + impl + tonic::server::ServerStreamingService + for GenerateSvc + { + type Response = super::GenerateStreamItem; + type ResponseStream = T::GenerateStream; + type Future = + BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::generate(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = GenerateSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.server_streaming(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.api.v1.SglangService/HealthCheck" => { + #[allow(non_camel_case_types)] + struct HealthCheckSvc(pub Arc); + impl tonic::server::UnaryService + for HealthCheckSvc + { + type Response = super::HealthCheckResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::health_check(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = HealthCheckSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.api.v1.SglangService/GetModelInfo" => { + #[allow(non_camel_case_types)] + struct GetModelInfoSvc(pub Arc); + impl tonic::server::UnaryService + for GetModelInfoSvc + { + type Response = super::GetModelInfoResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::get_model_info(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = GetModelInfoSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.api.v1.SglangService/GetServerInfo" => { + #[allow(non_camel_case_types)] + struct GetServerInfoSvc(pub Arc); + impl tonic::server::UnaryService + for GetServerInfoSvc + { + type Response = super::GetServerInfoResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::get_server_info(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = GetServerInfoSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => Box::pin(async move { + let mut response = http::Response::new(tonic::body::Body::default()); + let headers = response.headers_mut(); + headers.insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers.insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) + }), + } + } + } + impl Clone for SglangServiceServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { + inner, + accept_compression_encodings: self.accept_compression_encodings, + send_compression_encodings: self.send_compression_encodings, + max_decoding_message_size: self.max_decoding_message_size, + max_encoding_message_size: self.max_encoding_message_size, + } + } + } + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "sglang.api.v1.SglangService"; + impl tonic::server::NamedService for SglangServiceServer { + const NAME: &'static str = SERVICE_NAME; + } +} diff --git a/rust/sglang-api-types/src/generated/sglang.runtime.v1.rs b/rust/sglang-api-types/src/generated/sglang.runtime.v1.rs new file mode 100644 index 0000000000..61ffeed839 --- /dev/null +++ b/rust/sglang-api-types/src/generated/sglang.runtime.v1.rs @@ -0,0 +1,457 @@ +// This file is @generated by prost-build. +/// Disaggregated prefill/decode rendezvous parameters. +/// +/// Carried on Generate/TextGenerate requests when the worker is part of a +/// disaggregated deployment. The prefill leg writes its KV cache to the +/// bootstrap server identified by (bootstrap_host, bootstrap_port) under +/// the room id; the decode leg fetches it from the same room. Both legs +/// receive the same `bootstrap_room` from the router so they rendezvous +/// on the KV transfer. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct DisaggregatedParams { + #[prost(string, tag = "1")] + pub bootstrap_host: ::prost::alloc::string::String, + #[prost(int32, tag = "2")] + pub bootstrap_port: i32, + #[prost(int64, tag = "3")] + pub bootstrap_room: i64, +} +/// Sampling parameters shared across text and tokenized RPCs. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SamplingParams { + #[prost(float, optional, tag = "1")] + pub temperature: ::core::option::Option, + #[prost(float, optional, tag = "2")] + pub top_p: ::core::option::Option, + #[prost(int32, optional, tag = "3")] + pub top_k: ::core::option::Option, + #[prost(float, optional, tag = "4")] + pub min_p: ::core::option::Option, + #[prost(float, optional, tag = "5")] + pub frequency_penalty: ::core::option::Option, + #[prost(float, optional, tag = "6")] + pub presence_penalty: ::core::option::Option, + #[prost(float, optional, tag = "7")] + pub repetition_penalty: ::core::option::Option, + #[prost(int32, optional, tag = "8")] + pub max_new_tokens: ::core::option::Option, + #[prost(int32, optional, tag = "9")] + pub min_new_tokens: ::core::option::Option, + #[prost(string, repeated, tag = "10")] + pub stop: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(int32, repeated, tag = "11")] + pub stop_token_ids: ::prost::alloc::vec::Vec, + #[prost(bool, optional, tag = "12")] + pub ignore_eos: ::core::option::Option, + #[prost(int32, optional, tag = "13")] + pub n: ::core::option::Option, + #[deprecated] + #[prost(string, optional, tag = "14")] + pub json_schema: ::core::option::Option<::prost::alloc::string::String>, + #[deprecated] + #[prost(string, optional, tag = "15")] + pub regex: ::core::option::Option<::prost::alloc::string::String>, + #[prost(int64, optional, tag = "16")] + pub seed: ::core::option::Option, + #[prost(message, optional, tag = "17")] + pub guided_decoding: ::core::option::Option, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GuidedDecoding { + #[prost(oneof = "guided_decoding::Constraint", tags = "1, 2, 3, 4, 5")] + pub constraint: ::core::option::Option, +} +/// Nested message and enum types in `GuidedDecoding`. +pub mod guided_decoding { + #[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)] + pub enum Constraint { + #[prost(string, tag = "1")] + JsonSchema(::prost::alloc::string::String), + #[prost(string, tag = "2")] + Regex(::prost::alloc::string::String), + #[prost(string, tag = "3")] + Ebnf(::prost::alloc::string::String), + #[prost(message, tag = "4")] + Choice(super::ChoiceConstraint), + #[prost(string, tag = "5")] + StructuralTag(::prost::alloc::string::String), + } +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ChoiceConstraint { + #[prost(string, repeated, tag = "1")] + pub values: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TextGenerateRequest { + #[prost(string, tag = "1")] + pub text: ::prost::alloc::string::String, + #[prost(message, optional, tag = "2")] + pub sampling_params: ::core::option::Option, + #[prost(bool, optional, tag = "3")] + pub stream: ::core::option::Option, + #[prost(bool, optional, tag = "4")] + pub return_logprob: ::core::option::Option, + #[prost(int32, optional, tag = "5")] + pub top_logprobs_num: ::core::option::Option, + #[prost(int32, optional, tag = "6")] + pub logprob_start_len: ::core::option::Option, + #[prost(bool, optional, tag = "7")] + pub return_text_in_logprobs: ::core::option::Option, + #[prost(string, optional, tag = "8")] + pub rid: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "9")] + pub lora_path: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "10")] + pub routing_key: ::core::option::Option<::prost::alloc::string::String>, + #[prost(int32, optional, tag = "11")] + pub routed_dp_rank: ::core::option::Option, + #[prost(map = "string, string", tag = "12")] + pub trace_headers: + ::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>, + #[prost(string, optional, tag = "13")] + pub session_id: ::core::option::Option<::prost::alloc::string::String>, + #[prost(message, optional, tag = "14")] + pub disaggregated_params: ::core::option::Option, + #[prost(int32, optional, tag = "15")] + pub priority: ::core::option::Option, + #[prost(bool, optional, tag = "16")] + pub require_reasoning: ::core::option::Option, + #[prost(uint32, optional, tag = "17")] + pub max_thinking_tokens: ::core::option::Option, + #[prost(message, optional, tag = "18")] + pub kv_hints: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TextGenerateResponse { + #[prost(string, tag = "1")] + pub text: ::prost::alloc::string::String, + #[prost(map = "string, string", tag = "2")] + pub meta_info: + ::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>, + #[prost(bool, tag = "3")] + pub finished: bool, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GenerateRequest { + #[prost(int32, repeated, tag = "1")] + pub input_ids: ::prost::alloc::vec::Vec, + #[prost(message, optional, tag = "2")] + pub sampling_params: ::core::option::Option, + #[prost(bool, optional, tag = "3")] + pub stream: ::core::option::Option, + #[prost(bool, optional, tag = "4")] + pub return_logprob: ::core::option::Option, + #[prost(int32, optional, tag = "5")] + pub top_logprobs_num: ::core::option::Option, + #[prost(int32, optional, tag = "6")] + pub logprob_start_len: ::core::option::Option, + #[prost(string, optional, tag = "7")] + pub rid: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "8")] + pub lora_path: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "9")] + pub routing_key: ::core::option::Option<::prost::alloc::string::String>, + #[prost(int32, optional, tag = "10")] + pub routed_dp_rank: ::core::option::Option, + #[prost(map = "string, string", tag = "11")] + pub trace_headers: + ::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>, + #[prost(string, optional, tag = "12")] + pub session_id: ::core::option::Option<::prost::alloc::string::String>, + #[prost(message, optional, tag = "13")] + pub disaggregated_params: ::core::option::Option, + #[prost(int32, optional, tag = "14")] + pub priority: ::core::option::Option, + #[prost(bool, optional, tag = "15")] + pub require_reasoning: ::core::option::Option, + #[prost(uint32, optional, tag = "16")] + pub max_thinking_tokens: ::core::option::Option, + #[prost(message, optional, tag = "17")] + pub kv_hints: ::core::option::Option, +} +/// Versioned KV-hint envelope forwarded from a trusted orchestrator. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct KvHintsEnvelope { + #[prost(string, tag = "1")] + pub protocol_version: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub message_id: ::prost::alloc::string::String, + #[prost(message, repeated, tag = "3")] + pub actions: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct KvHintAction { + #[prost(string, tag = "1")] + pub action_id: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub action_type: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub action_version: ::prost::alloc::string::String, + /// Action-specific; validated by the component implementing action_type. + #[prost(message, optional, tag = "4")] + pub payload: ::core::option::Option<::prost_types::Struct>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GenerateResponse { + #[prost(int32, repeated, tag = "1")] + pub output_ids: ::prost::alloc::vec::Vec, + #[prost(map = "string, string", tag = "2")] + pub meta_info: + ::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>, + #[prost(bool, tag = "3")] + pub finished: bool, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TextEmbedRequest { + #[prost(string, tag = "1")] + pub text: ::prost::alloc::string::String, + #[prost(string, optional, tag = "2")] + pub rid: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "3")] + pub routing_key: ::core::option::Option<::prost::alloc::string::String>, + #[prost(map = "string, string", tag = "4")] + pub trace_headers: + ::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TextEmbedResponse { + #[prost(float, repeated, tag = "1")] + pub embedding: ::prost::alloc::vec::Vec, + #[prost(map = "string, string", tag = "2")] + pub meta_info: + ::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct EmbedRequest { + #[prost(int32, repeated, tag = "1")] + pub input_ids: ::prost::alloc::vec::Vec, + #[prost(string, optional, tag = "2")] + pub rid: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "3")] + pub routing_key: ::core::option::Option<::prost::alloc::string::String>, + #[prost(map = "string, string", tag = "4")] + pub trace_headers: + ::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct EmbedResponse { + #[prost(float, repeated, tag = "1")] + pub embedding: ::prost::alloc::vec::Vec, + #[prost(map = "string, string", tag = "2")] + pub meta_info: + ::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct HealthCheckRequest {} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct HealthCheckResponse { + #[prost(bool, tag = "1")] + pub healthy: bool, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct WatchEngineStateRequest {} +/// A complete discovery and lifecycle snapshot. The JSON discovery payloads +/// intentionally retain the same wire shape as GetModelInfo/GetServerInfo. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct EngineStateSnapshot { + /// Unix time in nanoseconds, captured once for this engine process. + #[prost(uint64, tag = "1")] + pub instance_id: u64, + /// Starts at one and increases for each snapshot from this instance. + #[prost(uint64, tag = "2")] + pub revision: u64, + #[prost(bool, tag = "3")] + pub healthy: bool, + #[prost(bool, tag = "4")] + pub is_pause: bool, + #[prost(message, optional, tag = "5")] + pub model_info: ::core::option::Option, + #[prost(message, optional, tag = "6")] + pub server_info: ::core::option::Option, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetModelInfoRequest {} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetModelInfoResponse { + #[prost(string, tag = "1")] + pub model_path: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub json_info: ::prost::alloc::string::String, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetServerInfoRequest {} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetServerInfoResponse { + #[prost(string, tag = "1")] + pub json_info: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct AbortRequest { + #[prost(string, tag = "1")] + pub rid: ::prost::alloc::string::String, + #[prost(bool, tag = "2")] + pub abort_all: bool, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct AbortResponse { + #[prost(bool, tag = "1")] + pub success: bool, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ClassifyRequest { + #[prost(string, tag = "1")] + pub text: ::prost::alloc::string::String, + #[prost(int32, repeated, tag = "2")] + pub input_ids: ::prost::alloc::vec::Vec, + #[prost(string, optional, tag = "3")] + pub rid: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "4")] + pub routing_key: ::core::option::Option<::prost::alloc::string::String>, + #[prost(map = "string, string", tag = "5")] + pub trace_headers: + ::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ClassifyResponse { + #[prost(float, repeated, tag = "1")] + pub embedding: ::prost::alloc::vec::Vec, + #[prost(map = "string, string", tag = "2")] + pub meta_info: + ::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct TokenizeRequest { + #[prost(string, tag = "1")] + pub text: ::prost::alloc::string::String, + #[prost(bool, optional, tag = "2")] + pub add_special_tokens: ::core::option::Option, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct TokenizeResponse { + #[prost(int32, repeated, tag = "1")] + pub tokens: ::prost::alloc::vec::Vec, + #[prost(int32, tag = "2")] + pub count: i32, + #[prost(int32, tag = "3")] + pub max_model_len: i32, + #[prost(string, tag = "4")] + pub input_text: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct DetokenizeRequest { + #[prost(int32, repeated, tag = "1")] + pub tokens: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct DetokenizeResponse { + #[prost(string, tag = "1")] + pub text: ::prost::alloc::string::String, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ListModelsRequest {} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ListModelsResponse { + #[prost(message, repeated, tag = "1")] + pub models: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ModelCard { + #[prost(string, tag = "1")] + pub id: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub root: ::prost::alloc::string::String, + #[prost(string, optional, tag = "3")] + pub parent: ::core::option::Option<::prost::alloc::string::String>, + #[prost(int32, optional, tag = "4")] + pub max_model_len: ::core::option::Option, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetLoadRequest { + #[prost(int32, optional, tag = "1")] + pub dp_rank: ::core::option::Option, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetLoadResponse { + #[prost(string, tag = "1")] + pub json_info: ::prost::alloc::string::String, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct FlushCacheRequest {} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct FlushCacheResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, tag = "2")] + pub message: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PauseGenerationRequest { + #[prost(string, tag = "1")] + pub mode: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PauseGenerationResponse { + #[prost(string, tag = "1")] + pub message: ::prost::alloc::string::String, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ContinueGenerationRequest {} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ContinueGenerationResponse { + #[prost(string, tag = "1")] + pub message: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct OpenAiRequest { + #[prost(bytes = "vec", tag = "1")] + pub json_body: ::prost::alloc::vec::Vec, + #[prost(map = "string, string", tag = "2")] + pub trace_headers: + ::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct OpenAiStreamChunk { + #[prost(bytes = "vec", tag = "1")] + pub json_chunk: ::prost::alloc::vec::Vec, + #[prost(bool, tag = "2")] + pub finished: bool, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct OpenAiResponse { + #[prost(bytes = "vec", tag = "1")] + pub json_body: ::prost::alloc::vec::Vec, + #[prost(int32, tag = "2")] + pub status_code: i32, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct StartProfileRequest { + #[prost(string, optional, tag = "1")] + pub output_dir: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct StartProfileResponse { + #[prost(string, tag = "1")] + pub message: ::prost::alloc::string::String, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct StopProfileRequest {} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct StopProfileResponse { + #[prost(string, tag = "1")] + pub message: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct UpdateWeightsRequest { + #[prost(string, tag = "1")] + pub model_path: ::prost::alloc::string::String, + #[prost(string, optional, tag = "2")] + pub load_format: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct UpdateWeightsResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, tag = "2")] + pub message: ::prost::alloc::string::String, +} diff --git a/rust/sglang-api-types/src/generated/sglang.runtime.v1.tonic.rs b/rust/sglang-api-types/src/generated/sglang.runtime.v1.tonic.rs new file mode 100644 index 0000000000..90a4053b1d --- /dev/null +++ b/rust/sglang-api-types/src/generated/sglang.runtime.v1.tonic.rs @@ -0,0 +1,1922 @@ +/// Generated client implementations. +pub mod sglang_service_client { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value + )] + use tonic::codegen::http::Uri; + use tonic::codegen::*; + #[derive(Debug, Clone)] + pub struct SglangServiceClient { + inner: tonic::client::Grpc, + } + impl SglangServiceClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl SglangServiceClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> SglangServiceClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + >>::Error: + Into + std::marker::Send + std::marker::Sync, + { + SglangServiceClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + /// SGLang-native RPCs (typed proto) + pub async fn text_generate( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response>, + tonic::Status, + > { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/sglang.runtime.v1.SglangService/TextGenerate", + ); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "sglang.runtime.v1.SglangService", + "TextGenerate", + )); + self.inner.server_streaming(req, path, codec).await + } + pub async fn generate( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response>, + tonic::Status, + > { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = + http::uri::PathAndQuery::from_static("/sglang.runtime.v1.SglangService/Generate"); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "sglang.runtime.v1.SglangService", + "Generate", + )); + self.inner.server_streaming(req, path, codec).await + } + pub async fn text_embed( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = + http::uri::PathAndQuery::from_static("/sglang.runtime.v1.SglangService/TextEmbed"); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "sglang.runtime.v1.SglangService", + "TextEmbed", + )); + self.inner.unary(req, path, codec).await + } + pub async fn embed( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = + http::uri::PathAndQuery::from_static("/sglang.runtime.v1.SglangService/Embed"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("sglang.runtime.v1.SglangService", "Embed")); + self.inner.unary(req, path, codec).await + } + pub async fn classify( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = + http::uri::PathAndQuery::from_static("/sglang.runtime.v1.SglangService/Classify"); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "sglang.runtime.v1.SglangService", + "Classify", + )); + self.inner.unary(req, path, codec).await + } + pub async fn tokenize( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = + http::uri::PathAndQuery::from_static("/sglang.runtime.v1.SglangService/Tokenize"); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "sglang.runtime.v1.SglangService", + "Tokenize", + )); + self.inner.unary(req, path, codec).await + } + pub async fn detokenize( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = + http::uri::PathAndQuery::from_static("/sglang.runtime.v1.SglangService/Detokenize"); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "sglang.runtime.v1.SglangService", + "Detokenize", + )); + self.inner.unary(req, path, codec).await + } + pub async fn health_check( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/sglang.runtime.v1.SglangService/HealthCheck", + ); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "sglang.runtime.v1.SglangService", + "HealthCheck", + )); + self.inner.unary(req, path, codec).await + } + pub async fn watch_engine_state( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response>, + tonic::Status, + > { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/sglang.runtime.v1.SglangService/WatchEngineState", + ); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "sglang.runtime.v1.SglangService", + "WatchEngineState", + )); + self.inner.server_streaming(req, path, codec).await + } + pub async fn get_model_info( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/sglang.runtime.v1.SglangService/GetModelInfo", + ); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "sglang.runtime.v1.SglangService", + "GetModelInfo", + )); + self.inner.unary(req, path, codec).await + } + pub async fn get_server_info( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/sglang.runtime.v1.SglangService/GetServerInfo", + ); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "sglang.runtime.v1.SglangService", + "GetServerInfo", + )); + self.inner.unary(req, path, codec).await + } + pub async fn list_models( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = + http::uri::PathAndQuery::from_static("/sglang.runtime.v1.SglangService/ListModels"); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "sglang.runtime.v1.SglangService", + "ListModels", + )); + self.inner.unary(req, path, codec).await + } + pub async fn get_load( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = + http::uri::PathAndQuery::from_static("/sglang.runtime.v1.SglangService/GetLoad"); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "sglang.runtime.v1.SglangService", + "GetLoad", + )); + self.inner.unary(req, path, codec).await + } + pub async fn abort( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = + http::uri::PathAndQuery::from_static("/sglang.runtime.v1.SglangService/Abort"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("sglang.runtime.v1.SglangService", "Abort")); + self.inner.unary(req, path, codec).await + } + pub async fn flush_cache( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = + http::uri::PathAndQuery::from_static("/sglang.runtime.v1.SglangService/FlushCache"); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "sglang.runtime.v1.SglangService", + "FlushCache", + )); + self.inner.unary(req, path, codec).await + } + pub async fn pause_generation( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/sglang.runtime.v1.SglangService/PauseGeneration", + ); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "sglang.runtime.v1.SglangService", + "PauseGeneration", + )); + self.inner.unary(req, path, codec).await + } + pub async fn continue_generation( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/sglang.runtime.v1.SglangService/ContinueGeneration", + ); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "sglang.runtime.v1.SglangService", + "ContinueGeneration", + )); + self.inner.unary(req, path, codec).await + } + /// OpenAI-compatible RPCs (JSON pass-through) + pub async fn chat_complete( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response>, + tonic::Status, + > { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/sglang.runtime.v1.SglangService/ChatComplete", + ); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "sglang.runtime.v1.SglangService", + "ChatComplete", + )); + self.inner.server_streaming(req, path, codec).await + } + pub async fn complete( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response>, + tonic::Status, + > { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = + http::uri::PathAndQuery::from_static("/sglang.runtime.v1.SglangService/Complete"); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "sglang.runtime.v1.SglangService", + "Complete", + )); + self.inner.server_streaming(req, path, codec).await + } + pub async fn open_ai_embed( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/sglang.runtime.v1.SglangService/OpenAIEmbed", + ); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "sglang.runtime.v1.SglangService", + "OpenAIEmbed", + )); + self.inner.unary(req, path, codec).await + } + pub async fn open_ai_classify( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/sglang.runtime.v1.SglangService/OpenAIClassify", + ); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "sglang.runtime.v1.SglangService", + "OpenAIClassify", + )); + self.inner.unary(req, path, codec).await + } + pub async fn score( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = + http::uri::PathAndQuery::from_static("/sglang.runtime.v1.SglangService/Score"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("sglang.runtime.v1.SglangService", "Score")); + self.inner.unary(req, path, codec).await + } + pub async fn rerank( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = + http::uri::PathAndQuery::from_static("/sglang.runtime.v1.SglangService/Rerank"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("sglang.runtime.v1.SglangService", "Rerank")); + self.inner.unary(req, path, codec).await + } + /// Admin/Ops RPCs + pub async fn start_profile( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/sglang.runtime.v1.SglangService/StartProfile", + ); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "sglang.runtime.v1.SglangService", + "StartProfile", + )); + self.inner.unary(req, path, codec).await + } + pub async fn stop_profile( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/sglang.runtime.v1.SglangService/StopProfile", + ); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "sglang.runtime.v1.SglangService", + "StopProfile", + )); + self.inner.unary(req, path, codec).await + } + pub async fn update_weights_from_disk( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/sglang.runtime.v1.SglangService/UpdateWeightsFromDisk", + ); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "sglang.runtime.v1.SglangService", + "UpdateWeightsFromDisk", + )); + self.inner.unary(req, path, codec).await + } + } +} +/// Generated server implementations. +pub mod sglang_service_server { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value + )] + use tonic::codegen::*; + /// Generated trait containing gRPC methods that should be implemented for use with SglangServiceServer. + #[async_trait] + pub trait SglangService: std::marker::Send + std::marker::Sync + 'static { + /// SGLang-native RPCs (typed proto) + async fn text_generate( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response>, + tonic::Status, + > { + Err(tonic::Status::unimplemented("Not yet implemented")) + } + async fn generate( + &self, + request: tonic::Request, + ) -> std::result::Result>, tonic::Status> + { + Err(tonic::Status::unimplemented("Not yet implemented")) + } + async fn text_embed( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + Err(tonic::Status::unimplemented("Not yet implemented")) + } + async fn embed( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + Err(tonic::Status::unimplemented("Not yet implemented")) + } + async fn classify( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + Err(tonic::Status::unimplemented("Not yet implemented")) + } + async fn tokenize( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + Err(tonic::Status::unimplemented("Not yet implemented")) + } + async fn detokenize( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> + { + Err(tonic::Status::unimplemented("Not yet implemented")) + } + async fn health_check( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> + { + Err(tonic::Status::unimplemented("Not yet implemented")) + } + async fn watch_engine_state( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response>, + tonic::Status, + > { + Err(tonic::Status::unimplemented("Not yet implemented")) + } + async fn get_model_info( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> + { + Err(tonic::Status::unimplemented("Not yet implemented")) + } + async fn get_server_info( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> + { + Err(tonic::Status::unimplemented("Not yet implemented")) + } + async fn list_models( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> + { + Err(tonic::Status::unimplemented("Not yet implemented")) + } + async fn get_load( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + Err(tonic::Status::unimplemented("Not yet implemented")) + } + async fn abort( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + Err(tonic::Status::unimplemented("Not yet implemented")) + } + async fn flush_cache( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> + { + Err(tonic::Status::unimplemented("Not yet implemented")) + } + async fn pause_generation( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> + { + Err(tonic::Status::unimplemented("Not yet implemented")) + } + async fn continue_generation( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> + { + Err(tonic::Status::unimplemented("Not yet implemented")) + } + /// OpenAI-compatible RPCs (JSON pass-through) + async fn chat_complete( + &self, + request: tonic::Request, + ) -> std::result::Result>, tonic::Status> + { + Err(tonic::Status::unimplemented("Not yet implemented")) + } + async fn complete( + &self, + request: tonic::Request, + ) -> std::result::Result>, tonic::Status> + { + Err(tonic::Status::unimplemented("Not yet implemented")) + } + async fn open_ai_embed( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + Err(tonic::Status::unimplemented("Not yet implemented")) + } + async fn open_ai_classify( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + Err(tonic::Status::unimplemented("Not yet implemented")) + } + async fn score( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + Err(tonic::Status::unimplemented("Not yet implemented")) + } + async fn rerank( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + Err(tonic::Status::unimplemented("Not yet implemented")) + } + /// Admin/Ops RPCs + async fn start_profile( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> + { + Err(tonic::Status::unimplemented("Not yet implemented")) + } + async fn stop_profile( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> + { + Err(tonic::Status::unimplemented("Not yet implemented")) + } + async fn update_weights_from_disk( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> + { + Err(tonic::Status::unimplemented("Not yet implemented")) + } + } + #[derive(Debug)] + pub struct SglangServiceServer { + inner: Arc, + accept_compression_encodings: EnabledCompressionEncodings, + send_compression_encodings: EnabledCompressionEncodings, + max_decoding_message_size: Option, + max_encoding_message_size: Option, + } + impl SglangServiceServer { + pub fn new(inner: T) -> Self { + Self::from_arc(Arc::new(inner)) + } + pub fn from_arc(inner: Arc) -> Self { + Self { + inner, + accept_compression_encodings: Default::default(), + send_compression_encodings: Default::default(), + max_decoding_message_size: None, + max_encoding_message_size: None, + } + } + pub fn with_interceptor(inner: T, interceptor: F) -> InterceptedService + where + F: tonic::service::Interceptor, + { + InterceptedService::new(Self::new(inner), interceptor) + } + /// Enable decompressing requests with the given encoding. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.accept_compression_encodings.enable(encoding); + self + } + /// Compress responses with the given encoding, if the client supports it. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.send_compression_encodings.enable(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.max_decoding_message_size = Some(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.max_encoding_message_size = Some(limit); + self + } + } + impl tonic::codegen::Service> for SglangServiceServer + where + T: SglangService, + B: Body + std::marker::Send + 'static, + B::Error: Into + std::marker::Send + 'static, + { + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + match req.uri().path() { + "/sglang.runtime.v1.SglangService/TextGenerate" => { + #[allow(non_camel_case_types)] + struct TextGenerateSvc(pub Arc); + impl + tonic::server::ServerStreamingService + for TextGenerateSvc + { + type Response = super::TextGenerateResponse; + type ResponseStream = BoxStream; + type Future = + BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::text_generate(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = TextGenerateSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.server_streaming(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.runtime.v1.SglangService/Generate" => { + #[allow(non_camel_case_types)] + struct GenerateSvc(pub Arc); + impl + tonic::server::ServerStreamingService + for GenerateSvc + { + type Response = super::GenerateResponse; + type ResponseStream = BoxStream; + type Future = + BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::generate(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = GenerateSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.server_streaming(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.runtime.v1.SglangService/TextEmbed" => { + #[allow(non_camel_case_types)] + struct TextEmbedSvc(pub Arc); + impl tonic::server::UnaryService for TextEmbedSvc { + type Response = super::TextEmbedResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::text_embed(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = TextEmbedSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.runtime.v1.SglangService/Embed" => { + #[allow(non_camel_case_types)] + struct EmbedSvc(pub Arc); + impl tonic::server::UnaryService for EmbedSvc { + type Response = super::EmbedResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = + async move { ::embed(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = EmbedSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.runtime.v1.SglangService/Classify" => { + #[allow(non_camel_case_types)] + struct ClassifySvc(pub Arc); + impl tonic::server::UnaryService for ClassifySvc { + type Response = super::ClassifyResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::classify(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = ClassifySvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.runtime.v1.SglangService/Tokenize" => { + #[allow(non_camel_case_types)] + struct TokenizeSvc(pub Arc); + impl tonic::server::UnaryService for TokenizeSvc { + type Response = super::TokenizeResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::tokenize(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = TokenizeSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.runtime.v1.SglangService/Detokenize" => { + #[allow(non_camel_case_types)] + struct DetokenizeSvc(pub Arc); + impl tonic::server::UnaryService for DetokenizeSvc { + type Response = super::DetokenizeResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::detokenize(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = DetokenizeSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.runtime.v1.SglangService/HealthCheck" => { + #[allow(non_camel_case_types)] + struct HealthCheckSvc(pub Arc); + impl tonic::server::UnaryService + for HealthCheckSvc + { + type Response = super::HealthCheckResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::health_check(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = HealthCheckSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.runtime.v1.SglangService/WatchEngineState" => { + #[allow(non_camel_case_types)] + struct WatchEngineStateSvc(pub Arc); + impl + tonic::server::ServerStreamingService + for WatchEngineStateSvc + { + type Response = super::EngineStateSnapshot; + type ResponseStream = BoxStream; + type Future = + BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::watch_engine_state(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = WatchEngineStateSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.server_streaming(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.runtime.v1.SglangService/GetModelInfo" => { + #[allow(non_camel_case_types)] + struct GetModelInfoSvc(pub Arc); + impl tonic::server::UnaryService + for GetModelInfoSvc + { + type Response = super::GetModelInfoResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::get_model_info(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = GetModelInfoSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.runtime.v1.SglangService/GetServerInfo" => { + #[allow(non_camel_case_types)] + struct GetServerInfoSvc(pub Arc); + impl tonic::server::UnaryService + for GetServerInfoSvc + { + type Response = super::GetServerInfoResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::get_server_info(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = GetServerInfoSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.runtime.v1.SglangService/ListModels" => { + #[allow(non_camel_case_types)] + struct ListModelsSvc(pub Arc); + impl tonic::server::UnaryService for ListModelsSvc { + type Response = super::ListModelsResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::list_models(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = ListModelsSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.runtime.v1.SglangService/GetLoad" => { + #[allow(non_camel_case_types)] + struct GetLoadSvc(pub Arc); + impl tonic::server::UnaryService for GetLoadSvc { + type Response = super::GetLoadResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::get_load(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = GetLoadSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.runtime.v1.SglangService/Abort" => { + #[allow(non_camel_case_types)] + struct AbortSvc(pub Arc); + impl tonic::server::UnaryService for AbortSvc { + type Response = super::AbortResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = + async move { ::abort(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = AbortSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.runtime.v1.SglangService/FlushCache" => { + #[allow(non_camel_case_types)] + struct FlushCacheSvc(pub Arc); + impl tonic::server::UnaryService for FlushCacheSvc { + type Response = super::FlushCacheResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::flush_cache(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = FlushCacheSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.runtime.v1.SglangService/PauseGeneration" => { + #[allow(non_camel_case_types)] + struct PauseGenerationSvc(pub Arc); + impl + tonic::server::UnaryService + for PauseGenerationSvc + { + type Response = super::PauseGenerationResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::pause_generation(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = PauseGenerationSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.runtime.v1.SglangService/ContinueGeneration" => { + #[allow(non_camel_case_types)] + struct ContinueGenerationSvc(pub Arc); + impl + tonic::server::UnaryService + for ContinueGenerationSvc + { + type Response = super::ContinueGenerationResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::continue_generation(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = ContinueGenerationSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.runtime.v1.SglangService/ChatComplete" => { + #[allow(non_camel_case_types)] + struct ChatCompleteSvc(pub Arc); + impl + tonic::server::ServerStreamingService + for ChatCompleteSvc + { + type Response = super::OpenAiStreamChunk; + type ResponseStream = BoxStream; + type Future = + BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::chat_complete(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = ChatCompleteSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.server_streaming(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.runtime.v1.SglangService/Complete" => { + #[allow(non_camel_case_types)] + struct CompleteSvc(pub Arc); + impl + tonic::server::ServerStreamingService + for CompleteSvc + { + type Response = super::OpenAiStreamChunk; + type ResponseStream = BoxStream; + type Future = + BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::complete(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = CompleteSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.server_streaming(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.runtime.v1.SglangService/OpenAIEmbed" => { + #[allow(non_camel_case_types)] + struct OpenAIEmbedSvc(pub Arc); + impl tonic::server::UnaryService for OpenAIEmbedSvc { + type Response = super::OpenAiResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::open_ai_embed(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = OpenAIEmbedSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.runtime.v1.SglangService/OpenAIClassify" => { + #[allow(non_camel_case_types)] + struct OpenAIClassifySvc(pub Arc); + impl tonic::server::UnaryService for OpenAIClassifySvc { + type Response = super::OpenAiResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::open_ai_classify(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = OpenAIClassifySvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.runtime.v1.SglangService/Score" => { + #[allow(non_camel_case_types)] + struct ScoreSvc(pub Arc); + impl tonic::server::UnaryService for ScoreSvc { + type Response = super::OpenAiResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = + async move { ::score(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = ScoreSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.runtime.v1.SglangService/Rerank" => { + #[allow(non_camel_case_types)] + struct RerankSvc(pub Arc); + impl tonic::server::UnaryService for RerankSvc { + type Response = super::OpenAiResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = + async move { ::rerank(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = RerankSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.runtime.v1.SglangService/StartProfile" => { + #[allow(non_camel_case_types)] + struct StartProfileSvc(pub Arc); + impl tonic::server::UnaryService + for StartProfileSvc + { + type Response = super::StartProfileResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::start_profile(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = StartProfileSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.runtime.v1.SglangService/StopProfile" => { + #[allow(non_camel_case_types)] + struct StopProfileSvc(pub Arc); + impl tonic::server::UnaryService + for StopProfileSvc + { + type Response = super::StopProfileResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::stop_profile(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = StopProfileSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.runtime.v1.SglangService/UpdateWeightsFromDisk" => { + #[allow(non_camel_case_types)] + struct UpdateWeightsFromDiskSvc(pub Arc); + impl tonic::server::UnaryService + for UpdateWeightsFromDiskSvc + { + type Response = super::UpdateWeightsResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::update_weights_from_disk(&inner, request) + .await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = UpdateWeightsFromDiskSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => Box::pin(async move { + let mut response = http::Response::new(tonic::body::Body::default()); + let headers = response.headers_mut(); + headers.insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers.insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) + }), + } + } + } + impl Clone for SglangServiceServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { + inner, + accept_compression_encodings: self.accept_compression_encodings, + send_compression_encodings: self.send_compression_encodings, + max_decoding_message_size: self.max_decoding_message_size, + max_encoding_message_size: self.max_encoding_message_size, + } + } + } + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "sglang.runtime.v1.SglangService"; + impl tonic::server::NamedService for SglangServiceServer { + const NAME: &'static str = SERVICE_NAME; + } +} diff --git a/rust/sglang-api-types/src/lib.rs b/rust/sglang-api-types/src/lib.rs new file mode 100644 index 0000000000..3967fe1a58 --- /dev/null +++ b/rust/sglang-api-types/src/lib.rs @@ -0,0 +1,28 @@ +//! Generated SGLang API types. `src/generated/` is written by +//! `cargo gen-api` from `proto/sglang/` -- the schema is the ground truth; +//! edit the .proto files and regenerate, never these files, and commit the +//! regenerated output with the .proto change. +//! +//! Module paths mirror the proto packages: `api::v1` is `sglang.api.v1` (the +//! HTTP/gRPC contract, with schema-driven JSON serde), `runtime::v1` is +//! `sglang.runtime.v1` (the gRPC bridge's protocol types, prost + tonic only). + +mod ext; + +pub mod api { + // Generated code optimizes for template uniformity, not lint cleanliness. + #[allow(clippy::all)] + pub mod v1 { + include!("generated/sglang.api.v1.rs"); + include!("generated/sglang.api.v1.serde.rs"); + include!("generated/sglang.api.v1.tonic.rs"); + } +} + +pub mod runtime { + #[allow(clippy::all)] + pub mod v1 { + include!("generated/sglang.runtime.v1.rs"); + include!("generated/sglang.runtime.v1.tonic.rs"); + } +} diff --git a/rust/sglang-api-types/testdata/json_parity.json b/rust/sglang-api-types/testdata/json_parity.json new file mode 100644 index 0000000000..369799ec73 --- /dev/null +++ b/rust/sglang-api-types/testdata/json_parity.json @@ -0,0 +1,373 @@ +{ + "_": "Cross-language JSON parity fixtures for sglang.api.v1. Each case: decoding `input` and re-encoding yields `canonical` byte for byte, and `canonical` re-encodes to itself. Checked by rust/sglang-api-types/tests/json_parity.rs and test/registered/unit/api/v1/test_parity.py.", + "cases": [ + { + "type": "GenerateRequest", + "name": "single text prompt with a sampling object; stream absent", + "input": { + "text": "hi", + "sampling_params": { + "temperature": 0.5, + "max_new_tokens": 8 + } + }, + "canonical": "{\"text\":\"hi\",\"stream\":false,\"sampling_params\":{\"max_new_tokens\":8,\"temperature\":0.5,\"top_p\":1.0,\"top_k\":1073741824,\"min_p\":0.0,\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"repetition_penalty\":1.0,\"min_new_tokens\":0,\"n\":1,\"ignore_eos\":false,\"skip_special_tokens\":true,\"spaces_between_special_tokens\":true,\"no_stop_trim\":false}}" + }, + { + "type": "GenerateRequest", + "name": "batch form: per-item sampling, broadcast flag, nullable PD columns", + "input": { + "text": [ + "a", + "b" + ], + "stream": true, + "sampling_params": [ + { + "top_p": 0.9 + }, + { + "temperature": 0.5 + } + ], + "return_logprob": true, + "bootstrap_host": [ + null, + "h" + ], + "bootstrap_port": [ + 8998, + null + ] + }, + "canonical": "{\"text\":[\"a\",\"b\"],\"stream\":true,\"sampling_params\":[{\"max_new_tokens\":128,\"temperature\":1.0,\"top_p\":0.9,\"top_k\":1073741824,\"min_p\":0.0,\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"repetition_penalty\":1.0,\"min_new_tokens\":0,\"n\":1,\"ignore_eos\":false,\"skip_special_tokens\":true,\"spaces_between_special_tokens\":true,\"no_stop_trim\":false},{\"max_new_tokens\":128,\"temperature\":0.5,\"top_p\":1.0,\"top_k\":1073741824,\"min_p\":0.0,\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"repetition_penalty\":1.0,\"min_new_tokens\":0,\"n\":1,\"ignore_eos\":false,\"skip_special_tokens\":true,\"spaces_between_special_tokens\":true,\"no_stop_trim\":false}],\"return_logprob\":true,\"bootstrap_host\":[null,\"h\"],\"bootstrap_port\":[8998,null]}" + }, + { + "type": "GenerateRequest", + "name": "token prompt with every logprob knob; stream null resets to false", + "input": { + "input_ids": [ + 1, + 2, + 3 + ], + "rid": "r-1", + "stream": null, + "return_logprob": true, + "logprob_start_len": 0, + "top_logprobs_num": 2, + "token_ids_logprob": [ + 5, + 6 + ], + "return_text_in_logprobs": true, + "return_hidden_states": true, + "routed_dp_rank": 1 + }, + "canonical": "{\"rid\":\"r-1\",\"input_ids\":[1,2,3],\"stream\":false,\"return_logprob\":true,\"logprob_start_len\":0,\"top_logprobs_num\":2,\"token_ids_logprob\":[5,6],\"return_hidden_states\":true,\"return_text_in_logprobs\":true,\"routed_dp_rank\":1}" + }, + { + "type": "GenerateRequest", + "name": "media containers: nested, ref object, bare source; unknown keys ignored", + "input": { + "text": "see", + "image_data": [ + [ + "a", + null + ], + null, + [] + ], + "video_data": { + "url": "v", + "fps": 2.0, + "use_audio": true + }, + "audio_data": "data:audio/wav;base64,AA==", + "mm_hashes": [ + "h1", + "h2" + ], + "priority": 3, + "session_id": "ignored" + }, + "canonical": "{\"text\":\"see\",\"stream\":false,\"image_data\":[[\"a\",null],null,[]],\"mm_hashes\":[\"h1\",\"h2\"],\"video_data\":{\"url\":\"v\",\"fps\":2.0,\"use_audio\":true},\"audio_data\":\"data:audio/wav;base64,AA==\"}" + }, + { + "type": "SamplingParams", + "name": "empty object: every schema default is emitted", + "input": {}, + "canonical": "{\"max_new_tokens\":128,\"temperature\":1.0,\"top_p\":1.0,\"top_k\":1073741824,\"min_p\":0.0,\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"repetition_penalty\":1.0,\"min_new_tokens\":0,\"n\":1,\"ignore_eos\":false,\"skip_special_tokens\":true,\"spaces_between_special_tokens\":true,\"no_stop_trim\":false}" + }, + { + "type": "SamplingParams", + "name": "explicit null resets a default; lists, maps and raw custom_params round-trip", + "input": { + "temperature": null, + "stop": [ + "a", + "b" + ], + "stop_token_ids": [ + 1, + 2 + ], + "logit_bias": { + "7": -1.5 + }, + "custom_params": { + "k": [ + 1, + "x", + null + ] + }, + "json_schema": "{}", + "beam_width": 2 + }, + "canonical": "{\"max_new_tokens\":128,\"stop\":[\"a\",\"b\"],\"stop_token_ids\":[1,2],\"temperature\":1.0,\"top_p\":1.0,\"top_k\":1073741824,\"min_p\":0.0,\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"repetition_penalty\":1.0,\"min_new_tokens\":0,\"n\":1,\"json_schema\":\"{}\",\"ignore_eos\":false,\"skip_special_tokens\":true,\"spaces_between_special_tokens\":true,\"no_stop_trim\":false,\"logit_bias\":{\"7\":-1.5},\"custom_params\":{\"k\":[1,\"x\",null]},\"beam_width\":2}" + }, + { + "type": "GenerateResponse", + "name": "plain frame: finish_reason null pre-terminal, output_ids present", + "input": { + "text": "ok", + "meta_info": { + "id": "client-rid", + "prompt_tokens": 5, + "completion_tokens": 2, + "finish_reason": null + }, + "output_ids": [ + 7, + 8 + ] + }, + "canonical": "{\"text\":\"ok\",\"meta_info\":{\"id\":\"client-rid\",\"prompt_tokens\":5,\"completion_tokens\":2,\"finish_reason\":null},\"output_ids\":[7,8]}" + }, + { + "type": "GenerateResponse", + "name": "terminal batch frame with every logprob family, null rows, hidden states, e2e_latency, index", + "input": { + "text": "ok", + "meta_info": { + "id": "r", + "prompt_tokens": 1, + "completion_tokens": 1, + "finish_reason": { + "type": "stop", + "matched": [ + 1, + 2 + ] + }, + "output_token_logprobs": [ + [ + -0.5, + 5, + "x" + ], + [ + null, + 6, + null + ] + ], + "input_token_logprobs": [], + "output_top_logprobs": [ + null, + [ + [ + -0.25, + 6, + null + ] + ] + ], + "input_top_logprobs": [ + [ + [ + -0.125, + 1, + "" + ] + ] + ], + "output_token_ids_logprobs": [ + [ + [ + -0.5, + 9, + null + ] + ] + ], + "input_token_ids_logprobs": [ + null + ], + "hidden_states": [ + [ + 0.5, + 0.25 + ], + [] + ], + "e2e_latency": 0.01 + }, + "output_ids": [ + 5 + ], + "index": 0 + }, + "canonical": "{\"text\":\"ok\",\"meta_info\":{\"id\":\"r\",\"prompt_tokens\":1,\"completion_tokens\":1,\"finish_reason\":{\"type\":\"stop\",\"matched\":[1,2]},\"output_token_logprobs\":[[-0.5,5,\"x\"],[null,6,null]],\"input_token_logprobs\":[],\"output_top_logprobs\":[null,[[-0.25,6,null]]],\"input_top_logprobs\":[[[-0.125,1,\"\"]]],\"output_token_ids_logprobs\":[[[-0.5,9,null]]],\"input_token_ids_logprobs\":[null],\"hidden_states\":[[0.5,0.25],[]],\"e2e_latency\":0.01},\"output_ids\":[5],\"index\":0}" + }, + { + "type": "GenerateResponse", + "name": "finish length", + "input": { + "text": "", + "meta_info": { + "id": "r", + "prompt_tokens": 1, + "completion_tokens": 9, + "finish_reason": { + "type": "length", + "length": 9 + } + } + }, + "canonical": "{\"text\":\"\",\"meta_info\":{\"id\":\"r\",\"prompt_tokens\":1,\"completion_tokens\":9,\"finish_reason\":{\"type\":\"length\",\"length\":9}}}" + }, + { + "type": "GenerateResponse", + "name": "finish abort emits its null keys", + "input": { + "text": "", + "meta_info": { + "id": "r", + "prompt_tokens": 1, + "completion_tokens": 0, + "finish_reason": { + "type": "abort", + "message": "over the limit", + "status_code": 400, + "err_type": null + } + } + }, + "canonical": "{\"text\":\"\",\"meta_info\":{\"id\":\"r\",\"prompt_tokens\":1,\"completion_tokens\":0,\"finish_reason\":{\"type\":\"abort\",\"message\":\"over the limit\",\"status_code\":400,\"err_type\":null}}}" + }, + { + "type": "GenerateResponse", + "name": "unknown finish type passes through unchanged", + "input": { + "text": "", + "meta_info": { + "id": "r", + "prompt_tokens": 1, + "completion_tokens": 0, + "finish_reason": { + "detail": 3, + "type": "novel" + } + } + }, + "canonical": "{\"text\":\"\",\"meta_info\":{\"id\":\"r\",\"prompt_tokens\":1,\"completion_tokens\":0,\"finish_reason\":{\"detail\":3,\"type\":\"novel\"}}}" + }, + { + "type": "GenerateStreamError", + "name": "in-stream error item with batch index", + "input": { + "error": { + "message": "bad sampling", + "code": 400 + }, + "index": 1 + }, + "canonical": "{\"error\":{\"message\":\"bad sampling\",\"code\":400},\"index\":1}" + }, + { + "type": "GetModelInfoResponse", + "name": "absent optionals are emitted as null", + "input": { + "model_path": "/m", + "served_model_name": "m", + "tokenizer_path": "/t", + "is_generation": true, + "weight_version": "v1" + }, + "canonical": "{\"model_path\":\"/m\",\"served_model_name\":\"m\",\"tokenizer_path\":\"/t\",\"is_generation\":true,\"preferred_sampling_params\":null,\"weight_version\":\"v1\",\"load_format\":null,\"reasoning_parser\":null,\"tool_call_parser\":null}" + }, + { + "type": "GetServerInfoResponse", + "name": "internal_states is a list of raw JSON documents", + "input": { + "model_path": "/m", + "served_model_name": "m", + "tokenizer_path": "/t", + "max_context_length": 4096, + "max_total_num_tokens": 8192, + "version": "0.5.0", + "internal_states": [ + { + "last_gen_throughput": 1.5, + "memory_usage": { + "weight": 2.0 + } + } + ] + }, + "canonical": "{\"model_path\":\"/m\",\"served_model_name\":\"m\",\"tokenizer_path\":\"/t\",\"max_context_length\":4096,\"max_total_num_tokens\":8192,\"version\":\"0.5.0\",\"internal_states\":[{\"last_gen_throughput\":1.5,\"memory_usage\":{\"weight\":2.0}}]}" + }, + { + "type": "SamplingParams", + "name": "null_is_none: an explicit max_new_tokens null (unbounded) survives the round trip as null", + "input": { + "max_new_tokens": null, + "n": 2 + }, + "canonical": "{\"max_new_tokens\":null,\"temperature\":1.0,\"top_p\":1.0,\"top_k\":1073741824,\"min_p\":0.0,\"frequency_penalty\":0.0,\"presence_penalty\":0.0,\"repetition_penalty\":1.0,\"min_new_tokens\":0,\"n\":2,\"ignore_eos\":false,\"skip_special_tokens\":true,\"spaces_between_special_tokens\":true,\"no_stop_trim\":false}" + }, + { + "type": "GetModelInfoResponse", + "name": "preferred_sampling_params is the launch object itself (raw JSON)", + "input": { + "model_path": "/m", + "served_model_name": "m", + "tokenizer_path": "/t", + "is_generation": true, + "preferred_sampling_params": { + "temperature": 0.5, + "max_new_tokens": 64 + }, + "weight_version": "v1" + }, + "canonical": "{\"model_path\":\"/m\",\"served_model_name\":\"m\",\"tokenizer_path\":\"/t\",\"is_generation\":true,\"preferred_sampling_params\":{\"temperature\":0.5,\"max_new_tokens\":64},\"weight_version\":\"v1\",\"load_format\":null,\"reasoning_parser\":null,\"tool_call_parser\":null}" + }, + { + "type": "GenerateResponse", + "name": "unknown finish type passes through with its key order", + "input": { + "text": "", + "meta_info": { + "id": "r", + "prompt_tokens": 1, + "completion_tokens": 0, + "finish_reason": { + "type": "novel", + "detail": 3, + "again": [ + 1, + { + "z": 1, + "a": 2 + } + ] + } + } + }, + "canonical": "{\"text\":\"\",\"meta_info\":{\"id\":\"r\",\"prompt_tokens\":1,\"completion_tokens\":0,\"finish_reason\":{\"type\":\"novel\",\"detail\":3,\"again\":[1,{\"z\":1,\"a\":2}]}}}" + } + ] +} diff --git a/rust/sglang-api-types/tests/json_contract.rs b/rust/sglang-api-types/tests/json_contract.rs new file mode 100644 index 0000000000..ac822f4469 --- /dev/null +++ b/rust/sglang-api-types/tests/json_contract.rs @@ -0,0 +1,346 @@ +//! Golden tests for the schema-driven JSON contract: each sglang.json.v1 +//! option template, pinned against the behavior of the hand-written types it +//! replaces (message/sampling.rs, message/request.rs, message/finish_reason.rs, +//! api_server/core/frame.rs). These fixtures are the contract's only gate, +//! so they keep the generator honest on their own. + +use sglang_api_types::api::v1::*; + +fn sp(json: &str) -> Result { + serde_json::from_str(json) +} + +/// Absent keys yield the schema defaults (sampling.rs `defaulted!` values). +#[test] +fn sampling_defaults_match_hand_written() { + let p = sp("{}").unwrap(); + assert_eq!(p.max_new_tokens, Some(128)); + assert_eq!(p.temperature, Some(1.0)); + assert_eq!(p.top_p, Some(1.0)); + assert_eq!(p.top_k, Some(1 << 30)); + assert_eq!(p.min_p, Some(0.0)); + assert_eq!(p.repetition_penalty, Some(1.0)); + assert_eq!(p.min_new_tokens, Some(0)); + assert_eq!(p.n, Some(1)); + assert_eq!(p.ignore_eos, Some(false)); + assert_eq!(p.skip_special_tokens, Some(true)); + assert_eq!(p.spaces_between_special_tokens, Some(true)); + assert_eq!(p.no_stop_trim, Some(false)); +} + +/// Protobuf arrival: a prost-decoded (all-unset) SamplingParams reads its +/// schema defaults through the accessors — the top_p=0 bug the first live +/// gRPC call found. +#[test] +fn proto_unset_reads_schema_defaults() { + let p = SamplingParams::default(); + assert_eq!(p.temperature_or_default(), 1.0); + assert_eq!(p.top_p_or_default(), 1.0); + assert_eq!(p.top_k_or_default(), 1 << 30); + assert_eq!(p.n_or_default(), 1); + assert!(p.skip_special_tokens_or_default()); + let mut p = SamplingParams::default(); + p.apply_absent_defaults(); + assert_eq!(p.max_new_tokens, Some(128)); + assert_eq!(p.top_p, Some(1.0)); +} + +/// null_resets_default: an explicit null is the default, not an error +/// (Python `__post_init__` parity). +#[test] +fn sampling_null_resets_default() { + let p = sp(r#"{"temperature": null, "top_k": null, "ignore_eos": null}"#).unwrap(); + assert_eq!(p.temperature, Some(1.0)); + assert_eq!(p.top_k, Some(1 << 30)); + assert_eq!(p.ignore_eos, Some(false)); +} + +/// max_new_tokens is the null_is_none field: absent = 128, null = unbounded. +#[test] +fn max_new_tokens_absent_vs_null() { + assert_eq!(sp("{}").unwrap().max_new_tokens, Some(128)); + assert_eq!( + sp(r#"{"max_new_tokens": null}"#).unwrap().max_new_tokens, + None + ); + assert_eq!( + sp(r#"{"max_new_tokens": 7}"#).unwrap().max_new_tokens, + Some(7) + ); +} + +/// SamplingParams DENIES unknown keys with serde's own error text (parity +/// with the hand-written `deny_unknown_fields`). +#[test] +fn sampling_denies_unknown_keys() { + let err = sp(r#"{"temperatur": 0.5}"#).unwrap_err().to_string(); // codespell:ignore temperatur + assert!( + err.starts_with("unknown field `temperatur`, expected one of"), // codespell:ignore temperatur + "{err}" + ); +} + +/// GenerateRequest IGNORES unknown keys (unported Python fields must not 400; +/// pinned in message/request.rs). +#[test] +fn generate_request_ignores_unknown_keys() { + let req: GenerateRequest = serde_json::from_str( + r#"{"text": "hi", "priority": 3, "session_id": "s", "custom_logit_processor": "x"}"#, + ) + .unwrap(); + assert!(matches!( + req.text, + Some(StringOrList { + value: Some(string_or_list::Value::One(ref s)) + }) if s == "hi" + )); +} + +/// one_or_many: a scalar and a list both parse; serialization flattens back. +#[test] +fn one_or_many_flattens() { + let one: StringOrList = serde_json::from_str(r#""a""#).unwrap(); + assert_eq!(serde_json::to_string(&one).unwrap(), r#""a""#); + let many: StringOrList = serde_json::from_str(r#"["a", "b"]"#).unwrap(); + assert_eq!(serde_json::to_string(&many).unwrap(), r#"["a","b"]"#); +} + +/// The TokenIds ambiguity: [1,2] is one id list, [[1],[2]] is a batch, [] is +/// one empty list (the hand-written untagged first-arm rule). +#[test] +fn token_ids_or_list_shape_dispatch() { + let one: TokenIdsOrList = serde_json::from_str("[1, 2]").unwrap(); + assert!(matches!( + one.value, + Some(token_ids_or_list::Value::One(ref t)) if t.ids == vec![1, 2] + )); + let many: TokenIdsOrList = serde_json::from_str("[[1], [2, 3]]").unwrap(); + assert!(matches!( + many.value, + Some(token_ids_or_list::Value::Many(ref l)) if l.items.len() == 2 + )); + let empty: TokenIdsOrList = serde_json::from_str("[]").unwrap(); + assert!(matches!( + empty.value, + Some(token_ids_or_list::Value::One(ref t)) if t.ids.is_empty() + )); +} + +/// Nullable list elements (PD bootstrap columns): [null, "h"] keeps per-item +/// presence; a bare scalar is the one-arm. +#[test] +fn optional_carriers_keep_element_nulls() { + let v: OptionalStringOrList = serde_json::from_str(r#"[null, "h"]"#).unwrap(); + assert_eq!(serde_json::to_string(&v).unwrap(), r#"[null,"h"]"#); + match v.value { + Some(optional_string_or_list::Value::Many(l)) => { + assert_eq!(l.items[0].value, None); + assert_eq!(l.items[1].value.as_deref(), Some("h")); + } + other => panic!("expected many, got {other:?}"), + } + let one: OptionalInt64OrList = serde_json::from_str("17000").unwrap(); + assert!(matches!( + one.value, + Some(optional_int64_or_list::Value::One(OptionalInt64 { + value: Some(17000) + })) + )); +} + +/// SamplingParams one-or-many keeps serde's field-level error texts (the +/// hand-written SamplingParamsInput contract: never "data did not match any +/// variant"). +#[test] +fn sampling_or_list_keeps_field_errors() { + let err = serde_json::from_str::(r#"{"temperatur": 1}"#) // codespell:ignore temperatur + .unwrap_err() + .to_string(); + assert!( + err.starts_with("unknown field `temperatur`"), // codespell:ignore temperatur + "field-level text lost: {err}" + ); +} + +/// finish_reason round-trips key-for-key, tagged by "type"; matched keeps its +/// wire shape (str / int / int list). +#[test] +fn finish_reason_round_trips() { + for json in [ + r#"{"type":"stop","matched":""}"#, + r#"{"type":"stop","matched":7}"#, + r#"{"type":"stop","matched":[1,2]}"#, + r#"{"type":"length","length":2}"#, + ] { + let parsed: FinishReason = serde_json::from_str(json).unwrap(); + let back: serde_json::Value = serde_json::to_value(&parsed).unwrap(); + let want: serde_json::Value = serde_json::from_str(json).unwrap(); + assert_eq!(back, want, "round trip of {json}"); + } +} + +/// An abort carries all three keys, nulls included (Python emits them). +#[test] +fn finish_abort_emits_null_keys() { + let parsed: FinishReason = serde_json::from_str(r#"{"type":"abort","message":"m"}"#).unwrap(); + let back = serde_json::to_string(&parsed).unwrap(); + let v: serde_json::Value = serde_json::from_str(&back).unwrap(); + assert_eq!(v["type"], "abort"); + assert_eq!(v["message"], "m"); + assert!(v.as_object().unwrap().contains_key("status_code")); + assert!(v["status_code"].is_null()); + assert!(v.as_object().unwrap().contains_key("err_type")); +} + +/// An unknown finish type round-trips unchanged (the forward-compat escape +/// hatch: a Python-side addition must not fail frame decode). +#[test] +fn unknown_finish_reason_passes_through() { + let json = r#"{"type":"paused","step":3}"#; + let parsed: FinishReason = serde_json::from_str(json).unwrap(); + assert!(matches!(parsed.kind, Some(finish_reason::Kind::Unknown(_)))); + let back: serde_json::Value = serde_json::to_value(&parsed).unwrap(); + let want: serde_json::Value = serde_json::from_str(json).unwrap(); + assert_eq!(back, want); +} + +/// A known tag with a malformed payload also falls back to passthrough (the +/// hand-written FinishReason's untagged-outer behavior). +#[test] +fn malformed_known_finish_reason_passes_through() { + let json = r#"{"type":"stop","matched":{"bad":"shape"}}"#; + let parsed: FinishReason = serde_json::from_str(json).unwrap(); + assert!(matches!(parsed.kind, Some(finish_reason::Kind::Unknown(_)))); +} + +/// Logprob entries are [logprob, token_id, text|null] tuples; the logprob +/// slot is nullable (Python emits null for the first prefill position). +#[test] +fn logprob_tuple_shape() { + let e = LogprobEntry { + logprob: Some(-0.25), + token_id: 7, + text: None, + }; + assert_eq!(serde_json::to_string(&e).unwrap(), "[-0.25,7,null]"); + let parsed: LogprobEntry = serde_json::from_str(r#"[-0.5, 9, "x"]"#).unwrap(); + assert_eq!(parsed.token_id, 9); + assert_eq!(parsed.text.as_deref(), Some("x")); + let first_prefill: LogprobEntry = serde_json::from_str(r#"[null, 10, null]"#).unwrap(); + assert_eq!(first_prefill.logprob, None); + assert_eq!( + serde_json::to_string(&first_prefill).unwrap(), + "[null,10,null]" + ); +} + +/// The /generate frame shape: key order (text, meta_info, output_ids), +/// conditional keys, finish_reason always present (null pre-terminal) -- +/// matching api_core/frame.rs's frame_value. +#[test] +fn generate_response_matches_frame_value_shape() { + let frame = GenerateResponse { + text: "ok".into(), + output_ids: Some(TokenIds { ids: vec![7, 8] }), + meta_info: Some(GenerateMetaInfo { + id: "client-rid".into(), + prompt_tokens: 5, + completion_tokens: 2, + finish_reason: None, + ..Default::default() + }), + index: None, + }; + assert_eq!( + serde_json::to_string(&frame).unwrap(), + r#"{"text":"ok","meta_info":{"id":"client-rid","prompt_tokens":5,"completion_tokens":2,"finish_reason":null},"output_ids":[7,8]}"# + ); +} + +/// Typed media input: every container form Python's `/generate` accepts, with +/// the value's shape picking the arm (string/object = one item, array = flat +/// list, array of arrays = per batch entry) and JSON null kept per element. +#[test] +fn media_input_shapes_dispatch() { + use media_input::Value; + let parse = |json: &str| -> MediaInput { serde_json::from_str(json).unwrap() }; + assert!(matches!(parse(r#""u""#).value, Some(Value::Item(_)))); + assert!(matches!( + parse(r#"{"url": "u", "detail": "high"}"#).value, + Some(Value::Item(_)) + )); + let flat = parse(r#"["a", null, {"url": "b"}]"#); + let Some(Value::Many(items)) = flat.value else { + panic!("flat list") + }; + assert_eq!(items.items.len(), 3); + assert!(items.items[1].item.is_none(), "null element kept"); + assert!( + matches!(parse("[]").value, Some(Value::Many(_))), + "[] is an empty flat list" + ); + let nested = parse(r#"[["a", null], null, []]"#); + let Some(Value::Nested(lists)) = nested.value else { + panic!("nested list") + }; + assert_eq!(lists.items.len(), 3); + assert!(lists.items[1].items.is_none(), "null entry kept"); + assert_eq!(lists.items[2].items.as_ref().unwrap().items.len(), 0); + // Round trip keeps the client's spelling. + for json in [ + r#""u""#, + r#"["a",null,{"url":"b"}]"#, + r#"[["a",null],null,[]]"#, + ] { + assert_eq!(serde_json::to_string(&parse(json)).unwrap(), json); + } +} + +/// A media object is a MediaRef with named hints; an unknown hint is an error +/// (it used to be silently dropped), and a preprocessed-input dict is not an +/// HTTP shape at all. +#[test] +fn media_ref_is_typed_and_strict() { + let r: MediaRef = + serde_json::from_str(r#"{"url": "u", "max_dynamic_patch": 6, "fps": 2.5}"#).unwrap(); + assert_eq!( + (r.url.as_str(), r.max_dynamic_patch, r.fps), + ("u", Some(6), Some(2.5)) + ); + let err = serde_json::from_str::(r#"{"url": "u", "max_dynamic_patc": 6}"#) + .unwrap_err() + .to_string(); + assert!(err.contains("unknown field `max_dynamic_patc`"), "{err}"); + let err = serde_json::from_str::(r#"[{"format": "processor_output"}]"#) + .unwrap_err() + .to_string(); + assert!(err.contains("unknown field `format`"), "{err}"); + let err = serde_json::from_str::(r#"["a", ["b"]]"#) + .unwrap_err() + .to_string(); + assert!(!err.is_empty(), "mixed flat/nested is rejected"); +} + +/// mm_hashes: a flat string list for one request, nested for a batch; the +/// first element decides (the bare-scalar-list rule, generalized past ids). +#[test] +fn string_list_or_list_shape_dispatch() { + use string_list_or_list::Value; + let parse = |json: &str| -> StringListOrList { serde_json::from_str(json).unwrap() }; + assert!(matches!(parse(r#"["a", "b"]"#).value, Some(Value::One(_)))); + assert!(matches!( + parse(r#"[["a"], ["b", "c"]]"#).value, + Some(Value::Many(_)) + )); + assert!(matches!(parse("[]").value, Some(Value::One(_)))); + let req: GenerateRequest = + serde_json::from_str(r#"{"mm_hashes": ["a1b2", "0xff"], "image_data": "u"}"#).unwrap(); + assert!(req.mm_hashes.is_some() && req.image_data.is_some()); +} + +/// custom_params is the one remaining raw_json passthrough on a request. +#[test] +fn custom_params_raw_json_passthrough() { + let p: SamplingParams = serde_json::from_str(r#"{"custom_params": {"k": 3}}"#).unwrap(); + assert_eq!(p.custom_params.as_deref(), Some(r#"{"k":3}"#)); +} diff --git a/rust/sglang-api-types/tests/json_parity.rs b/rust/sglang-api-types/tests/json_parity.rs new file mode 100644 index 0000000000..9a0ae0a700 --- /dev/null +++ b/rust/sglang-api-types/tests/json_parity.rs @@ -0,0 +1,50 @@ +//! Cross-language parity: the fixtures in `testdata/json_parity.json` are +//! checked here against the generated Rust types and, case for case, against +//! the generated Python types by `test/registered/unit/api/v1/test_parity.py`. +//! Both emitters must decode each `input` to the same `canonical` bytes (field +//! set, key order, presence and null rules) and keep `canonical` fixed under +//! a second round trip. + +use serde::{Serialize, de::DeserializeOwned}; +use sglang_api_types::api::v1 as api; + +const FIXTURES: &str = include_str!("../testdata/json_parity.json"); + +fn check(name: &str, input: &serde_json::Value, canonical: &str) { + let decoded: T = serde_json::from_value(input.clone()) + .unwrap_or_else(|error| panic!("{name}: input does not decode: {error}")); + assert_eq!( + serde_json::to_string(&decoded).unwrap(), + canonical, + "{name}: input -> canonical" + ); + let again: T = serde_json::from_str(canonical) + .unwrap_or_else(|error| panic!("{name}: canonical does not decode: {error}")); + assert_eq!( + serde_json::to_string(&again).unwrap(), + canonical, + "{name}: canonical is not a fixed point" + ); +} + +#[test] +fn fixtures_decode_to_their_canonical_bytes() { + let fixtures: serde_json::Value = serde_json::from_str(FIXTURES).unwrap(); + let cases = fixtures["cases"].as_array().expect("cases array"); + assert!(!cases.is_empty()); + for case in cases { + let ty = case["type"].as_str().expect("type"); + let name = format!("{ty}: {}", case["name"].as_str().expect("name")); + let input = &case["input"]; + let canonical = case["canonical"].as_str().expect("canonical"); + match ty { + "GenerateRequest" => check::(&name, input, canonical), + "SamplingParams" => check::(&name, input, canonical), + "GenerateResponse" => check::(&name, input, canonical), + "GenerateStreamError" => check::(&name, input, canonical), + "GetModelInfoResponse" => check::(&name, input, canonical), + "GetServerInfoResponse" => check::(&name, input, canonical), + other => panic!("fixture type {other} has no Rust mapping; add it here"), + } + } +} diff --git a/rust/sglang-grpc/Cargo.toml b/rust/sglang-grpc/Cargo.toml index 9be39b42b5..fca96206a9 100644 --- a/rust/sglang-grpc/Cargo.toml +++ b/rust/sglang-grpc/Cargo.toml @@ -9,9 +9,6 @@ license.workspace = true # of the main sglang wheel at the given import path. [package.metadata.sglang] python-module = "sglang.srt.rust_extensions._grpc" -# The shared protocol dependency lives outside rust/; include it in the -# source-loader artifact fingerprint. -source-inputs = ["../../proto"] # Always build optimized, even for an editable install. debug = false @@ -26,7 +23,7 @@ crate-type = ["cdylib"] [dependencies] async-stream = { workspace = true } pyo3 = { workspace = true } -sglang-grpc-types = { path = "../../proto" } +sglang-api-types = { path = "../sglang-api-types" } serde_json = { workspace = true } tokio = { workspace = true } tokio-stream = { workspace = true } @@ -38,5 +35,9 @@ uuid = { workspace = true } prost-types = { workspace = true } tokenizers = { version = "0.21", default-features = false, features = ["onig"] } +[dev-dependencies] +# `ProstCodec` for the hand-rolled unary call in the metadata-service test. +tonic-prost = { workspace = true } + [features] default = ["pyo3/extension-module"] diff --git a/rust/sglang-grpc/pyproject.toml b/rust/sglang-grpc/pyproject.toml index 0ab76d3cef..83fc607dea 100644 --- a/rust/sglang-grpc/pyproject.toml +++ b/rust/sglang-grpc/pyproject.toml @@ -1,9 +1,9 @@ # Standalone dev builds only (`maturin build` / `maturin develop`); the # extension ships to users inside the main sglang wheel via setuptools-rust # (python/pyproject.toml, target sglang.srt.rust_extensions._grpc). Its shared -# protocol dependency lives in ../../proto, so standalone builds require a -# repository checkout. A system protoc is unnecessary; that crate uses a -# vendored binary. +# protocol dependency is the sibling sglang-api-types crate, so standalone +# builds require a repository checkout. No protoc is needed: the bindings are +# generated ahead of time and checked in. [build-system] requires = ["maturin>=1.5,<2"] build-backend = "maturin" diff --git a/rust/sglang-grpc/src/server.rs b/rust/sglang-grpc/src/server.rs index ce5e5763df..ba7cd6aead 100644 --- a/rust/sglang-grpc/src/server.rs +++ b/rust/sglang-grpc/src/server.rs @@ -17,7 +17,7 @@ use crate::utils::{ build_classify_dict, build_embed_dict, build_generate_dict, build_text_embed_dict, build_text_generate_dict, extract_model_path, }; -use sglang_grpc_types::sglang::runtime::v1 as proto; +use sglang_api_types::runtime::v1 as proto; pub struct SglangServiceImpl { pub bridge: Arc, diff --git a/rust/sglang-grpc/src/server/tests.rs b/rust/sglang-grpc/src/server/tests.rs index 53b0b0e794..a197ef02bf 100644 --- a/rust/sglang-grpc/src/server/tests.rs +++ b/rust/sglang-grpc/src/server/tests.rs @@ -81,8 +81,8 @@ fn resolve_max_message_size_honors_env_var() { #[tokio::test] async fn follower_metadata_server_exposes_only_server_info_and_shuts_down() { - use sglang_grpc_types::sglang::runtime::v1::{GetServerInfoRequest, GetServerInfoResponse}; - use tonic::codec::ProstCodec; + use sglang_api_types::runtime::v1::{GetServerInfoRequest, GetServerInfoResponse}; + use tonic_prost::ProstCodec; let _guard = PAYLOAD_ENV_LOCK.lock().await; diff --git a/rust/sglang-grpc/src/utils/request_utils.rs b/rust/sglang-grpc/src/utils/request_utils.rs index 8697d8d716..06f7e9e915 100644 --- a/rust/sglang-grpc/src/utils/request_utils.rs +++ b/rust/sglang-grpc/src/utils/request_utils.rs @@ -1,6 +1,6 @@ use std::collections::HashMap; -use sglang_grpc_types::sglang::runtime::v1 as proto; +use sglang_api_types::runtime::v1 as proto; fn regex_escape_literal(value: &str) -> String { let mut escaped = String::with_capacity(value.len()); diff --git a/rust/sglang-server/Cargo.toml b/rust/sglang-server/Cargo.toml index 58a0feb28e..ffcecfe7b4 100644 --- a/rust/sglang-server/Cargo.toml +++ b/rust/sglang-server/Cargo.toml @@ -10,9 +10,6 @@ license.workspace = true python-module = "sglang.srt.rust_extensions._server" # Always build optimized, even for an editable install. debug = false -# The shared runtime.v1 crate lives outside rust/; include it in extension -# source fingerprints so protobuf changes cannot reuse a stale _server build. -source-inputs = ["../../proto"] [lib] name = "sglang_server" @@ -51,6 +48,7 @@ dynamo-renderer = "5.1.2" minijinja = "2.24" flume = "0.12.0" hf-hub = { version = "0.4", default-features = false } +http = "1.5.0" itertools = "0.14" # POSIX shm (`shm_open` family) for the MM feature fan-out (`utils::shm`). rustix = { version = "1", features = ["fs", "mm", "shm"] } @@ -63,13 +61,13 @@ numpy = "0.29.0" regex-syntax = "=0.8.11" rmp-serde = "1" rmpv = { version = "1", features = ["with-serde"] } -sglang-grpc-types = { path = "../../proto" } # The pure-Rust core of the MM pipeline. `default-features = false` drops the # pyo3 bindings so it links as a plain rlib; renamed so `use sglang_mm::…` reads # naturally while the crate keeps its own artifact name in the shared target/. sglang_mm = { package = "sglang-mm", path = "../sglang-mm", default-features = false } tonic = { workspace = true } tokio-stream = { workspace = true } +sglang_api_types = { package = "sglang-api-types", path = "../sglang-api-types" } [dev-dependencies] # `Router::oneshot` for handler-level router tests. diff --git a/rust/sglang-server/src/api_server.rs b/rust/sglang-server/src/api_server.rs index c23ced6f3a..e4db9447cc 100644 --- a/rust/sglang-server/src/api_server.rs +++ b/rust/sglang-server/src/api_server.rs @@ -1,83 +1,10 @@ -//! API server (axum / tokio). I/O-bound; own pinned multi-thread runtime. Only -//! this module knows HTTP. Its handlers use the transport-neutral -//! [`crate::frontend::FrontendHandle`] to enter the shared runtime pipeline. -//! Generation handlers render semantic frontend events as unary JSON or SSE; -//! control handlers serialize typed frontend results such as server metadata. -pub mod app; -mod common; +//! API server (axum / tokio). I/O-bound; own pinned multi-thread runtime. The +//! transport-neutral [`core::CoreHandle`] is the shared entry into the runtime +//! pipeline; `http` and `grpc` are the wire adapters on top of it. Generation +//! handlers render semantic core events as unary JSON, SSE, or gRPC frames; +//! control handlers serialize typed core results such as server metadata. +pub(crate) mod core; mod disaggregation; -mod frame; +pub(crate) mod grpc; +pub(crate) mod http; mod log; -mod native_api; -mod openai; - -use axum::http::StatusCode; - -use crate::frontend::{FrontendError, FrontendErrorKind}; - -/// Map transport-neutral failures into HTTP status codes. Response bodies and -/// stream framing remain the responsibility of each HTTP API surface. -pub(crate) fn frontend_error_status(error: &FrontendError) -> StatusCode { - if let FrontendError::RuntimeRejected { - legacy_http_status, .. - } = error - { - return StatusCode::from_u16(*legacy_http_status) - .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); - } - - match error.kind() { - FrontendErrorKind::InvalidArgument => StatusCode::BAD_REQUEST, - FrontendErrorKind::NotFound => StatusCode::NOT_FOUND, - FrontendErrorKind::FailedPrecondition => StatusCode::PRECONDITION_FAILED, - // Existing intake backpressure has historically surfaced as 503. - FrontendErrorKind::ResourceExhausted | FrontendErrorKind::Unavailable => { - StatusCode::SERVICE_UNAVAILABLE - } - FrontendErrorKind::Cancelled => StatusCode::from_u16(499).expect("499 is valid"), - FrontendErrorKind::DeadlineExceeded => StatusCode::GATEWAY_TIMEOUT, - FrontendErrorKind::Internal => StatusCode::INTERNAL_SERVER_ERROR, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn frontend_error_categories_map_at_the_http_boundary() { - assert_eq!( - frontend_error_status(&FrontendError::InvalidArgument("bad".into())), - StatusCode::BAD_REQUEST - ); - assert_eq!( - frontend_error_status(&FrontendError::Unavailable), - StatusCode::SERVICE_UNAVAILABLE - ); - assert_eq!( - frontend_error_status(&FrontendError::Overloaded("full".into())), - StatusCode::SERVICE_UNAVAILABLE - ); - assert_eq!( - frontend_error_status(&FrontendError::Cancelled("gone".into())).as_u16(), - 499 - ); - assert_eq!( - frontend_error_status(&FrontendError::RuntimeRejected { - kind: FrontendErrorKind::InvalidArgument, - message: "runtime rejected request".into(), - legacy_http_status: 432, - }) - .as_u16(), - 432 - ); - assert_eq!( - frontend_error_status(&FrontendError::InvalidResponse("bad reply".into())), - StatusCode::INTERNAL_SERVER_ERROR - ); - assert_eq!( - frontend_error_status(&FrontendError::Internal("bug".into())), - StatusCode::INTERNAL_SERVER_ERROR - ); - } -} diff --git a/rust/sglang-server/src/api_server/core.rs b/rust/sglang-server/src/api_server/core.rs new file mode 100644 index 0000000000..ff63f3be47 --- /dev/null +++ b/rust/sglang-server/src/api_server/core.rs @@ -0,0 +1,18 @@ +//! Transport-neutral core shared by the API adapters: the entry point into the Rust request pipeline. +//! +//! Wire adapters normalize their protocol into the in-process +//! [`GenerateRequest`](crate::message::request::GenerateRequest), submit it +//! through [`CoreHandle`], and render semantic [`CoreEvent`]s for their +//! own transport. This module owns shared preprocessing, runtime translation, +//! capabilities, request lifetime, and the rendering of outputs into the +//! generated `api.v1` response types that both adapters put on the wire; it +//! deliberately knows nothing about Axum, SSE framing, or Tonic. + +mod error; +mod frame; +mod handle; +mod prefetch; + +pub(crate) use error::{CoreError, CoreErrorKind}; +pub(crate) use frame::{CoreEvent, CoreOutput, HealthStatus}; +pub(crate) use handle::{CoreCall, CoreConfig, CoreHandle, CoreMetadata, recv_indexed}; diff --git a/rust/sglang-server/src/api_server/core/error.rs b/rust/sglang-server/src/api_server/core/error.rs new file mode 100644 index 0000000000..804a59aeae --- /dev/null +++ b/rust/sglang-server/src/api_server/core/error.rs @@ -0,0 +1,186 @@ +//! Transport-neutral failure categories, and how each wire adapter maps them +//! into its own status space. + +use http::StatusCode; +use sglang_api_types::api::v1 as api; + +/// Protocol-independent error category used by transport adapters. +/// +/// The set intentionally follows operation semantics rather than HTTP or gRPC +/// status spaces. An adapter maps this category into its own wire status. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum CoreErrorKind { + InvalidArgument, + NotFound, + FailedPrecondition, + ResourceExhausted, + Cancelled, + DeadlineExceeded, + Unavailable, + Internal, +} + +/// A transport-neutral failure category. +/// +/// HTTP and gRPC adapters map these categories into their own status spaces; +/// neither adapter needs to understand errors emitted by runtime stages. +#[derive(Clone, Debug, thiserror::Error)] +pub(crate) enum CoreError { + #[error("{0}")] + InvalidArgument(String), + + /// The intake loop has shut down. A caller may retry elsewhere. + #[error("service unavailable")] + Unavailable, + + /// Runtime capacity was exhausted before the request could make progress. + #[error("{0}")] + Overloaded(String), + + #[error("{0}")] + Cancelled(String), + + /// The internal producer disappeared before returning a terminal result. + /// + /// This is a server-side failure, not evidence that the caller cancelled. + #[error("response truncated before completion")] + ResponseTruncated, + + /// A scheduler-side abort that carried an error status. + /// + /// The current Python scheduler communicates this category as an HTTP + /// status code inside its finish reason. The semantic event is still + /// `Failed`; retaining the legacy code only lets the HTTP adapter preserve + /// existing behavior while another adapter maps it into its own status + /// space. New core failures should use the semantic variants above. + #[error("{message}")] + RuntimeRejected { + kind: CoreErrorKind, + message: String, + legacy_http_status: u16, + }, + + /// The runtime replied with an unexpected or malformed result. + #[error("{0}")] + InvalidResponse(String), + + #[error("{0}")] + Internal(String), +} + +impl CoreError { + /// Semantic status for adapters that do not use the scheduler's legacy + /// HTTP code space (notably the future gRPC adapter). + pub(crate) fn kind(&self) -> CoreErrorKind { + match self { + Self::InvalidArgument(_) => CoreErrorKind::InvalidArgument, + Self::Unavailable => CoreErrorKind::Unavailable, + Self::Overloaded(_) => CoreErrorKind::ResourceExhausted, + Self::Cancelled(_) => CoreErrorKind::Cancelled, + Self::ResponseTruncated => CoreErrorKind::Internal, + Self::RuntimeRejected { kind, .. } => *kind, + Self::InvalidResponse(_) | Self::Internal(_) => CoreErrorKind::Internal, + } + } + + /// Translate the scheduler's inherited HTTP-coded failure into a semantic + /// category once, at the runtime boundary. HTTP can retain the exact legacy + /// status while other adapters consume [`Self::kind`]. + pub(super) fn from_runtime_rejection(message: String, legacy_http_status: u16) -> Self { + let kind = match legacy_http_status { + 404 => CoreErrorKind::NotFound, + 408 | 504 => CoreErrorKind::DeadlineExceeded, + 412 => CoreErrorKind::FailedPrecondition, + 413 | 429 => CoreErrorKind::ResourceExhausted, + 499 => CoreErrorKind::Cancelled, + 502 | 503 => CoreErrorKind::Unavailable, + 400..=499 => CoreErrorKind::InvalidArgument, + _ => CoreErrorKind::Internal, + }; + Self::RuntimeRejected { + kind, + message, + legacy_http_status, + } + } + + /// The HTTP status for this failure. Response bodies and stream framing + /// remain the responsibility of each HTTP API surface. + /// This failure as one `/generate` item: the native error body with the + /// HTTP status as `code`, plus the batch position where there is one. + pub(crate) fn stream_error(&self, index: Option) -> api::GenerateStreamError { + api::GenerateStreamError { + error: Some(api::ErrorBody { + message: self.to_string(), + code: u32::from(self.http_status().as_u16()), + }), + index, + } + } + + pub(crate) fn http_status(&self) -> StatusCode { + if let CoreError::RuntimeRejected { + legacy_http_status, .. + } = self + { + return StatusCode::from_u16(*legacy_http_status) + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + } + + match self.kind() { + CoreErrorKind::InvalidArgument => StatusCode::BAD_REQUEST, + CoreErrorKind::NotFound => StatusCode::NOT_FOUND, + CoreErrorKind::FailedPrecondition => StatusCode::PRECONDITION_FAILED, + // Existing intake backpressure has historically surfaced as 503. + CoreErrorKind::ResourceExhausted | CoreErrorKind::Unavailable => { + StatusCode::SERVICE_UNAVAILABLE + } + CoreErrorKind::Cancelled => StatusCode::from_u16(499).expect("499 is valid"), + CoreErrorKind::DeadlineExceeded => StatusCode::GATEWAY_TIMEOUT, + CoreErrorKind::Internal => StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn core_error_categories_map_at_the_http_boundary() { + assert_eq!( + CoreError::InvalidArgument("bad".into()).http_status(), + StatusCode::BAD_REQUEST + ); + assert_eq!( + CoreError::Unavailable.http_status(), + StatusCode::SERVICE_UNAVAILABLE + ); + assert_eq!( + CoreError::Overloaded("full".into()).http_status(), + StatusCode::SERVICE_UNAVAILABLE + ); + assert_eq!( + CoreError::Cancelled("gone".into()).http_status().as_u16(), + 499 + ); + assert_eq!( + CoreError::RuntimeRejected { + kind: CoreErrorKind::InvalidArgument, + message: "runtime rejected request".into(), + legacy_http_status: 432, + } + .http_status() + .as_u16(), + 432 + ); + assert_eq!( + CoreError::InvalidResponse("bad reply".into()).http_status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + assert_eq!( + CoreError::Internal("bug".into()).http_status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + } +} diff --git a/rust/sglang-server/src/api_server/core/frame.rs b/rust/sglang-server/src/api_server/core/frame.rs new file mode 100644 index 0000000000..90a82c9bb4 --- /dev/null +++ b/rust/sglang-server/src/api_server/core/frame.rs @@ -0,0 +1,457 @@ +//! The semantic events a call yields (incremental and terminal outputs, a +//! request-scoped failure, the health probe's result) and the rendering of an +//! output into the `api.v1` response types. The generated schema is the wire +//! shape for both adapters: HTTP serializes it through serde, gRPC encodes it +//! as protobuf, so a column added here reaches both the same way. No Axum, +//! SSE, Tonic, or scheduler-channel details. + +use sglang_api_types::api::v1 as api; + +use super::CoreError; +use crate::message::finish_reason::{FinishKind, FinishReason, Matched}; +use crate::message::response::{ChunkEvent, ChunkExtras}; +use crate::message::types::TokenIds; + +/// Canonical transport-neutral output payload for generation events. +/// +/// The runtime correlation ID is deliberately absent: it is an implementation +/// detail used for scheduler routing, while adapters obtain the client-visible +/// identity from [`crate::api_server::core::CoreCall::public_id`]. Runtime-only +/// response variants such as raw control bytes are likewise hidden by +/// [`crate::api_server::core::CoreCall`]. +#[derive(Clone, Debug, Default)] +pub(crate) struct CoreOutput { + pub(crate) token_ids: TokenIds, + pub(crate) finish_reason: Option, + pub(crate) prompt_tokens: u32, + pub(crate) text: String, + pub(crate) completion_tokens: u64, + pub(crate) extras: Option>, +} + +impl From for CoreOutput { + fn from(output: ChunkEvent) -> Self { + let ChunkEvent { + rid: _, + token_ids, + finish_reason, + prompt_tokens, + text, + completion_tokens, + extras, + } = output; + Self { + token_ids, + finish_reason, + prompt_tokens, + text, + completion_tokens, + extras, + } + } +} + +impl CoreOutput { + /// Fold one runtime delta into this cumulative native-generation output. + /// + /// Runtime events are always incremental. Native HTTP and `api.v1` + /// gRPC independently decide whether to expose those deltas or the + /// cumulative result, so the fold itself lives at their shared semantic + /// boundary rather than in either wire adapter. + pub(crate) fn append_delta(&mut self, delta: &Self) { + self.text.push_str(&delta.text); + self.token_ids.extend_from_slice(&delta.token_ids); + self.completion_tokens += delta.completion_tokens; + self.prompt_tokens = delta.prompt_tokens; + if delta.finish_reason.is_some() { + self.finish_reason = delta.finish_reason.clone(); + } + + let Some(delta_extras) = delta.extras.as_deref() else { + return; + }; + let extras = self + .extras + .get_or_insert_with(|| Box::new(ChunkExtras::default())); + + extras + .out_lp_val + .extend_from_slice(&delta_extras.out_lp_val); + extras + .out_lp_idx + .extend_from_slice(&delta_extras.out_lp_idx); + extras + .out_top_val + .extend_from_slice(&delta_extras.out_top_val); + extras + .out_top_idx + .extend_from_slice(&delta_extras.out_top_idx); + extras + .out_top_lens + .extend_from_slice(&delta_extras.out_top_lens); + extras + .out_tid_val + .extend_from_slice(&delta_extras.out_tid_val); + extras + .out_tid_idx + .extend_from_slice(&delta_extras.out_tid_idx); + extras + .out_tid_lens + .extend_from_slice(&delta_extras.out_tid_lens); + extras + .out_lp_txt + .extend_from_slice(&delta_extras.out_lp_txt); + extras + .out_top_txt + .extend_from_slice(&delta_extras.out_top_txt); + extras + .out_tid_txt + .extend_from_slice(&delta_extras.out_tid_txt); + + // Input logprobs arrive once (during prefill). A later non-empty value + // replaces the earlier one, matching the existing native HTTP fold. + if !delta_extras.in_lp_val.is_empty() { + extras.in_lp_val = delta_extras.in_lp_val.clone(); + extras.in_lp_idx = delta_extras.in_lp_idx.clone(); + extras.in_lp_txt = delta_extras.in_lp_txt.clone(); + } + if !delta_extras.in_top_lens.is_empty() { + extras.in_top_val = delta_extras.in_top_val.clone(); + extras.in_top_idx = delta_extras.in_top_idx.clone(); + extras.in_top_lens = delta_extras.in_top_lens.clone(); + extras.in_top_txt = delta_extras.in_top_txt.clone(); + } + if !delta_extras.in_tid_lens.is_empty() { + extras.in_tid_val = delta_extras.in_tid_val.clone(); + extras.in_tid_idx = delta_extras.in_tid_idx.clone(); + extras.in_tid_lens = delta_extras.in_tid_lens.clone(); + extras.in_tid_txt = delta_extras.in_tid_txt.clone(); + } + // Hidden states are non-cumulative: the latest complete set wins. + if !delta_extras.hidden_lens.is_empty() { + extras.hidden_val = delta_extras.hidden_val.clone(); + extras.hidden_lens = delta_extras.hidden_lens.clone(); + } + } +} + +/// Semantic result of a deep core health check. +/// +/// Expected lifecycle states are values rather than transport errors so each +/// adapter can render them according to its own protocol. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum HealthStatus { + Healthy, + NotReady, + Stalled, +} + +/// A semantic generation event before any transport-specific framing. +#[derive(Debug)] +pub(crate) enum CoreEvent { + /// One incremental model-output update. + Delta(CoreOutput), + /// The final model-output update for this request. + Finished(CoreOutput), + /// A request-scoped failure. Other requests in a batch may continue. + Failed(CoreError), +} + +impl CoreEvent { + pub(crate) fn is_terminal(&self) -> bool { + matches!(self, Self::Finished(_) | Self::Failed(_)) + } +} + +impl CoreOutput { + /// One `/generate` frame. `index` is the batch position (batch form only); + /// `e2e_latency` rides the terminal frame only. + /// + /// Consumes the output so `text` and `output_ids` move onto the wire: a + /// delta or terminal frame never copies them. A cumulative intermediate + /// frame is the one caller that must clone first, since the accumulated + /// output has to stay behind for the next frame. + pub(crate) fn frame( + self, + id: &str, + index: Option, + e2e_latency: Option, + ) -> api::GenerateResponse { + let meta_info = Some(self.meta_info(id, e2e_latency)); + api::GenerateResponse { + text: self.text, + meta_info, + // Omitted while empty (a text-only frame). + output_ids: (!self.token_ids.is_empty()).then_some(api::TokenIds { + ids: self.token_ids, + }), + index, + } + } + + pub(crate) fn meta_info(&self, id: &str, e2e_latency: Option) -> api::GenerateMetaInfo { + let mut meta = api::GenerateMetaInfo { + id: id.to_owned(), + prompt_tokens: self.prompt_tokens, + completion_tokens: self.completion_tokens, + finish_reason: self.finish_reason.as_ref().map(finish_reason), + e2e_latency, + ..Default::default() + }; + let Some(extras) = self.extras.as_deref() else { + return meta; + }; + + // Python always sets input and output token logprobs together, including + // the empty input list emitted by a disaggregated decode node. + if !extras.out_lp_val.is_empty() || !extras.in_lp_val.is_empty() { + meta.output_token_logprobs = Some(logprob_entries( + &extras.out_lp_val, + &extras.out_lp_idx, + opt_texts(&extras.out_lp_txt), + )); + meta.input_token_logprobs = Some(logprob_entries( + &extras.in_lp_val, + &extras.in_lp_idx, + opt_texts(&extras.in_lp_txt), + )); + } + if !extras.out_top_lens.is_empty() { + meta.output_top_logprobs = ragged_logprob_rows( + &extras.out_top_val, + &extras.out_top_idx, + &extras.out_top_lens, + opt_texts(&extras.out_top_txt), + ); + } + if !extras.in_top_lens.is_empty() { + meta.input_top_logprobs = ragged_logprob_rows( + &extras.in_top_val, + &extras.in_top_idx, + &extras.in_top_lens, + opt_texts(&extras.in_top_txt), + ); + } + if !extras.out_tid_lens.is_empty() { + meta.output_token_ids_logprobs = ragged_logprob_rows( + &extras.out_tid_val, + &extras.out_tid_idx, + &extras.out_tid_lens, + opt_texts(&extras.out_tid_txt), + ); + } + if !extras.in_tid_lens.is_empty() { + meta.input_token_ids_logprobs = ragged_logprob_rows( + &extras.in_tid_val, + &extras.in_tid_idx, + &extras.in_tid_lens, + opt_texts(&extras.in_tid_txt), + ); + } + if !extras.hidden_lens.is_empty() { + meta.hidden_states = hidden_state_rows(&extras.hidden_val, &extras.hidden_lens); + } + meta + } +} + +/// A decoded-text column becomes the entries' text source only when populated. +fn opt_texts(texts: &[String]) -> Option<&[String]> { + (!texts.is_empty()).then_some(texts) +} + +/// One `[logprob, token_id, text]` entry. +fn logprob_entry(value: f32, token_id: i32, text: Option<&String>) -> api::LogprobEntry { + api::LogprobEntry { + // The scheduler's NaN sentinel means "no logprob at this position". + logprob: (!value.is_nan()).then(|| f64::from(value)), + token_id: i64::from(token_id), + text: text.cloned(), + } +} + +fn logprob_entries( + values: &[f32], + token_ids: &[i32], + texts: Option<&[String]>, +) -> api::LogprobEntries { + let entries = values + .iter() + .zip(token_ids) + .enumerate() + .map(|(j, (&value, &token_id))| { + logprob_entry(value, token_id, texts.and_then(|texts| texts.get(j))) + }) + .collect(); + api::LogprobEntries { entries } +} + +/// Ragged top-k / requested-token rows: one per position, null where that +/// position's length is zero. +fn ragged_logprob_rows( + values: &[f32], + token_ids: &[i32], + lengths: &[u32], + texts: Option<&[String]>, +) -> Vec { + let mut rows = Vec::with_capacity(lengths.len()); + let mut offset = 0usize; + for &length in lengths { + let length = length as usize; + let row = (length > 0).then(|| { + // Bounds checks turn malformed parallel columns into a shortened + // row instead of panicking a transport worker. + let entries = (offset..offset + length) + .filter_map(|j| { + Some(logprob_entry( + *values.get(j)?, + *token_ids.get(j)?, + texts.and_then(|texts| texts.get(j)), + )) + }) + .collect(); + api::TopLogprobRow { entries } + }); + rows.push(api::NullableTopLogprobs { row }); + offset += length; + } + rows +} + +/// Reshape flat hidden-state values and per-row lengths into rows. +fn hidden_state_rows(values: &[f32], lengths: &[u32]) -> Vec { + let mut rows = Vec::with_capacity(lengths.len()); + let mut offset = 0usize; + for &length in lengths { + let length = length as usize; + let values = values + .get(offset..offset + length) + .unwrap_or(&[]) + .iter() + .copied() + .map(f64::from) + .collect(); + rows.push(api::HiddenStateRow { values }); + offset += length; + } + rows +} + +fn finish_reason(reason: &FinishReason) -> api::FinishReason { + use api::finish_reason::Kind; + let kind = match reason { + FinishReason::Known(FinishKind::Stop { matched }) => Kind::Stop(api::FinishStop { + matched: matched.as_ref().map(matched_value), + }), + FinishReason::Known(FinishKind::Length { length }) => { + Kind::Length(api::FinishLength { length: *length }) + } + FinishReason::Known(FinishKind::Abort(abort)) => Kind::Abort(Box::new(api::FinishAbort { + message: abort.message.clone(), + status_code: abort.status_code.map(u32::from), + err_type: abort.err_type.clone(), + })), + // The passthrough arm keeps the raw map as JSON text, the proto's own + // encoding for a `type` this build does not know. + FinishReason::Unknown(fields) => { + Kind::Unknown(serde_json::to_string(fields.as_ref()).unwrap_or_default()) + } + }; + api::FinishReason { kind: Some(kind) } +} + +fn matched_value(matched: &Matched) -> api::Matched { + use api::matched::Value; + let value = match matched { + Matched::Token(token) => Value::Token(*token), + Matched::Str(text) => Value::Str(text.clone()), + Matched::Tokens(ids) => Value::Tokens(api::MatchedTokens { ids: ids.clone() }), + }; + api::Matched { value: Some(value) } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::message::response::ChunkExtras; + + fn json(value: &T) -> serde_json::Value { + serde_json::to_value(value).unwrap() + } + + /// Python parity pinned on the typed shape: a NaN logprob is a null slot, + /// input logprobs are set together with output ones, and a zero-length + /// top-k position is a null row rather than an empty one. + #[test] + fn meta_info_keeps_the_native_null_conventions() { + let extras = ChunkExtras { + out_lp_val: vec![f32::NAN, -0.5], + out_lp_idx: vec![7, 8], + out_top_val: vec![-0.1], + out_top_idx: vec![9], + out_top_lens: vec![0, 1], + ..Default::default() + }; + let output = CoreOutput { + extras: Some(Box::new(extras)), + ..Default::default() + }; + + let meta = output.meta_info("rid", None); + let entries = meta.output_token_logprobs.unwrap().entries; + assert_eq!(entries[0].logprob, None); + assert_eq!(entries[1].logprob, Some(f64::from(-0.5_f32))); + assert_eq!(entries[1].token_id, 8); + assert_eq!(meta.input_token_logprobs.unwrap().entries, []); + assert!(meta.output_top_logprobs[0].row.is_none()); + let row = meta.output_top_logprobs[1].row.as_ref().unwrap(); + assert_eq!(row.entries[0].token_id, 9); + assert!(meta.input_top_logprobs.is_empty()); + } + + /// The JSON the schema renders is SGLang's `[logprob, token_id, text]` + /// tuple list, with the text slot null unless `return_text_in_logprobs` + /// supplied a text column, and null positions in the ragged families. + #[test] + fn logprob_json_keeps_the_tuple_shape() { + let texts = vec!["a".to_string(), "b".to_string()]; + assert_eq!( + json(&logprob_entries(&[-0.5, -1.5], &[10, 20], None)), + serde_json::json!([[-0.5f32, 10, null], [-1.5f32, 20, null]]) + ); + assert_eq!( + json(&logprob_entries(&[-0.5, -1.5], &[10, 20], Some(&texts))), + serde_json::json!([[-0.5f32, 10, "a"], [-1.5f32, 20, "b"]]) + ); + assert_eq!( + json(&ragged_logprob_rows(&[-0.3], &[9], &[0, 1], None)), + serde_json::json!([null, [[-0.3f32, 9, null]]]) + ); + // The NaN sentinel (the Python `None` logprob for the first prompt + // token) is a null logprob with its token id preserved. + assert_eq!( + json(&ragged_logprob_rows(&[f32::NAN], &[7], &[1], None)), + serde_json::json!([[[null, 7, null]]]) + ); + assert!(opt_texts(&[]).is_none()); + assert_eq!(opt_texts(&texts), Some(texts.as_slice())); + } + + /// A finish type this build does not know still reaches the client, as + /// the JSON text the proto's passthrough arm carries. + #[test] + fn unknown_finish_reason_passes_through_as_json_text() { + let reason: FinishReason = + serde_json::from_value(serde_json::json!({"type": "novel", "detail": 3})).unwrap(); + let api::FinishReason { + kind: Some(api::finish_reason::Kind::Unknown(text)), + } = finish_reason(&reason) + else { + panic!("unknown type takes the passthrough arm"); + }; + assert_eq!( + serde_json::from_str::(&text).unwrap(), + serde_json::json!({"type": "novel", "detail": 3}) + ); + } +} diff --git a/rust/sglang-server/src/frontend.rs b/rust/sglang-server/src/api_server/core/handle.rs similarity index 84% rename from rust/sglang-server/src/frontend.rs rename to rust/sglang-server/src/api_server/core/handle.rs index c7f1ea220c..4cecaa397b 100644 --- a/rust/sglang-server/src/frontend.rs +++ b/rust/sglang-server/src/api_server/core/handle.rs @@ -1,10 +1,6 @@ -//! Transport-neutral entry point into the Rust frontend pipeline. -//! -//! Wire adapters normalize their protocol into [`FrontendRequest`], submit it -//! through [`FrontendHandle`], and render semantic [`FrontendEvent`]s for their -//! own transport. This module owns shared preprocessing, runtime translation, -//! capabilities, and request lifetime; it deliberately knows nothing about -//! Axum, HTTP response shapes, Tonic, or protobuf. +//! The shared handle: submission with admission and cancellation ownership, +//! runtime-to-semantic event translation, and the typed control operations +//! (model info, server info, detokenize, health probe). use std::collections::BTreeMap; use std::sync::{ @@ -18,7 +14,7 @@ use tokio::sync::mpsc; use crate::message::config::{DisaggregationMode, ServerArgs}; use crate::message::ids::Rid; use crate::message::io_struct::{ControlRequest, GetInternalStateReq}; -use crate::message::request::{Request, RequestKind}; +use crate::message::request::{GenerateRequest, Request, RequestKind}; use crate::message::response::{ResponseItem, ResponseSink}; use crate::message::sampling::SamplingParams; use crate::message::types::TokenIds; @@ -27,32 +23,27 @@ use crate::tokenizer_manager::wiring::{AbortSource, RequestAdmission, TmEvent}; use crate::utils::error::Error; use crate::utils::fsm::RequestState; -mod contract; -mod prefetch; - -pub(crate) use contract::{ - FrontendError, FrontendErrorKind, FrontendEvent, FrontendOutput, FrontendRequest, HealthStatus, - InternalState, ModelInfo, ServerInfo, -}; +use super::{CoreError, CoreEvent, HealthStatus, prefetch}; +use crate::message::info::{InternalState, ModelInfo, ServerInfo}; /// Sentinel host that makes the KV connector no-op. Parity with /// `sglang.srt.disaggregation.utils.FAKE_BOOTSTRAP_HOST`. const FAKE_BOOTSTRAP_HOST: &str = "2.2.2.2"; -pub(crate) struct FrontendConfig { +pub(crate) struct CoreConfig { pub(crate) response_capacity: usize, pub(crate) response_activity: ActivityCounter, pub(crate) startup_ready: bool, pub(crate) is_disaggregation: bool, pub(crate) mm_limits: BTreeMap, - pub(crate) metadata: FrontendMetadata, + pub(crate) metadata: CoreMetadata, } -/// Immutable, transport-independent metadata retained by the frontend. +/// Immutable, transport-independent metadata retained by the core. /// Keeping this snapshot narrow avoids making adapters or the shared handle /// depend on the full launch-configuration object. #[derive(Clone, Debug, Default)] -pub(crate) struct FrontendMetadata { +pub(crate) struct CoreMetadata { model_path: String, served_model_name: String, tokenizer_path: String, @@ -67,7 +58,7 @@ pub(crate) struct FrontendMetadata { version: String, } -impl From<&ServerArgs> for FrontendMetadata { +impl From<&ServerArgs> for CoreMetadata { fn from(args: &ServerArgs) -> Self { Self { model_path: args.model_path.clone(), @@ -86,16 +77,16 @@ impl From<&ServerArgs> for FrontendMetadata { } } -/// Cloneable capability for submitting work to the existing frontend runtime. +/// Cloneable capability for submitting work to the runtime pipeline. /// /// Only the runtime constructs this handle. Protocol adapters receive clones; /// they cannot reach the raw stage wiring directly. #[derive(Clone)] -pub(crate) struct FrontendHandle { - inner: Arc, +pub(crate) struct CoreHandle { + inner: Arc, } -struct FrontendInner { +struct CoreInner { intake_tx: flume::Sender, abort_tx: flume::Sender, response_capacity: usize, @@ -103,17 +94,17 @@ struct FrontendInner { startup_ready: AtomicBool, is_disaggregation: bool, mm_limits: BTreeMap, - metadata: FrontendMetadata, + metadata: CoreMetadata, } -impl FrontendHandle { +impl CoreHandle { pub(crate) fn new( intake_tx: flume::Sender, abort_tx: flume::Sender, - config: FrontendConfig, + config: CoreConfig, ) -> Self { Self { - inner: Arc::new(FrontendInner { + inner: Arc::new(CoreInner { intake_tx, abort_tx, response_capacity: config.response_capacity, @@ -143,13 +134,13 @@ impl FrontendHandle { pub(crate) async fn generate( &self, - mut request: FrontendRequest, - ) -> Result { + mut request: GenerateRequest, + ) -> Result { self.prepare_generations(std::slice::from_mut(&mut request)) .await?; self.submit(RequestKind::Generate(Box::new(request))) .await - .map(FrontendCall::new) + .map(CoreCall::new) } /// Preprocess and submit a group atomically from the adapter's perspective: @@ -158,8 +149,8 @@ impl FrontendHandle { /// the already-created calls aborts those accepted requests. pub(crate) async fn generate_batch( &self, - mut requests: Vec, - ) -> Result, FrontendError> { + mut requests: Vec, + ) -> Result, CoreError> { self.prepare_generations(&mut requests).await?; let mut calls = Vec::with_capacity(requests.len()); @@ -167,19 +158,16 @@ impl FrontendHandle { calls.push( self.submit(RequestKind::Generate(Box::new(request))) .await - .map(FrontendCall::new)?, + .map(CoreCall::new)?, ); } Ok(calls) } - async fn prepare_generations( - &self, - requests: &mut [FrontendRequest], - ) -> Result<(), FrontendError> { + async fn prepare_generations(&self, requests: &mut [GenerateRequest]) -> Result<(), CoreError> { prefetch::prefetch_all(requests, &self.inner.mm_limits) .await - .map_err(FrontendError::InvalidArgument) + .map_err(CoreError::InvalidArgument) } /// Return static model metadata without exposing the full launch config. @@ -201,8 +189,8 @@ impl FrontendHandle { /// Return public server metadata with the current scheduler metrics. /// Raw control bytes and the scheduler's full launch-argument dump never - /// cross the frontend boundary. - pub(crate) async fn server_info(&self) -> Result { + /// cross the core boundary. + pub(crate) async fn server_info(&self) -> Result { let internal_state = self.internal_state().await?; let metadata = &self.inner.metadata; Ok(ServerInfo { @@ -217,7 +205,7 @@ impl FrontendHandle { }) } - async fn internal_state(&self) -> Result { + async fn internal_state(&self) -> Result { let bytes = self .control(ControlRequest::GetInternalStateReq( GetInternalStateReq::new(Rid::new().to_string()), @@ -226,58 +214,54 @@ impl FrontendHandle { rmp_serde::from_slice::(&bytes) .map(|response| response.internal_state) .map_err(|error| { - FrontendError::InvalidResponse(format!("invalid internal-state response: {error}")) + CoreError::InvalidResponse(format!("invalid internal-state response: {error}")) }) } - async fn control(&self, request: ControlRequest) -> Result { + async fn control(&self, request: ControlRequest) -> Result { let mut call = self.submit(RequestKind::Control(Box::new(request))).await?; match call.recv().await { Some(ResponseItem::Control(bytes)) => Ok(bytes), Some(ResponseItem::Error(error)) => Err(translate_runtime_error(error)), - Some(_) => Err(FrontendError::InvalidResponse( + Some(_) => Err(CoreError::InvalidResponse( "unexpected generation output for control request".into(), )), - None => Err(FrontendError::ResponseTruncated), + None => Err(CoreError::ResponseTruncated), } } /// Decode token IDs into text without exposing the runtime's byte payload. - pub(crate) async fn detokenize(&self, token_ids: TokenIds) -> Result { + pub(crate) async fn detokenize(&self, token_ids: TokenIds) -> Result { let mut call = self.submit(RequestKind::Detokenize { token_ids }).await?; let bytes = match call.recv().await { Some(ResponseItem::Data(bytes)) => bytes, Some(ResponseItem::Error(Error::Validation(message))) => { - return Err(FrontendError::InvalidArgument(message)); + return Err(CoreError::InvalidArgument(message)); } Some(ResponseItem::Error(error)) => return Err(translate_runtime_error(error)), Some(_) => { - return Err(FrontendError::InvalidResponse( + return Err(CoreError::InvalidResponse( "unexpected output for detokenize request".into(), )); } None => { - return Err(FrontendError::Internal("reply channel closed".into())); + return Err(CoreError::Internal("reply channel closed".into())); } }; - String::from_utf8(bytes.to_vec()).map_err(|_| { - FrontendError::InvalidResponse("detokenized prompt is not valid UTF-8".into()) - }) + String::from_utf8(bytes.to_vec()) + .map_err(|_| CoreError::InvalidResponse("detokenized prompt is not valid UTF-8".into())) } /// Confirm that scheduler output is moving. The response heartbeat is the /// signal rather than this probe's own result, matching Python's /// `last_receive_tstamp` behavior on a busy server. - pub(crate) async fn probe_health( - &self, - timeout: Duration, - ) -> Result { + pub(crate) async fn probe_health(&self, timeout: Duration) -> Result { if !self.is_ready() { return Ok(HealthStatus::NotReady); } let baseline = self.response_activity(); - let probe = FrontendRequest { + let probe = GenerateRequest { rid: Rid::new_health_check(), input_ids: Some(vec![0]), sampling_params: SamplingParams { @@ -315,7 +299,7 @@ impl FrontendHandle { /// The async send preserves TokenizerManager inbox backpressure. Once the /// request is accepted, the returned call owns cancellation until it /// observes a terminal response. - async fn submit(&self, kind: RequestKind) -> Result { + async fn submit(&self, kind: RequestKind) -> Result { let rid = request_rid(&kind); let expected_response = ExpectedResponse::for_request(&kind); let (response_tx, response_rx) = mpsc::channel(self.inner.response_capacity); @@ -349,21 +333,21 @@ impl FrontendHandle { // Disarm before returning the transport-neutral rejection. call.in_flight = false; tracing::error!(%rid, "tm inbox closed; request rejected"); - return Err(FrontendError::Unavailable); + return Err(CoreError::Unavailable); } Ok(call) } } -fn translate_runtime_error(error: Error) -> FrontendError { +fn translate_runtime_error(error: Error) -> CoreError { match error { Error::Validation(message) => { - FrontendError::InvalidArgument(format!("validation failed: {message}")) + CoreError::InvalidArgument(format!("validation failed: {message}")) } - Error::QueueFull => FrontendError::Overloaded("to_scheduler channel full".into()), - Error::Disconnected => FrontendError::Cancelled("client disconnected".into()), - error => FrontendError::Internal(error.to_string()), + Error::QueueFull => CoreError::Overloaded("to_scheduler channel full".into()), + Error::Disconnected => CoreError::Cancelled("client disconnected".into()), + error => CoreError::Internal(error.to_string()), } } @@ -372,12 +356,12 @@ fn translate_runtime_error(error: Error) -> FrontendError { /// Runtime response variants are translated here so adapters cannot depend on /// scheduler/control encodings. Dropping an unfinished call retains the /// cancellation guarantees owned by [`RuntimeCall`]. -pub(crate) struct FrontendCall { +pub(crate) struct CoreCall { runtime: RuntimeCall, events_finished: bool, } -impl FrontendCall { +impl CoreCall { fn new(runtime: RuntimeCall) -> Self { Self { runtime, @@ -393,14 +377,14 @@ impl FrontendCall { /// Receive the next semantic event. A runtime channel that closes before a /// terminal response is converted into one terminal failure event; adapters /// never need to infer whether an empty runtime channel means success. - pub(crate) async fn recv(&mut self) -> Option { + pub(crate) async fn recv(&mut self) -> Option { if self.events_finished { return None; } let event = match self.runtime.recv().await { Some(item) => generation_event(item), - None => FrontendEvent::Failed(FrontendError::Internal( + None => CoreEvent::Failed(CoreError::Internal( "response truncated before completion".into(), )), }; @@ -410,7 +394,7 @@ impl FrontendCall { /// Append every semantic event that is ready without waiting. Channel /// emptiness and closure remain private runtime details. - pub(crate) fn drain_ready(&mut self, events: &mut Vec) { + pub(crate) fn drain_ready(&mut self, events: &mut Vec) { while !self.events_finished { let event = match self.runtime.try_recv() { Ok(item) => generation_event(item), @@ -446,39 +430,56 @@ impl FrontendCall { } } -fn generation_event(item: ResponseItem) -> FrontendEvent { +/// Await the next event from `call`, then drain whatever queued behind it (so the +/// caller can coalesce a backlog, as Python's `state.out_list` does), handing the +/// call back for `FuturesUnordered` to re-poll. An empty result means the semantic +/// stream was already exhausted. Shared by the HTTP and gRPC batch multiplexers. +pub(crate) async fn recv_indexed( + index: usize, + mut call: CoreCall, +) -> (usize, CoreCall, Vec) { + let mut items = Vec::new(); + match call.recv().await { + Some(item) => items.push(item), + None => return (index, call, items), // already exhausted + } + call.drain_ready(&mut items); + (index, call, items) +} + +fn generation_event(item: ResponseItem) -> CoreEvent { match item { - ResponseItem::Frame(output) => FrontendEvent::Delta(output.into()), + ResponseItem::Frame(output) => CoreEvent::Delta(output.into()), ResponseItem::Done(output) => { if let Some((legacy_http_status, message)) = output .finish_reason .as_ref() .and_then(|reason| reason.abort_status()) { - FrontendEvent::Failed(FrontendError::from_runtime_rejection( + CoreEvent::Failed(CoreError::from_runtime_rejection( message.to_owned(), legacy_http_status, )) } else { - FrontendEvent::Finished(output.into()) + CoreEvent::Finished(output.into()) } } - ResponseItem::Error(error) => FrontendEvent::Failed(translate_runtime_error(error)), - ResponseItem::Control(_) | ResponseItem::Data(_) => FrontendEvent::Failed( - FrontendError::InvalidResponse("unexpected non-generation response".into()), + ResponseItem::Error(error) => CoreEvent::Failed(translate_runtime_error(error)), + ResponseItem::Control(_) | ResponseItem::Data(_) => CoreEvent::Failed( + CoreError::InvalidResponse("unexpected non-generation response".into()), ), } } /// Private scheduler response envelope. This MessagePack shape is decoded at -/// the runtime boundary and deliberately is not part of the frontend contract. +/// the runtime boundary and deliberately is not part of the core contract. #[derive(serde::Deserialize)] struct InternalStateEnvelope { #[serde(default)] internal_state: InternalState, } -/// Runtime-facing request guard kept private behind [`FrontendCall`]. +/// Runtime-facing request guard kept private behind [`CoreCall`]. /// /// This type is the only owner of the raw response channel and admission/abort /// capabilities. It is also used directly by typed unary operations such as @@ -569,13 +570,14 @@ mod tests { use bytes::Bytes; use super::*; + use crate::api_server::core::{CoreErrorKind, CoreOutput}; use crate::message::io_struct::GetInternalStateReq; use crate::message::multimodal::MmItem; - use crate::message::request::{GenerateRequest, MmData}; + use crate::message::request::MmData; use crate::message::response::ChunkEvent; struct Harness { - handle: FrontendHandle, + handle: CoreHandle, intake_rx: flume::Receiver, abort_rx: flume::Receiver, } @@ -596,7 +598,7 @@ mod tests { intake_tx: flume::Sender, abort_tx: flume::Sender, response_capacity: usize, - ) -> FrontendHandle { + ) -> CoreHandle { configured_handle( intake_tx, abort_tx, @@ -616,17 +618,17 @@ mod tests { startup_ready: bool, is_disaggregation: bool, mm_limits: BTreeMap, - ) -> FrontendHandle { - FrontendHandle::new( + ) -> CoreHandle { + CoreHandle::new( intake_tx, abort_tx, - FrontendConfig { + CoreConfig { response_capacity, response_activity, startup_ready, is_disaggregation, mm_limits, - metadata: FrontendMetadata::default(), + metadata: CoreMetadata::default(), }, ) } @@ -700,44 +702,44 @@ mod tests { fn runtime_errors_become_transport_neutral_categories() { assert!(matches!( translate_runtime_error(Error::Validation("bad input".into())), - FrontendError::InvalidArgument(message) + CoreError::InvalidArgument(message) if message == "validation failed: bad input" )); assert!(matches!( translate_runtime_error(Error::QueueFull), - FrontendError::Overloaded(message) if message == "to_scheduler channel full" + CoreError::Overloaded(message) if message == "to_scheduler channel full" )); assert!(matches!( translate_runtime_error(Error::Disconnected), - FrontendError::Cancelled(message) if message == "client disconnected" + CoreError::Cancelled(message) if message == "client disconnected" )); assert!(matches!( translate_runtime_error(Error::Internal("bug".into())), - FrontendError::Internal(message) if message == "internal error: bug" + CoreError::Internal(message) if message == "internal error: bug" )); } #[test] fn scheduler_status_is_classified_once_for_non_http_adapters() { for (status, expected) in [ - (400, FrontendErrorKind::InvalidArgument), - (404, FrontendErrorKind::NotFound), - (408, FrontendErrorKind::DeadlineExceeded), - (412, FrontendErrorKind::FailedPrecondition), - (413, FrontendErrorKind::ResourceExhausted), - (429, FrontendErrorKind::ResourceExhausted), - (432, FrontendErrorKind::InvalidArgument), - (499, FrontendErrorKind::Cancelled), - (500, FrontendErrorKind::Internal), - (503, FrontendErrorKind::Unavailable), - (504, FrontendErrorKind::DeadlineExceeded), + (400, CoreErrorKind::InvalidArgument), + (404, CoreErrorKind::NotFound), + (408, CoreErrorKind::DeadlineExceeded), + (412, CoreErrorKind::FailedPrecondition), + (413, CoreErrorKind::ResourceExhausted), + (429, CoreErrorKind::ResourceExhausted), + (432, CoreErrorKind::InvalidArgument), + (499, CoreErrorKind::Cancelled), + (500, CoreErrorKind::Internal), + (503, CoreErrorKind::Unavailable), + (504, CoreErrorKind::DeadlineExceeded), ] { - let error = FrontendError::from_runtime_rejection("rejected".into(), status); + let error = CoreError::from_runtime_rejection("rejected".into(), status); assert_eq!(error.kind(), expected, "legacy status {status}"); assert!(matches!( error, - FrontendError::RuntimeRejected { + CoreError::RuntimeRejected { legacy_http_status, .. } if legacy_http_status == status @@ -777,11 +779,11 @@ mod tests { assert!(matches!( call.recv().await, - Some(FrontendEvent::Delta(FrontendOutput { text, .. })) if text == "delta" + Some(CoreEvent::Delta(CoreOutput { text, .. })) if text == "delta" )); assert!(matches!( call.recv().await, - Some(FrontendEvent::Finished(FrontendOutput { text, .. })) if text == "final" + Some(CoreEvent::Finished(CoreOutput { text, .. })) if text == "final" )); assert!(call.recv().await.is_none()); drop(call); @@ -816,7 +818,7 @@ mod tests { assert!(matches!( call.recv().await, - Some(FrontendEvent::Finished(FrontendOutput { + Some(CoreEvent::Finished(CoreOutput { finish_reason: Some(_), .. })) @@ -855,7 +857,7 @@ mod tests { drop(intake_rx); assert!(matches!( submitter.await.unwrap(), - Err(FrontendError::Unavailable) + Err(CoreError::Unavailable) )); assert!(matches!( @@ -891,7 +893,7 @@ mod tests { assert!(matches!( handle.generate(request).await, - Err(FrontendError::InvalidArgument(message)) + Err(CoreError::InvalidArgument(message)) if message == "Image count 2 exceeds limit 1 per request." )); assert!(intake_rx.try_recv().is_err()); diff --git a/rust/sglang-server/src/frontend/prefetch.rs b/rust/sglang-server/src/api_server/core/prefetch.rs similarity index 99% rename from rust/sglang-server/src/frontend/prefetch.rs rename to rust/sglang-server/src/api_server/core/prefetch.rs index 606d28481a..5875838761 100644 --- a/rust/sglang-server/src/frontend/prefetch.rs +++ b/rust/sglang-server/src/api_server/core/prefetch.rs @@ -1,4 +1,4 @@ -//! Resolve I/O-backed media sources on the async frontend runtime, before MM dispatch. +//! Resolve I/O-backed media sources on the async API runtime, before MM dispatch. //! //! The MM worker pool is fixed, core-pinned CPU capacity: a slow image host — or //! a file on a hanging network mount — must never occupy it, and a request's diff --git a/rust/sglang-server/src/api_server/frame.rs b/rust/sglang-server/src/api_server/frame.rs deleted file mode 100644 index eae8c18c2d..0000000000 --- a/rust/sglang-server/src/api_server/frame.rs +++ /dev/null @@ -1,697 +0,0 @@ -//! Frame shaping for the native `/generate` protocol: the cumulative -//! [`OutputAccumulator`] plus the functions that render [`FrontendOutput`]s / -//! accumulated state into wire JSON (`meta_info`, logprob tuples, error and -//! abort frames). No HTTP here — the sibling `native_api` module owns the handlers -//! and streams; it calls these per frame. - -use crate::frontend::FrontendOutput; -#[cfg(test)] -use crate::message::response::ChunkExtras; -use crate::native_generation::{ - hidden_states_rows, lp_value, meta_info_value, opt_texts, text_slot, -}; -#[cfg(test)] -use crate::native_generation::{logprob_tuples, ragged_logprob_tuples}; - -/// Append a flat family's `[logprob, token_id, text]` tuples to `dst`, comma -/// separated and WITHOUT the enclosing brackets, so a cumulative frame can -/// concatenate each delta instead of re-rendering every accumulated position. -/// -/// Byte-identical to [`logprob_tuples`]'s serialization: `serde_json` writes an -/// array as `[`, elements joined by `,`, `]` with no spaces, and each element here -/// is rendered by the same `Value` Display. -fn push_logprob_tuples(dst: &mut String, vals: &[f32], idxs: &[i32], texts: Option<&[String]>) { - use std::fmt::Write; - for (j, (&v, &tid)) in vals.iter().zip(idxs.iter()).enumerate() { - if !dst.is_empty() { - dst.push(','); - } - let _ = write!(dst, "[{},{tid},{}]", lp_value(v), text_slot(texts, j)); - } -} - -/// Ragged counterpart of [`push_logprob_tuples`] — one entry per position, `null` -/// where `lens[p] == 0`. Mirrors [`ragged_logprob_tuples`] including its -/// bounds-checked skip, so a header whose `lens` run past the value buffer yields -/// the same (shortened) row rather than panicking. -fn push_ragged_tuples( - dst: &mut String, - vals: &[f32], - idxs: &[i32], - lens: &[u32], - texts: Option<&[String]>, -) { - use std::fmt::Write; - let mut off = 0usize; - for &l in lens { - let l = l as usize; - if !dst.is_empty() { - dst.push(','); - } - if l == 0 { - dst.push_str("null"); - } else { - dst.push('['); - let mut first = true; - for j in off..off + l { - let (Some(&v), Some(&tid)) = (vals.get(j), idxs.get(j)) else { - continue; - }; - if !first { - dst.push(','); - } - first = false; - let _ = write!(dst, "[{},{tid},{}]", lp_value(v), text_slot(texts, j)); - } - dst.push(']'); - } - off += l; - } -} - -/// Render one set-once family (they ride the prefill or the final chunk) into a -/// standalone JSON array, so it is serialized when it arrives rather than on every -/// subsequent frame. -fn ragged_array_json(vals: &[f32], idxs: &[i32], lens: &[u32], texts: Option<&[String]>) -> String { - let mut body = String::new(); - push_ragged_tuples(&mut body, vals, idxs, lens, texts); - format!("[{body}]") -} - -/// Format a decoded [`FrontendOutput`] as one SGLang `/generate` frame's JSON. `rid` -/// (response `meta_info.id`) is passed as a string; the event's numeric `rid` is -/// just the shard routing key. -pub(super) fn frame_value(out: &FrontendOutput, rid: &str) -> serde_json::Value { - let mut v = serde_json::json!({ - "text": out.text, - "meta_info": meta_info_value(out, rid), - }); - if !out.token_ids.is_empty() { - v["output_ids"] = serde_json::json!(out.token_ids); - } - v -} - -/// Cumulative frame JSON from the accumulator's memoized parts — O(1) in the -/// accumulated length, where rebuilding the `Value` is O(T) per frame and so O(T²) -/// per request. -/// -/// Encodes the same JSON document as `frame_value(..).to_string()` — same keys, -/// same values, same escaping — pinned for both the plain and the logprob shapes -/// by `cumulative_frame_json_matches_serde`. -pub(super) fn cumulative_frame_json( - acc: &OutputAccumulator, - rid: &str, - index: Option, -) -> Option { - use std::fmt::Write; - - if acc.extras_memo_broken { - return None; - } - let o = acc.snapshot(); - // Through `Value` rather than `to_string` on the struct: it is the same - // encoder the slow path runs the finish reason through, so any representation - // quirk is reproduced instead of re-derived. - let finish = serde_json::to_value(&o.finish_reason).ok()?.to_string(); - - // Alphabetical by convention only — a stable order that is easy to extend and - // diff. - let mut m = String::new(); - let _ = write!(m, "{{\"completion_tokens\":{}", o.completion_tokens); - let _ = write!(m, ",\"finish_reason\":{finish}"); - if let Some(h) = &acc.hidden_json { - let _ = write!(m, ",\"hidden_states\":{h}"); - } - let _ = write!(m, ",\"id\":{}", serde_json::Value::String(rid.to_string())); - if let Some(v) = &acc.in_tid_json { - let _ = write!(m, ",\"input_token_ids_logprobs\":{v}"); - } - // Input+output token logprobs are emitted as a PAIR whenever either side has - // data (empty list included), matching `frame_value` byte for byte — see the - // PD-router rationale there. - let lp_pair = acc.in_lp_json.is_some() || !acc.out_lp_json.is_empty(); - if lp_pair { - let v = acc.in_lp_json.as_deref().unwrap_or("[]"); - let _ = write!(m, ",\"input_token_logprobs\":{v}"); - } - if let Some(v) = &acc.in_top_json { - let _ = write!(m, ",\"input_top_logprobs\":{v}"); - } - // The `Value` path keys these off the source columns being non-empty; an empty - // source renders to an empty body, so the two guards coincide. - if !acc.out_tid_json.is_empty() { - let _ = write!(m, ",\"output_token_ids_logprobs\":[{}]", acc.out_tid_json); - } - if lp_pair { - let _ = write!(m, ",\"output_token_logprobs\":[{}]", acc.out_lp_json); - } - if !acc.out_top_json.is_empty() { - let _ = write!(m, ",\"output_top_logprobs\":[{}]", acc.out_top_json); - } - let _ = write!(m, ",\"prompt_tokens\":{}}}", o.prompt_tokens); - - let mut s = String::with_capacity(acc.text_json.len() + acc.ids_json.len() + m.len() + 40); - s.push('{'); - if let Some(i) = index { - let _ = write!(s, "\"index\":{i},"); - } - s.push_str("\"meta_info\":"); - s.push_str(&m); - if !acc.ids_json.is_empty() { - s.push_str(",\"output_ids\":["); - s.push_str(&acc.ids_json); - s.push(']'); - } - s.push_str(",\"text\":\""); - s.push_str(&acc.text_json); - s.push_str("\"}"); - Some(s) -} - -/// Attach the batch `index` (batch streams only) and render to the SSE `data` text. -pub(super) fn tag_value(mut v: serde_json::Value, index: Option) -> String { - if let Some(i) = index { - v["index"] = serde_json::json!(i); - } - v.to_string() -} - -/// One streaming frame's JSON: cumulative ignores `delta`, incremental ships it. -pub(super) fn stream_frame_string( - delta: FrontendOutput, - acc: &OutputAccumulator, - incremental: bool, - rid_str: &str, - index: Option, -) -> String { - if !incremental { - return cumulative_frame_string(acc, rid_str, index); - } - tag_value(stream_frame_value(delta, acc, true, rid_str), index) -} - -/// A cumulative frame's JSON, built purely from the accumulator (which is why a -/// backlog can coalesce to its last); falls back to the `Value` builder on extras. -pub(super) fn cumulative_frame_string( - acc: &OutputAccumulator, - rid_str: &str, - index: Option, -) -> String { - cumulative_frame_json(acc, rid_str, index) - .unwrap_or_else(|| tag_value(frame_value(acc.snapshot(), rid_str), index)) -} - -/// Format one streaming frame: the accumulator's cumulative view (default), or this -/// step's delta with the cumulative token count in `meta_info` (matching Python). -pub(super) fn stream_frame_value( - delta: FrontendOutput, - acc: &OutputAccumulator, - incremental: bool, - rid_str: &str, -) -> serde_json::Value { - if incremental { - let mut d = delta; - d.completion_tokens = acc.snapshot().completion_tokens; - frame_value(&d, rid_str) - } else { - frame_value(acc.snapshot(), rid_str) - } -} - -/// Folds per-chunk [`FrontendOutput`] deltas into a cumulative view — used by the drain -/// loops needing cumulative output (every unary response + the cumulative SGLang -/// stream; OpenAI streaming forwards deltas and skips this). Holds a single -/// [`FrontendOutput`] so `snapshot` hands back a **borrow** per frame — no per-frame -/// clone of the growing buffers (that added O(T²) atop the wire's inherent O(T²)). -/// Shared with the [`openai`] submodule. -#[derive(Default)] -pub(super) struct OutputAccumulator { - out: FrontendOutput, - /// Serialized cumulative `output_ids` body (`"1,2,3"`, no brackets), appended per - /// delta so a frame memcpy's it instead of rebuilding the array — O(T), not O(T²). - ids_json: String, - /// JSON-escaped cumulative text, without the surrounding quotes. Escaping is - /// per-character, so `escape(a + b) == escape(a) + escape(b)` and deltas append. - text_json: String, - /// Memoized bodies (no enclosing brackets) of the three CUMULATIVE logprob - /// families, appended per delta — the same O(T) trick `ids_json` uses, extended - /// to the families that made a cumulative stream with logprobs O(T²). Cumulative - /// is SGLang's default, so that path re-rendered every accumulated position on - /// every frame: measured 117 ms for one 500-token top-5 request, versus 1.2 ms - /// incremental. - out_lp_json: String, - out_top_json: String, - out_tid_json: String, - /// Set-once families — they ride the prefill or the final chunk, so they are - /// rendered when they arrive rather than on every frame after. - in_lp_json: Option, - in_top_json: Option, - in_tid_json: Option, - hidden_json: Option, - /// Set once a family's text column falls out of lockstep with its values, at - /// which point the memo is abandoned for the `Value` path. - /// - /// Appending per delta assumes `text_slot(accumulated, global_j)` equals - /// `text_slot(delta, local_j)`, which holds only while every delta supplies - /// either a text per value or none at all. That is what a real request does — - /// `return_text_in_logprobs` is per-request, so the detok shard fills `*_txt` - /// for all deltas or none — but a mixed sequence would silently diverge from - /// `frame_value`, so it is detected rather than assumed. - extras_memo_broken: bool, -} - -/// Append `s` JSON-escaped (no surrounding quotes) — `serde_json` quotes it, and the -/// quotes are the first and last bytes of a string encoding. -fn push_escaped(dst: &mut String, s: &str) { - if s.is_empty() { - return; - } - let quoted = serde_json::to_string(s).expect("str-to-json should never fail"); - dst.push_str("ed[1..quoted.len() - 1]); -} - -impl OutputAccumulator { - /// Fold one delta frame in. Output families concatenate; input families and - /// hidden states are set-once / last-writer-wins (they ride the prefill/final - /// chunk), matching the Python `meta_info` assignment. - pub(super) fn fold(&mut self, d: &FrontendOutput) { - use std::fmt::Write; - - // Grow the memoized serializations alongside the raw cumulative buffers. - push_escaped(&mut self.text_json, &d.text); - for &id in &d.token_ids { - if !self.ids_json.is_empty() { - self.ids_json.push(','); - } - let _ = write!(self.ids_json, "{id}"); - } - - let Some(de) = d.extras.as_deref() else { - self.out.append_delta(d); - return; - }; - // Append THIS delta's tuples, indexed within the delta — equivalent to - // indexing the accumulated arrays only while texts stay in lockstep, which - // the guard below verifies. - push_logprob_tuples( - &mut self.out_lp_json, - &de.out_lp_val, - &de.out_lp_idx, - opt_texts(&de.out_lp_txt), - ); - push_ragged_tuples( - &mut self.out_top_json, - &de.out_top_val, - &de.out_top_idx, - &de.out_top_lens, - opt_texts(&de.out_top_txt), - ); - push_ragged_tuples( - &mut self.out_tid_json, - &de.out_tid_val, - &de.out_tid_idx, - &de.out_tid_lens, - opt_texts(&de.out_tid_txt), - ); - if !de.in_lp_val.is_empty() { - let mut body = String::new(); - push_logprob_tuples( - &mut body, - &de.in_lp_val, - &de.in_lp_idx, - opt_texts(&de.in_lp_txt), - ); - self.in_lp_json = Some(format!("[{body}]")); - } - // Input families ride once (prefill); `lens` non-empty marks their arrival. - if !de.in_top_lens.is_empty() { - self.in_top_json = Some(ragged_array_json( - &de.in_top_val, - &de.in_top_idx, - &de.in_top_lens, - opt_texts(&de.in_top_txt), - )); - } - if !de.in_tid_lens.is_empty() { - self.in_tid_json = Some(ragged_array_json( - &de.in_tid_val, - &de.in_tid_idx, - &de.in_tid_lens, - opt_texts(&de.in_tid_txt), - )); - } - // Hidden states are non-cumulative: the latest non-empty set wins. - if !de.hidden_lens.is_empty() { - self.hidden_json = - Some(hidden_states_rows(&de.hidden_val, &de.hidden_lens).to_string()); - } - - self.out.append_delta(d); - let oe = self - .out - .extras - .as_deref() - .expect("a delta with extras creates cumulative extras"); - let lockstep = |txt: &Vec, val: &Vec| txt.is_empty() || txt.len() == val.len(); - if !lockstep(&oe.out_lp_txt, &oe.out_lp_val) - || !lockstep(&oe.out_top_txt, &oe.out_top_val) - || !lockstep(&oe.out_tid_txt, &oe.out_tid_val) - { - self.extras_memo_broken = true; - } - } - - /// Borrow the cumulative output for an intermediate streaming frame. - pub(super) fn snapshot(&self) -> &FrontendOutput { - &self.out - } - - /// Consume into the final cumulative output. - pub(super) fn into_output(self) -> FrontendOutput { - self.out - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn flat_logprob_tuples_shape() { - let v = logprob_tuples(&[-0.5, -1.5], &[10, 20], None); - assert_eq!( - v, - serde_json::json!([ - [-0.5f32, 10, serde_json::Value::Null], - [-1.5f32, 20, serde_json::Value::Null] - ]) - ); - } - - /// With a text buffer, the tuple's third slot carries the decoded token. - #[test] - fn flat_logprob_tuples_with_text() { - let texts = vec!["a".to_string(), "b".to_string()]; - let v = logprob_tuples(&[-0.5, -1.5], &[10, 20], Some(&texts)); - assert_eq!( - v, - serde_json::json!([[-0.5f32, 10, "a"], [-1.5f32, 20, "b"]]) - ); - } - - /// Ragged reshape restores null positions (len 0) — mirrors - /// detokenize_top_logprobs_tokens emitting None for empty positions. - #[test] - fn ragged_logprob_tuples_restores_null_positions() { - // 2 positions: first null (len 0), second k=1. - let v = ragged_logprob_tuples(&[-0.3], &[9], &[0, 1], None); - assert_eq!( - v, - serde_json::json!([ - serde_json::Value::Null, - [[-0.3f32, 9, serde_json::Value::Null]] - ]) - ); - } - - /// The `NaN` sentinel (the Python `None` logprob for the first prompt token) - /// becomes a JSON `null` logprob, while its token id in the parallel `idx` - /// column is preserved. Guards the scheduler-killing prompt-logprob crash. - #[test] - fn nan_sentinel_becomes_null_logprob() { - // Flat (input/output logprobs): first value absent, second present. - let flat = logprob_tuples(&[f32::NAN, -0.5], &[10, 20], None); - assert_eq!( - flat, - serde_json::json!([ - [serde_json::Value::Null, 10, serde_json::Value::Null], - [-0.5f32, 20, serde_json::Value::Null], - ]) - ); - // Ragged (top-k / token-ids logprobs): a NaN inside a position → null. - let ragged = ragged_logprob_tuples(&[f32::NAN], &[7], &[1], None); - assert_eq!( - ragged, - serde_json::json!([[[serde_json::Value::Null, 7, serde_json::Value::Null]]]) - ); - } - - /// End-to-end: a `FrontendOutput` carrying a prompt-logprob request (first input - /// logprob is the `NaN` sentinel) formats without panicking and emits - /// `input_token_logprobs` with a leading `[null, token_id, text]`. - #[test] - fn prompt_logprob_frame_emits_null_first() { - let out = FrontendOutput { - extras: Some(Box::new(ChunkExtras { - in_lp_val: vec![f32::NAN, -0.5], - in_lp_idx: vec![10, 20], - in_lp_txt: vec!["".into(), "hi".into()], - ..Default::default() - })), - ..Default::default() - }; - let frame = frame_value(&out, "1"); - assert_eq!( - frame["meta_info"]["input_token_logprobs"], - serde_json::json!([[serde_json::Value::Null, 10, ""], [-0.5f32, 20, "hi"]]) - ); - } - - /// The accumulator folds deltas cumulatively and `snapshot` borrows the - /// running state (no per-frame clone); `into_output` moves the same state. - #[test] - fn accumulator_snapshot_is_cumulative() { - let mut acc = OutputAccumulator::default(); - acc.fold(&FrontendOutput { - text: "he".into(), - token_ids: vec![1, 2], - completion_tokens: 2, - ..Default::default() - }); - { - let s = acc.snapshot(); - assert_eq!(s.text, "he"); - assert_eq!(s.token_ids, vec![1, 2]); - } - acc.fold(&FrontendOutput { - text: "llo".into(), - token_ids: vec![3], - completion_tokens: 1, - ..Default::default() - }); - { - let s = acc.snapshot(); - assert_eq!(s.text, "hello"); // cumulative - assert_eq!(s.token_ids, vec![1, 2, 3]); - assert_eq!(s.completion_tokens, 3); - } - let out = acc.into_output(); - assert_eq!(out.text, "hello"); - } - - /// A populated text column (decoded on the detok shard) → `Some`; empty - /// (`return_text_in_logprobs` off) → `None` → null text slots. - #[test] - fn opt_texts_gates_on_population() { - assert!(opt_texts(&[]).is_none()); - let t = vec!["x".to_string()]; - assert_eq!(opt_texts(&t), Some(t.as_slice())); - } - - /// Parse a frame so the fast and slow paths can be compared as documents. - fn as_json(frame: &str) -> serde_json::Value { - serde_json::from_str(frame).expect("a frame must be valid JSON") - } - - /// The memoized cumulative fast path must emit the **same JSON document** as the - /// `serde_json::Value` builder it replaces — same keys, same values, same - /// escaping. Covers unicode and control chars, an empty-ids first frame, a - /// finish_reason, and the batch `index`. Guards the O(T) rewrite of the O(T²) - /// `output_ids` serialization. - #[test] - fn cumulative_frame_json_matches_serde() { - let deltas = [ - FrontendOutput { - text: String::new(), - token_ids: vec![], - completion_tokens: 0, - prompt_tokens: 128, - ..Default::default() - }, - FrontendOutput { - text: "He\"llo\n\t".into(), - token_ids: vec![1000], - completion_tokens: 1, - prompt_tokens: 128, - ..Default::default() - }, - FrontendOutput { - text: " 世界 🌍 \\".into(), - token_ids: vec![-2, 3], - completion_tokens: 2, - prompt_tokens: 128, - ..Default::default() - }, - FrontendOutput { - text: "!".into(), - token_ids: vec![9], - completion_tokens: 1, - prompt_tokens: 128, - finish_reason: serde_json::from_value( - serde_json::json!({"type": "stop", "matched": 9}), - ) - .expect("finish reason must parse"), - ..Default::default() - }, - ]; - - for index in [None, Some(3usize)] { - let mut acc = OutputAccumulator::default(); - for d in &deltas { - acc.fold(d); - let fast = cumulative_frame_json(&acc, "7", index).expect("no extras → fast path"); - let slow = tag_value(frame_value(acc.snapshot(), "7"), index); - println!("fast={fast:?}"); - println!("slow={slow:?}"); - assert_eq!( - as_json(&fast), - as_json(&slow), - "index={index:?} text={:?}", - acc.snapshot().text - ); - } - } - } - - /// The same equivalence, for the shape that made cumulative streaming O(T²): - /// every logprob family at once, across several deltas, with and without - /// `return_text_in_logprobs` texts and with a null ragged position. - #[test] - fn cumulative_frame_json_matches_serde_with_logprobs() { - for with_texts in [false, true] { - let txt = |v: &[&str]| -> Vec { - if with_texts { - v.iter().map(|s| (*s).to_string()).collect() - } else { - Vec::new() - } - }; - let deltas = [ - // Prefill: the set-once input families and a null top-k position. - FrontendOutput { - prompt_tokens: 4, - extras: Some(Box::new(ChunkExtras { - in_lp_val: vec![f32::NAN, -1.5], - in_lp_idx: vec![10, 11], - in_lp_txt: txt(&["a", "b"]), - in_top_val: vec![-0.25], - in_top_idx: vec![12], - in_top_lens: vec![0, 1], - in_top_txt: txt(&["c"]), - in_tid_val: vec![-2.0], - in_tid_idx: vec![13], - in_tid_lens: vec![1], - in_tid_txt: txt(&["d"]), - ..Default::default() - })), - ..Default::default() - }, - FrontendOutput { - text: "He\"llo".into(), - token_ids: vec![100], - completion_tokens: 1, - prompt_tokens: 4, - extras: Some(Box::new(ChunkExtras { - out_lp_val: vec![-0.5], - out_lp_idx: vec![100], - out_lp_txt: txt(&["He\"llo"]), - out_top_val: vec![-0.5, -3.0], - out_top_idx: vec![100, 7], - out_top_lens: vec![2], - out_top_txt: txt(&["He\"llo", "x"]), - out_tid_val: vec![-0.5], - out_tid_idx: vec![100], - out_tid_lens: vec![1], - out_tid_txt: txt(&["He\"llo"]), - ..Default::default() - })), - ..Default::default() - }, - FrontendOutput { - text: " 世界".into(), - token_ids: vec![-2, 3], - completion_tokens: 2, - prompt_tokens: 4, - finish_reason: serde_json::from_value( - serde_json::json!({"type": "stop", "matched": 3}), - ) - .expect("finish reason must parse"), - extras: Some(Box::new(ChunkExtras { - out_lp_val: vec![f32::NAN, -0.125], - out_lp_idx: vec![-2, 3], - out_lp_txt: txt(&[" 世", "界"]), - // A zero-length position must render as `null`, not `[]`. - out_top_val: vec![-0.125], - out_top_idx: vec![3], - out_top_lens: vec![0, 1], - out_top_txt: txt(&["界"]), - out_tid_val: vec![], - out_tid_idx: vec![], - out_tid_lens: vec![0, 0], - out_tid_txt: txt(&[]), - hidden_val: vec![0.5, -0.25, 1.0], - hidden_lens: vec![2, 1], - ..Default::default() - })), - }, - ]; - - for index in [None, Some(2usize)] { - let mut acc = OutputAccumulator::default(); - for d in &deltas { - acc.fold(d); - let fast = cumulative_frame_json(&acc, "9", index) - .expect("the extras memo must stay valid for a well-formed request"); - let slow = tag_value(frame_value(acc.snapshot(), "9"), index); - assert_eq!( - as_json(&fast), - as_json(&slow), - "with_texts={with_texts} index={index:?}" - ); - } - } - } - } - - /// A delta sequence that supplies texts for some values and not others breaks - /// the append-equivalence the memo rests on (`text_slot` is indexed globally, - /// so a gap shifts every later text). The accumulator must notice and defer to - /// the `Value` builder rather than emit a frame that disagrees with it. - #[test] - fn mismatched_logprob_texts_fall_back_to_the_value_path() { - let mut acc = OutputAccumulator::default(); - acc.fold(&FrontendOutput { - extras: Some(Box::new(ChunkExtras { - out_lp_val: vec![-0.5], - out_lp_idx: vec![5], - ..Default::default() // no texts - })), - ..Default::default() - }); - assert!(cumulative_frame_json(&acc, "1", None).is_some()); - acc.fold(&FrontendOutput { - extras: Some(Box::new(ChunkExtras { - out_lp_val: vec![-0.25], - out_lp_idx: vec![6], - out_lp_txt: vec!["b".into()], // …now texts: out of lockstep - ..Default::default() - })), - ..Default::default() - }); - assert!( - cumulative_frame_json(&acc, "1", None).is_none(), - "a text column out of lockstep must invalidate the memo" - ); - } -} diff --git a/rust/sglang-server/src/api_server/grpc.rs b/rust/sglang-server/src/api_server/grpc.rs new file mode 100644 index 0000000000..a1e94e442e --- /dev/null +++ b/rust/sglang-server/src/api_server/grpc.rs @@ -0,0 +1,15 @@ +//! Tonic adapter for the native `sglang.api.v1` service. +//! +//! The service is the gRPC rendering of the contract the native HTTP endpoints +//! serve: `Generate` is `/generate`, `HealthCheck` is `/health_generate`, and +//! `GetModelInfo` / `GetServerInfo` are their HTTP namesakes. Requests decode +//! through the shared `/generate` fan-out and responses stream through the +//! shared [`CoreHandle`], which owns preprocessing, admission, runtime +//! communication, and request cancellation. +//! +//! The thin Tonic-service structure, streamed response approach. + +pub(crate) mod app; +mod info; +mod native_api; +pub(crate) mod service; diff --git a/rust/sglang-server/src/grpc/server.rs b/rust/sglang-server/src/api_server/grpc/app.rs similarity index 85% rename from rust/sglang-server/src/grpc/server.rs rename to rust/sglang-server/src/api_server/grpc/app.rs index 2b440b70ab..8775ceda53 100644 --- a/rust/sglang-server/src/grpc/server.rs +++ b/rust/sglang-server/src/api_server/grpc/app.rs @@ -1,18 +1,17 @@ //! Tonic listener lifecycle for the native Rust frontend. //! //! The pre-bound socket, shared API runtime, and shutdown shape build on Rain -//! Jiang's multi-protocol prototype in `sgl-project/sglang#36923`. Unlike that -//! prototype, the listener mounts the canonical `runtime.v1` adapter on the -//! transport-neutral [`crate::frontend::FrontendHandle`] contract. +//! Jiang's multi-protocol prototype in `sgl-project/sglang#36923`. The listener +//! mounts the `api.v1` adapter on the transport-neutral +//! [`crate::api_server::core::CoreHandle`] contract. -use sglang_grpc_types::sglang::runtime::v1::sglang_service_server::SglangServiceServer; +use sglang_api_types::api::v1::sglang_service_server::SglangServiceServer; use tokio_stream::wrappers::TcpListenerStream; -use super::GrpcService; +use super::service::GrpcService; -/// Keep the native Rust listener aligned with the existing Python-backed -/// runtime.v1 server. Tonic's 4 MiB default is too small for large token or -/// multimodal request bodies. +/// Tonic's 4 MiB default is too small for large token or multimodal request +/// bodies; keep the limit the Python-backed gRPC server established. const DEFAULT_GRPC_MAX_MESSAGE_SIZE: usize = 64 * 1024 * 1024; fn max_message_size() -> usize { diff --git a/rust/sglang-server/src/api_server/grpc/info.rs b/rust/sglang-server/src/api_server/grpc/info.rs new file mode 100644 index 0000000000..d6f40244a7 --- /dev/null +++ b/rust/sglang-server/src/api_server/grpc/info.rs @@ -0,0 +1,68 @@ +//! Frontend metadata to the `api.v1` info responses. +//! +//! Both protos carry the JSON-typed members (`preferred_sampling_params`, the +//! allowlisted `internal_states`) as JSON text, so the gRPC reply holds the +//! same documents the HTTP reply embeds. + +use sglang_api_types::api::v1 as api; + +use crate::message::info::{ModelInfo, ServerInfo}; + +pub(super) fn model_info(info: ModelInfo) -> Result { + let ModelInfo { + model_path, + served_model_name, + tokenizer_path, + is_generation, + preferred_sampling_params, + weight_version, + load_format, + reasoning_parser, + tool_call_parser, + disaggregation_mode: _, + } = info; + let preferred_sampling_params = preferred_sampling_params + .map(|params| serde_json::to_string(¶ms.0)) + .transpose() + .map_err(|error| format!("preferred_sampling_params is not JSON: {error}"))?; + Ok(api::GetModelInfoResponse { + model_path, + served_model_name, + tokenizer_path, + is_generation, + preferred_sampling_params, + weight_version, + load_format, + reasoning_parser, + tool_call_parser, + }) +} + +pub(super) fn server_info(info: ServerInfo) -> Result { + let ServerInfo { + model_path, + served_model_name, + tokenizer_path, + max_context_length, + max_total_num_tokens, + version, + frontend: _, + internal_states, + } = info; + let max_context_length = u32::try_from(max_context_length) + .map_err(|_| format!("max_context_length {max_context_length} exceeds uint32"))?; + let internal_states = internal_states + .iter() + .map(serde_json::to_string) + .collect::, _>>() + .map_err(|error| format!("internal_states are not JSON: {error}"))?; + Ok(api::GetServerInfoResponse { + model_path, + served_model_name, + tokenizer_path, + max_context_length, + max_total_num_tokens: Some(max_total_num_tokens), + version, + internal_states, + }) +} diff --git a/rust/sglang-server/src/api_server/grpc/native_api.rs b/rust/sglang-server/src/api_server/grpc/native_api.rs new file mode 100644 index 0000000000..cf04d9edef --- /dev/null +++ b/rust/sglang-server/src/api_server/grpc/native_api.rs @@ -0,0 +1,215 @@ +//! Core events to `api.v1` stream items and statuses. + +use std::time::{Duration, Instant}; + +use futures::StreamExt; +use futures::stream::FuturesUnordered; +use sglang_api_types::api::v1 as api; +use tonic::{Code, Status}; + +use super::service::ResponseStream; +use crate::api_server::core::{ + CoreCall, CoreError, CoreErrorKind, CoreEvent, CoreOutput, recv_indexed, +}; + +pub(super) struct StreamOptions { + /// `stream` as sent: false folds each item to its one terminal frame. + pub stream: bool, + /// Streamed frames are deltas (true) or cumulative (false, the default). + pub incremental: bool, + /// Batch form: every item carries its position. + pub with_index: bool, + pub response_timeout: Duration, + pub created_at: Instant, +} + +/// Multiplex the admitted calls into one `GenerateStreamItem` stream, the +/// gRPC twin of the native SSE multiplexer. Each call aborts itself if the +/// stream is dropped while it is unfinished. +pub(super) fn generate_stream( + calls: Vec, + options: StreamOptions, +) -> ResponseStream { + let stream = async_stream::stream! { + let public_ids: Vec = calls + .iter() + .map(|call| call.public_id().to_owned()) + .collect(); + let mut cumulative: Vec = + calls.iter().map(|_| CoreOutput::default()).collect(); + let index = |i: usize| options.with_index.then_some(i as u32); + + // Poll all receivers concurrently; re-arm a receiver's future after each + // non-terminal event so its stream keeps flowing. + let mut pending = FuturesUnordered::new(); + for (i, call) in calls.into_iter().enumerate() { + pending.push(recv_indexed(i, call)); + } + + loop { + let next = tokio::time::timeout(options.response_timeout, pending.next()).await; + let (i, call, events) = match next { + Ok(Some(next)) => next, + // Every call has reached its terminal event. + Ok(None) => break, + Err(_) => { + yield Err(Status::deadline_exceeded("Stream chunk timed out")); + break; + } + }; + if events.is_empty() { + // Defensive fallback: CoreCall normally turns a premature + // runtime close into a Failed event. + yield Ok(error_item(&CoreError::ResponseTruncated, index(i))); + continue; + } + + let acc = &mut cumulative[i]; + // Cumulative frames supersede one another, so a drained backlog + // collapses to its last (Python's `out_list[-1]`); deltas can't be + // dropped. + let mut coalesced = false; + let mut terminal = None; + let mut failed = None; + for event in events { + match event { + CoreEvent::Delta(mut delta) => { + acc.append_delta(&delta); + if !options.stream { + continue; + } + if options.incremental { + // Python's incremental stream still reports the + // cumulative completion-token count in every frame. + delta.completion_tokens = acc.completion_tokens; + yield Ok(frame_item(delta, &public_ids[i], index(i), None)); + } else { + coalesced = true; + } + } + CoreEvent::Finished(delta) => { + acc.append_delta(&delta); + terminal = Some(delta); + } + CoreEvent::Failed(error) => failed = Some(error), + } + } + + if let Some(error) = failed { + yield Ok(error_item(&error, index(i))); + } else if let Some(mut delta) = terminal { + let e2e_latency = Some(options.created_at.elapsed().as_secs_f64()); + // A unary reply is the cumulative result whatever the stream + // policy. The terminal frame is the accumulated output's last + // use, so it moves out instead of cloning. + let output = if options.stream && options.incremental { + delta.completion_tokens = acc.completion_tokens; + delta + } else { + std::mem::take(acc) + }; + yield Ok(frame_item(output, &public_ids[i], index(i), e2e_latency)); + } else { + if coalesced { + // An intermediate cumulative frame has to clone: the + // accumulated output stays behind for the next one. + yield Ok(frame_item(acc.clone(), &public_ids[i], index(i), None)); + } + pending.push(recv_indexed(i, call)); + } + } + // Unfinished calls are still owned by `pending`: dropping the stream + // drops them and uses the core abort lane; a terminal event has + // already disarmed its call. + }; + Box::pin(stream) +} + +fn frame_item( + output: CoreOutput, + public_id: &str, + index: Option, + e2e_latency: Option, +) -> api::GenerateStreamItem { + api::GenerateStreamItem { + item: Some(api::generate_stream_item::Item::Frame(output.frame( + public_id, + index, + e2e_latency, + ))), + } +} + +/// One item's failure inside the stream. `code` is the native error body's +/// HTTP status, as on the SSE error frame. +fn error_item(error: &CoreError, index: Option) -> api::GenerateStreamItem { + api::GenerateStreamItem { + item: Some(api::generate_stream_item::Item::Error( + error.stream_error(index), + )), + } +} + +pub(super) fn status(error: CoreError) -> Status { + let code = match error.kind() { + CoreErrorKind::InvalidArgument => Code::InvalidArgument, + CoreErrorKind::NotFound => Code::NotFound, + CoreErrorKind::FailedPrecondition => Code::FailedPrecondition, + CoreErrorKind::ResourceExhausted => Code::ResourceExhausted, + CoreErrorKind::Cancelled => Code::Cancelled, + CoreErrorKind::DeadlineExceeded => Code::DeadlineExceeded, + CoreErrorKind::Unavailable => Code::Unavailable, + CoreErrorKind::Internal => Code::Internal, + }; + Status::new(code, error.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn semantic_errors_map_to_canonical_grpc_codes() { + let cases = [ + ( + CoreError::InvalidArgument("bad".into()), + Code::InvalidArgument, + ), + ( + CoreError::RuntimeRejected { + kind: CoreErrorKind::NotFound, + message: "missing".into(), + legacy_http_status: 404, + }, + Code::NotFound, + ), + ( + CoreError::RuntimeRejected { + kind: CoreErrorKind::FailedPrecondition, + message: "not ready".into(), + legacy_http_status: 412, + }, + Code::FailedPrecondition, + ), + ( + CoreError::Overloaded("full".into()), + Code::ResourceExhausted, + ), + (CoreError::Cancelled("gone".into()), Code::Cancelled), + ( + CoreError::RuntimeRejected { + kind: CoreErrorKind::DeadlineExceeded, + message: "late".into(), + legacy_http_status: 504, + }, + Code::DeadlineExceeded, + ), + (CoreError::Unavailable, Code::Unavailable), + (CoreError::Internal("bug".into()), Code::Internal), + ]; + + for (error, expected) in cases { + assert_eq!(status(error).code(), expected); + } + } +} diff --git a/rust/sglang-server/src/api_server/grpc/service.rs b/rust/sglang-server/src/api_server/grpc/service.rs new file mode 100644 index 0000000000..46f36d8c35 --- /dev/null +++ b/rust/sglang-server/src/api_server/grpc/service.rs @@ -0,0 +1,718 @@ +//! The `SglangService` implementation: per-RPC translation between the +//! `api.v1` messages and the shared core. + +use std::pin::Pin; +use std::time::{Duration, Instant}; + +use futures::Stream; +use sglang_api_types::api::v1 as api; +use sglang_api_types::api::v1::sglang_service_server::SglangService; +use tonic::{Request, Response, Status}; + +use super::{info, native_api}; +use crate::api_server::core::{CoreHandle, HealthStatus}; +use crate::message::config::{PreferredSamplingParams, ServerArgs}; +use crate::message::request::into_requests; +use crate::message::wire::fill_preferred_sampling; +use crate::utils::environ; + +const DEFAULT_RESPONSE_TIMEOUT: Duration = Duration::from_secs(300); + +pub(super) type ResponseStream = Pin> + Send + 'static>>; + +/// Narrow snapshot of launch policy the adapter needs per call. It +/// intentionally excludes listener, authentication, TLS, and lifecycle +/// configuration. +#[derive(Clone)] +struct AdapterConfig { + preferred_sampling_params: Option, + incremental_streaming_output: bool, + /// Longest wait for the next event of any in-flight call before the RPC + /// ends with DEADLINE_EXCEEDED. + response_timeout: Duration, + /// `/health_generate`'s heartbeat window (`SGLANG_HEALTH_CHECK_TIMEOUT`). + health_timeout: Duration, +} + +/// Tonic-facing implementation backed by the transport-neutral core. +pub(crate) struct GrpcService { + core: CoreHandle, + config: AdapterConfig, +} + +impl GrpcService { + pub(crate) fn new(core: CoreHandle, server_args: &ServerArgs) -> Self { + Self { + core, + config: AdapterConfig { + preferred_sampling_params: server_args.preferred_sampling_params.clone(), + incremental_streaming_output: server_args.incremental_streaming_output, + response_timeout: DEFAULT_RESPONSE_TIMEOUT, + health_timeout: Duration::from_secs( + environ::env_i64("SGLANG_HEALTH_CHECK_TIMEOUT", 20).max(0) as u64, + ), + }, + } + } + + #[cfg(test)] + pub(super) fn for_test( + core: CoreHandle, + preferred_sampling_params: Option, + incremental_streaming_output: bool, + response_timeout: Duration, + ) -> Self { + Self { + core, + config: AdapterConfig { + preferred_sampling_params, + incremental_streaming_output, + response_timeout, + health_timeout: Duration::from_millis(20), + }, + } + } +} + +#[tonic::async_trait] +impl SglangService for GrpcService { + type GenerateStream = ResponseStream; + + /// `/generate` over gRPC: one schema, one fan-out, one admission path. A + /// body that fails normalization ends the RPC with INVALID_ARGUMENT before + /// anything reaches the scheduler, as the HTTP 400 does. + async fn generate( + &self, + request: Request, + ) -> Result, Status> { + let mut request = request.into_inner(); + if let Some(preferred) = &self.config.preferred_sampling_params { + request.sampling_params = + fill_preferred_sampling(request.sampling_params.take(), &preferred.0) + .map_err(Status::internal)?; + } + // Protobuf has no null: a field the client left unset takes the + // schema default, as an absent JSON key does. + request.apply_absent_defaults(); + let stream = request.stream.unwrap_or(false); + let (payloads, is_batch) = + into_requests(request).map_err(|error| Status::invalid_argument(error.to_string()))?; + // Python starts its request clock after normalization and before + // tokenization / multimodal preprocessing; `e2e_latency` measures from + // the same boundary here. + let created_at = Instant::now(); + let calls = self + .core + .generate_batch(payloads) + .await + .map_err(native_api::status)?; + Ok(Response::new(native_api::generate_stream( + calls, + native_api::StreamOptions { + stream, + incremental: self.config.incremental_streaming_output, + with_index: is_batch, + response_timeout: self.config.response_timeout, + created_at, + }, + ))) + } + + async fn health_check( + &self, + _request: Request, + ) -> Result, Status> { + match self.core.probe_health(self.config.health_timeout).await { + Ok(HealthStatus::Healthy) => { + Ok(Response::new(api::HealthCheckResponse { healthy: true })) + } + Ok(HealthStatus::NotReady) => Err(Status::unavailable( + "server is still completing its startup warmup", + )), + Ok(HealthStatus::Stalled) => Err(Status::unavailable( + "no scheduler output within the health-check timeout", + )), + Err(error) => Err(native_api::status(error)), + } + } + + async fn get_model_info( + &self, + _request: Request, + ) -> Result, Status> { + info::model_info(self.core.model_info()) + .map(Response::new) + .map_err(Status::internal) + } + + async fn get_server_info( + &self, + _request: Request, + ) -> Result, Status> { + let server_info = self.core.server_info().await.map_err(native_api::status)?; + info::server_info(server_info) + .map(Response::new) + .map_err(Status::internal) + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use futures::StreamExt; + use sglang_api_types::api::v1 as api; + use sglang_api_types::api::v1::generate_stream_item::Item; + use sglang_api_types::api::v1::sglang_service_server::SglangService; + use tonic::{Code, Request}; + + use super::GrpcService; + use crate::api_server::core::{CoreConfig, CoreHandle, CoreMetadata}; + use crate::message::config::PreferredSamplingParams; + use crate::message::finish_reason::FinishReason; + use crate::message::ids::Rid; + use crate::message::request::{GenerateRequest, Request as RuntimeRequest, RequestKind}; + use crate::message::response::{ChunkEvent, ResponseItem, ResponseSink}; + use crate::tokenizer_manager::wiring::{AbortSource, RequestAdmission, TmEvent}; + + struct Harness { + service: GrpcService, + intake_rx: flume::Receiver, + abort_rx: flume::Receiver, + } + + struct GenerationIntake { + request: Box, + rid: Rid, + sink: ResponseSink, + admission: RequestAdmission, + } + + impl Harness { + fn new(response_capacity: usize, incremental: bool, response_timeout: Duration) -> Self { + Self::build(response_capacity, incremental, response_timeout, true, None) + } + + fn build( + response_capacity: usize, + incremental: bool, + response_timeout: Duration, + startup_ready: bool, + preferred: Option, + ) -> Self { + let (intake_tx, intake_rx) = flume::unbounded(); + let (abort_tx, abort_rx) = flume::unbounded(); + let core = CoreHandle::new( + intake_tx, + abort_tx, + CoreConfig { + response_capacity, + response_activity: Default::default(), + startup_ready, + is_disaggregation: false, + mm_limits: Default::default(), + metadata: CoreMetadata::default(), + }, + ); + Self { + service: GrpcService::for_test(core, preferred, incremental, response_timeout), + intake_rx, + abort_rx, + } + } + + async fn next_generation(&self) -> GenerationIntake { + let TmEvent::Intake { request, admission } = self + .intake_rx + .recv_async() + .await + .expect("request submitted") + else { + panic!("expected generation intake"); + }; + let RuntimeRequest { + rid, + sink, + kind, + state: _, + } = request; + let RequestKind::Generate(request) = kind else { + panic!("expected generation request"); + }; + GenerationIntake { + request, + rid, + sink, + admission, + } + } + } + + fn chunk(rid: &Rid, text: &str, token_id: i64, finished: bool) -> ChunkEvent { + ChunkEvent { + rid: rid.clone(), + token_ids: vec![token_id], + finish_reason: finished.then(stop_reason), + prompt_tokens: 3, + text: text.into(), + completion_tokens: 1, + extras: None, + } + } + + fn stop_reason() -> FinishReason { + serde_json::from_value(serde_json::json!({"type": "stop"})).unwrap() + } + + fn one_string(value: &str) -> api::StringOrList { + api::StringOrList { + value: Some(api::string_or_list::Value::One(value.into())), + } + } + + fn many_strings(values: &[&str]) -> api::StringOrList { + api::StringOrList { + value: Some(api::string_or_list::Value::Many(api::StringList { + items: values.iter().map(|value| (*value).to_owned()).collect(), + })), + } + } + + fn one_token_ids(ids: &[i64]) -> api::TokenIdsOrList { + api::TokenIdsOrList { + value: Some(api::token_ids_or_list::Value::One(api::TokenIds { + ids: ids.to_vec(), + })), + } + } + + fn text_request(text: &str, rid: &str) -> api::GenerateRequest { + api::GenerateRequest { + text: Some(one_string(text)), + rid: Some(one_string(rid)), + stream: Some(true), + ..Default::default() + } + } + + fn ids_request(ids: &[i64], rid: Option<&str>) -> api::GenerateRequest { + api::GenerateRequest { + input_ids: Some(one_token_ids(ids)), + rid: rid.map(one_string), + stream: Some(true), + ..Default::default() + } + } + + fn frame(item: api::GenerateStreamItem) -> api::GenerateResponse { + match item.item { + Some(Item::Frame(frame)) => frame, + other => panic!("expected a frame, got {other:?}"), + } + } + + fn stream_error(item: api::GenerateStreamItem) -> api::GenerateStreamError { + match item.item { + Some(Item::Error(error)) => error, + other => panic!("expected an error item, got {other:?}"), + } + } + + fn meta(frame: &api::GenerateResponse) -> &api::GenerateMetaInfo { + frame + .meta_info + .as_ref() + .expect("every frame carries meta_info") + } + + fn output_ids(frame: &api::GenerateResponse) -> &[i64] { + frame + .output_ids + .as_ref() + .map_or(&[], |ids| ids.ids.as_slice()) + } + + #[tokio::test] + async fn generate_streams_cumulative_frames_for_a_text_prompt() { + let harness = Harness::new(2, false, Duration::from_secs(1)); + let mut stream = harness + .service + .generate(Request::new(text_request("prompt", "request-1"))) + .await + .unwrap() + .into_inner(); + + let intake = harness.next_generation().await; + assert!(intake.admission.try_accept()); + assert_eq!(intake.request.text.as_deref(), Some("prompt")); + assert!(intake.request.input_ids.is_none()); + assert!(intake.request.stream); + assert_eq!(intake.rid.client_facing(), "request-1"); + // One chunk at a time: a queued backlog of cumulative frames collapses to + // its last, so sending both up front would yield only the terminal frame. + intake + .sink + .try_send(ResponseItem::Frame(chunk(&intake.rid, "Hel", 1, false))) + .unwrap(); + let first = frame(stream.next().await.unwrap().unwrap()); + assert_eq!(first.text, "Hel"); + assert_eq!(output_ids(&first), [1]); + assert_eq!(first.index, None); + let first_meta = meta(&first); + assert_eq!(first_meta.id, "request-1"); + assert_eq!(first_meta.prompt_tokens, 3); + assert_eq!(first_meta.completion_tokens, 1); + assert!(first_meta.finish_reason.is_none()); + assert!(first_meta.e2e_latency.is_none()); + + intake + .sink + .try_send(ResponseItem::Done(chunk(&intake.rid, "lo", 2, true))) + .unwrap(); + let finished = frame(stream.next().await.unwrap().unwrap()); + assert_eq!(finished.text, "Hello"); + assert_eq!(output_ids(&finished), [1, 2]); + let finished_meta = meta(&finished); + assert_eq!(finished_meta.completion_tokens, 2); + assert!(matches!( + finished_meta.finish_reason, + Some(api::FinishReason { + kind: Some(api::finish_reason::Kind::Stop(_)) + }) + )); + assert!(finished_meta.e2e_latency.is_some()); + assert!(stream.next().await.is_none()); + assert!(harness.abort_rx.try_recv().is_err()); + } + + #[tokio::test] + async fn incremental_frames_carry_deltas_with_cumulative_count() { + let harness = Harness::new(2, true, Duration::from_secs(1)); + let mut stream = harness + .service + .generate(Request::new(ids_request(&[4, 5], Some("tokens")))) + .await + .unwrap() + .into_inner(); + + let intake = harness.next_generation().await; + assert!(intake.admission.try_accept()); + assert_eq!(intake.request.input_ids.as_deref(), Some(&[4, 5][..])); + assert!(intake.request.text.is_none()); + intake + .sink + .try_send(ResponseItem::Frame(chunk(&intake.rid, "A", 10, false))) + .unwrap(); + intake + .sink + .try_send(ResponseItem::Done(chunk(&intake.rid, "B", 11, true))) + .unwrap(); + + let first = frame(stream.next().await.unwrap().unwrap()); + let finished = frame(stream.next().await.unwrap().unwrap()); + assert_eq!(output_ids(&first), [10]); + assert_eq!(output_ids(&finished), [11]); + assert_eq!(finished.text, "B"); + assert_eq!(meta(&finished).completion_tokens, 2); + assert!(stream.next().await.is_none()); + } + + /// `stream: false` is a one-frame stream: deltas fold into the terminal + /// result, and the server's incremental policy does not split a unary reply. + #[tokio::test] + async fn unary_request_yields_only_the_cumulative_terminal_frame() { + let harness = Harness::new(2, true, Duration::from_secs(1)); + let mut request = text_request("prompt", "unary"); + request.stream = Some(false); + let mut stream = harness + .service + .generate(Request::new(request)) + .await + .unwrap() + .into_inner(); + + let intake = harness.next_generation().await; + assert!(intake.admission.try_accept()); + assert!(!intake.request.stream); + intake + .sink + .try_send(ResponseItem::Frame(chunk(&intake.rid, "Hel", 1, false))) + .unwrap(); + intake + .sink + .try_send(ResponseItem::Done(chunk(&intake.rid, "lo", 2, true))) + .unwrap(); + + let only = frame(stream.next().await.unwrap().unwrap()); + assert_eq!(only.text, "Hello"); + assert_eq!(output_ids(&only), [1, 2]); + assert_eq!(meta(&only).completion_tokens, 2); + assert!(stream.next().await.is_none()); + } + + #[tokio::test] + async fn stream_drop_aborts_admitted_request() { + let harness = Harness::new(1, false, Duration::from_secs(1)); + let mut stream = harness + .service + .generate(Request::new(ids_request(&[1], Some("cancel-me")))) + .await + .unwrap() + .into_inner(); + let intake = harness.next_generation().await; + assert!(intake.admission.try_accept()); + intake + .sink + .try_send(ResponseItem::Frame(chunk(&intake.rid, "partial", 1, false))) + .unwrap(); + assert!(stream.next().await.unwrap().is_ok()); + drop(stream); + let abort = harness.abort_rx.recv_async().await.unwrap(); + assert_eq!(abort.rid().client_facing(), "cancel-me"); + assert!(harness.abort_rx.try_recv().is_err()); + } + + /// A runtime failure after admission is this request's in-stream error item + /// (the SSE error frame's twin), not an RPC status; the call is disarmed. + #[tokio::test] + async fn runtime_failure_rides_the_stream_as_an_error_item() { + let harness = Harness::new(1, false, Duration::from_secs(1)); + let mut stream = harness + .service + .generate(Request::new(ids_request(&[1], None))) + .await + .unwrap() + .into_inner(); + let intake = harness.next_generation().await; + assert!(intake.admission.try_accept()); + intake + .sink + .try_send(ResponseItem::Error(crate::utils::error::Error::Validation( + "bad sampling".into(), + ))) + .unwrap(); + + let error = stream_error(stream.next().await.unwrap().unwrap()); + assert_eq!(error.index, None); + let body = error.error.unwrap(); + assert_eq!(body.code, 400); + assert!(body.message.contains("bad sampling")); + assert!(stream.next().await.is_none()); + drop(stream); + assert!(harness.abort_rx.try_recv().is_err()); + } + + #[tokio::test] + async fn per_chunk_timeout_after_progress_returns_deadline_exceeded_and_aborts() { + let harness = Harness::new(1, false, Duration::from_millis(10)); + let mut stream = harness + .service + .generate(Request::new(ids_request(&[1], Some("too-slow")))) + .await + .unwrap() + .into_inner(); + let intake = harness.next_generation().await; + assert!(intake.admission.try_accept()); + intake + .sink + .try_send(ResponseItem::Frame(chunk(&intake.rid, "partial", 1, false))) + .unwrap(); + assert!(stream.next().await.unwrap().is_ok()); + + let error = stream.next().await.unwrap().unwrap_err(); + assert_eq!(error.code(), Code::DeadlineExceeded); + drop(stream); + let abort = harness.abort_rx.recv_async().await.unwrap(); + assert_eq!(abort.rid().client_facing(), "too-slow"); + } + + #[tokio::test] + async fn closed_intake_is_a_top_level_unavailable_status() { + let (intake_tx, intake_rx) = flume::unbounded(); + drop(intake_rx); + let (abort_tx, _abort_rx) = flume::unbounded(); + let core = CoreHandle::new( + intake_tx, + abort_tx, + CoreConfig { + response_capacity: 1, + response_activity: Default::default(), + startup_ready: true, + is_disaggregation: false, + mm_limits: Default::default(), + metadata: CoreMetadata::default(), + }, + ); + let service = GrpcService::for_test(core, None, false, Duration::from_secs(1)); + + let result = service + .generate(Request::new(ids_request(&[1], None))) + .await; + let error = match result { + Ok(_) => panic!("closed intake must reject the RPC"), + Err(error) => error, + }; + assert_eq!(error.code(), Code::Unavailable); + } + + /// Normalization failures end the RPC before anything is submitted, with the + /// status that mirrors the HTTP 400. + #[tokio::test] + async fn malformed_request_is_rejected_before_submission() { + let harness = Harness::new(1, false, Duration::from_secs(1)); + let mut both = text_request("prompt", "both"); + both.input_ids = Some(one_token_ids(&[1])); + let error = harness + .service + .generate(Request::new(both)) + .await + .err() + .expect("text and input_ids together must be rejected"); + assert_eq!(error.code(), Code::InvalidArgument); + assert!(error.message().contains("text")); + + let error = harness + .service + .generate(Request::new(ids_request(&[], None))) + .await + .err() + .expect("empty input_ids must be rejected"); + assert_eq!(error.code(), Code::InvalidArgument); + assert!(harness.intake_rx.try_recv().is_err()); + } + + /// A list-form request fans out like the HTTP batch: every item is tagged + /// with its position, and one item's failure neither ends the stream nor + /// touches its siblings. + #[tokio::test] + async fn batch_items_are_indexed_and_fail_independently() { + let harness = Harness::new(2, false, Duration::from_secs(1)); + let mut stream = harness + .service + .generate(Request::new(api::GenerateRequest { + text: Some(many_strings(&["a", "b"])), + stream: Some(true), + ..Default::default() + })) + .await + .unwrap() + .into_inner(); + + let first = harness.next_generation().await; + let second = harness.next_generation().await; + assert!(first.admission.try_accept()); + assert!(second.admission.try_accept()); + assert_eq!(first.request.text.as_deref(), Some("a")); + assert_eq!(second.request.text.as_deref(), Some("b")); + first + .sink + .try_send(ResponseItem::Error(crate::utils::error::Error::Validation( + "first failed".into(), + ))) + .unwrap(); + second + .sink + .try_send(ResponseItem::Done(chunk(&second.rid, "ok", 5, true))) + .unwrap(); + + let mut items = Vec::new(); + while let Some(item) = stream.next().await { + items.push(item.unwrap()); + } + assert_eq!(items.len(), 2); + let (errors, frames): (Vec<_>, Vec<_>) = items + .into_iter() + .partition(|item| matches!(item.item, Some(Item::Error(_)))); + let error = stream_error(errors.into_iter().next().unwrap()); + assert_eq!(error.index, Some(0)); + assert!(error.error.unwrap().message.contains("first failed")); + let ok = frame(frames.into_iter().next().unwrap()); + assert_eq!(ok.index, Some(1)); + assert_eq!(ok.text, "ok"); + assert_eq!(meta(&ok).id, second.rid.client_facing()); + assert!(harness.abort_rx.try_recv().is_err()); + } + + /// Launch-time preferred sampling params fill the fields a protobuf request + /// left unset, beneath the ones it set: the HTTP precedence at the gRPC entry. + #[tokio::test] + async fn preferred_sampling_params_fill_unset_fields() { + let preferred = PreferredSamplingParams(serde_json::json!({ + "temperature": 0.25, + "max_new_tokens": 32, + })); + let harness = Harness::build(1, false, Duration::from_secs(1), true, Some(preferred)); + let mut request = ids_request(&[1], None); + request.sampling_params = Some(api::SamplingParamsOrList { + value: Some(api::sampling_params_or_list::Value::One( + api::SamplingParams { + temperature: Some(0.8), + ..Default::default() + }, + )), + }); + let _stream = harness + .service + .generate(Request::new(request)) + .await + .unwrap() + .into_inner(); + + let intake = harness.next_generation().await; + assert_eq!(intake.request.sampling_params.temperature, 0.8); + assert_eq!(intake.request.sampling_params.max_new_tokens, Some(32)); + } + + /// Protobuf has no null: a sampling field the client left unset takes the + /// schema default, as an absent JSON key does. `max_new_tokens` is the one + /// field whose Rust `None` means "unbounded", so without the absent-defaults + /// pass an unset value would lift the limit instead of applying 128. + #[tokio::test] + async fn unset_protobuf_sampling_fields_take_schema_defaults() { + let harness = Harness::new(1, false, Duration::from_secs(1)); + let mut request = ids_request(&[1], None); + request.sampling_params = Some(api::SamplingParamsOrList { + value: Some(api::sampling_params_or_list::Value::One( + api::SamplingParams { + temperature: Some(0.8), + ..Default::default() + }, + )), + }); + let _stream = harness + .service + .generate(Request::new(request)) + .await + .unwrap() + .into_inner(); + + let intake = harness.next_generation().await; + assert_eq!(intake.request.sampling_params.temperature, 0.8); + assert_eq!(intake.request.sampling_params.max_new_tokens, Some(128)); + assert_eq!(intake.request.sampling_params.top_p, 1.0); + } + + /// `HealthCheck` is `/health_generate`: not ready and stalled are both + /// UNAVAILABLE, and the stalled probe is a real generation submission. + #[tokio::test] + async fn health_check_reports_not_ready_and_stalled_as_unavailable() { + let not_ready = Harness::build(1, false, Duration::from_secs(1), false, None); + let error = not_ready + .service + .health_check(Request::new(api::HealthCheckRequest {})) + .await + .expect_err("warmup must not report healthy"); + assert_eq!(error.code(), Code::Unavailable); + assert!(not_ready.intake_rx.try_recv().is_err()); + + let stalled = Harness::new(1, false, Duration::from_secs(1)); + let error = stalled + .service + .health_check(Request::new(api::HealthCheckRequest {})) + .await + .expect_err("no heartbeat within the window must not report healthy"); + assert_eq!(error.code(), Code::Unavailable); + let probe = stalled.next_generation().await; + assert_eq!(probe.request.input_ids.as_deref(), Some(&[0][..])); + } +} diff --git a/rust/sglang-server/src/api_server/http.rs b/rust/sglang-server/src/api_server/http.rs new file mode 100644 index 0000000000..10b8f780f2 --- /dev/null +++ b/rust/sglang-server/src/api_server/http.rs @@ -0,0 +1,4 @@ +pub(crate) mod app; +mod info; +mod native_api; +mod openai; diff --git a/rust/sglang-server/src/api_server/app.rs b/rust/sglang-server/src/api_server/http/app.rs similarity index 79% rename from rust/sglang-server/src/api_server/app.rs rename to rust/sglang-server/src/api_server/http/app.rs index 2fa0deb2a9..9d16ad9205 100644 --- a/rust/sglang-server/src/api_server/app.rs +++ b/rust/sglang-server/src/api_server/http/app.rs @@ -12,20 +12,21 @@ use axum::{ response::Response, }; -use super::disaggregation::bootstrap as pd_bootstrap; -use super::{common, log, native_api, openai}; -use crate::frontend::FrontendHandle; +use super::{info, native_api, openai}; +use crate::api_server::core::CoreHandle; +use crate::api_server::disaggregation::bootstrap as pd_bootstrap; +use crate::api_server::log; use crate::message::config::ServerArgs; -/// HTTP adapter state: the shared frontend capability, immutable server +/// HTTP adapter state: the shared core capability, immutable server /// configuration needed for HTTP request preparation, and the chat formatter. /// /// axum clones the router state into **every** request, so it is mounted as /// `Arc` — one refcount bump per request instead of cloning the -/// frontend handle and chat formatter. Deliberately not `Clone`, so it +/// core handle and chat formatter. Deliberately not `Clone`, so it /// can only be shared through that `Arc`. pub(super) struct AppState { - pub(super) frontend: FrontendHandle, + pub(super) core: CoreHandle, pub(super) server_args: Arc, pub(super) chat_formatter: Option, } @@ -45,23 +46,19 @@ async fn mark_startup_ready( "/generate" | "/encode" | "/v1/chat/completions" ); let response = next.run(req).await; - record_startup_warmup_status(&state.frontend, is_startup_warmup, response.status()); + record_startup_warmup_status(&state.core, is_startup_warmup, response.status()); response } -fn record_startup_warmup_status( - frontend: &FrontendHandle, - is_startup_warmup: bool, - status: StatusCode, -) { +fn record_startup_warmup_status(core: &CoreHandle, is_startup_warmup: bool, status: StatusCode) { if is_startup_warmup && status.is_success() { - frontend.mark_ready(); + core.mark_ready(); } } pub async fn serve( listener: std::net::TcpListener, - frontend: FrontendHandle, + core: CoreHandle, server_args: Arc, // The runtime's shutdown signal, shared with every worker stage: it fires // (disconnects) when `Runtime::request_shutdown` drops the sender, at @@ -71,13 +68,13 @@ pub async fn serve( ) { let chat_formatter = openai::load_chat_support(&server_args); let state = Arc::new(AppState { - frontend, + core, server_args: server_args.clone(), chat_formatter, }); // Each endpoint module registers its own routes and merges here. let router = Router::new() - .merge(common::routes()) + .merge(info::routes()) .merge(native_api::routes()) .merge(openai::routes()); @@ -138,32 +135,32 @@ pub async fn serve( mod tests { use super::*; - fn frontend() -> FrontendHandle { - FrontendHandle::new( + fn core_handle() -> CoreHandle { + CoreHandle::new( flume::unbounded().0, flume::unbounded().0, - crate::frontend::FrontendConfig { + crate::api_server::core::CoreConfig { response_capacity: 8, response_activity: Default::default(), startup_ready: false, is_disaggregation: false, mm_limits: Default::default(), - metadata: crate::frontend::FrontendMetadata::default(), + metadata: crate::api_server::core::CoreMetadata::default(), }, ) } #[test] fn only_a_successful_recognized_warmup_marks_frontend_ready() { - let frontend = frontend(); + let core = core_handle(); - record_startup_warmup_status(&frontend, false, StatusCode::OK); - assert!(!frontend.is_ready()); + record_startup_warmup_status(&core, false, StatusCode::OK); + assert!(!core.is_ready()); - record_startup_warmup_status(&frontend, true, StatusCode::INTERNAL_SERVER_ERROR); - assert!(!frontend.is_ready()); + record_startup_warmup_status(&core, true, StatusCode::INTERNAL_SERVER_ERROR); + assert!(!core.is_ready()); - record_startup_warmup_status(&frontend, true, StatusCode::OK); - assert!(frontend.is_ready()); + record_startup_warmup_status(&core, true, StatusCode::OK); + assert!(core.is_ready()); } } diff --git a/rust/sglang-server/src/api_server/common.rs b/rust/sglang-server/src/api_server/http/info.rs similarity index 92% rename from rust/sglang-server/src/api_server/common.rs rename to rust/sglang-server/src/api_server/http/info.rs index 20f38780d9..df798c45a6 100644 --- a/rust/sglang-server/src/api_server/common.rs +++ b/rust/sglang-server/src/api_server/http/info.rs @@ -1,6 +1,6 @@ //! Common control-plane endpoints — `/server_info`, `/get_model_info` //! (+ `/model_info` alias), backed by typed operations on -//! [`crate::frontend::FrontendHandle`]. Data-plane endpoints (incl. `/health*`, +//! [`crate::api_server::core::CoreHandle`]. Data-plane endpoints (incl. `/health*`, //! which round-trips a generate probe) live in the sibling `native_api` and //! `openai` modules; the shared `AppState` lives in the parent //! `api_server` module. @@ -15,14 +15,14 @@ use axum::{ use std::sync::Arc; use super::app::AppState; -use super::frontend_error_status; use super::native_api::native_error; -use crate::frontend::{FrontendError, ServerInfo}; +use crate::api_server::core::CoreError; +use crate::message::info::ServerInfo; /// The routes this module owns, mounted by `api_server::serve`. pub(super) fn routes() -> Router> { Router::new() - // Control-plane: the frontend performs the typed operation and this + // Control-plane: the core performs the typed operation and this // adapter renders its result as one non-streamed JSON response. .route("/server_info", get(server_info)) // Static config, no scheduler round-trip. `/get_model_info` (+ `/model_info` @@ -31,23 +31,23 @@ pub(super) fn routes() -> Router> { .route("/model_info", get(model_info)) } -/// Await the frontend's complete semantic server-info result. HTTP status and +/// Await the core's complete semantic server-info result. HTTP status and /// serialization stay here; scheduler control bytes stay behind the contract. async fn await_server_info(state: &AppState) -> Result { - match state.frontend.server_info().await { + match state.core.server_info().await { Ok(server_info) => Ok(server_info), - Err(FrontendError::Unavailable) => Err(native_error( + Err(CoreError::Unavailable) => Err(native_error( StatusCode::SERVICE_UNAVAILABLE, "service unavailable", false, )), // Preserve the established HTTP behavior for a scheduler-side control // stream that disappears, while non-HTTP adapters see Internal via - // FrontendError::kind(). - Err(FrontendError::ResponseTruncated) => { + // CoreError::kind(). + Err(CoreError::ResponseTruncated) => { Err((StatusCode::from_u16(499).unwrap(), "request aborted").into_response()) } - Err(error @ FrontendError::InvalidResponse(_)) => { + Err(error @ CoreError::InvalidResponse(_)) => { tracing::error!(%error, "server_info: invalid runtime response"); Err(( StatusCode::INTERNAL_SERVER_ERROR, @@ -55,14 +55,14 @@ async fn await_server_info(state: &AppState) -> Result { ) .into_response()) } - Err(error) => Err((frontend_error_status(&error), error.to_string()).into_response()), + Err(error) => Err((error.http_status(), error.to_string()).into_response()), } } /// `GET /get_model_info` (+ `/model_info` alias) — serialize the shared static /// model metadata (no scheduler round-trip). async fn model_info(State(state): State>) -> Response { - let info = state.frontend.model_info(); + let info = state.core.model_info(); ( StatusCode::OK, [("content-type", "application/json")], @@ -104,7 +104,7 @@ mod tests { use tower::ServiceExt; use super::*; - use crate::frontend::{FrontendConfig, FrontendHandle, FrontendMetadata}; + use crate::api_server::core::{CoreConfig, CoreHandle, CoreMetadata}; use crate::message::config::{DisaggregationMode, ModelConfig, ServerArgs}; use crate::message::request::RequestKind; use crate::message::response::ResponseItem; @@ -120,21 +120,21 @@ mod tests { let server_args = Arc::new(server_args); let (intake_tx, intake_rx) = flume::unbounded(); let (abort_tx, abort_rx) = flume::unbounded(); - let frontend = FrontendHandle::new( + let core = CoreHandle::new( intake_tx, abort_tx, - FrontendConfig { + CoreConfig { response_capacity: 8, response_activity: Default::default(), startup_ready: true, is_disaggregation: false, mm_limits: Default::default(), - metadata: FrontendMetadata::from(server_args.as_ref()), + metadata: CoreMetadata::from(server_args.as_ref()), }, ); ( Arc::new(AppState { - frontend, + core, server_args, chat_formatter: None, }), diff --git a/rust/sglang-server/src/api_server/native_api.rs b/rust/sglang-server/src/api_server/http/native_api.rs similarity index 81% rename from rust/sglang-server/src/api_server/native_api.rs rename to rust/sglang-server/src/api_server/http/native_api.rs index ca06f7bd80..1b63b66185 100644 --- a/rust/sglang-server/src/api_server/native_api.rs +++ b/rust/sglang-server/src/api_server/http/native_api.rs @@ -4,23 +4,20 @@ //! `http_server.generate_request`) and `/health` + `/health_generate` (which //! round-trip a 1-token generate probe). Frame shaping (`meta_info`, logprob //! tuples, cumulative vs incremental streams) lives here; submission and -//! request lifetime belong to the shared [`crate::frontend::FrontendHandle`]. +//! request lifetime belong to the shared [`crate::api_server::core::CoreHandle`]. use std::convert::Infallible; use std::sync::Arc; use std::time::{Duration, Instant}; use super::app::AppState; -use super::frame::{ - OutputAccumulator, cumulative_frame_string, frame_value, stream_frame_string, tag_value, -}; -use super::frontend_error_status; -use crate::frontend::{ - FrontendCall, FrontendError, FrontendEvent, FrontendOutput, FrontendRequest, HealthStatus, +use crate::api_server::core::{ + CoreCall, CoreError, CoreEvent, CoreOutput, HealthStatus, recv_indexed, }; #[cfg(test)] use crate::message::ids::Rid; -use crate::message::request::GenerateBody; +use crate::message::request::{GenerateRequest, into_requests}; +use crate::message::wire::merge_preferred_sampling; use crate::utils::{ environ, response::{error_response, error_value}, @@ -36,6 +33,7 @@ use axum::{ }, routing::{get, post}, }; +use sglang_api_types::api::v1 as api; /// API-local timing for one request. /// @@ -89,8 +87,8 @@ pub(super) fn native_error(code: StatusCode, message: &str, stream: bool) -> Res error_response(code, error_value(code.as_u16(), message), stream) } -fn native_frontend_error(error: FrontendError, stream: bool) -> Response { - native_error(frontend_error_status(&error), &error.to_string(), stream) +fn native_core_error(error: CoreError, stream: bool) -> Response { + native_error(error.http_status(), &error.to_string(), stream) } /// `/health` + `/health_generate`. Both env knobs are resolved ONCE here, at @@ -116,7 +114,7 @@ fn health_routes() -> Router> { } async fn health_without_generation(State(state): State>) -> Response { - if state.frontend.is_ready() { + if state.core.is_ready() { StatusCode::OK.into_response() } else { StatusCode::SERVICE_UNAVAILABLE.into_response() @@ -131,19 +129,19 @@ async fn health_without_generation(State(state): State>) -> Respon /// /// Fires a pre-tokenized 1-token probe (`input_ids = [0]`, skips the tokenizer) so /// an idle pipeline produces a frame, then watches the *global* -/// frontend response-activity counter (not the probe's own rid) — so a busy +/// core response-activity counter (not the probe's own rid) — so a busy /// server passes immediately and a backlog never false-503s (the analogue of /// Python's `last_receive_tstamp`). async fn health_generate( State(state): State>, timeout: std::time::Duration, ) -> Response { - match state.frontend.probe_health(timeout).await { + match state.core.probe_health(timeout).await { Ok(HealthStatus::Healthy) => StatusCode::OK.into_response(), Ok(HealthStatus::NotReady | HealthStatus::Stalled) => { StatusCode::SERVICE_UNAVAILABLE.into_response() } - Err(FrontendError::Unavailable) => native_error( + Err(CoreError::Unavailable) => native_error( StatusCode::SERVICE_UNAVAILABLE, "service unavailable", false, @@ -162,30 +160,35 @@ async fn health_generate( /// message, instead of axum's default 422. async fn generate( State(state): State>, - body: Result, JsonRejection>, + body: Result, JsonRejection>, ) -> Response { let mut body = match body { Ok(Json(body)) => body, // A body that fails to parse has no readable `stream` flag, so this one - // can only answer unary — as Python's does (FastAPI rejects before its + // can only answer unary -- as Python's does (FastAPI rejects before its // handler runs). Err(rejection) => { return native_error(StatusCode::BAD_REQUEST, &rejection.body_text(), false); } }; - let stream = body.stream; + // Launch-time preferred sampling params fill in beneath the request's own + // keys before decoding, so an explicit request value (even a default or + // null) wins: Python TokenizerManager's precedence. if let Some(preferred) = &state.server_args.preferred_sampling_params - && let Err(error) = body.apply_preferred_sampling(&preferred.0) + && let Err(error) = merge_preferred_sampling(&mut body, &preferred.0) { - return native_error( - StatusCode::INTERNAL_SERVER_ERROR, - &error.to_string(), - stream, - ); + return native_error(StatusCode::INTERNAL_SERVER_ERROR, &error, false); } + // The schema (sglang_api_types) is the contract: decode errors carry + // serde's field-level text, as Python's 400s do. + let body: api::GenerateRequest = match serde_json::from_value(body) { + Ok(body) => body, + Err(error) => return native_error(StatusCode::BAD_REQUEST, &error.to_string(), false), + }; + let stream = body.stream.unwrap_or(false); // Fan `text`/`input_ids`/`sampling_params` (scalar or list) into per-request - // payloads. `is_batch` = list form → the response is a JSON array. - let (payloads, is_batch) = match body.into_requests() { + // payloads. `is_batch` = list form -> the response is a JSON array. + let (payloads, is_batch) = match into_requests(body) { Ok(v) => v, // Request normalization only returns validation failures. Err(e) => { @@ -197,7 +200,7 @@ async fn generate( // equivalent boundary: into_requests() has normalized the body, while prefetch // and every downstream stage are still ahead of us. let timing = RequestTiming::new(); - // Shared frontend admission performs media I/O (URL downloads, file reads) + // Shared core admission performs media I/O (URL downloads, file reads) // on this async runtime before any request reaches the MM worker pool. if !is_batch { // `into_requests` guarantees exactly one payload for a non-batch body. @@ -215,15 +218,15 @@ async fn generate( /// SSE frames or fold to one unary response. async fn generate_single( state: &AppState, - req: FrontendRequest, + req: GenerateRequest, stream: bool, timing: RequestTiming, ) -> Response { // `return_text_in_logprobs` is decoded on the detok shard into `*_txt`, so - // `frame_value` just reads them — no tokenizer needed here. - let mut call = match state.frontend.generate(req).await { + // the frame builder just reads them — no tokenizer needed here. + let mut call = match state.core.generate(req).await { Ok(call) => call, - Err(error) => return native_frontend_error(error, stream), + Err(error) => return native_core_error(error, stream), }; let rid = call.public_id().to_owned(); // Cumulative frames (SGLang default) vs per-step deltas. @@ -237,47 +240,59 @@ async fn generate_single( .map(|data| Ok::<_, Infallible>(Event::default().data(data))); Sse::new(s).into_response() } else { - // Unary: fold to the terminal and respond once. `FrontendCall` disarms + // Unary: fold to the terminal and respond once. `CoreCall` disarms // itself on a terminal item; truncation leaves it armed for drop cleanup. - let (status, value) = drain_unary(&mut call, &rid, timing).await; - (status, Json(value)).into_response() + let (status, item) = drain_unary(&mut call, &rid, timing).await; + (status, Json(item)).into_response() } } +/// One unary `/generate` result: the frame, or the item's own error body (a +/// failed batch item is its own `{ "error": ... }` entry in the array). +#[derive(Debug, serde::Serialize)] +#[serde(untagged)] +enum GenerateItem { + // Boxed: a frame is ~300 bytes against the error's ~40, and a batch + // array holds one per item. + Frame(Box), + Error(api::GenerateStreamError), +} + /// Fold a unary request to its terminal HTTP result. Shared by single + batch. async fn drain_unary( - call: &mut FrontendCall, + call: &mut CoreCall, rid_str: &str, mut timing: RequestTiming, -) -> (StatusCode, serde_json::Value) { - let mut acc = OutputAccumulator::default(); +) -> (StatusCode, GenerateItem) { + let mut accumulated = CoreOutput::default(); while let Some(item) = call.recv().await { match item { - FrontendEvent::Delta(out) => { + CoreEvent::Delta(out) => { timing.observe_first_output(); - acc.fold(&out); + accumulated.append_delta(&out); } - FrontendEvent::Finished(out) => { + CoreEvent::Finished(out) => { timing.observe_first_output(); timing.finish(); - acc.fold(&out); - let final_out = acc.into_output(); - let mut value = frame_value(&final_out, rid_str); - add_e2e_latency(&mut value, &timing); - return (StatusCode::OK, value); + accumulated.append_delta(&out); + let frame = accumulated.frame(rid_str, None, Some(e2e_latency(&timing))); + return (StatusCode::OK, GenerateItem::Frame(Box::new(frame))); } - FrontendEvent::Failed(error) => { + CoreEvent::Failed(error) => { timing.finish(); - let status = frontend_error_status(&error); - return (status, error_value(status.as_u16(), &error.to_string())); + return ( + error.http_status(), + GenerateItem::Error(error.stream_error(None)), + ); } } } - // Defensive fallback: FrontendCall normally turns premature runtime + // Defensive fallback: CoreCall normally turns premature runtime // termination into a terminal Failed event before exposing stream end. + let truncated = CoreError::ResponseTruncated; ( - StatusCode::INTERNAL_SERVER_ERROR, - error_value(500, "response truncated before completion"), + truncated.http_status(), + GenerateItem::Error(truncated.stream_error(None)), ) } @@ -289,19 +304,19 @@ async fn drain_unary( /// own `{ "error": … }` entry; the batch response is 200. async fn generate_batch( state: &AppState, - requests: Vec, + requests: Vec, stream: bool, timing: RequestTiming, ) -> Response { // No cross-item rid collision to worry about: `into_requests` rejected duplicate // rids within this batch, and `Rid::from_client` made each one unique against // every other in-flight request. - let calls = match state.frontend.generate_batch(requests).await { + let calls = match state.core.generate_batch(requests).await { Ok(calls) => calls .into_iter() .map(|call| (call, timing.clone())) .collect(), - Err(error) => return native_frontend_error(error, stream), + Err(error) => return native_core_error(error, stream), }; if stream { @@ -319,37 +334,20 @@ async fn generate_batch( let drained = futures::future::join_all(calls.into_iter().map( |(mut call, request_timing)| async move { let client_rid = call.public_id().to_owned(); - let (_status, value) = drain_unary(&mut call, &client_rid, request_timing).await; - value + let (_status, item) = drain_unary(&mut call, &client_rid, request_timing).await; + item }, )) .await; - (StatusCode::OK, Json(serde_json::Value::Array(drained))).into_response() + (StatusCode::OK, Json(drained)).into_response() } } -/// Await the next event from `call`, then drain whatever queued behind it (so the -/// caller can coalesce a backlog, as Python's `state.out_list` does), handing the -/// call back for `FuturesUnordered` to re-poll. An empty result means the semantic -/// stream was already exhausted. -async fn recv_indexed( - index: usize, - mut call: FrontendCall, -) -> (usize, FrontendCall, Vec) { - let mut items = Vec::new(); - match call.recv().await { - Some(item) => items.push(item), - None => return (index, call, items), // already exhausted - } - call.drain_ready(&mut items); - (index, call, items) -} - /// Multiplex calls into SSE `data` strings + a final `[DONE]`; `with_index` tags /// each frame (batch only), `incremental` = delta vs cumulative. Each call /// aborts itself if the stream is dropped while it is unfinished. fn generation_event_stream( - calls: Vec<(FrontendCall, RequestTiming)>, + calls: Vec<(CoreCall, RequestTiming)>, incremental: bool, with_index: bool, ) -> impl futures::Stream { @@ -365,11 +363,10 @@ fn generation_event_stream( .iter() .map(|(_, timing)| timing.clone()) .collect(); - let mut accs: Vec = - (0..n).map(|_| OutputAccumulator::default()).collect(); + let mut accumulated: Vec = (0..n).map(|_| CoreOutput::default()).collect(); // Batch position, tagged onto every frame (a single request omits it). - let idx = |i: usize| with_index.then_some(i); + let idx = |i: usize| with_index.then_some(i as u32); // Poll all receivers concurrently; re-arm a receiver's future after each // non-terminal frame so its stream keeps flowing. @@ -381,9 +378,9 @@ fn generation_event_stream( while let Some((i, call, items)) = futs.next().await { if items.is_empty() { // Defensive fallback: a premature runtime close is normally a - // Failed event produced by FrontendCall. Keep this item's call + // Failed event produced by CoreCall. Keep this item's call // armed if its semantic stream somehow ends unexpectedly. - yield tag_value(error_value(500, "response truncated before completion"), idx(i)); + yield json_string(&CoreError::ResponseTruncated.stream_error(idx(i))); continue; } @@ -395,22 +392,22 @@ fn generation_event_stream( for item in items { match item { - FrontendEvent::Delta(out) => { + CoreEvent::Delta(out) => { timings[i].observe_first_output(); - accs[i].fold(&out); + accumulated[i].append_delta(&out); if incremental { - yield stream_frame_string(out, &accs[i], true, &rid_strs[i], idx(i)); + yield json_string(&delta_frame(out, &accumulated[i], &rid_strs[i], idx(i), None)); } else { coalesced = true; } } - FrontendEvent::Finished(out) => { + CoreEvent::Finished(out) => { timings[i].observe_first_output(); timings[i].finish(); - accs[i].fold(&out); + accumulated[i].append_delta(&out); terminal = Some(out); } - FrontendEvent::Failed(error) => { + CoreEvent::Failed(error) => { timings[i].finish(); failed = Some(error); } @@ -418,22 +415,24 @@ fn generation_event_stream( } if let Some(error) = failed { - let status = frontend_error_status(&error); - yield tag_value(error_value(status.as_u16(), &error.to_string()), idx(i)); + yield json_string(&error.stream_error(idx(i))); } else if let Some(out) = terminal { // The final frame carries the full cumulative state, so any // coalesced non-terminal frames are moot. - yield terminal_stream_frame_string( - out, - &accs[i], - incremental, - &rid_strs[i], - idx(i), - &timings[i], - ); + let e2e_latency = Some(e2e_latency(&timings[i])); + // The terminal frame is the accumulated output's last use, so + // the cumulative form moves it out instead of cloning. + let frame = if incremental { + delta_frame(out, &accumulated[i], &rid_strs[i], idx(i), e2e_latency) + } else { + std::mem::take(&mut accumulated[i]).frame(&rid_strs[i], idx(i), e2e_latency) + }; + yield json_string(&frame); } else { if coalesced { - yield cumulative_frame_string(&accs[i], &rid_strs[i], idx(i)); + // An intermediate cumulative frame has to clone: the accumulated + // output stays behind for the next one. + yield json_string(&accumulated[i].clone().frame(&rid_strs[i], idx(i), None)); } futs.push(recv_indexed(i, call)); // keep this item flowing } @@ -444,30 +443,33 @@ fn generation_event_stream( /// Python's `e2e_latency` is `finished_time - created_time`, in seconds, and is /// attached only when the request finishes. The Rust native API owns the same -/// lifecycle boundary, so it adds the value while handling the terminal egress -/// item rather than putting API-only timing onto every shared frontend output. -fn add_e2e_latency(value: &mut serde_json::Value, timing: &RequestTiming) { +/// lifecycle boundary, so it sets the value while handling the terminal egress +/// item rather than putting API-only timing onto every shared core output. +fn e2e_latency(timing: &RequestTiming) -> f64 { let (time_to_first_token, e2e_latency) = timing .terminal_latencies() .expect("a successful terminal output has complete request timing"); debug_assert!(time_to_first_token <= e2e_latency); - value["meta_info"]["e2e_latency"] = serde_json::json!(e2e_latency.as_secs_f64()); + e2e_latency.as_secs_f64() } -/// Render a terminal streaming frame. Intermediate cumulative frames keep the -/// memoized fast path; the one terminal frame uses the Value path so it can carry -/// the request-local `e2e_latency`, exactly as Python does. -fn terminal_stream_frame_string( - out: FrontendOutput, - acc: &OutputAccumulator, - incremental: bool, +/// One incremental streaming frame: this step's delta with the cumulative +/// token count in `meta_info` (matching Python). Only the terminal frame +/// carries `e2e_latency`. +fn delta_frame( + mut delta: CoreOutput, + accumulated: &CoreOutput, rid_str: &str, - index: Option, - timing: &RequestTiming, -) -> String { - let mut value = super::frame::stream_frame_value(out, acc, incremental, rid_str); - add_e2e_latency(&mut value, timing); - tag_value(value, index) + index: Option, + e2e_latency: Option, +) -> api::GenerateResponse { + delta.completion_tokens = accumulated.completion_tokens; + delta.frame(rid_str, index, e2e_latency) +} + +/// The SSE `data` text of one `api.v1` item (a frame or an error). +fn json_string(value: &T) -> String { + serde_json::to_string(value).expect("api.v1 types serialize to JSON") } #[cfg(test)] @@ -509,7 +511,7 @@ mod tests { serde_json::from_str(s).expect("frame is JSON") } - fn timed_call(rid: u64, rx: mpsc::Receiver) -> (FrontendCall, RequestTiming) { + fn timed_call(rid: u64, rx: mpsc::Receiver) -> (CoreCall, RequestTiming) { timed_call_with_abort(rid, rx, flume::unbounded().0) } @@ -517,9 +519,9 @@ mod tests { rid: u64, rx: mpsc::Receiver, abort_tx: flume::Sender, - ) -> (FrontendCall, RequestTiming) { + ) -> (CoreCall, RequestTiming) { ( - FrontendCall::from_test_generation_parts(Rid::from(rid.to_string()), rx, abort_tx), + CoreCall::from_test_generation_parts(Rid::from(rid.to_string()), rx, abort_tx), RequestTiming { created_at: Instant::now() - Duration::from_millis(10), time_to_first_token: None, @@ -533,16 +535,16 @@ mod tests { let (intake_tx, intake_rx) = flume::unbounded(); let (abort_tx, abort_rx) = flume::unbounded(); let state = Arc::new(AppState { - frontend: crate::frontend::FrontendHandle::new( + core: crate::api_server::core::CoreHandle::new( intake_tx, abort_tx, - crate::frontend::FrontendConfig { + crate::api_server::core::CoreConfig { response_capacity: 8, response_activity: Default::default(), startup_ready: false, is_disaggregation: false, mm_limits: Default::default(), - metadata: crate::frontend::FrontendMetadata::default(), + metadata: crate::api_server::core::CoreMetadata::default(), }, ), server_args: Arc::new(crate::message::config::ServerArgs::default()), @@ -559,20 +561,20 @@ mod tests { async fn unary_http_generate_round_trips_through_frontend_handle() { let (intake_tx, intake_rx) = flume::unbounded(); let (abort_tx, abort_rx) = flume::unbounded(); - let frontend = crate::frontend::FrontendHandle::new( + let core = crate::api_server::core::CoreHandle::new( intake_tx, abort_tx, - crate::frontend::FrontendConfig { + crate::api_server::core::CoreConfig { response_capacity: 8, response_activity: Default::default(), startup_ready: true, is_disaggregation: false, mm_limits: Default::default(), - metadata: crate::frontend::FrontendMetadata::default(), + metadata: crate::api_server::core::CoreMetadata::default(), }, ); let state = Arc::new(AppState { - frontend, + core, server_args: Arc::new(crate::message::config::ServerArgs::default()), chat_formatter: None, }); @@ -580,7 +582,7 @@ mod tests { let responder = tokio::spawn(async move { let TmEvent::Intake { request, admission } = intake_rx.recv_async().await.unwrap() else { - panic!("HTTP generation must enter through frontend intake"); + panic!("HTTP generation must enter through core intake"); }; assert!(admission.try_accept()); let RequestKind::Generate(generate) = &request.kind else { @@ -681,18 +683,16 @@ mod tests { })) .await .unwrap(); - let mut call = FrontendCall::from_test_generation_parts( - "internal-rid".into(), - rx, - flume::unbounded().0, - ); + let mut call = + CoreCall::from_test_generation_parts("internal-rid".into(), rx, flume::unbounded().0); let timing = RequestTiming { created_at: Instant::now() - Duration::from_millis(20), time_to_first_token: None, e2e_latency: None, }; - let (status, value) = drain_unary(&mut call, "client-rid", timing).await; + let (status, item) = drain_unary(&mut call, "client-rid", timing).await; + let value = serde_json::to_value(&item).unwrap(); assert_eq!(status, StatusCode::OK); assert_eq!(value["meta_info"]["id"], "client-rid"); assert_eq!(value["meta_info"]["prompt_tokens"], 5); diff --git a/rust/sglang-server/src/api_server/openai.rs b/rust/sglang-server/src/api_server/http/openai.rs similarity index 84% rename from rust/sglang-server/src/api_server/openai.rs rename to rust/sglang-server/src/api_server/http/openai.rs index 589968065c..8cba188568 100644 --- a/rust/sglang-server/src/api_server/openai.rs +++ b/rust/sglang-server/src/api_server/http/openai.rs @@ -1,7 +1,7 @@ //! OpenAI-compatible generation endpoints. //! //! The HTTP adapter stays deliberately thin: Dynamo owns the standard OpenAI -//! request and response primitives. [`FrontendOutput`] remains the one backend +//! request and response primitives. [`CoreOutput`] remains the one backend //! output type for both unary and streaming responses. use axum::{Router, http::StatusCode, response::Response}; @@ -21,12 +21,9 @@ mod tools; pub(super) use template::{ChatFormatter, ChatTemplateKwargs}; use super::app::AppState; -use super::frame::OutputAccumulator; -use super::frontend_error_status; -use crate::frontend::{ - FrontendCall, FrontendError, FrontendEvent, FrontendOutput, FrontendRequest, -}; +use crate::api_server::core::{CoreCall, CoreError, CoreEvent, CoreOutput}; use crate::message::config::ServerArgs; +use crate::message::request::GenerateRequest; use crate::tokenizer_manager::tokenizer; use crate::utils::response::error_response; @@ -118,17 +115,17 @@ pub(super) fn openai_error(code: StatusCode, message: impl Into, stream: /// Drain one submitted request to its terminal output, fold frames, and map /// semantic failures / truncation to `(status, message)` for the OpenAI error /// shape. The call owns cancellation and disarms itself. -async fn collect_output(mut call: FrontendCall) -> Result { - let mut accumulator = OutputAccumulator::default(); +async fn collect_output(mut call: CoreCall) -> Result { + let mut accumulated = CoreOutput::default(); let output = loop { match call.recv().await { - Some(FrontendEvent::Delta(output)) => accumulator.fold(&output), - Some(FrontendEvent::Finished(output)) => { - accumulator.fold(&output); - break accumulator.into_output(); + Some(CoreEvent::Delta(output)) => accumulated.append_delta(&output), + Some(CoreEvent::Finished(output)) => { + accumulated.append_delta(&output); + break accumulated; } - Some(FrontendEvent::Failed(error)) => { - let status = frontend_error_status(&error); + Some(CoreEvent::Failed(error)) => { + let status = error.http_status(); return Err((status, error.to_string())); } None => { @@ -144,15 +141,15 @@ async fn collect_output(mut call: FrontendCall) -> Result Result { - match state.frontend.generate(request).await { +) -> Result { + match state.core.generate(request).await { Ok(call) => Ok(call), // Same `error_response` rule: a committed stream gets 200 plus an // SSE error frame + `[DONE]`, not a unary 503 — but with the OpenAI // error shape, since this is the OpenAI frontend. - Err(FrontendError::Unavailable) => Err(openai_error( + Err(CoreError::Unavailable) => Err(openai_error( StatusCode::SERVICE_UNAVAILABLE, "service unavailable", stream, @@ -167,8 +164,8 @@ async fn submit_generation( fn indexed_decode_stream( index: usize, - call: FrontendCall, -) -> futures::stream::BoxStream<'static, (usize, FrontendEvent)> { + call: CoreCall, +) -> futures::stream::BoxStream<'static, (usize, CoreEvent)> { futures::stream::unfold((call, false), move |(mut call, finished)| async move { if finished { return None; diff --git a/rust/sglang-server/src/api_server/openai/chat.rs b/rust/sglang-server/src/api_server/http/openai/chat.rs similarity index 98% rename from rust/sglang-server/src/api_server/openai/chat.rs rename to rust/sglang-server/src/api_server/http/openai/chat.rs index 76aabbe03e..28aa63cea2 100644 --- a/rust/sglang-server/src/api_server/openai/chat.rs +++ b/rust/sglang-server/src/api_server/http/openai/chat.rs @@ -36,10 +36,10 @@ use super::{ AppState, ChatFormatter, ChatTemplateKwargs, collect_output, contains_media, error_payload, indexed_decode_stream, openai_error, submit_generation, unix_seconds_u32, }; -use crate::api_server::frontend_error_status; -use crate::frontend::{FrontendCall, FrontendEvent, FrontendRequest}; +use crate::api_server::core::{CoreCall, CoreEvent}; use crate::message::config::{DefaultSamplingParams, ServerArgs}; use crate::message::ids::Rid; +use crate::message::request::GenerateRequest; use crate::message::response::ChunkExtras; use crate::message::sampling::SamplingParams; use crate::message::types::OneOrMany; @@ -204,7 +204,7 @@ async fn chat_completions( .expect("chat prompt exists until the last choice") .clone() }; - let native = FrontendRequest { + let native = GenerateRequest { rid: rid.clone(), text: Some(choice_prompt), // Rendered templates own their special tokens — the pool must not @@ -438,7 +438,7 @@ pub(super) fn chat_sampling_params( #[allow(clippy::too_many_arguments)] pub(super) async fn unary_chat( - submitted: Vec<(usize, FrontendCall)>, + submitted: Vec<(usize, CoreCall)>, response_id: String, model: String, created: u32, @@ -521,7 +521,7 @@ pub(super) async fn unary_chat( #[allow(clippy::too_many_arguments)] pub(super) fn chat_event_stream( - submitted: Vec<(usize, FrontendCall)>, + submitted: Vec<(usize, CoreCall)>, response_id: String, model: String, created: u32, @@ -582,14 +582,14 @@ pub(super) fn chat_event_stream( let mut events = futures::stream::select_all(streams); while let Some((index, event)) = events.next().await { let output = match event { - FrontendEvent::Delta(output) | FrontendEvent::Finished(output) => output, - FrontendEvent::Failed(error) => { + CoreEvent::Delta(output) | CoreEvent::Finished(output) => output, + CoreEvent::Failed(error) => { yield Annotated { data: None, id: None, event: None, comment: None, - error: Some(error_payload(frontend_error_status(&error), error.to_string()).to_string()), + error: Some(error_payload(error.http_status(), error.to_string()).to_string()), }; continue; } diff --git a/rust/sglang-server/src/api_server/openai/completions.rs b/rust/sglang-server/src/api_server/http/openai/completions.rs similarity index 96% rename from rust/sglang-server/src/api_server/openai/completions.rs rename to rust/sglang-server/src/api_server/http/openai/completions.rs index dc2a4b50b9..a668b714ef 100644 --- a/rust/sglang-server/src/api_server/openai/completions.rs +++ b/rust/sglang-server/src/api_server/http/openai/completions.rs @@ -24,12 +24,10 @@ use super::{ AppState, MAX_OPENAI_CHOICES, collect_output, error_payload, indexed_decode_stream, openai_error, submit_generation, unix_seconds_u32, }; -use crate::api_server::frontend_error_status; -use crate::frontend::{ - FrontendCall, FrontendError, FrontendEvent, FrontendOutput, FrontendRequest, -}; +use crate::api_server::core::{CoreCall, CoreError, CoreEvent, CoreOutput}; use crate::message::finish_reason::Matched; use crate::message::ids::Rid; +use crate::message::request::GenerateRequest; use crate::message::response::ChunkExtras; use crate::message::sampling::SamplingParams; use crate::message::types::{OneOrMany, TokenIds}; @@ -48,7 +46,7 @@ pub(super) struct SubmittedChoice { pub(super) index: usize, pub(super) prompt_index: usize, pub(super) echo: String, - pub(super) call: FrontendCall, + pub(super) call: CoreCall, } #[derive(Debug, Default)] pub(super) struct ChoiceExtensions { @@ -82,7 +80,7 @@ async fn completions( if request.prompt_embeds.is_some() { return openai_error( StatusCode::BAD_REQUEST, - "prompt_embeds is not supported by the Rust frontend", + "prompt_embeds is not supported by the Rust server", false, ); } @@ -164,7 +162,7 @@ async fn completions( Err(response) => return response, }; } - let native = FrontendRequest { + let native = GenerateRequest { rid: rid.clone(), text: text.clone(), input_ids: input_ids.clone(), @@ -230,29 +228,29 @@ async fn completions( } /// Decode a token-id prompt back to text for `echo=true`, via a -/// detokenize operation through the shared frontend handle — the +/// detokenize operation through the shared core handle — the /// detok stage answers it with a single `Data` payload (the raw UTF-8 text), /// or an `Error` (e.g. out-of-range ids → `Validation` → 400). async fn decode_prompt_echo(state: &AppState, token_ids: TokenIds) -> Result { - match state.frontend.detokenize(token_ids).await { + match state.core.detokenize(token_ids).await { Ok(text) => Ok(text), // Same rule as `submit_generation`: build the refusal in the OpenAI // error shape rather than forwarding the native-shaped response. - Err(FrontendError::Unavailable) => Err(openai_error( + Err(CoreError::Unavailable) => Err(openai_error( StatusCode::SERVICE_UNAVAILABLE, "service unavailable", false, )), - Err(FrontendError::InvalidArgument(message)) => { + Err(CoreError::InvalidArgument(message)) => { Err(openai_error(StatusCode::BAD_REQUEST, &message, false)) } - Err(FrontendError::InvalidResponse(message)) => Err(openai_error( + Err(CoreError::InvalidResponse(message)) => Err(openai_error( StatusCode::INTERNAL_SERVER_ERROR, message, false, )), Err(error) => { - let status = frontend_error_status(&error); + let status = error.http_status(); Err(openai_error( status, format!("failed to decode prompt for echo: {error}"), @@ -406,7 +404,7 @@ pub(super) async fn unary_completion( fn completion_choice( index: usize, text: String, - output: &FrontendOutput, + output: &CoreOutput, want_logprobs: bool, include_input_logprobs: bool, ) -> (Choice, ChoiceExtensions) { @@ -521,9 +519,9 @@ pub(super) fn completion_event_stream( while let Some((index, event)) = events.next().await { let output = match event { - FrontendEvent::Delta(output) | FrontendEvent::Finished(output) => output, - FrontendEvent::Failed(error) => { - yield error_payload(frontend_error_status(&error), error.to_string()).to_string(); + CoreEvent::Delta(output) | CoreEvent::Finished(output) => output, + CoreEvent::Failed(error) => { + yield error_payload(error.http_status(), error.to_string()).to_string(); continue; } }; diff --git a/rust/sglang-server/src/api_server/openai/models.rs b/rust/sglang-server/src/api_server/http/openai/models.rs similarity index 100% rename from rust/sglang-server/src/api_server/openai/models.rs rename to rust/sglang-server/src/api_server/http/openai/models.rs diff --git a/rust/sglang-server/src/api_server/openai/reasoning.rs b/rust/sglang-server/src/api_server/http/openai/reasoning.rs similarity index 100% rename from rust/sglang-server/src/api_server/openai/reasoning.rs rename to rust/sglang-server/src/api_server/http/openai/reasoning.rs diff --git a/rust/sglang-server/src/api_server/openai/template.rs b/rust/sglang-server/src/api_server/http/openai/template.rs similarity index 100% rename from rust/sglang-server/src/api_server/openai/template.rs rename to rust/sglang-server/src/api_server/http/openai/template.rs diff --git a/rust/sglang-server/src/api_server/openai/template_builtins.rs b/rust/sglang-server/src/api_server/http/openai/template_builtins.rs similarity index 100% rename from rust/sglang-server/src/api_server/openai/template_builtins.rs rename to rust/sglang-server/src/api_server/http/openai/template_builtins.rs diff --git a/rust/sglang-server/src/api_server/openai/template_legacy.rs b/rust/sglang-server/src/api_server/http/openai/template_legacy.rs similarity index 100% rename from rust/sglang-server/src/api_server/openai/template_legacy.rs rename to rust/sglang-server/src/api_server/http/openai/template_legacy.rs diff --git a/rust/sglang-server/src/api_server/openai/template_loader.rs b/rust/sglang-server/src/api_server/http/openai/template_loader.rs similarity index 100% rename from rust/sglang-server/src/api_server/openai/template_loader.rs rename to rust/sglang-server/src/api_server/http/openai/template_loader.rs diff --git a/rust/sglang-server/src/api_server/openai/test_utils.rs b/rust/sglang-server/src/api_server/http/openai/test_utils.rs similarity index 89% rename from rust/sglang-server/src/api_server/openai/test_utils.rs rename to rust/sglang-server/src/api_server/http/openai/test_utils.rs index 7b3d5bcab8..8bc557eab7 100644 --- a/rust/sglang-server/src/api_server/openai/test_utils.rs +++ b/rust/sglang-server/src/api_server/http/openai/test_utils.rs @@ -3,7 +3,7 @@ //! Submodule tests live next to the code they cover: `chat`, `completions`, //! `tools`, and `reasoning` each carry their own //! `#[cfg(test)] mod tests`. This module keeps the fixtures they all share — -//! frontend/call fixtures (`frontend`, `chunk`, `submitted`, `chat_submitted`) and the +//! core/call fixtures (`core_handle`, `chunk`, `submitted`, `chat_submitted`) and the //! full-router harness (`server_args`, `app_state`, //! `oneshot`, `post_json`, `body_json`) — plus the handler-level tests that //! exercise [`routes`] end to end. The helpers are `pub(super)` so sibling @@ -20,20 +20,20 @@ use serde_json::json; use tower::util::ServiceExt; use super::{indexed_decode_stream, openai_error, routes}; -use crate::frontend::{FrontendCall, FrontendEvent, FrontendHandle}; +use crate::api_server::core::{CoreCall, CoreEvent, CoreHandle}; use crate::message::config::ServerArgs; use crate::message::response::{ChunkEvent, ResponseItem}; -pub(super) fn frontend() -> FrontendHandle { - FrontendHandle::new( +pub(super) fn core_handle() -> CoreHandle { + CoreHandle::new( flume::unbounded().0, flume::unbounded().0, - crate::frontend::FrontendConfig { + crate::api_server::core::CoreConfig { response_capacity: 8, response_activity: Default::default(), startup_ready: false, is_disaggregation: false, mm_limits: Default::default(), - metadata: crate::frontend::FrontendMetadata::from(server_args().as_ref()), + metadata: crate::api_server::core::CoreMetadata::from(server_args().as_ref()), }, ) } @@ -76,7 +76,7 @@ pub(super) fn submitted( index, prompt_index, echo: String::new(), - call: FrontendCall::from_test_generation_parts(rid.into(), rx, flume::unbounded().0), + call: CoreCall::from_test_generation_parts(rid.into(), rx, flume::unbounded().0), }, tx, ) @@ -86,15 +86,12 @@ pub(super) fn submitted( pub(super) fn chat_submitted( index: usize, rid: &str, -) -> ( - (usize, FrontendCall), - tokio::sync::mpsc::Sender, -) { +) -> ((usize, CoreCall), tokio::sync::mpsc::Sender) { let (tx, rx) = tokio::sync::mpsc::channel(8); ( ( index, - FrontendCall::from_test_generation_parts(rid.into(), rx, flume::unbounded().0), + CoreCall::from_test_generation_parts(rid.into(), rx, flume::unbounded().0), ), tx, ) @@ -107,31 +104,31 @@ pub(super) fn server_args() -> Arc { }) } -pub(super) fn app_state(frontend: FrontendHandle) -> Arc { +pub(super) fn app_state(core: CoreHandle) -> Arc { Arc::new(super::AppState { - frontend, + core, server_args: server_args(), chat_formatter: None, }) } -pub(super) fn frontend_closed() -> FrontendHandle { - // Dropping the receivers makes frontend admission report the shutdown +pub(super) fn frontend_closed() -> CoreHandle { + // Dropping the receivers makes core admission report the shutdown // state as a 503. let (tm_tx, tm_rx) = flume::unbounded(); drop(tm_rx); let (abort_tx, abort_rx) = flume::unbounded(); drop(abort_rx); - FrontendHandle::new( + CoreHandle::new( tm_tx, abort_tx, - crate::frontend::FrontendConfig { + crate::api_server::core::CoreConfig { response_capacity: 8, response_activity: Default::default(), startup_ready: false, is_disaggregation: false, mm_limits: Default::default(), - metadata: crate::frontend::FrontendMetadata::from(server_args().as_ref()), + metadata: crate::api_server::core::CoreMetadata::from(server_args().as_ref()), }, ) } @@ -166,13 +163,13 @@ async fn dropping_indexed_stream_aborts_its_live_call() { let (response_tx, response_rx) = tokio::sync::mpsc::channel(8); let (abort_tx, abort_rx) = flume::unbounded(); - let call = FrontendCall::from_test_generation_parts("live".into(), response_rx, abort_tx); + let call = CoreCall::from_test_generation_parts("live".into(), response_rx, abort_tx); let mut stream = indexed_decode_stream(0, call); response_tx.send(chunk("live", "x", false)).await.unwrap(); assert!(matches!( stream.next().await, - Some((0, FrontendEvent::Delta(_))) + Some((0, CoreEvent::Delta(_))) )); drop(stream); @@ -217,7 +214,7 @@ async fn openai_error_response_covers_unary_and_sse() { #[tokio::test] async fn completions_handler_validates_before_submit() { - let app = routes().with_state(app_state(frontend())); + let app = routes().with_state(app_state(core_handle())); let cases = [ (json!({"model": "other", "prompt": "hi"}), "unknown model"), (json!({"model": "model", "prompt": "hi", "n": 0}), "n=0"), @@ -265,7 +262,7 @@ async fn completions_handler_validates_before_submit() { #[tokio::test] async fn chat_handler_validates_before_submit() { - let app = routes().with_state(app_state(frontend())); + let app = routes().with_state(app_state(core_handle())); let cases = [ ( json!({"model": "other", "messages": [{"role": "user", "content": "hi"}]}), @@ -309,7 +306,7 @@ async fn chat_handler_validates_before_submit() { #[tokio::test] async fn basic_openai_router_excludes_responses_api() { - let app = routes().with_state(app_state(frontend())); + let app = routes().with_state(app_state(core_handle())); let response = post_json(app, "/v1/responses", json!({"input": "hi"})).await; assert_eq!(response.status(), StatusCode::NOT_FOUND); } diff --git a/rust/sglang-server/src/api_server/openai/tools.rs b/rust/sglang-server/src/api_server/http/openai/tools.rs similarity index 99% rename from rust/sglang-server/src/api_server/openai/tools.rs rename to rust/sglang-server/src/api_server/http/openai/tools.rs index ae2353f2eb..296181e4d1 100644 --- a/rust/sglang-server/src/api_server/openai/tools.rs +++ b/rust/sglang-server/src/api_server/http/openai/tools.rs @@ -36,7 +36,7 @@ use dynamo_protocols::types::{ Role, }; -use crate::frontend::FrontendOutput; +use crate::api_server::core::CoreOutput; use crate::message::sampling::SamplingParams; /// Canonicalize a tool-call parser name onto the dynamo-parsers registry keys. @@ -246,7 +246,7 @@ pub(super) async fn parse_chat_tool_calls( /// Map the scheduler's finish kind onto the OpenAI wire values. Length and /// content-filter keep their names; everything else (including a bare abort) /// reports as `stop`, matching Python's fallback. -pub(super) fn chat_finish_reason(output: &FrontendOutput) -> Option { +pub(super) fn chat_finish_reason(output: &CoreOutput) -> Option { let kind = output .finish_reason .as_ref() @@ -264,7 +264,7 @@ mod tests { apply_tool_constraint, chat_delta, chat_finish_reason, dynamo_parser_name, dynamo_tool_choice, parse_chat_tool_calls, }; - use crate::frontend::FrontendOutput; + use crate::api_server::core::CoreOutput; use crate::message::sampling::SamplingParams; use dynamo_parsers::tool_calling::jail::{Annotated, apply_tool_calling_jail}; use dynamo_parsers::{ToolChoice as DynamoToolChoice, ToolDefinition}; @@ -766,7 +766,7 @@ mod tests { #[test] fn chat_finish_reason_maps_scheduler_kinds() { - let output = |finish: serde_json::Value| FrontendOutput { + let output = |finish: serde_json::Value| CoreOutput { text: "x".into(), token_ids: vec![1], prompt_tokens: 1, @@ -794,7 +794,7 @@ mod tests { Some(OpenAIFinishReason::Stop) ); assert_eq!( - chat_finish_reason(&FrontendOutput { + chat_finish_reason(&CoreOutput { finish_reason: None, ..Default::default() }), diff --git a/rust/sglang-server/src/frontend/contract.rs b/rust/sglang-server/src/frontend/contract.rs deleted file mode 100644 index 037c3b7583..0000000000 --- a/rust/sglang-server/src/frontend/contract.rs +++ /dev/null @@ -1,356 +0,0 @@ -//! Transport-neutral types shared by frontend adapters. -//! -//! These types describe what an inference operation means after a wire adapter -//! has normalized its request. They intentionally contain no Axum, HTTP, SSE, -//! Tonic, protobuf, or scheduler-channel details. - -use std::collections::BTreeMap; - -use serde::{Deserialize, Serialize}; - -use crate::message::config::{DisaggregationMode, PreferredSamplingParams}; -use crate::message::finish_reason::FinishReason; -use crate::message::request::GenerateRequest; -use crate::message::response::{ChunkEvent, ChunkExtras}; -use crate::message::types::TokenIds; - -/// Canonical transport-neutral input for one generation operation. -/// -/// The existing in-process request already contains semantic generation fields -/// and has no Serde, Axum, Tonic, or protobuf representation attached to it. -/// Re-exporting it through the frontend contract gives every adapter one public -/// boundary without duplicating the runtime's request model. -pub(crate) type FrontendRequest = GenerateRequest; - -/// Canonical transport-neutral output payload for generation events. -/// -/// The runtime correlation ID is deliberately absent: it is an implementation -/// detail used for scheduler routing, while adapters obtain the client-visible -/// identity from [`crate::frontend::FrontendCall::public_id`]. Runtime-only -/// response variants such as raw control bytes are likewise hidden by -/// [`crate::frontend::FrontendCall`]. -#[derive(Clone, Debug, Default)] -pub(crate) struct FrontendOutput { - pub(crate) token_ids: TokenIds, - pub(crate) finish_reason: Option, - pub(crate) prompt_tokens: u32, - pub(crate) text: String, - pub(crate) completion_tokens: u64, - pub(crate) extras: Option>, -} - -impl From for FrontendOutput { - fn from(output: ChunkEvent) -> Self { - let ChunkEvent { - rid: _, - token_ids, - finish_reason, - prompt_tokens, - text, - completion_tokens, - extras, - } = output; - Self { - token_ids, - finish_reason, - prompt_tokens, - text, - completion_tokens, - extras, - } - } -} - -impl FrontendOutput { - /// Fold one runtime delta into this cumulative native-generation output. - /// - /// Runtime events are always incremental. Native HTTP and `runtime.v1` - /// gRPC independently decide whether to expose those deltas or the - /// cumulative result, so the fold itself lives at their shared semantic - /// boundary rather than in either wire adapter. - pub(crate) fn append_delta(&mut self, delta: &Self) { - self.text.push_str(&delta.text); - self.token_ids.extend_from_slice(&delta.token_ids); - self.completion_tokens += delta.completion_tokens; - self.prompt_tokens = delta.prompt_tokens; - if delta.finish_reason.is_some() { - self.finish_reason = delta.finish_reason.clone(); - } - - let Some(delta_extras) = delta.extras.as_deref() else { - return; - }; - let extras = self - .extras - .get_or_insert_with(|| Box::new(ChunkExtras::default())); - - extras - .out_lp_val - .extend_from_slice(&delta_extras.out_lp_val); - extras - .out_lp_idx - .extend_from_slice(&delta_extras.out_lp_idx); - extras - .out_top_val - .extend_from_slice(&delta_extras.out_top_val); - extras - .out_top_idx - .extend_from_slice(&delta_extras.out_top_idx); - extras - .out_top_lens - .extend_from_slice(&delta_extras.out_top_lens); - extras - .out_tid_val - .extend_from_slice(&delta_extras.out_tid_val); - extras - .out_tid_idx - .extend_from_slice(&delta_extras.out_tid_idx); - extras - .out_tid_lens - .extend_from_slice(&delta_extras.out_tid_lens); - extras - .out_lp_txt - .extend_from_slice(&delta_extras.out_lp_txt); - extras - .out_top_txt - .extend_from_slice(&delta_extras.out_top_txt); - extras - .out_tid_txt - .extend_from_slice(&delta_extras.out_tid_txt); - - // Input logprobs arrive once (during prefill). A later non-empty value - // replaces the earlier one, matching the existing native HTTP fold. - if !delta_extras.in_lp_val.is_empty() { - extras.in_lp_val = delta_extras.in_lp_val.clone(); - extras.in_lp_idx = delta_extras.in_lp_idx.clone(); - extras.in_lp_txt = delta_extras.in_lp_txt.clone(); - } - if !delta_extras.in_top_lens.is_empty() { - extras.in_top_val = delta_extras.in_top_val.clone(); - extras.in_top_idx = delta_extras.in_top_idx.clone(); - extras.in_top_lens = delta_extras.in_top_lens.clone(); - extras.in_top_txt = delta_extras.in_top_txt.clone(); - } - if !delta_extras.in_tid_lens.is_empty() { - extras.in_tid_val = delta_extras.in_tid_val.clone(); - extras.in_tid_idx = delta_extras.in_tid_idx.clone(); - extras.in_tid_lens = delta_extras.in_tid_lens.clone(); - extras.in_tid_txt = delta_extras.in_tid_txt.clone(); - } - // Hidden states are non-cumulative: the latest complete set wins. - if !delta_extras.hidden_lens.is_empty() { - extras.hidden_val = delta_extras.hidden_val.clone(); - extras.hidden_lens = delta_extras.hidden_lens.clone(); - } - } -} - -/// Semantic result of a deep frontend health check. -/// -/// Expected lifecycle states are values rather than transport errors so each -/// adapter can render them according to its own protocol. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) enum HealthStatus { - Healthy, - NotReady, - Stalled, -} - -/// A semantic generation event before any transport-specific framing. -#[derive(Debug)] -pub(crate) enum FrontendEvent { - /// One incremental model-output update. - Delta(FrontendOutput), - /// The final model-output update for this request. - Finished(FrontendOutput), - /// A request-scoped failure. Other requests in a batch may continue. - Failed(FrontendError), -} - -/// Protocol-independent error category used by transport adapters. -/// -/// The set intentionally follows operation semantics rather than HTTP or gRPC -/// status spaces. An adapter maps this category into its own wire status. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) enum FrontendErrorKind { - InvalidArgument, - NotFound, - FailedPrecondition, - ResourceExhausted, - Cancelled, - DeadlineExceeded, - Unavailable, - Internal, -} - -impl FrontendEvent { - pub(crate) fn is_terminal(&self) -> bool { - matches!(self, Self::Finished(_) | Self::Failed(_)) - } -} - -/// A transport-neutral failure category. -/// -/// HTTP and gRPC adapters map these categories into their own status spaces; -/// neither adapter needs to understand errors emitted by runtime stages. -#[derive(Clone, Debug, thiserror::Error)] -pub(crate) enum FrontendError { - #[error("{0}")] - InvalidArgument(String), - - /// The frontend intake loop has shut down. A caller may retry elsewhere. - #[error("service unavailable")] - Unavailable, - - /// Runtime capacity was exhausted before the request could make progress. - #[error("{0}")] - Overloaded(String), - - #[error("{0}")] - Cancelled(String), - - /// The internal producer disappeared before returning a terminal result. - /// - /// This is a server-side failure, not evidence that the caller cancelled. - #[error("response truncated before completion")] - ResponseTruncated, - - /// A scheduler-side abort that carried an error status. - /// - /// The current Python scheduler communicates this category as an HTTP - /// status code inside its finish reason. The semantic event is still - /// `Failed`; retaining the legacy code only lets the HTTP adapter preserve - /// existing behavior while another adapter maps it into its own status - /// space. New frontend failures should use the semantic variants above. - #[error("{message}")] - RuntimeRejected { - kind: FrontendErrorKind, - message: String, - legacy_http_status: u16, - }, - - /// The runtime replied with an unexpected or malformed result. - #[error("{0}")] - InvalidResponse(String), - - #[error("{0}")] - Internal(String), -} - -impl FrontendError { - /// Semantic status for adapters that do not use the scheduler's legacy - /// HTTP code space (notably the future gRPC adapter). - pub(crate) fn kind(&self) -> FrontendErrorKind { - match self { - Self::InvalidArgument(_) => FrontendErrorKind::InvalidArgument, - Self::Unavailable => FrontendErrorKind::Unavailable, - Self::Overloaded(_) => FrontendErrorKind::ResourceExhausted, - Self::Cancelled(_) => FrontendErrorKind::Cancelled, - Self::ResponseTruncated => FrontendErrorKind::Internal, - Self::RuntimeRejected { kind, .. } => *kind, - Self::InvalidResponse(_) | Self::Internal(_) => FrontendErrorKind::Internal, - } - } - - /// Translate the scheduler's inherited HTTP-coded failure into a semantic - /// category once, at the runtime boundary. HTTP can retain the exact legacy - /// status while other adapters consume [`Self::kind`]. - pub(super) fn from_runtime_rejection(message: String, legacy_http_status: u16) -> Self { - let kind = match legacy_http_status { - 404 => FrontendErrorKind::NotFound, - 408 | 504 => FrontendErrorKind::DeadlineExceeded, - 412 => FrontendErrorKind::FailedPrecondition, - 413 | 429 => FrontendErrorKind::ResourceExhausted, - 499 => FrontendErrorKind::Cancelled, - 502 | 503 => FrontendErrorKind::Unavailable, - 400..=499 => FrontendErrorKind::InvalidArgument, - _ => FrontendErrorKind::Internal, - }; - Self::RuntimeRejected { - kind, - message, - legacy_http_status, - } - } -} - -/// Static model metadata shared by every public transport. -#[derive(Clone, Debug, Serialize)] -pub(crate) struct ModelInfo { - pub(crate) model_path: String, - pub(crate) served_model_name: String, - pub(crate) tokenizer_path: String, - pub(crate) is_generation: bool, - pub(crate) preferred_sampling_params: Option, - pub(crate) weight_version: Option, - pub(crate) load_format: Option, - pub(crate) reasoning_parser: Option, - pub(crate) tool_call_parser: Option, - pub(crate) disaggregation_mode: DisaggregationMode, -} - -/// Public server metadata plus scheduler-owned runtime metrics. -#[derive(Clone, Debug, Serialize)] -pub(crate) struct ServerInfo { - pub(crate) model_path: String, - pub(crate) served_model_name: String, - pub(crate) tokenizer_path: String, - pub(crate) max_context_length: u64, - pub(crate) max_total_num_tokens: u64, - pub(crate) version: String, - pub(crate) frontend: &'static str, - pub(crate) internal_states: Vec, -} - -/// Scheduler-owned runtime metrics used by the public server-info operation. -/// -/// Deserializing into this allowlisted shape is intentional: the scheduler's -/// raw state also contains its full launch arguments, including credentials. -/// Unknown fields are discarded before any transport receives the result. -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] -#[serde(default)] -pub(crate) struct InternalState { - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) last_gen_throughput: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) memory_usage: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) effective_max_running_requests_per_dp: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) avg_spec_accept_length: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) step_time_dict: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) rust_mm_transport: Option>, -} - -/// Public subset of scheduler memory metrics. -/// -/// This nested allowlist is intentional: adding a scheduler field does not -/// silently add different public data to HTTP before the gRPC contract can add -/// the same field. New metrics should be promoted here deliberately. -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] -#[serde(default)] -pub(crate) struct MemoryUsage { - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) weight: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) kvcache: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) startup_available: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) token_capacity: Option, - pub(crate) token_capacity_swa: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) graph: Option>, -} - -/// The scheduler's KV-cache measurement can be a native float or a NumPy -/// scalar stringified by the MessagePack bridge. Preserve that representation -/// so extracting the typed frontend contract does not change public responses. -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] -#[serde(untagged)] -pub(crate) enum MemoryMeasurement { - Number(f64), - String(String), -} diff --git a/rust/sglang-server/src/grpc/convert.rs b/rust/sglang-server/src/grpc/convert.rs deleted file mode 100644 index ff1aa9884e..0000000000 --- a/rust/sglang-server/src/grpc/convert.rs +++ /dev/null @@ -1,531 +0,0 @@ -//! `runtime.v1` request conversion for the Rust frontend. -//! -//! The canonical protobuf is wider than the Rust frontend today. Fields with -//! no internal representation are rejected explicitly instead of being -//! silently ignored or expanding frontend behavior in this wiring change. - -use sglang_grpc_types::sglang::runtime::v1 as proto; -use tonic::Status; - -use crate::frontend::FrontendRequest; -use crate::message::config::PreferredSamplingParams; -use crate::message::ids::Rid; -use crate::message::sampling::SamplingParams; - -type ConvertResult = Result; - -#[derive(Debug, thiserror::Error)] -pub(super) enum ConvertError { - #[error("{0}")] - InvalidArgument(String), - #[error("{0}")] - Unsupported(String), - #[error("{0}")] - Internal(String), -} - -impl ConvertError { - fn invalid_argument(message: impl Into) -> Self { - Self::InvalidArgument(message.into()) - } - - fn internal(message: impl Into) -> Self { - Self::Internal(message.into()) - } -} - -impl From for Status { - fn from(error: ConvertError) -> Self { - match error { - ConvertError::InvalidArgument(message) => Self::invalid_argument(message), - ConvertError::Unsupported(message) => Self::unimplemented(message), - ConvertError::Internal(message) => Self::internal(message), - } - } -} - -pub(super) fn text_generate( - request: proto::TextGenerateRequest, - preferred: Option<&PreferredSamplingParams>, -) -> ConvertResult { - let proto::TextGenerateRequest { - text, - sampling_params, - stream, - return_logprob, - top_logprobs_num, - logprob_start_len, - return_text_in_logprobs, - rid, - lora_path, - routing_key, - routed_dp_rank, - trace_headers, - session_id, - disaggregated_params, - priority, - require_reasoning, - max_thinking_tokens, - kv_hints, - } = request; - - reject_unsupported([ - ("kv_hints", kv_hints.is_some()), - ("lora_path", lora_path.is_some()), - ("routing_key", routing_key.is_some()), - ("trace_headers", !trace_headers.is_empty()), - ("session_id", session_id.is_some()), - ("priority", priority.is_some()), - ("require_reasoning", require_reasoning.is_some()), - ("max_thinking_tokens", max_thinking_tokens.is_some()), - ])?; - - let (bootstrap_host, bootstrap_port, bootstrap_room) = - disaggregated_fields(disaggregated_params); - Ok(FrontendRequest { - rid: request_id(rid), - text: Some(text), - input_ids: None, - skip_special_tokens: false, - sampling_params: convert_sampling_params(sampling_params, preferred)?, - stream: stream.unwrap_or(false), - return_logprob: return_logprob.unwrap_or(false), - logprob_start_len: logprob_start_len.map(i64::from).unwrap_or(-1), - top_logprobs_num: top_logprobs_num.map(i64::from).unwrap_or(0), - token_ids_logprob: None, - return_sampling_mask: false, - return_hidden_states: false, - return_text_in_logprobs, - bootstrap_host, - bootstrap_port, - bootstrap_room, - bootstrap_pair_key: None, - decode_tp_size: None, - routed_dp_rank: routed_dp_rank.map(i64::from), - disagg_prefill_dp_rank: None, - mm: None, - mm_buffers: Vec::new(), - }) -} - -pub(super) fn generate( - request: proto::GenerateRequest, - preferred: Option<&PreferredSamplingParams>, -) -> ConvertResult { - let proto::GenerateRequest { - input_ids, - sampling_params, - stream, - return_logprob, - top_logprobs_num, - logprob_start_len, - rid, - lora_path, - routing_key, - routed_dp_rank, - trace_headers, - session_id, - disaggregated_params, - priority, - require_reasoning, - max_thinking_tokens, - kv_hints, - } = request; - - reject_unsupported([ - ("kv_hints", kv_hints.is_some()), - ("lora_path", lora_path.is_some()), - ("routing_key", routing_key.is_some()), - ("trace_headers", !trace_headers.is_empty()), - ("session_id", session_id.is_some()), - ("priority", priority.is_some()), - ("require_reasoning", require_reasoning.is_some()), - ("max_thinking_tokens", max_thinking_tokens.is_some()), - ])?; - if input_ids.is_empty() { - return Err(ConvertError::invalid_argument("input_ids cannot be empty")); - } - - let (bootstrap_host, bootstrap_port, bootstrap_room) = - disaggregated_fields(disaggregated_params); - Ok(FrontendRequest { - rid: request_id(rid), - text: None, - input_ids: Some(input_ids.into_iter().map(i64::from).collect()), - skip_special_tokens: false, - sampling_params: convert_sampling_params(sampling_params, preferred)?, - stream: stream.unwrap_or(false), - return_logprob: return_logprob.unwrap_or(false), - logprob_start_len: logprob_start_len.map(i64::from).unwrap_or(-1), - top_logprobs_num: top_logprobs_num.map(i64::from).unwrap_or(0), - token_ids_logprob: None, - return_sampling_mask: false, - return_hidden_states: false, - return_text_in_logprobs: None, - bootstrap_host, - bootstrap_port, - bootstrap_room, - bootstrap_pair_key: None, - decode_tp_size: None, - routed_dp_rank: routed_dp_rank.map(i64::from), - disagg_prefill_dp_rank: None, - mm: None, - mm_buffers: Vec::new(), - }) -} - -fn request_id(rid: Option) -> Rid { - rid.map_or_else(Rid::new, |rid| Rid::from_client(&rid)) -} - -fn reject_unsupported(fields: [(&'static str, bool); N]) -> ConvertResult<()> { - let unsupported = fields - .into_iter() - .filter_map(|(name, present)| present.then_some(name)) - .collect::>(); - if unsupported.is_empty() { - return Ok(()); - } - Err(ConvertError::Unsupported(format!( - "the Rust frontend does not yet support runtime.v1 field(s): {}", - unsupported.join(", ") - ))) -} - -fn disaggregated_fields( - params: Option, -) -> (Option, Option, Option) { - params.map_or((None, None, None), |params| { - ( - Some(params.bootstrap_host), - Some(i64::from(params.bootstrap_port)), - Some(params.bootstrap_room), - ) - }) -} - -/// Merge launch-time preferred values first, then overwrite only fields that -/// are explicitly present in protobuf. This is the same precedence used by -/// the Python-backed gRPC path; proto3 repeated fields cannot distinguish an -/// omitted list from an explicit empty list, so empty stop lists remain absent. -#[allow(deprecated)] -fn convert_sampling_params( - params: Option, - preferred: Option<&PreferredSamplingParams>, -) -> ConvertResult { - let mut output = SamplingParams::default(); - - let Some(params) = params else { - if let Some(preferred) = preferred { - output.apply_preferred(&preferred.0).map_err(|error| { - ConvertError::internal(format!("invalid preferred_sampling_params: {error}")) - })?; - } - return Ok(output); - }; - if params.guided_decoding.is_some() && (params.json_schema.is_some() || params.regex.is_some()) - { - return Err(ConvertError::invalid_argument( - "legacy json_schema/regex cannot be combined with guided_decoding", - )); - } - - if let Some(value) = params.temperature { - output.temperature = f64::from(value); - output.mark_explicit("temperature"); - } - if let Some(value) = params.top_p { - output.top_p = f64::from(value); - output.mark_explicit("top_p"); - } - if let Some(value) = params.top_k { - output.top_k = i64::from(value); - output.mark_explicit("top_k"); - } - if let Some(value) = params.min_p { - output.min_p = f64::from(value); - output.mark_explicit("min_p"); - } - if let Some(value) = params.frequency_penalty { - output.frequency_penalty = f64::from(value); - output.mark_explicit("frequency_penalty"); - } - if let Some(value) = params.presence_penalty { - output.presence_penalty = f64::from(value); - output.mark_explicit("presence_penalty"); - } - if let Some(value) = params.repetition_penalty { - output.repetition_penalty = f64::from(value); - output.mark_explicit("repetition_penalty"); - } - if let Some(value) = params.max_new_tokens { - output.max_new_tokens = Some(i64::from(value)); - output.mark_explicit("max_new_tokens"); - } - if let Some(value) = params.min_new_tokens { - output.min_new_tokens = i64::from(value); - output.mark_explicit("min_new_tokens"); - } - if !params.stop.is_empty() { - output.stop = Some(crate::message::types::OneOrMany::Many(params.stop)); - output.mark_explicit("stop"); - } - if !params.stop_token_ids.is_empty() { - output.stop_token_ids = Some(params.stop_token_ids.into_iter().map(i64::from).collect()); - output.mark_explicit("stop_token_ids"); - } - if let Some(value) = params.ignore_eos { - output.ignore_eos = value; - output.mark_explicit("ignore_eos"); - } - if let Some(value) = params.n { - output.n = i64::from(value); - output.mark_explicit("n"); - } - if let Some(value) = params.seed { - output.sampling_seed = Some(value); - output.mark_explicit("sampling_seed"); - } - - if let Some(guided) = params.guided_decoding { - apply_guided_decoding(&mut output, guided)?; - } else { - if let Some(value) = params.json_schema { - if value.is_empty() { - return Err(ConvertError::invalid_argument( - "legacy json_schema must not be empty", - )); - } - output.json_schema = Some(value); - output.mark_explicit("json_schema"); - } - if let Some(value) = params.regex { - if value.is_empty() { - return Err(ConvertError::invalid_argument( - "legacy regex must not be empty", - )); - } - output.regex = Some(value); - output.mark_explicit("regex"); - } - } - - if let Some(preferred) = preferred { - output.apply_preferred(&preferred.0).map_err(|error| { - ConvertError::internal(format!("invalid preferred_sampling_params: {error}")) - })?; - } - - Ok(output) -} - -fn apply_guided_decoding( - output: &mut SamplingParams, - guided: proto::GuidedDecoding, -) -> ConvertResult<()> { - use proto::guided_decoding::Constraint; - - match guided.constraint { - Some(Constraint::JsonSchema(value)) if !value.is_empty() => { - output.json_schema = Some(value); - output.mark_explicit("json_schema"); - } - Some(Constraint::Regex(value)) if !value.is_empty() => { - output.regex = Some(value); - output.mark_explicit("regex"); - } - Some(Constraint::Ebnf(value)) if !value.is_empty() => { - output.ebnf = Some(value); - output.mark_explicit("ebnf"); - } - Some(Constraint::StructuralTag(value)) if !value.is_empty() => { - output.structural_tag = Some(value); - output.mark_explicit("structural_tag"); - } - Some(Constraint::Choice(choice)) - if !choice.values.is_empty() && choice.values.iter().all(|value| !value.is_empty()) => - { - let alternatives = choice - .values - .iter() - .map(|value| regex_escape_literal(value)) - .collect::>() - .join("|"); - output.regex = Some(format!("(?:{alternatives})")); - output.mark_explicit("regex"); - } - Some(Constraint::Choice(_)) => { - return Err(ConvertError::invalid_argument( - "guided choice must contain only non-empty values", - )); - } - Some(_) => { - return Err(ConvertError::invalid_argument( - "guided decoding constraint must not be empty", - )); - } - None => { - return Err(ConvertError::invalid_argument( - "guided decoding constraint must be specified", - )); - } - } - Ok(()) -} - -fn regex_escape_literal(value: &str) -> String { - let mut escaped = String::with_capacity(value.len()); - for character in value.chars() { - if matches!( - character, - '.' | '+' | '*' | '?' | '^' | '$' | '(' | ')' | '[' | ']' | '{' | '}' | '|' | '\\' - ) { - escaped.push('\\'); - } - escaped.push(character); - } - escaped -} - -#[cfg(test)] -mod tests { - use super::*; - use tonic::Code; - - #[test] - fn text_request_maps_supported_fields_and_preferred_sampling() { - let preferred = PreferredSamplingParams(serde_json::json!({ - "temperature": 0.2, - "top_p": 0.7, - "max_new_tokens": 64, - })); - let request = proto::TextGenerateRequest { - text: "hello".into(), - sampling_params: Some(proto::SamplingParams { - temperature: Some(0.8), - seed: Some(7), - ..Default::default() - }), - stream: Some(true), - return_logprob: Some(true), - top_logprobs_num: Some(3), - logprob_start_len: Some(2), - return_text_in_logprobs: Some(true), - rid: Some("client-id".into()), - routed_dp_rank: Some(4), - disaggregated_params: Some(proto::DisaggregatedParams { - bootstrap_host: "10.0.0.1".into(), - bootstrap_port: 8998, - bootstrap_room: 12, - }), - ..Default::default() - }; - - let request = text_generate(request, Some(&preferred)).unwrap(); - assert_eq!(request.rid.client_facing(), "client-id"); - assert_eq!(request.text.as_deref(), Some("hello")); - assert!(request.input_ids.is_none()); - assert!(request.stream); - assert!(request.return_logprob); - assert_eq!(request.top_logprobs_num, 3); - assert_eq!(request.logprob_start_len, 2); - assert_eq!(request.return_text_in_logprobs, Some(true)); - assert_eq!(request.routed_dp_rank, Some(4)); - assert_eq!(request.bootstrap_host.as_deref(), Some("10.0.0.1")); - assert_eq!(request.bootstrap_port, Some(8998)); - assert_eq!(request.bootstrap_room, Some(12)); - assert_eq!(request.sampling_params.temperature, f64::from(0.8_f32)); - assert_eq!(request.sampling_params.top_p, 0.7); - assert_eq!(request.sampling_params.max_new_tokens, Some(64)); - assert_eq!(request.sampling_params.sampling_seed, Some(7)); - } - - #[test] - fn token_request_validates_input_and_rejects_unported_fields() { - let empty = Status::from(generate(proto::GenerateRequest::default(), None).unwrap_err()); - assert_eq!(empty.code(), Code::InvalidArgument); - - let unsupported = Status::from( - generate( - proto::GenerateRequest { - input_ids: vec![1], - routing_key: Some("worker-a".into()), - priority: Some(1), - ..Default::default() - }, - None, - ) - .unwrap_err(), - ); - assert_eq!(unsupported.code(), Code::Unimplemented); - assert!(unsupported.message().contains("routing_key, priority")); - } - - #[test] - fn absent_sampling_params_still_uses_preferred_values() { - let preferred = PreferredSamplingParams(serde_json::json!({ - "temperature": 0.25, - "max_new_tokens": 32, - })); - let request = generate( - proto::GenerateRequest { - input_ids: vec![1], - ..Default::default() - }, - Some(&preferred), - ) - .unwrap(); - - assert_eq!(request.sampling_params.temperature, 0.25); - assert_eq!(request.sampling_params.max_new_tokens, Some(32)); - } - - #[test] - fn guided_choice_preserves_existing_runtime_v1_mapping() { - let request = generate( - proto::GenerateRequest { - input_ids: vec![1], - sampling_params: Some(proto::SamplingParams { - guided_decoding: Some(proto::GuidedDecoding { - constraint: Some(proto::guided_decoding::Constraint::Choice( - proto::ChoiceConstraint { - values: vec!["a+b".into(), "x.y".into()], - }, - )), - }), - ..Default::default() - }), - ..Default::default() - }, - None, - ) - .unwrap(); - assert_eq!( - request.sampling_params.regex.as_deref(), - Some("(?:a\\+b|x\\.y)") - ); - } - - #[test] - #[allow(deprecated)] - fn guided_and_legacy_constraints_cannot_be_combined() { - let error = Status::from( - generate( - proto::GenerateRequest { - input_ids: vec![1], - sampling_params: Some(proto::SamplingParams { - regex: Some("a".into()), - guided_decoding: Some(proto::GuidedDecoding { - constraint: Some(proto::guided_decoding::Constraint::Regex("b".into())), - }), - ..Default::default() - }), - ..Default::default() - }, - None, - ) - .unwrap_err(), - ); - assert_eq!(error.code(), Code::InvalidArgument); - } -} diff --git a/rust/sglang-server/src/grpc/mod.rs b/rust/sglang-server/src/grpc/mod.rs deleted file mode 100644 index 963c9ebe19..0000000000 --- a/rust/sglang-server/src/grpc/mod.rs +++ /dev/null @@ -1,303 +0,0 @@ -//! Tonic adapter for the canonical `sglang.runtime.v1` API. -//! -//! This module translates protobuf requests and responses and mounts that thin -//! adapter on Tonic. The shared [`FrontendHandle`] owns preprocessing, -//! admission, runtime communication, and request cancellation. -//! -//! The thin Tonic-service structure, streamed response approach, and test -//! strategy build on Rain Jiang's multi-protocol prototype in -//! `sgl-project/sglang#36923`. This stack keeps those ideas while using the -//! canonical `runtime.v1` contract and the narrower [`FrontendHandle`] seam. - -use std::pin::Pin; -use std::time::Duration; - -use futures::Stream; -use sglang_grpc_types::sglang::runtime::v1 as proto; -use sglang_grpc_types::sglang::runtime::v1::sglang_service_server::SglangService; -use tonic::{Request, Response, Status}; - -use crate::frontend::FrontendHandle; -use crate::message::config::{PreferredSamplingParams, ServerArgs}; - -mod convert; -mod response; -mod server; - -pub(crate) use server::serve; - -#[cfg(test)] -mod tests; - -const DEFAULT_RESPONSE_TIMEOUT: Duration = Duration::from_secs(300); - -type ResponseStream = Pin> + Send + 'static>>; - -/// Narrow snapshot of launch policy required to preserve the existing -/// `runtime.v1` generation behavior. It intentionally excludes listener, -/// authentication, TLS, and lifecycle configuration. -#[derive(Clone)] -struct AdapterConfig { - preferred_sampling_params: Option, - incremental_streaming_output: bool, - response_timeout: Duration, -} - -/// Tonic-facing implementation backed by the transport-neutral Rust frontend. -pub(crate) struct GrpcService { - frontend: FrontendHandle, - config: AdapterConfig, -} - -impl GrpcService { - pub(crate) fn new(frontend: FrontendHandle, server_args: &ServerArgs) -> Self { - Self { - frontend, - config: AdapterConfig { - preferred_sampling_params: server_args.preferred_sampling_params.clone(), - incremental_streaming_output: server_args.incremental_streaming_output, - response_timeout: DEFAULT_RESPONSE_TIMEOUT, - }, - } - } - - #[cfg(test)] - fn for_test( - frontend: FrontendHandle, - preferred_sampling_params: Option, - incremental_streaming_output: bool, - response_timeout: Duration, - ) -> Self { - Self { - frontend, - config: AdapterConfig { - preferred_sampling_params, - incremental_streaming_output, - response_timeout, - }, - } - } -} - -fn unimplemented_rpc(name: &'static str) -> Status { - Status::unimplemented(format!( - "{name} is not implemented by the Rust frontend gRPC adapter" - )) -} - -#[tonic::async_trait] -impl SglangService for GrpcService { - async fn text_generate( - &self, - request: Request, - ) -> Result>, Status> { - let request = convert::text_generate( - request.into_inner(), - self.config.preferred_sampling_params.as_ref(), - ) - .map_err(Status::from)?; - let call = self - .frontend - .generate(request) - .await - .map_err(response::status)?; - Ok(Response::new(response::text_generate_stream( - call, - self.config.incremental_streaming_output, - self.config.response_timeout, - ))) - } - - async fn generate( - &self, - request: Request, - ) -> Result>, Status> { - let request = convert::generate( - request.into_inner(), - self.config.preferred_sampling_params.as_ref(), - ) - .map_err(Status::from)?; - let call = self - .frontend - .generate(request) - .await - .map_err(response::status)?; - Ok(Response::new(response::generate_stream( - call, - self.config.incremental_streaming_output, - self.config.response_timeout, - ))) - } - - // runtime.v1 is shared with the broader Python-backed service, so its - // generated trait contains more operations than this initial Rust adapter. - // Keep that support boundary explicit: adding an RPC is a deliberate future - // change, never an accidental empty success. - async fn text_embed( - &self, - _request: Request, - ) -> Result, Status> { - Err(unimplemented_rpc("text_embed")) - } - - async fn embed( - &self, - _request: Request, - ) -> Result, Status> { - Err(unimplemented_rpc("embed")) - } - - async fn classify( - &self, - _request: Request, - ) -> Result, Status> { - Err(unimplemented_rpc("classify")) - } - - async fn tokenize( - &self, - _request: Request, - ) -> Result, Status> { - Err(unimplemented_rpc("tokenize")) - } - - async fn detokenize( - &self, - _request: Request, - ) -> Result, Status> { - Err(unimplemented_rpc("detokenize")) - } - - async fn health_check( - &self, - _request: Request, - ) -> Result, Status> { - Err(unimplemented_rpc("health_check")) - } - - async fn get_model_info( - &self, - _request: Request, - ) -> Result, Status> { - Err(unimplemented_rpc("get_model_info")) - } - - async fn get_server_info( - &self, - _request: Request, - ) -> Result, Status> { - Err(unimplemented_rpc("get_server_info")) - } - - async fn list_models( - &self, - _request: Request, - ) -> Result, Status> { - Err(unimplemented_rpc("list_models")) - } - - async fn get_load( - &self, - _request: Request, - ) -> Result, Status> { - Err(unimplemented_rpc("get_load")) - } - - async fn abort( - &self, - _request: Request, - ) -> Result, Status> { - Err(unimplemented_rpc("abort")) - } - - async fn flush_cache( - &self, - _request: Request, - ) -> Result, Status> { - Err(unimplemented_rpc("flush_cache")) - } - - async fn pause_generation( - &self, - _request: Request, - ) -> Result, Status> { - Err(unimplemented_rpc("pause_generation")) - } - - async fn watch_engine_state( - &self, - _request: Request, - ) -> Result>, Status> { - Err(unimplemented_rpc("watch_engine_state")) - } - - async fn continue_generation( - &self, - _request: Request, - ) -> Result, Status> { - Err(unimplemented_rpc("continue_generation")) - } - - async fn chat_complete( - &self, - _request: Request, - ) -> Result>, Status> { - Err(unimplemented_rpc("chat_complete")) - } - - async fn complete( - &self, - _request: Request, - ) -> Result>, Status> { - Err(unimplemented_rpc("complete")) - } - - async fn open_ai_embed( - &self, - _request: Request, - ) -> Result, Status> { - Err(unimplemented_rpc("open_ai_embed")) - } - - async fn open_ai_classify( - &self, - _request: Request, - ) -> Result, Status> { - Err(unimplemented_rpc("open_ai_classify")) - } - - async fn score( - &self, - _request: Request, - ) -> Result, Status> { - Err(unimplemented_rpc("score")) - } - - async fn rerank( - &self, - _request: Request, - ) -> Result, Status> { - Err(unimplemented_rpc("rerank")) - } - - async fn start_profile( - &self, - _request: Request, - ) -> Result, Status> { - Err(unimplemented_rpc("start_profile")) - } - - async fn stop_profile( - &self, - _request: Request, - ) -> Result, Status> { - Err(unimplemented_rpc("stop_profile")) - } - - async fn update_weights_from_disk( - &self, - _request: Request, - ) -> Result, Status> { - Err(unimplemented_rpc("update_weights_from_disk")) - } -} diff --git a/rust/sglang-server/src/grpc/response.rs b/rust/sglang-server/src/grpc/response.rs deleted file mode 100644 index 93ee25b2c9..0000000000 --- a/rust/sglang-server/src/grpc/response.rs +++ /dev/null @@ -1,200 +0,0 @@ -//! Semantic frontend events to `runtime.v1` stream items and statuses. - -use std::collections::HashMap; -use std::time::Duration; - -use sglang_grpc_types::sglang::runtime::v1 as proto; -use tonic::{Code, Status}; - -use super::ResponseStream; -use crate::frontend::{ - FrontendCall, FrontendError, FrontendErrorKind, FrontendEvent, FrontendOutput, -}; -use crate::native_generation::meta_info_value; - -pub(super) fn text_generate_stream( - call: FrontendCall, - incremental: bool, - response_timeout: Duration, -) -> ResponseStream { - generation_stream( - call, - incremental, - response_timeout, - |output, meta_info, finished| { - Ok(proto::TextGenerateResponse { - text: output.text.clone(), - meta_info, - finished, - }) - }, - ) -} - -pub(super) fn generate_stream( - call: FrontendCall, - incremental: bool, - response_timeout: Duration, -) -> ResponseStream { - generation_stream( - call, - incremental, - response_timeout, - |output, meta_info, finished| { - Ok(proto::GenerateResponse { - output_ids: output - .token_ids - .iter() - .copied() - .map(i32::try_from) - .collect::, _>>() - .map_err(|_| { - Status::internal("output token ID exceeds runtime.v1 int32 range") - })?, - meta_info, - finished, - }) - }, - ) -} - -fn generation_stream( - mut call: FrontendCall, - incremental: bool, - response_timeout: Duration, - build: F, -) -> ResponseStream -where - T: Send + 'static, - F: Fn(&FrontendOutput, HashMap, bool) -> Result + Send + 'static, -{ - let public_id = call.public_id().to_owned(); - let stream = async_stream::stream! { - let mut cumulative = FrontendOutput::default(); - let mut cumulative_completion_tokens = 0; - loop { - let event = match tokio::time::timeout(response_timeout, call.recv()).await { - Ok(Some(event)) => event, - // The loop stops on terminal events, so this is defensive only. - Ok(None) => break, - Err(_) => { - yield Err(Status::deadline_exceeded("Stream chunk timed out")); - break; - } - }; - - let (mut delta, finished) = match event { - FrontendEvent::Delta(output) => (output, false), - FrontendEvent::Finished(output) => (output, true), - FrontendEvent::Failed(error) => { - yield Err(status(error)); - break; - } - }; - - let output = if incremental { - // Python's incremental native stream still reports the cumulative - // completion-token count in every frame. Do not retain the full - // generated output just to calculate that one cumulative field. - cumulative_completion_tokens += delta.completion_tokens; - delta.completion_tokens = cumulative_completion_tokens; - &delta - } else { - cumulative.append_delta(&delta); - &cumulative - }; - let meta_info = grpc_meta_info(output, &public_id); - match build(output, meta_info, finished) { - Ok(response) => yield Ok(response), - Err(error) => { - yield Err(error); - break; - } - } - if finished { - break; - } - } - // `call` remains owned by this stream. Dropping a non-terminal stream or - // timing out drops the armed FrontendCall and uses the core abort lane; - // observing a terminal event has already disarmed it. - }; - Box::pin(stream) -} - -/// runtime.v1 represents heterogeneous metadata as strings. Match the existing -/// Python-backed server by JSON-encoding each value independently—including -/// string values such as the request ID. -fn grpc_meta_info(output: &FrontendOutput, public_id: &str) -> HashMap { - let serde_json::Value::Object(meta_info) = meta_info_value(output, public_id) else { - unreachable!("native meta_info is always a JSON object"); - }; - meta_info - .into_iter() - .map(|(key, value)| (key, value.to_string())) - .collect() -} - -pub(super) fn status(error: FrontendError) -> Status { - let code = match error.kind() { - FrontendErrorKind::InvalidArgument => Code::InvalidArgument, - FrontendErrorKind::NotFound => Code::NotFound, - FrontendErrorKind::FailedPrecondition => Code::FailedPrecondition, - FrontendErrorKind::ResourceExhausted => Code::ResourceExhausted, - FrontendErrorKind::Cancelled => Code::Cancelled, - FrontendErrorKind::DeadlineExceeded => Code::DeadlineExceeded, - FrontendErrorKind::Unavailable => Code::Unavailable, - FrontendErrorKind::Internal => Code::Internal, - }; - Status::new(code, error.to_string()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn semantic_errors_map_to_canonical_grpc_codes() { - let cases = [ - ( - FrontendError::InvalidArgument("bad".into()), - Code::InvalidArgument, - ), - ( - FrontendError::RuntimeRejected { - kind: FrontendErrorKind::NotFound, - message: "missing".into(), - legacy_http_status: 404, - }, - Code::NotFound, - ), - ( - FrontendError::RuntimeRejected { - kind: FrontendErrorKind::FailedPrecondition, - message: "not ready".into(), - legacy_http_status: 412, - }, - Code::FailedPrecondition, - ), - ( - FrontendError::Overloaded("full".into()), - Code::ResourceExhausted, - ), - (FrontendError::Cancelled("gone".into()), Code::Cancelled), - ( - FrontendError::RuntimeRejected { - kind: FrontendErrorKind::DeadlineExceeded, - message: "late".into(), - legacy_http_status: 504, - }, - Code::DeadlineExceeded, - ), - (FrontendError::Unavailable, Code::Unavailable), - (FrontendError::Internal("bug".into()), Code::Internal), - ]; - - for (error, expected) in cases { - assert_eq!(status(error).code(), expected); - } - } -} diff --git a/rust/sglang-server/src/grpc/tests.rs b/rust/sglang-server/src/grpc/tests.rs deleted file mode 100644 index ff6b718297..0000000000 --- a/rust/sglang-server/src/grpc/tests.rs +++ /dev/null @@ -1,361 +0,0 @@ -use std::time::Duration; - -use futures::StreamExt; -use sglang_grpc_types::sglang::runtime::v1 as proto; -use sglang_grpc_types::sglang::runtime::v1::sglang_service_server::SglangService; -use tonic::{Code, Request}; - -use super::GrpcService; -use crate::frontend::{FrontendConfig, FrontendHandle, FrontendMetadata}; -use crate::message::finish_reason::FinishReason; -use crate::message::ids::Rid; -use crate::message::request::{GenerateRequest, Request as RuntimeRequest, RequestKind}; -use crate::message::response::{ChunkEvent, ResponseItem, ResponseSink}; -use crate::tokenizer_manager::wiring::{AbortSource, RequestAdmission, TmEvent}; - -struct Harness { - service: GrpcService, - intake_rx: flume::Receiver, - abort_rx: flume::Receiver, -} - -struct GenerationIntake { - request: Box, - rid: Rid, - sink: ResponseSink, - admission: RequestAdmission, -} - -impl Harness { - fn new(response_capacity: usize, incremental: bool, response_timeout: Duration) -> Self { - let (intake_tx, intake_rx) = flume::unbounded(); - let (abort_tx, abort_rx) = flume::unbounded(); - let frontend = FrontendHandle::new( - intake_tx, - abort_tx, - FrontendConfig { - response_capacity, - response_activity: Default::default(), - startup_ready: true, - is_disaggregation: false, - mm_limits: Default::default(), - metadata: FrontendMetadata::default(), - }, - ); - Self { - service: GrpcService::for_test(frontend, None, incremental, response_timeout), - intake_rx, - abort_rx, - } - } - - async fn next_generation(&self) -> GenerationIntake { - let TmEvent::Intake { request, admission } = self - .intake_rx - .recv_async() - .await - .expect("request submitted") - else { - panic!("expected generation intake"); - }; - let RuntimeRequest { - rid, - sink, - kind, - state: _, - } = request; - let RequestKind::Generate(request) = kind else { - panic!("expected generation request"); - }; - GenerationIntake { - request, - rid, - sink, - admission, - } - } -} - -fn chunk(rid: &Rid, text: &str, token_id: i64, finished: bool) -> ChunkEvent { - ChunkEvent { - rid: rid.clone(), - token_ids: vec![token_id], - finish_reason: finished.then(stop_reason), - prompt_tokens: 3, - text: text.into(), - completion_tokens: 1, - extras: None, - } -} - -fn stop_reason() -> FinishReason { - serde_json::from_value(serde_json::json!({"type": "stop"})).unwrap() -} - -#[tokio::test] -async fn text_generate_maps_request_and_streams_cumulative_responses() { - let harness = Harness::new(2, false, Duration::from_secs(1)); - let response = harness - .service - .text_generate(Request::new(proto::TextGenerateRequest { - text: "prompt".into(), - stream: Some(true), - rid: Some("request-1".into()), - ..Default::default() - })) - .await - .unwrap(); - let mut stream = response.into_inner(); - - let intake = harness.next_generation().await; - assert!(intake.admission.try_accept()); - assert_eq!(intake.request.text.as_deref(), Some("prompt")); - assert!(intake.request.input_ids.is_none()); - assert!(intake.request.stream); - assert_eq!(intake.rid.client_facing(), "request-1"); - intake - .sink - .try_send(ResponseItem::Frame(chunk(&intake.rid, "Hel", 1, false))) - .unwrap(); - intake - .sink - .try_send(ResponseItem::Done(chunk(&intake.rid, "lo", 2, true))) - .unwrap(); - - let first = stream.next().await.unwrap().unwrap(); - assert_eq!(first.text, "Hel"); - assert!(!first.finished); - assert_eq!(first.meta_info["id"], r#""request-1""#); - assert_eq!(first.meta_info["prompt_tokens"], "3"); - assert_eq!(first.meta_info["completion_tokens"], "1"); - assert_eq!(first.meta_info["finish_reason"], "null"); - - let finished = stream.next().await.unwrap().unwrap(); - assert_eq!(finished.text, "Hello"); - assert!(finished.finished); - assert_eq!(finished.meta_info["completion_tokens"], "2"); - assert!(finished.meta_info["finish_reason"].contains(r#""type":"stop""#)); - assert!(stream.next().await.is_none()); - assert!(harness.abort_rx.try_recv().is_err()); -} - -#[tokio::test] -async fn generate_streams_incremental_token_ids_with_cumulative_count() { - let harness = Harness::new(2, true, Duration::from_secs(1)); - let response = harness - .service - .generate(Request::new(proto::GenerateRequest { - input_ids: vec![4, 5], - stream: Some(true), - rid: Some("tokens".into()), - ..Default::default() - })) - .await - .unwrap(); - let mut stream = response.into_inner(); - - let intake = harness.next_generation().await; - assert!(intake.admission.try_accept()); - assert_eq!(intake.request.input_ids.as_deref(), Some(&[4, 5][..])); - assert!(intake.request.text.is_none()); - intake - .sink - .try_send(ResponseItem::Frame(chunk(&intake.rid, "A", 10, false))) - .unwrap(); - intake - .sink - .try_send(ResponseItem::Done(chunk(&intake.rid, "B", 11, true))) - .unwrap(); - - let first = stream.next().await.unwrap().unwrap(); - let finished = stream.next().await.unwrap().unwrap(); - assert_eq!(first.output_ids, vec![10]); - assert_eq!(finished.output_ids, vec![11]); - assert_eq!(finished.meta_info["completion_tokens"], "2"); - assert!(finished.finished); -} - -#[tokio::test] -async fn stream_drop_aborts_admitted_request() { - let harness = Harness::new(1, false, Duration::from_secs(1)); - let mut stream = harness - .service - .generate(Request::new(proto::GenerateRequest { - input_ids: vec![1], - rid: Some("cancel-me".into()), - ..Default::default() - })) - .await - .unwrap() - .into_inner(); - let intake = harness.next_generation().await; - assert!(intake.admission.try_accept()); - intake - .sink - .try_send(ResponseItem::Frame(chunk(&intake.rid, "partial", 1, false))) - .unwrap(); - assert!(stream.next().await.unwrap().is_ok()); - drop(stream); - let abort = harness.abort_rx.recv_async().await.unwrap(); - assert_eq!(abort.rid().client_facing(), "cancel-me"); - assert!(harness.abort_rx.try_recv().is_err()); -} - -#[tokio::test] -async fn runtime_failure_is_an_in_stream_status_and_disarms_abort() { - let harness = Harness::new(1, false, Duration::from_secs(1)); - let mut stream = harness - .service - .generate(Request::new(proto::GenerateRequest { - input_ids: vec![1], - ..Default::default() - })) - .await - .unwrap() - .into_inner(); - let intake = harness.next_generation().await; - assert!(intake.admission.try_accept()); - intake - .sink - .try_send(ResponseItem::Error(crate::utils::error::Error::Validation( - "bad sampling".into(), - ))) - .unwrap(); - - let error = stream.next().await.unwrap().unwrap_err(); - assert_eq!(error.code(), Code::InvalidArgument); - drop(stream); - assert!(harness.abort_rx.try_recv().is_err()); -} - -#[tokio::test] -async fn per_chunk_timeout_after_progress_returns_deadline_exceeded_and_aborts() { - let harness = Harness::new(1, false, Duration::from_millis(10)); - let mut stream = harness - .service - .generate(Request::new(proto::GenerateRequest { - input_ids: vec![1], - rid: Some("too-slow".into()), - ..Default::default() - })) - .await - .unwrap() - .into_inner(); - let intake = harness.next_generation().await; - assert!(intake.admission.try_accept()); - intake - .sink - .try_send(ResponseItem::Frame(chunk(&intake.rid, "partial", 1, false))) - .unwrap(); - assert!(stream.next().await.unwrap().is_ok()); - - let error = stream.next().await.unwrap().unwrap_err(); - assert_eq!(error.code(), Code::DeadlineExceeded); - drop(stream); - let abort = harness.abort_rx.recv_async().await.unwrap(); - assert_eq!(abort.rid().client_facing(), "too-slow"); -} - -#[tokio::test] -async fn closed_intake_is_a_top_level_unavailable_status() { - let (intake_tx, intake_rx) = flume::unbounded(); - drop(intake_rx); - let (abort_tx, _abort_rx) = flume::unbounded(); - let frontend = FrontendHandle::new( - intake_tx, - abort_tx, - FrontendConfig { - response_capacity: 1, - response_activity: Default::default(), - startup_ready: true, - is_disaggregation: false, - mm_limits: Default::default(), - metadata: FrontendMetadata::default(), - }, - ); - let service = GrpcService::for_test(frontend, None, false, Duration::from_secs(1)); - - let result = service - .generate(Request::new(proto::GenerateRequest { - input_ids: vec![1], - ..Default::default() - })) - .await; - let error = match result { - Ok(_) => panic!("closed intake must reject the RPC"), - Err(error) => error, - }; - assert_eq!(error.code(), Code::Unavailable); -} - -#[tokio::test] -async fn new_proto_fields_and_engine_watch_are_explicitly_unsupported() { - let harness = Harness::new(1, false, Duration::from_secs(1)); - let text_error = harness - .service - .text_generate(Request::new(proto::TextGenerateRequest { - text: "hello".into(), - kv_hints: Some(Default::default()), - ..Default::default() - })) - .await - .err() - .expect("KV hints must not be silently ignored"); - assert_eq!(text_error.code(), Code::Unimplemented); - let token_error = harness - .service - .generate(Request::new(proto::GenerateRequest { - input_ids: vec![1], - kv_hints: Some(Default::default()), - ..Default::default() - })) - .await - .err() - .expect("KV hints must not be silently ignored"); - assert_eq!(token_error.code(), Code::Unimplemented); - let watch_error = harness - .service - .watch_engine_state(Request::new(proto::WatchEngineStateRequest {})) - .await - .err() - .expect("engine watch is outside the generation adapter"); - assert_eq!(watch_error.code(), Code::Unimplemented); - assert!(harness.intake_rx.try_recv().is_err()); -} - -#[tokio::test] -async fn output_token_overflow_returns_error_instead_of_truncating() { - let harness = Harness::new(1, true, Duration::from_secs(1)); - let mut stream = harness - .service - .generate(Request::new(proto::GenerateRequest { - input_ids: vec![i32::MAX], - ..Default::default() - })) - .await - .unwrap() - .into_inner(); - let intake = harness.next_generation().await; - assert!(intake.admission.try_accept()); - assert_eq!( - intake.request.input_ids.as_deref(), - Some(&[i64::from(i32::MAX)][..]) - ); - intake - .sink - .try_send(ResponseItem::Frame(chunk( - &intake.rid, - "", - i64::from(i32::MAX) + 1, - false, - ))) - .unwrap(); - assert_eq!( - stream.next().await.unwrap().unwrap_err().code(), - Code::Internal - ); - assert!(stream.next().await.is_none()); - assert!( - matches!(harness.abort_rx.recv_async().await.unwrap(), AbortSource::Guard(rid) if rid == intake.rid) - ); -} diff --git a/rust/sglang-server/src/lib.rs b/rust/sglang-server/src/lib.rs index 64116e14a8..a23eae8867 100644 --- a/rust/sglang-server/src/lib.rs +++ b/rust/sglang-server/src/lib.rs @@ -12,11 +12,8 @@ //! a `PyObject`. mod api_server; -mod frontend; -mod grpc; mod message; mod multi_modality; -mod native_generation; mod tokenizer_manager; mod utils; @@ -25,7 +22,7 @@ pub use message::config::{ RustServerServerArgs, ServerArgs, }; pub use message::multimodal::MmItem; -pub use message::request::{MmData, ProcessorExtensions}; +pub use message::request::MmData; pub use message::types::TokenIds; pub use multi_modality::encoded::{ MRope, MmEncodedEntry, MmEncodedItem, MmMetaValue, MmModality, MmTokenIds, diff --git a/rust/sglang-server/src/message.rs b/rust/sglang-server/src/message.rs index 5680b68231..0c1772547f 100644 --- a/rust/sglang-server/src/message.rs +++ b/rust/sglang-server/src/message.rs @@ -7,9 +7,11 @@ pub mod config; pub mod detok; pub mod finish_reason; pub mod ids; +pub mod info; pub mod io_struct; pub mod multimodal; pub mod request; pub mod response; pub mod sampling; pub mod types; +pub mod wire; diff --git a/rust/sglang-server/src/message/config.rs b/rust/sglang-server/src/message/config.rs index 59fabea7ca..152e54a71e 100644 --- a/rust/sglang-server/src/message/config.rs +++ b/rust/sglang-server/src/message/config.rs @@ -572,8 +572,12 @@ impl ServerArgs { } } if let Some(preferred) = &self.preferred_sampling_params { - super::sampling::SamplingParamsInput::from_preferred(&preferred.0) - .map_err(|e| format!("invalid preferred_sampling_params: {e}"))?; + // The wire schema is the contract: a preferred value the decoder + // rejects would 400 every request, so reject it at boot instead. + serde_json::from_value::( + preferred.0.clone(), + ) + .map_err(|e| format!("invalid preferred_sampling_params: {e}"))?; } Ok(()) } diff --git a/rust/sglang-server/src/message/finish_reason.rs b/rust/sglang-server/src/message/finish_reason.rs index 8050cd6b87..106afed188 100644 --- a/rust/sglang-server/src/message/finish_reason.rs +++ b/rust/sglang-server/src/message/finish_reason.rs @@ -98,7 +98,7 @@ impl FinishReason { } /// `Some((status, message))` when this is an abort carrying the scheduler's - /// legacy HTTP status — a request failure the frontend boundary converts + /// legacy HTTP status — a request failure the core boundary converts /// into a semantic error event. A plain abort (no code) reads as `None`. pub fn abort_status(&self) -> Option<(u16, &str)> { match self { @@ -128,7 +128,7 @@ mod tests { Some((code, message.to_string())) } - /// The classifier the frontend boundary uses: a validation abort yields its + /// The classifier the core boundary uses: a validation abort yields its /// `(code, message)` so it can become a semantic failure event; anything /// else yields `None`. #[test] diff --git a/rust/sglang-server/src/message/info.rs b/rust/sglang-server/src/message/info.rs new file mode 100644 index 0000000000..cf50f20eb4 --- /dev/null +++ b/rust/sglang-server/src/message/info.rs @@ -0,0 +1,89 @@ +//! Public model and server metadata: the typed results of `/get_model_info` +//! and `/server_info` on every transport. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +use super::config::{DisaggregationMode, PreferredSamplingParams}; + +/// Static model metadata shared by every public transport. +#[derive(Clone, Debug, Serialize)] +pub(crate) struct ModelInfo { + pub(crate) model_path: String, + pub(crate) served_model_name: String, + pub(crate) tokenizer_path: String, + pub(crate) is_generation: bool, + pub(crate) preferred_sampling_params: Option, + pub(crate) weight_version: Option, + pub(crate) load_format: Option, + pub(crate) reasoning_parser: Option, + pub(crate) tool_call_parser: Option, + pub(crate) disaggregation_mode: DisaggregationMode, +} + +/// Public server metadata plus scheduler-owned runtime metrics. +#[derive(Clone, Debug, Serialize)] +pub(crate) struct ServerInfo { + pub(crate) model_path: String, + pub(crate) served_model_name: String, + pub(crate) tokenizer_path: String, + pub(crate) max_context_length: u64, + pub(crate) max_total_num_tokens: u64, + pub(crate) version: String, + pub(crate) frontend: &'static str, + pub(crate) internal_states: Vec, +} + +/// Scheduler-owned runtime metrics used by the public server-info operation. +/// +/// Deserializing into this allowlisted shape is intentional: the scheduler's +/// raw state also contains its full launch arguments, including credentials. +/// Unknown fields are discarded before any transport receives the result. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] +#[serde(default)] +pub(crate) struct InternalState { + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) last_gen_throughput: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) memory_usage: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) effective_max_running_requests_per_dp: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) avg_spec_accept_length: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) step_time_dict: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) rust_mm_transport: Option>, +} + +/// Public subset of scheduler memory metrics. +/// +/// This nested allowlist is intentional: adding a scheduler field does not +/// silently add different public data to HTTP before the gRPC contract can add +/// the same field. New metrics should be promoted here deliberately. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] +#[serde(default)] +pub(crate) struct MemoryUsage { + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) weight: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) kvcache: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) startup_available: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) token_capacity: Option, + pub(crate) token_capacity_swa: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) graph: Option>, +} + +/// The scheduler's KV-cache measurement can be a native float or a NumPy +/// scalar stringified by the MessagePack bridge. Preserve that representation +/// so extracting the typed contract does not change public responses. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(untagged)] +pub(crate) enum MemoryMeasurement { + Number(f64), + String(String), +} diff --git a/rust/sglang-server/src/message/multimodal.rs b/rust/sglang-server/src/message/multimodal.rs index e7da19d46b..0730dea755 100644 --- a/rust/sglang-server/src/message/multimodal.rs +++ b/rust/sglang-server/src/message/multimodal.rs @@ -1,12 +1,6 @@ //! Typed multimodal inputs of the `/generate` body — the Rust form of Python //! `MultimodalDataInputFormat` (`io_struct.py`) — and their per-request fan-out. -use std::fmt; - -use serde::de::value::{MapAccessDeserializer, SeqAccessDeserializer}; -use serde::de::{MapAccess, SeqAccess, Visitor}; -use serde::{Deserialize, Deserializer}; - use super::request::{HeapBytes, check_broadcast_budget}; use crate::utils::error::Error; @@ -22,19 +16,13 @@ pub enum MmItem { /// are read by model families this pipeline does not run, and Python's /// `load_image` itself reduces the item to `.url`. Ref { url: String }, - /// A preprocessed item (`{"format": "processor_output" | "precomputed_embedding", …}`). - /// Parsed only far enough to be rejected by name at the MM stage; Python - /// ignores it the same way on a text-only model. - Preprocessed { format: String }, } impl MmItem { - /// The raw source string for the modality pipeline, `None` for a - /// preprocessed item. + /// The raw source string for the modality pipeline. pub fn source(&self) -> Option<&str> { match self { MmItem::Source(source) | MmItem::Ref { url: source } => Some(source), - MmItem::Preprocessed { .. } => None, } } } @@ -42,68 +30,11 @@ impl MmItem { impl HeapBytes for MmItem { fn heap_bytes(&self) -> usize { match self { - MmItem::Source(s) | MmItem::Ref { url: s } | MmItem::Preprocessed { format: s } => { - s.len() - } + MmItem::Source(s) | MmItem::Ref { url: s } => s.len(), } } } -/// The object form of an item, as Python's `Dict[str, Any]`: `format` marks a -/// preprocessed item (checked first, as `glm4v` does), `url` an `ImageData`. -#[derive(Deserialize)] -struct ItemObject { - #[serde(default)] - url: Option, - #[serde(default)] - format: Option, -} - -impl TryFrom for MmItem { - type Error = &'static str; - - fn try_from(object: ItemObject) -> Result { - match (object.format, object.url) { - (Some(format), _) => Ok(MmItem::Preprocessed { format }), - (None, Some(url)) => Ok(MmItem::Ref { url }), - (None, None) => Err("a multimodal item object needs a `url` or a `format` key"), - } - } -} - -/// Hand-written rather than `#[serde(untagged)]` so a bad item is reported as -/// what it is ("expected a source string or an item object"), not as "did not -/// match any variant". -impl<'de> Deserialize<'de> for MmItem { - fn deserialize>(deserializer: D) -> Result { - struct ItemVisitor; - - impl<'de> Visitor<'de> for ItemVisitor { - type Value = MmItem; - - fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str("a media source string or a multimodal item object") - } - - fn visit_str(self, value: &str) -> Result { - Ok(MmItem::Source(value.to_owned())) - } - - fn visit_string(self, value: String) -> Result { - Ok(MmItem::Source(value)) - } - - fn visit_map>(self, map: A) -> Result { - ItemObject::deserialize(MapAccessDeserializer::new(map))? - .try_into() - .map_err(serde::de::Error::custom) - } - } - - deserializer.deserialize_any(ItemVisitor) - } -} - /// One `image_data` / `video_data` / `audio_data` field as sent: Python /// `MultimodalDataInputFormat`, whose three shapes read differently for a single /// request and a batch (see [`fan_out`]). @@ -117,119 +48,6 @@ pub enum MmDataInput { Nested(Vec>>>), } -/// One element of a list-form field, before the list is known to be flat or nested. -enum ListElement { - Null, - Item(MmItem), - List(Vec>), -} - -impl<'de> Deserialize<'de> for ListElement { - fn deserialize>(deserializer: D) -> Result { - struct ElementVisitor; - - impl<'de> Visitor<'de> for ElementVisitor { - type Value = ListElement; - - fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str("null, a media source string, an item object, or a list of items") - } - - fn visit_unit(self) -> Result { - Ok(ListElement::Null) - } - - fn visit_none(self) -> Result { - Ok(ListElement::Null) - } - - fn visit_str(self, value: &str) -> Result { - Ok(ListElement::Item(MmItem::Source(value.to_owned()))) - } - - fn visit_string(self, value: String) -> Result { - Ok(ListElement::Item(MmItem::Source(value))) - } - - fn visit_map>(self, map: A) -> Result { - ItemObject::deserialize(MapAccessDeserializer::new(map))? - .try_into() - .map(ListElement::Item) - .map_err(serde::de::Error::custom) - } - - fn visit_seq>(self, seq: A) -> Result { - Vec::>::deserialize(SeqAccessDeserializer::new(seq)) - .map(ListElement::List) - } - } - - deserializer.deserialize_any(ElementVisitor) - } -} - -impl<'de> Deserialize<'de> for MmDataInput { - fn deserialize>(deserializer: D) -> Result { - struct InputVisitor; - - impl<'de> Visitor<'de> for InputVisitor { - type Value = MmDataInput; - - fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str("a media item, a list of items, or a list of item lists") - } - - fn visit_str(self, value: &str) -> Result { - Ok(MmDataInput::One(MmItem::Source(value.to_owned()))) - } - - fn visit_string(self, value: String) -> Result { - Ok(MmDataInput::One(MmItem::Source(value))) - } - - fn visit_map>(self, map: A) -> Result { - ItemObject::deserialize(MapAccessDeserializer::new(map))? - .try_into() - .map(MmDataInput::One) - .map_err(serde::de::Error::custom) - } - - fn visit_seq>(self, seq: A) -> Result { - let elements = Vec::::deserialize(SeqAccessDeserializer::new(seq))?; - let nested = elements - .iter() - .any(|element| matches!(element, ListElement::List(_))); - if !nested { - return Ok(MmDataInput::Many( - elements - .into_iter() - .map(|element| match element { - ListElement::Item(item) => Some(item), - ListElement::Null => None, - ListElement::List(_) => unreachable!("checked above"), - }) - .collect(), - )); - } - elements - .into_iter() - .map(|element| match element { - ListElement::List(items) => Ok(Some(items)), - ListElement::Null => Ok(None), - ListElement::Item(_) => Err(serde::de::Error::custom( - "a nested list cannot mix bare items with item lists", - )), - }) - .collect::>() - .map(MmDataInput::Nested) - } - } - - deserializer.deserialize_any(InputVisitor) - } -} - -/// The items of one modality for one request, `null` entries dropped. fn present(items: Vec>) -> Vec { items.into_iter().flatten().collect() } @@ -300,8 +118,12 @@ fn check_len(len: usize, n: usize, name: &str) -> Result<(), Error> { mod tests { use super::*; - fn parse(json: &str) -> Result { - serde_json::from_str(json) + /// Parsed as the wire type (the schema's `MediaInput`), then converted; + /// either step's rejection is an `Err`. + fn parse(json: &str) -> Result { + let wire: sglang_api_types::api::v1::MediaInput = + serde_json::from_str(json).map_err(|e| e.to_string())?; + crate::message::wire::media_input(wire).map_err(|e| e.to_string()) } fn src(s: &str) -> MmItem { @@ -324,39 +146,28 @@ mod tests { ); } - /// Object items: `format` wins over `url` (a preprocessed item may carry - /// both), and an object with neither is named in the error. + /// Object items are typed `MediaRef`s: only `url` reaches this pipeline, a + /// hint the schema does not name is an error, and a preprocessed-input + /// dict is not a wire shape at all (its values are tensors; Engine-only). #[test] fn parses_item_objects() { assert_eq!( parse(r#"{"url": "u", "detail": "high"}"#).unwrap(), MmDataInput::One(MmItem::Ref { url: "u".into() }) ); - assert_eq!( - parse(r#"[{"format": "processor_output", "url": "u", "pixel_values": [1]}]"#).unwrap(), - MmDataInput::Many(vec![Some(MmItem::Preprocessed { - format: "processor_output".into() - })]) - ); - let err = parse(r#"{"detail": "high"}"#).unwrap_err().to_string(); - assert!(err.contains("`url` or a `format`"), "{err}"); + let err = parse(r#"{"url": "u", "detai": "high"}"#) + .unwrap_err() + .to_string(); // codespell:ignore detai + assert!(err.contains("unknown field"), "{err}"); + assert!(parse(r#"[{"format": "processor_output", "url": "u"}]"#).is_err()); + assert!(parse(r#"{"detail": "high"}"#).is_err(), "a ref needs a url"); } - /// Anything Python's item union does not cover is rejected up front, with - /// the expected shape in the message. + /// Anything Python's item union does not cover is rejected up front. #[test] fn rejects_non_items() { - for (json, expect) in [ - ("5", "expected a media item, a list of items"), - (r#"["a", 5]"#, "expected null, a media source string"), - (r#"["a", ["b"]]"#, "cannot mix"), - ( - r#"[[["a"]]]"#, - "expected a media source string or a multimodal item object", - ), - ] { - let err = parse(json).unwrap_err().to_string(); - assert!(err.contains(expect), "{json}: {err}"); + for json in ["5", r#"["a", 5]"#, r#"["a", ["b"]]"#, r#"[[["a"]]]"#] { + assert!(parse(json).is_err(), "{json}"); } } diff --git a/rust/sglang-server/src/message/request.rs b/rust/sglang-server/src/message/request.rs index d1e2fe1763..4e9de7c704 100644 --- a/rust/sglang-server/src/message/request.rs +++ b/rust/sglang-server/src/message/request.rs @@ -1,24 +1,23 @@ //! The `/generate` request path: the HTTP body and its per-request fan-out //! ([`GenerateBody`] → [`GenerateRequest`]s). -use std::collections::{BTreeMap, HashSet}; -use std::fmt; +use std::collections::HashSet; use std::sync::LazyLock; use bytes::Bytes; use itertools::izip; -use serde::de::{DeserializeOwned, SeqAccess, Visitor, value::SeqAccessDeserializer}; -use serde::{Deserialize, Deserializer}; use super::buffers::Buffer; use super::io_struct::{ControlRequest, TokenizedGenerateReqInput}; -use super::multimodal::{self, MmDataInput, MmItem}; +use super::multimodal::{self, MmItem}; use super::response::ResponseSink; -use super::sampling::{SamplingParams, SamplingParamsInput}; +use super::sampling::SamplingParams; use super::types::{OneOrMany, OneOrManyItem, TokenIds}; +use super::wire; use crate::message::ids::Rid; use crate::utils::fsm::RequestState; use crate::utils::{environ::env_i64, error::Error}; +use sglang_api_types::api::v1 as api_v1; /// Hard cap on how many scheduler requests one `/generate` HTTP call may expand /// into. Every column below is allocated per item before anything is dispatched, @@ -47,232 +46,325 @@ const MAX_BROADCAST_CLONE_BYTES: usize = 64 << 20; /// the wire form does not); 8 is the ceiling of that range, not a worst case. const JSON_TO_HEAP_FACTOR: usize = 8; -/// Top-level fields in this namespace belong to the selected multimodal -/// processor. Everything else unknown to [`GenerateBody`] keeps Python's -/// accepted-but-ignored behavior. -const PROCESSOR_EXTENSION_PREFIX: &str = "multimodal_"; - -/// Model-owned request fields. The shared server preserves and batches their -/// MessagePack value representation; the selected processor deserializes that -/// map into its own concrete schema. -#[derive(Debug, Clone, Default, Deserialize)] -#[serde(transparent)] -pub struct ProcessorExtensions(BTreeMap); - -impl ProcessorExtensions { - /// Deserialize the model-agnostic value tree directly into the selected - /// processor's schema. This does not encode or decode MessagePack bytes. - pub fn deserialize(self) -> Result { - let fields = self - .0 - .into_iter() - .map(|(name, value)| (rmpv::Value::from(name), value)) - .collect(); - rmpv::ext::from_value(rmpv::Value::Map(fields)) - .map_err(|error| format!("invalid processor extensions: {error}")) - } - - pub(crate) fn is_empty(&self) -> bool { - self.0.is_empty() - } - - fn values(&self) -> impl Iterator { - self.0.values() - } -} - -impl FromIterator<(String, rmpv::Value)> for ProcessorExtensions { - fn from_iter>(iter: T) -> Self { - Self(iter.into_iter().collect()) - } -} - -/// The `/generate` wire body before batch splitting: `text`/`input_ids`/`sampling_params` -/// each scalar-or-list, fanned into per-request [`GenerateRequest`]s by -/// [`into_requests`](GenerateBody::into_requests). +/// Validate, normalize and fan the `/generate` wire request into one +/// [`GenerateRequest`] per prompt + `is_batch` (list form -- a 1-element list +/// is still a batch -> JSON array response). The Rust counterpart of Python +/// `GenerateReqInput.normalize_batch_and_arguments`; an invalid/inconsistent +/// batch is [`Error::Validation`], which the HTTP adapter surfaces as 400. /// -/// Unknown keys are ignored, matching Python, except `multimodal_*` fields. Those -/// are opaque processor extensions: this layer only fans them out with the -/// request batch and passes them to the selected multimodal processor. -#[derive(Debug, Clone, Default, Deserialize)] -pub struct GenerateBody { - /// Optional client-supplied request id(s): a single string (a batch fans it - /// out as `{rid}_{i}`, mirroring Python `_normalize_batch`) or one per item. - pub rid: Option>, - pub text: Option>, - #[serde(default, deserialize_with = "deserialize_input_ids")] - pub input_ids: Option>, - #[serde(default)] - pub stream: bool, - /// One params object (broadcast) or a list of them (per item); see - /// [`SamplingParamsInput`]. - pub sampling_params: Option, - /// Logprob / hidden-state options: a scalar broadcasts to every prompt, a - /// list is per-prompt (Python `_normalize_logprob_params`). - pub return_logprob: Option>, - pub logprob_start_len: Option>, - pub top_logprobs_num: Option>, - /// Token ids to report logprobs for: one list (broadcast to every prompt) or - /// one list per prompt, mirroring Python's - /// `Union[List[int], List[List[int]]]` fan-out in `_normalize_batch`. - pub token_ids_logprob: Option>, - pub return_hidden_states: Option>, - /// Scalar-only in Python too (`return_text_in_logprobs: bool`). - pub return_text_in_logprobs: Option, - // PD-disaggregation routing, injected per request by the PD router - // (mini_lb / sgl-model-gateway): a scalar for a single prompt, one-per-item - // lists for a batch. Elements are nullable (`List[Optional[...]]` in - // Python) — the router sends `bootstrap_port: [null, …]` when deferring to - // the scheduler's `--disaggregation-bootstrap-port` default. - pub bootstrap_host: Option>>, - pub bootstrap_port: Option>>, - /// `bootstrap_room` fits in i64: the PD routers draw it from `[0, 2^63)`. - pub bootstrap_room: Option>>, - pub bootstrap_pair_key: Option>>, - pub decode_tp_size: Option>>, - /// DP routing hints — per-request scalars even for batches, as in Python. - pub routed_dp_rank: Option, - pub disagg_prefill_dp_rank: Option, - // Multimodal inputs (Python `MultimodalDataInputFormat`), fanned out per - // request by `multimodal::fan_out`. - pub image_data: Option, - /// Caller-supplied per-item content hashes (hex) overriding the computed - /// ones, so an external router's keys align with the prefix cache. Single - /// requests only: Python declares the batched (nested) shape but - /// `__getitem__` never forwards it, so a batch is rejected here rather than - /// answered with hashes it did not ask for. - pub mm_hashes: Option>>, - pub video_data: Option, - pub audio_data: Option, - /// Model-specific multimodal fields, retained without teaching the shared - /// request schema their contents. Other unknown fields remain ignored. - #[serde(flatten)] - processor_extensions: ProcessorExtensions, -} +/// The wire type is the generated `sglang_api_types` request (the schema is +/// the contract); its carriers become the internal ones through `message::wire` +/// on the way in. +pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec, bool), Error> { + let api_v1::GenerateRequest { + rid, + text, + input_ids, + stream, + sampling_params, + return_logprob, + logprob_start_len, + top_logprobs_num, + token_ids_logprob, + return_hidden_states, + return_text_in_logprobs, + bootstrap_host, + bootstrap_port, + bootstrap_room, + bootstrap_pair_key, + decode_tp_size, + routed_dp_rank, + disagg_prefill_dp_rank, + image_data, + mm_hashes, + video_data, + audio_data, + } = req; + let rid = rid.and_then(wire::string_or_list); + let text = text.and_then(wire::string_or_list); + let input_ids = input_ids.and_then(wire::token_ids_or_list); + let stream = stream.unwrap_or(false); + let sampling_params = sampling_params + .map(wire::sampling_params_or_list) + .transpose() + .map_err(Error::Validation)? + .flatten(); + let return_logprob = return_logprob.and_then(wire::bool_or_list); + let logprob_start_len = logprob_start_len.and_then(wire::int64_or_list); + let top_logprobs_num = top_logprobs_num.and_then(wire::int64_or_list); + let token_ids_logprob = token_ids_logprob.and_then(wire::token_ids_or_list); + let return_hidden_states = return_hidden_states.and_then(wire::bool_or_list); + let bootstrap_host = bootstrap_host.and_then(wire::optional_string_or_list); + let bootstrap_port = bootstrap_port.and_then(wire::optional_int64_or_list); + let bootstrap_room = bootstrap_room.and_then(wire::optional_int64_or_list); + let bootstrap_pair_key = bootstrap_pair_key.and_then(wire::optional_string_or_list); + let decode_tp_size = decode_tp_size.and_then(wire::optional_int64_or_list); + let image_data = image_data.map(wire::media_input).transpose()?; + let video_data = video_data.map(wire::media_input).transpose()?; + let audio_data = audio_data.map(wire::media_input).transpose()?; + let mm_hashes = mm_hashes.and_then(wire::string_list_or_list); -/// Decode `input_ids` directly into token vectors. The first element selects -/// flat vs. batched input, avoiding the untagged enum's intermediate value tree. -fn deserialize_input_ids<'de, D>(deserializer: D) -> Result>, D::Error> -where - D: Deserializer<'de>, -{ - struct InputIds(OneOrMany); - - // Token width follows `TokenIds` (i64 in this crate); main's parser was - // written against its i32 alias and is ported here. - enum FirstElement { - Token(i64), - Tokens(TokenIds), + // Cap the batch BEFORE the columns below allocate anything. Reading the + // declared length off the input costs nothing; the previous placement (after + // the match) had already allocated ~1.7 GiB for a 114 MiB body, most of it + // the `vec![None; n]` twin column. + let declared_n = match (&text, &input_ids) { + (Some(OneOrMany::Many(v)), None) => v.len(), + (None, Some(OneOrMany::Many(v))) => v.len(), + _ => 1, + }; + if batch_size_exceeds_limit(declared_n, *MAX_BATCH_REQS_PER_HTTP_REQ) { + return Err(Error::Validation(format!( + "batch size {declared_n} exceeds the maximum of {}", + *MAX_BATCH_REQS_PER_HTTP_REQ + ))); } - impl<'de> Deserialize<'de> for FirstElement { - fn deserialize>(deserializer: D) -> Result { - struct FirstVisitor; + // Per-item (text, input_ids) columns + whether the input used list form. + type Columns = (Vec>, Vec>, bool); + // Exactly one of text / input_ids (Python `_validate_inputs`), and no + // empty id list (Python `_determine_batch_size`). + let (texts, id_lists, is_batch): Columns = match (text, input_ids) { + (Some(_), Some(_)) => { + return Err(Error::Validation( + "provide either `text` or `input_ids`, not both".into(), + )); + } + (None, None) => { + return Err(Error::Validation( + "either `text` or `input_ids` must be provided".into(), + )); + } + (Some(OneOrMany::One(s)), None) => (vec![Some(s)], vec![None], false), + (Some(OneOrMany::Many(v)), None) => { + let n = v.len(); + (v.into_iter().map(Some).collect(), vec![None; n], true) + } + // `[]` parses as `One(vec![])` (one prompt with no ids), so the + // `n == 0` guard below never sees it — reject it here, as Python's + // `_determine_batch_size` does. + (None, Some(OneOrMany::One(x))) => { + if x.is_empty() { + return Err(Error::Validation("input_ids cannot be empty".into())); + } + (vec![None], vec![Some(x)], false) + } + (None, Some(OneOrMany::Many(vv))) => { + if vv.iter().any(|ids| ids.is_empty()) { + return Err(Error::Validation( + "input_ids cannot be empty for any prompt in the batch".into(), + )); + } + let n = vv.len(); + (vec![None; n], vv.into_iter().map(Some).collect(), true) + } + }; + let n = texts.len(); + if n == 0 { + return Err(Error::Validation( + "batch must contain at least one item".into(), + )); + } - impl<'de> Visitor<'de> for FirstVisitor { - type Value = FirstElement; + // A list is per-item; a single object broadcasts to every item. + let sps: Vec = match sampling_params { + None => vec![SamplingParams::default(); n], + Some(wire::SamplingInput::Many(v)) => { + if v.len() != n { + return Err(Error::Validation(format!( + "sampling_params list length {} does not match batch size {n}", + v.len() + ))); + } + v + } + Some(wire::SamplingInput::One(sp)) => { + // Broadcasting deep-clones the client's params once per prompt, + // heap and all — `stop`, `logit_bias` and `custom_params` (arbitrary + // JSON) are still unnormalized client data here. The blow-up is + // quadratic in the body: ~1 MB of `custom_params` broadcast to 200k + // prompts is ~200 GB of clones, and a Rust allocation failure calls + // `abort()`, which is uncatchable and takes the scheduler process + // with it. Bound the product, not just `n`. + // `n == 1` is not a broadcast, so skip the sizing entirely: measuring + // it means serializing the client's whole `custom_params` to a + // throwaway `String` on every single request. The callee's own + // `n > 1` guard cannot prevent that — the cost is in the argument. + if n > 1 { + // Serialized bytes are NOT the clone cost: measured, 63.7 MiB of + // JSON became ~1008 MiB of live heap once parsed into `Value` + // nodes, `String`s and map entries. Scale by that measured factor + // so the budget bounds memory rather than wire size. + let per_clone = serde_json::to_string(&*sp) + .map_or(0, |s| s.len()) + .saturating_mul(JSON_TO_HEAP_FACTOR); + check_broadcast_budget(per_clone, n, "sampling_params")?; + } + vec![*sp; n] + } + }; - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("a token id or a list of token ids") - } - - fn visit_i64(self, value: i64) -> Result { - Ok(FirstElement::Token(value)) - } - - fn visit_u64(self, value: u64) -> Result { - i64::try_from(value) - .map(FirstElement::Token) - .map_err(E::custom) - } - - fn visit_seq>( - self, - sequence: A, - ) -> Result { - TokenIds::deserialize(SeqAccessDeserializer::new(sequence)) - .map(FirstElement::Tokens) + // rid: absent → mint one uuid per item here, so every request carries its + // final rid from this point on; a single string fans out as `{rid}_{i}` + // for a batch (Python `_normalize_batch`); a list is per-item. + // + // Every CLIENT-supplied rid goes through `Rid::from_client`, which appends a + // uniquifier so two concurrent requests sharing an rid cannot collide on the + // detok table. `client_facing` strips it back off for `meta_info.id`, so the + // client sees exactly what it sent. Minted rids (`Rid::default`) are already + // unique and are left bare. + let rids: Vec = match rid { + None => (0..n).map(|_| Rid::default()).collect(), + Some(OneOrMany::One(r)) if !is_batch => vec![Rid::from_client(&r)], + Some(OneOrMany::One(r)) => { + check_broadcast_budget(r.len(), n, "rid")?; + // Uniquify AFTER the `_{i}` split, so the split index stays part of + // the rid the client gets back. + (0..n) + .map(|i| Rid::from_client(&format!("{r}_{i}"))) + .collect() + } + Some(OneOrMany::Many(v)) => { + if !is_batch || v.len() != n { + return Err(Error::Validation(format!( + "rid list length {} does not match batch size {n}", + v.len() + ))); + } + // Python `_validate_rid_uniqueness`. `from_client` below would make + // even these unique, so this is parity rather than safety: Python + // 400s a request that names one id twice, and echoing the same + // `meta_info.id` on two entries of one batch response is useless to + // the client regardless. Checked on the RAW strings, before the + // uniquifier hides the duplication. + { + let mut seen = HashSet::with_capacity(v.len()); + let duplicates: Vec<&String> = v.iter().filter(|r| !seen.insert(*r)).collect(); + if !duplicates.is_empty() { + return Err(Error::Validation(format!( + "duplicate request IDs detected within the request: {duplicates:?}" + ))); } } - - deserializer.deserialize_any(FirstVisitor) + v.iter().map(|r| Rid::from_client(r)).collect() } - } + }; - impl<'de> Deserialize<'de> for InputIds { - fn deserialize>(deserializer: D) -> Result { - struct TokensVisitor; + // Fans out exactly like the scalar options: one list broadcasts, a list of + // lists is per item (Python `_normalize_batch`'s nested branch). Empties + // are collapsed per item below, not here. + let tid_logprobs = fan_out(token_ids_logprob, n, "token_ids_logprob")?; - impl<'de> Visitor<'de> for TokensVisitor { - type Value = InputIds; + // Each logprob/hidden opt: absent → None for every item, a scalar + // broadcasts, a list is per-item (Python `normalize_param`, plus a length + // check Python lacks — it would `IndexError` later instead). + let return_logprobs = fan_out(return_logprob, n, "return_logprob")?; + let logprob_start_lens = fan_out(logprob_start_len, n, "logprob_start_len")?; + let top_logprobs_nums = fan_out(top_logprobs_num, n, "top_logprobs_num")?; + let return_hidden = fan_out(return_hidden_states, n, "return_hidden_states")?; - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("a token list or a list of token lists") - } - - fn visit_seq>( - self, - mut sequence: A, - ) -> Result { - let tokens = match sequence.next_element::()? { - // Preserve OneOrMany's first-variant choice for []. - None => OneOrMany::One(Vec::new()), - Some(FirstElement::Token(first)) => { - let mut tokens = vec![first]; - while let Some(token) = sequence.next_element::()? { - tokens.push(token); - } - OneOrMany::One(tokens) - } - Some(FirstElement::Tokens(first)) => { - let mut prompts = vec![first]; - while let Some(tokens) = sequence.next_element::()? { - prompts.push(tokens); - } - OneOrMany::Many(prompts) - } - }; - Ok(InputIds(tokens)) - } - } - - deserializer.deserialize_seq(TokensVisitor) + // PD fields fan out like Python `_normalize_bootstrap_params`: scalars + // broadcast — except a scalar `bootstrap_room`, which becomes `room + i` + // (each item needs a distinct room; rooms are the P↔D pairing key). + // `fan_out` yields `Option>` for these nullable elements + // (outer: absent, inner: an explicit `null` element) — flatten, both + // mean "not set" downstream. + let bootstrap_hosts = flatten_column(fan_out(bootstrap_host, n, "bootstrap_host")?); + let bootstrap_ports = flatten_column(fan_out(bootstrap_port, n, "bootstrap_port")?); + let bootstrap_rooms = match bootstrap_room { + // `wrapping_add`, not `checked_`: rooms are drawn from `[0, 2^63)`, + // so a batch can only overflow by starting within `n` of `i64::MAX` + // — and distinct-but-wrapped still pairs P↔D, where saturating + // would collide every item onto one room. + Some(OneOrMany::One(Some(room))) => { + (0..n).map(|i| Some(room.wrapping_add(i as i64))).collect() } - } - - Option::::deserialize(deserializer).map(|value| value.map(|tokens| tokens.0)) -} - -impl GenerateBody { - /// Merge operator-provided sampling defaults beneath request values, - /// matching Python TokenizerManager's preferred/request precedence. - pub fn apply_preferred_sampling(&mut self, preferred: &serde_json::Value) -> Result<(), Error> { - match &mut self.sampling_params { - Some(params) => params.apply_preferred(preferred), - None => SamplingParamsInput::from_preferred(preferred).map(|params| { - self.sampling_params = Some(params); - }), + other => flatten_column(fan_out(other, n, "bootstrap_room")?), + }; + let bootstrap_pair_keys = flatten_column(fan_out(bootstrap_pair_key, n, "bootstrap_pair_key")?); + let decode_tp_sizes = flatten_column(fan_out(decode_tp_size, n, "decode_tp_size")?); + // `mm_hashes` has no batch form: honoring it only here would give the two + // servers different prefix-cache keys for the same body. Reject instead of + // dropping it silently as Python does — the field exists to align a + // caller's keys, so ignoring it returns subtly wrong ones. + let mm_hashes: Vec = match mm_hashes { + None => Vec::new(), + Some(OneOrMany::One(hashes)) if hashes.is_empty() => Vec::new(), + Some(_) if is_batch => { + return Err(Error::Validation( + "mm_hashes is not supported for batch requests; send one request per prompt".into(), + )); } - .map_err(|e| Error::Validation(format!("invalid preferred_sampling_params: {e}"))) - } + Some(OneOrMany::One(hashes)) => hashes, + Some(OneOrMany::Many(_)) => { + return Err(Error::Validation( + "mm_hashes must be a flat list of hex strings for a single request".into(), + )); + } + }; + // Multimodal columns; see `multimodal::fan_out` for the Python parity rules. + let images = multimodal::fan_out(image_data, n, is_batch, "image_data")?; + let videos = multimodal::fan_out(video_data, n, is_batch, "video_data")?; + let audios = multimodal::fan_out(audio_data, n, is_batch, "audio_data")?; - /// Validate, normalize and fan the body into one [`GenerateRequest`] per - /// prompt + `is_batch` (list form — a 1-element list is still a batch → JSON - /// array response). The Rust counterpart of Python - /// `GenerateReqInput.normalize_batch_and_arguments`; an invalid/inconsistent - /// batch is [`Error::Validation`], which the HTTP adapter surfaces as 400. - pub fn into_requests(self) -> Result<(Vec, bool), Error> { - let GenerateBody { + // Every column above is exactly `n` long, so zip them by value: each + // request takes ownership of its cell, with no indexing or bounds checks. + let mut requests: Vec = izip!( + rids, + texts, + id_lists, + sps, + return_logprobs, + logprob_start_lens, + top_logprobs_nums, + tid_logprobs, + return_hidden, + bootstrap_hosts, + bootstrap_ports, + bootstrap_rooms, + bootstrap_pair_keys, + decode_tp_sizes, + images, + videos, + audios, + ) + .map( + |( rid, text, input_ids, - stream, sampling_params, return_logprob, logprob_start_len, top_logprobs_num, token_ids_logprob, return_hidden_states, + bootstrap_host, + bootstrap_port, + bootstrap_room, + bootstrap_pair_key, + decode_tp_size, + image_data, + video_data, + audio_data, + )| GenerateRequest { + rid, + text, + input_ids, + // Plain text prompts keep the post-processor specials; the + // chat flow sets this explicitly. + skip_special_tokens: false, + sampling_params, + stream, + // Python `GenerateReqInput` defaults. + return_logprob: return_logprob.unwrap_or(false), + logprob_start_len: logprob_start_len.unwrap_or(-1), + top_logprobs_num: top_logprobs_num.unwrap_or(0), + // `Some` here means "these ids were requested", so an empty list + // collapses to None. + token_ids_logprob: token_ids_logprob.filter(|ids| !ids.is_empty()), + return_sampling_mask: false, // TODO: port Python's `return_sampling_mask` + return_hidden_states: return_hidden_states.unwrap_or(false), return_text_in_logprobs, bootstrap_host, bootstrap_port, @@ -281,299 +373,17 @@ impl GenerateBody { decode_tp_size, routed_dp_rank, disagg_prefill_dp_rank, - image_data, - video_data, - audio_data, - mm_hashes, - processor_extensions, - } = self; - - // Cap the batch BEFORE the columns below allocate anything. Reading the - // declared length off the input costs nothing; the previous placement (after - // the match) had already allocated ~1.7 GiB for a 114 MiB body, most of it - // the `vec![None; n]` twin column. - let declared_n = match (&text, &input_ids) { - (Some(OneOrMany::Many(v)), None) => v.len(), - (None, Some(OneOrMany::Many(v))) => v.len(), - _ => 1, - }; - if batch_size_exceeds_limit(declared_n, *MAX_BATCH_REQS_PER_HTTP_REQ) { - return Err(Error::Validation(format!( - "batch size {declared_n} exceeds the maximum of {}", - *MAX_BATCH_REQS_PER_HTTP_REQ - ))); - } - - // Per-item (text, input_ids) columns + whether the input used list form. - type Columns = (Vec>, Vec>, bool); - // Exactly one of text / input_ids (Python `_validate_inputs`), and no - // empty id list (Python `_determine_batch_size`). - let (texts, id_lists, is_batch): Columns = match (text, input_ids) { - (Some(_), Some(_)) => { - return Err(Error::Validation( - "provide either `text` or `input_ids`, not both".into(), - )); - } - (None, None) => { - return Err(Error::Validation( - "either `text` or `input_ids` must be provided".into(), - )); - } - (Some(OneOrMany::One(s)), None) => (vec![Some(s)], vec![None], false), - (Some(OneOrMany::Many(v)), None) => { - let n = v.len(); - (v.into_iter().map(Some).collect(), vec![None; n], true) - } - // `[]` parses as `One(vec![])` (one prompt with no ids), so the - // `n == 0` guard below never sees it — reject it here, as Python's - // `_determine_batch_size` does. - (None, Some(OneOrMany::One(x))) => { - if x.is_empty() { - return Err(Error::Validation("input_ids cannot be empty".into())); - } - (vec![None], vec![Some(x)], false) - } - (None, Some(OneOrMany::Many(vv))) => { - if vv.iter().any(|ids| ids.is_empty()) { - return Err(Error::Validation( - "input_ids cannot be empty for any prompt in the batch".into(), - )); - } - let n = vv.len(); - (vec![None; n], vv.into_iter().map(Some).collect(), true) - } - }; - let n = texts.len(); - if n == 0 { - return Err(Error::Validation( - "batch must contain at least one item".into(), - )); - } - - // A list is per-item; a single object broadcasts to every item. - let sps: Vec = match sampling_params { - None => vec![SamplingParams::default(); n], - Some(SamplingParamsInput::Many(v)) => { - if v.len() != n { - return Err(Error::Validation(format!( - "sampling_params list length {} does not match batch size {n}", - v.len() - ))); - } - v - } - Some(SamplingParamsInput::One(sp)) => { - // Broadcasting deep-clones the client's params once per prompt, - // heap and all — `stop`, `logit_bias` and `custom_params` (arbitrary - // JSON) are still unnormalized client data here. The blow-up is - // quadratic in the body: ~1 MB of `custom_params` broadcast to 200k - // prompts is ~200 GB of clones, and a Rust allocation failure calls - // `abort()`, which is uncatchable and takes the scheduler process - // with it. Bound the product, not just `n`. - // `n == 1` is not a broadcast, so skip the sizing entirely: measuring - // it means serializing the client's whole `custom_params` to a - // throwaway `String` on every single request. The callee's own - // `n > 1` guard cannot prevent that — the cost is in the argument. - if n > 1 { - // Serialized bytes are NOT the clone cost: measured, 63.7 MiB of - // JSON became ~1008 MiB of live heap once parsed into `Value` - // nodes, `String`s and map entries. Scale by that measured factor - // so the budget bounds memory rather than wire size. - let per_clone = serde_json::to_string(&*sp) - .map_or(0, |s| s.len()) - .saturating_mul(JSON_TO_HEAP_FACTOR); - check_broadcast_budget(per_clone, n, "sampling_params")?; - } - vec![*sp; n] - } - }; - - // rid: absent → mint one uuid per item here, so every request carries its - // final rid from this point on; a single string fans out as `{rid}_{i}` - // for a batch (Python `_normalize_batch`); a list is per-item. - // - // Every CLIENT-supplied rid goes through `Rid::from_client`, which appends a - // uniquifier so two concurrent requests sharing an rid cannot collide on the - // detok table. `client_facing` strips it back off for `meta_info.id`, so the - // client sees exactly what it sent. Minted rids (`Rid::default`) are already - // unique and are left bare. - let rids: Vec = match rid { - None => (0..n).map(|_| Rid::default()).collect(), - Some(OneOrMany::One(r)) if !is_batch => vec![Rid::from_client(&r)], - Some(OneOrMany::One(r)) => { - check_broadcast_budget(r.len(), n, "rid")?; - // Uniquify AFTER the `_{i}` split, so the split index stays part of - // the rid the client gets back. - (0..n) - .map(|i| Rid::from_client(&format!("{r}_{i}"))) - .collect() - } - Some(OneOrMany::Many(v)) => { - if !is_batch || v.len() != n { - return Err(Error::Validation(format!( - "rid list length {} does not match batch size {n}", - v.len() - ))); - } - // Python `_validate_rid_uniqueness`. `from_client` below would make - // even these unique, so this is parity rather than safety: Python - // 400s a request that names one id twice, and echoing the same - // `meta_info.id` on two entries of one batch response is useless to - // the client regardless. Checked on the RAW strings, before the - // uniquifier hides the duplication. - { - let mut seen = HashSet::with_capacity(v.len()); - let duplicates: Vec<&String> = v.iter().filter(|r| !seen.insert(*r)).collect(); - if !duplicates.is_empty() { - return Err(Error::Validation(format!( - "duplicate request IDs detected within the request: {duplicates:?}" - ))); - } - } - v.iter().map(|r| Rid::from_client(r)).collect() - } - }; - - // Fans out exactly like the scalar options: one list broadcasts, a list of - // lists is per item (Python `_normalize_batch`'s nested branch). Empties - // are collapsed per item below, not here. - let tid_logprobs = fan_out(token_ids_logprob, n, "token_ids_logprob")?; - - // Each logprob/hidden opt: absent → None for every item, a scalar - // broadcasts, a list is per-item (Python `normalize_param`, plus a length - // check Python lacks — it would `IndexError` later instead). - let return_logprobs = fan_out(return_logprob, n, "return_logprob")?; - let logprob_start_lens = fan_out(logprob_start_len, n, "logprob_start_len")?; - let top_logprobs_nums = fan_out(top_logprobs_num, n, "top_logprobs_num")?; - let return_hidden = fan_out(return_hidden_states, n, "return_hidden_states")?; - - // PD fields fan out like Python `_normalize_bootstrap_params`: scalars - // broadcast — except a scalar `bootstrap_room`, which becomes `room + i` - // (each item needs a distinct room; rooms are the P↔D pairing key). - // `fan_out` yields `Option>` for these nullable elements - // (outer: absent, inner: an explicit `null` element) — flatten, both - // mean "not set" downstream. - let bootstrap_hosts = flatten_column(fan_out(bootstrap_host, n, "bootstrap_host")?); - let bootstrap_ports = flatten_column(fan_out(bootstrap_port, n, "bootstrap_port")?); - let bootstrap_rooms = match bootstrap_room { - // `wrapping_add`, not `checked_`: rooms are drawn from `[0, 2^63)`, - // so a batch can only overflow by starting within `n` of `i64::MAX` - // — and distinct-but-wrapped still pairs P↔D, where saturating - // would collide every item onto one room. - Some(OneOrMany::One(Some(room))) => { - (0..n).map(|i| Some(room.wrapping_add(i as i64))).collect() - } - other => flatten_column(fan_out(other, n, "bootstrap_room")?), - }; - let bootstrap_pair_keys = - flatten_column(fan_out(bootstrap_pair_key, n, "bootstrap_pair_key")?); - let decode_tp_sizes = flatten_column(fan_out(decode_tp_size, n, "decode_tp_size")?); - // `mm_hashes` has no batch form: honoring it only here would give the two - // servers different prefix-cache keys for the same body. Reject instead of - // dropping it silently as Python does — the field exists to align a - // caller's keys, so ignoring it returns subtly wrong ones. - let mm_hashes: Vec = match mm_hashes { - None => Vec::new(), - Some(OneOrMany::One(hashes)) if hashes.is_empty() => Vec::new(), - Some(_) if is_batch => { - return Err(Error::Validation( - "mm_hashes is not supported for batch requests; send one request per prompt" - .into(), - )); - } - Some(OneOrMany::One(hashes)) => hashes, - Some(OneOrMany::Many(_)) => { - return Err(Error::Validation( - "mm_hashes must be a flat list of hex strings for a single request".into(), - )); - } - }; - // Multimodal columns; see `multimodal::fan_out` for the Python parity rules. - let images = multimodal::fan_out(image_data, n, is_batch, "image_data")?; - let videos = multimodal::fan_out(video_data, n, is_batch, "video_data")?; - let audios = multimodal::fan_out(audio_data, n, is_batch, "audio_data")?; - let processor_extensions = split_extension_columns(processor_extensions, n, is_batch)?; - - // Every column above is exactly `n` long, so zip them by value: each - // request takes ownership of its cell, with no indexing or bounds checks. - let mut requests: Vec = izip!( - rids, - texts, - id_lists, - sps, - return_logprobs, - logprob_start_lens, - top_logprobs_nums, - tid_logprobs, - return_hidden, - bootstrap_hosts, - bootstrap_ports, - bootstrap_rooms, - bootstrap_pair_keys, - decode_tp_sizes, - images, - videos, - audios, - processor_extensions, - ) - .map( - |( - rid, - text, - input_ids, - sampling_params, - return_logprob, - logprob_start_len, - top_logprobs_num, - token_ids_logprob, - return_hidden_states, - bootstrap_host, - bootstrap_port, - bootstrap_room, - bootstrap_pair_key, - decode_tp_size, - image_data, - video_data, - audio_data, - processor_extensions, - )| GenerateRequest { - rid, - text, - input_ids, - // Plain text prompts keep the post-processor specials; the - // chat flow sets this explicitly. - skip_special_tokens: false, - sampling_params, - stream, - // Python `GenerateReqInput` defaults. - return_logprob: return_logprob.unwrap_or(false), - logprob_start_len: logprob_start_len.unwrap_or(-1), - top_logprobs_num: top_logprobs_num.unwrap_or(0), - // `Some` here means "these ids were requested", so an empty list - // collapses to None. - token_ids_logprob: token_ids_logprob.filter(|ids| !ids.is_empty()), - return_sampling_mask: false, // TODO: port Python's `return_sampling_mask` - return_hidden_states: return_hidden_states.unwrap_or(false), - return_text_in_logprobs, - bootstrap_host, - bootstrap_port, - bootstrap_room, - bootstrap_pair_key, - decode_tp_size, - routed_dp_rank, - disagg_prefill_dp_rank, - mm: pack_mm(image_data, video_data, audio_data, processor_extensions), - mm_buffers: Vec::new(), - }, - ) - .collect(); - // Single requests only (batches rejected above). Malformed entries are - // dropped here and warned about in `mm::apply_caller_hashes`, never a 400. - if let Some(mm) = requests.first_mut().and_then(|req| req.mm.as_deref_mut()) { - mm.mm_hashes = mm_hashes; - } - Ok((requests, is_batch)) + mm: pack_mm(image_data, video_data, audio_data), + mm_buffers: Vec::new(), + }, + ) + .collect(); + // Single requests only (batches rejected above). Malformed entries are + // dropped here and warned about in `mm::apply_caller_hashes`, never a 400. + if let Some(mm) = requests.first_mut().and_then(|req| req.mm.as_deref_mut()) { + mm.mm_hashes = mm_hashes; } + Ok((requests, is_batch)) } /// Box the per-item mm values, `None` when the item has none — the common @@ -582,78 +392,24 @@ fn pack_mm( image_data: Vec, video_data: Vec, audio_data: Vec, - processor_extensions: ProcessorExtensions, ) -> Option> { - if image_data.is_empty() - && video_data.is_empty() - && audio_data.is_empty() - && processor_extensions.is_empty() - { + if image_data.is_empty() && video_data.is_empty() && audio_data.is_empty() { return None; } Some(Box::new(MmData { image_data, video_data, audio_data, - processor_extensions, ..Default::default() })) } -fn split_extension_columns( - fields: ProcessorExtensions, - n: usize, - is_batch: bool, -) -> Result, Error> { - let mut requests = vec![ProcessorExtensions::default(); n]; - for (name, value) in fields.0 { - if !name.starts_with(PROCESSOR_EXTENSION_PREFIX) || value.is_nil() { - continue; - } - if !is_batch { - requests[0].0.insert(name, value); - continue; - } - let rmpv::Value::Array(values) = value else { - return Err(Error::Validation(format!( - "{name} must be a list for batch processing" - ))); - }; - if values.is_empty() { - for request in &mut requests { - request - .0 - .insert(name.clone(), rmpv::Value::Array(Vec::new())); - } - continue; - } - if values.len() != n { - return Err(Error::Validation(format!( - "{name} list length {} does not match batch size {n}", - values.len() - ))); - } - for (request, value) in requests.iter_mut().zip(values) { - request.0.insert(name.clone(), value); - } - } - Ok(requests) -} - -fn extension_value_present(value: &rmpv::Value) -> bool { - match value { - rmpv::Value::Nil => false, - rmpv::Value::Array(values) => values.iter().any(extension_value_present), - _ => true, - } -} - /// The owned request as it travels request stages (single owner, so `state` is /// mutated lock-free). Common fields here; variant data in [`RequestKind`]. #[derive(Debug)] pub struct Request { /// Runtime correlation ID. Client-provided IDs carry a private incarnation - /// suffix; adapters recover the public ID through `FrontendCall::public_id`. + /// suffix; adapters recover the public ID through `CoreCall::public_id`. pub rid: Rid, pub state: RequestState, /// Back-channel to the client connection for response frames. @@ -679,7 +435,7 @@ pub enum RequestKind { /// Generation: tokenize (if needed) then push a `TokenizedGenerateReqInput`. Generate(Box), /// Internal runtime control operation: no tokenization, with one serialized - /// result that the frontend boundary converts into a typed response. + /// result that the core boundary converts into a typed response. Control(Box), /// Internal service call: decode a complete token-id sequence to text. Walks /// the same FSM as every request (validate → register → Queued), but the @@ -692,7 +448,7 @@ pub enum RequestKind { /// A single in-flight generation request (one item after adapter fan-out), /// serialized to the scheduler wire once tokenized (see `to_header_msgpack`). Not a -/// wire type — built by frontend adapters, never (de)serialized; `input_ids` is +/// wire type — built by the API adapters, never (de)serialized; `input_ids` is /// supplied by an adapter or filled by the Tokenizer stage. #[derive(Debug, Default)] pub struct GenerateRequest { @@ -775,16 +531,15 @@ pub struct GenerateRequest { /// processor's input: the MM worker moves it out of the request whole, and /// `payload::to_mm_input` converts it to the driver's. /// -/// Constructed directly only by tests: `frontend::prefetch` fills its +/// Constructed directly only by tests: `core::prefetch` fills its /// `prefetched` field, everything else gets it packed inside a `GenerateRequest`. #[derive(Debug, Default)] pub struct MmData { pub image_data: Vec, pub video_data: Vec, pub audio_data: Vec, - pub processor_extensions: ProcessorExtensions, /// Bytes of `image_data`'s I/O-backed sources, resolved by - /// `frontend::prefetch` in `payload::io_sources` order so MM workers + /// `core::prefetch` in `payload::io_sources` order so MM workers /// never block on I/O. Out-of-band: the values above stay as the client /// sent them. pub prefetched: Vec, @@ -802,13 +557,7 @@ impl GenerateRequest { /// Python `GenerateReqInput.contains_mm_input()`. pub fn has_multimodal(&self) -> bool { self.mm.as_ref().is_some_and(|mm| { - !mm.image_data.is_empty() - || !mm.video_data.is_empty() - || !mm.audio_data.is_empty() - || mm - .processor_extensions - .values() - .any(extension_value_present) + !mm.image_data.is_empty() || !mm.video_data.is_empty() || !mm.audio_data.is_empty() }) } @@ -920,26 +669,14 @@ fn fan_out( #[cfg(test)] mod tests { use super::*; - - #[derive(Debug, Deserialize, PartialEq)] - struct TestProcessorExtensions { - multimodal_custom: TestProcessorExtension, - } - - #[derive(Debug, Deserialize, PartialEq)] - #[serde(deny_unknown_fields)] - struct TestProcessorExtension { - value: i64, - } + use crate::message::multimodal::MmDataInput; /// Vocab size for tests that aren't about the vocab bound (see /// `sampling::tests::TEST_VOCAB`). const TEST_VOCAB: u64 = 1000; fn requests(body: &str) -> Result<(Vec, bool), Error> { - serde_json::from_str::(body) - .unwrap() - .into_requests() + into_requests(serde_json::from_str::(body).unwrap()) } /// Scalar `text` → one item, not a batch (response stays a single object). @@ -1000,72 +737,33 @@ mod tests { assert_eq!(ps[1].input_ids, Some(vec![3])); } + /// A bad element inside `input_ids` is reported against the field with + /// serde's type text (the wire decoder settles the list form from its + /// first element, so the element index is not part of the path), and a + /// malformed body is a rejection, not a panic. #[test] - fn input_ids_deserialization_matches_untagged() { - assert!( - serde_json::from_str::(r#"{"text":"hi"}"#) - .unwrap() - .input_ids - .is_none() - ); - for value in [ - "null", - "[]", - "[0]", - "[-2147483648,2147483647]", - "[[]]", - "[[],[1,-2]]", - "[[1,2],[3]]", - "[[-2147483648],[2147483647]]", - "1", - "true", - "{}", - r#""1""#, - "[null]", - "[true]", - r#"["1"]"#, - "[1.0]", - "[1e0]", - "[-0]", - "[2147483648]", - "[-2147483649]", - "[18446744073709551616]", - "[0,2147483648]", - "[0,-0]", - "[1,[2]]", - "[[1],2]", - "[[[1]]]", - "[[1.0]]", - "[[0],[2147483648]]", + fn input_ids_rejections_name_the_field() { + for body in [ + r#"{"input_ids":[true]}"#, + r#"{"input_ids":[[1,true]]}"#, + r#"{"input_ids":[[1],[2,true]]}"#, ] { - let expected = serde_json::from_str::>>(value); - let body = format!(r#"{{"input_ids":{value}}}"#); - let actual = serde_json::from_str::(&body).map(|body| body.input_ids); - match (expected, actual) { - (Ok(expected), Ok(actual)) => assert_eq!(actual, expected, "{value}"), - (Err(_), Err(_)) => {} - (expected, actual) => panic!("{value}: expected {expected:?}, got {actual:?}"), - } - } - - for (body, path) in [ - (r#"{"input_ids":[true]}"#, "input_ids[0]"), - (r#"{"input_ids":[[1,true]]}"#, "input_ids[0][1]"), - (r#"{"input_ids":[[1],[2,true]]}"#, "input_ids[1][1]"), - ] { - let error = axum::Json::::from_bytes(body.as_bytes()).unwrap_err(); - assert!(error.body_text().contains(path), "{error}"); + let error = + axum::Json::::from_bytes(body.as_bytes()).unwrap_err(); + let text = error.body_text(); + assert!( + text.contains("input_ids") && text.contains("boolean"), + "{text}" + ); } for body in [ r#"{"input_ids":null,"input_ids":[1]}"#, r#"{"input_ids":[1],"input_ids":[2]}"#, r#"{"input_ids":[1]} {}"#, ] { - assert!(axum::Json::::from_bytes(body.as_bytes()).is_err()); + assert!(axum::Json::::from_bytes(body.as_bytes()).is_err()); } } - - /// Both / neither of text+input_ids is a 400. #[test] fn split_validates_inputs() { assert!(requests(r#"{"text": "a", "input_ids": [1]}"#).is_err()); @@ -1218,92 +916,6 @@ mod tests { assert!(ps[1].has_multimodal()); } - #[test] - fn multimodal_extensions_follow_request_batch_shape() { - let single = r#"{"input_ids":[9],"image_data":"u","multimodal_placeholders":[{"type":"image","token_index":0,"item_index":0}]}"#; - let (reqs, is_batch) = requests(single).unwrap(); - assert!(!is_batch); - let value = reqs[0] - .mm - .as_ref() - .unwrap() - .processor_extensions - .0 - .get("multimodal_placeholders") - .unwrap(); - assert_eq!(value.as_array().unwrap().len(), 1); - - let batched = r#"{"input_ids":[[9],[8]],"image_data":["u","v"],"multimodal_placeholders":[[{"type":"image","token_index":0,"item_index":0}],[{"type":"image","token_index":0,"item_index":0}]]}"#; - let (reqs, is_batch) = requests(batched).unwrap(); - assert!(is_batch); - assert_eq!(reqs.len(), 2); - assert!(reqs.iter().all(GenerateRequest::has_multimodal)); - assert!(reqs.iter().all(|request| { - request - .mm - .as_ref() - .and_then(|mm| mm.processor_extensions.0.get("multimodal_placeholders")) - .and_then(rmpv::Value::as_array) - .is_some_and(|placeholders| placeholders.len() == 1) - })); - - let invalid = r#"{"input_ids":[[9],[8]],"image_data":["u","v"],"multimodal_placeholders":[{"type":"image","token_index":0,"item_index":0}]}"#; - assert!(requests(invalid).is_err()); - - let generic = r#"{"input_ids":[[9],[8]],"image_data":["u","v"],"multimodal_custom":[{"value":1},{"value":2}]}"#; - let (reqs, _) = requests(generic).unwrap(); - assert_eq!( - reqs[1] - .mm - .as_ref() - .unwrap() - .processor_extensions - .0 - .get("multimodal_custom") - .unwrap() - .as_map() - .unwrap()[0] - .1 - .as_i64(), - Some(2) - ); - - let extensions: TestProcessorExtensions = - requests(r#"{"input_ids":[9],"multimodal_custom":{"value":3}}"#) - .unwrap() - .0 - .pop() - .unwrap() - .mm - .unwrap() - .processor_extensions - .deserialize() - .unwrap(); - assert_eq!(extensions.multimodal_custom.value, 3); - - for fields in [ - r#"{"multimodal_custom":{"value":true}}"#, - r#"{"multimodal_custom":{"value":"3"}}"#, - r#"{"multimodal_custom":{"value":3,"unknown":0}}"#, - r#"{"multimodal_custom":{}}"#, - ] { - let extensions: ProcessorExtensions = serde_json::from_str(fields).unwrap(); - assert!( - extensions.deserialize::().is_err(), - "{fields}" - ); - } - - let (reqs, _) = requests(r#"{"text":"hi","totally_made_up":1}"#).unwrap(); - assert!(reqs[0].mm.is_none()); - - let (reqs, _) = requests(r#"{"input_ids":[9],"multimodal_custom":null}"#).unwrap(); - assert!(!reqs[0].has_multimodal()); - } - - /// A scalar broadcast is budget-checked before the deep clones (16 MiB × - /// 4096 prompts would be 64 GiB and an abort); per-item lists clone nothing - /// and are never charged. #[test] fn oversized_mm_broadcast_rejected() { let big = MmItem::Source("x".repeat(MAX_BROADCAST_CLONE_BYTES / 2 + 1)); diff --git a/rust/sglang-server/src/message/response.rs b/rust/sglang-server/src/message/response.rs index 807389b5a1..c2cc222bb0 100644 --- a/rust/sglang-server/src/message/response.rs +++ b/rust/sglang-server/src/message/response.rs @@ -1,4 +1,4 @@ -//! The runtime response direction: per-request channels into the frontend +//! The runtime response direction: per-request channels into the core //! boundary ([`ResponseSink`] / [`ResponseItem`]), response frame encodings, //! and columnar batch decoding into per-request [`ChunkEvent`]s. @@ -12,7 +12,7 @@ use crate::message::types::TokenIds; use crate::utils::error::Error; /// Per-request back-channel the detok shard writes runtime outputs to and the -/// frontend boundary drains; bounded, and receiver-drop cancels the producer. +/// core boundary drains; bounded, and receiver-drop cancels the producer. #[derive(Clone, Debug)] pub enum ResponseSink { Local(mpsc::Sender), @@ -38,12 +38,12 @@ impl ResponseSink { } } -#[allow(dead_code)] // the receiver half is owned inside the frontend boundary. +#[allow(dead_code)] // the receiver half is owned inside the core boundary. pub type ResponseSource = mpsc::Receiver; -/// Runtime-facing output delivered to the frontend boundary: a detok-decoded +/// Runtime-facing output delivered to the core boundary: a detok-decoded /// [`ChunkEvent`], a serialized internal-operation result, or a stage error. -/// [`crate::frontend::FrontendHandle`] translates these variants before a +/// [`crate::api_server::core::CoreHandle`] translates these variants before a /// transport adapter sees them. #[derive(Debug)] pub enum ResponseItem { @@ -54,10 +54,10 @@ pub enum ResponseItem { /// A control-request result encoded by the Python scheduler. Control(Bytes), /// Reply to an internal service request (`RequestKind::Detokenize`): raw - /// bytes for the frontend to decode (e.g. detokenized text), not a + /// bytes for the core to decode (e.g. detokenized text), not a /// client-facing wire payload and not a generation frame. Data(Bytes), - /// Terminal runtime failure; the frontend maps it to a semantic error. + /// Terminal runtime failure; the core maps it to a semantic error. Error(Error), } @@ -558,7 +558,7 @@ pub fn frame_error(rid: &str, message: &str) -> Bytes { #[derive(Debug, Clone, Default)] pub struct ChunkEvent { /// Runtime correlation ID. Moved out of the frame header (which owns it and - /// drops it), so carrying it costs no allocation. The frontend strips this + /// drops it), so carrying it costs no allocation. The core strips this /// field before exposing semantic output; the shard routes it via `Rid::shard`. pub rid: Rid, /// New token ids for this step, widened from the scheduler's int32 wire diff --git a/rust/sglang-server/src/message/sampling.rs b/rust/sglang-server/src/message/sampling.rs index 7c8c9a2b6b..79b21a4752 100644 --- a/rust/sglang-server/src/message/sampling.rs +++ b/rust/sglang-server/src/message/sampling.rs @@ -3,12 +3,9 @@ //! `__post_init__` → `normalize` → `verify` pipeline (run in that order, as //! `TokenizerManager._create_tokenized_object` does). -use std::collections::{BTreeMap, BTreeSet}; -use std::fmt; +use std::collections::BTreeMap; -use serde::de::value::{MapAccessDeserializer, SeqAccessDeserializer}; -use serde::de::{MapAccess, SeqAccess, Visitor}; -use serde::{Deserialize, Deserializer, Serialize}; +use serde::{Deserialize, Serialize}; use super::types::OneOrMany; use crate::utils::{error::Error, regex::RegexPattern}; @@ -241,90 +238,6 @@ pub struct SamplingParams { /// strip that wrapper from its own grammar, so it is a pipeline output only. #[serde(skip_deserializing)] pub ebnf_full_assistant: bool, - /// API fields present in the request object. Serde defaults erase this - /// distinction, but preferred sampling parameters must not overwrite an - /// explicit request value, including an explicit default or null. - #[serde(skip)] - pub(crate) explicit_fields: BTreeSet, -} - -/// The `/generate` body's `sampling_params`: one object (broadcast to every -/// prompt) or a list of them (one per prompt), fanned out by `GenerateBody::into_requests`. -/// -/// Hand-written `Deserialize` rather than `#[serde(untagged)]`: untagged buffers -/// the input and, on failure, reports only "data did not match any variant" — -/// losing the field-level message ("unknown field `temperature`, expected one of -/// …") that makes a typo actionable. Object-vs-list is unambiguous here, so a -/// single `deserialize_any` dispatch keeps the inner error verbatim. -#[derive(Debug, Clone, PartialEq)] -pub enum SamplingParamsInput { - /// Boxed: `SamplingParams` is ~440 bytes, so an inline variant would make - /// every `GenerateBody` that big regardless of which form arrived. - One(Box), - Many(Vec), -} - -impl<'de> Deserialize<'de> for SamplingParamsInput { - fn deserialize>(deserializer: D) -> Result { - struct InputVisitor; - - impl<'de> Visitor<'de> for InputVisitor { - type Value = SamplingParamsInput; - - fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str("a sampling_params object, or a list of them (one per prompt)") - } - - fn visit_map>(self, map: A) -> Result { - let value = serde_json::Value::deserialize(MapAccessDeserializer::new(map))?; - sampling_params_from_value(value) - .map(|p| SamplingParamsInput::One(Box::new(p))) - .map_err(serde::de::Error::custom) - } - - fn visit_seq>(self, seq: A) -> Result { - let values = - Vec::::deserialize(SeqAccessDeserializer::new(seq))?; - values - .into_iter() - .map(sampling_params_from_value) - .collect::, _>>() - .map(SamplingParamsInput::Many) - .map_err(serde::de::Error::custom) - } - } - - deserializer.deserialize_any(InputVisitor) - } -} - -fn sampling_params_from_value(value: serde_json::Value) -> Result { - let explicit_fields = value - .as_object() - .ok_or_else(|| "sampling_params must be an object".to_string())? - .keys() - .cloned() - .collect(); - let mut params: SamplingParams = serde_json::from_value(value).map_err(|e| e.to_string())?; - params.explicit_fields = explicit_fields; - Ok(params) -} - -impl SamplingParamsInput { - /// Merge launch-time preferred params beneath request params. A request key - /// wins even when it explicitly carries the type's default or null. - pub fn apply_preferred(&mut self, preferred: &serde_json::Value) -> Result<(), String> { - match self { - Self::One(params) => params.apply_preferred(preferred), - Self::Many(params) => params - .iter_mut() - .try_for_each(|params| params.apply_preferred(preferred)), - } - } - - pub fn from_preferred(preferred: &serde_json::Value) -> Result { - sampling_params_from_value(preferred.clone()).map(|params| Self::One(Box::new(params))) - } } impl Default for SamplingParams { @@ -366,39 +279,11 @@ impl Default for SamplingParams { stop_regex_max_len: 0, is_normalized: false, ebnf_full_assistant: false, - explicit_fields: BTreeSet::new(), } } } impl SamplingParams { - /// Record a field supplied by a non-Serde adapter. Preferred launch values - /// are merged underneath these fields just as they are underneath keys - /// present in an HTTP sampling object. - pub(crate) fn mark_explicit(&mut self, field: &'static str) { - self.explicit_fields.insert(field.to_owned()); - } - - /// Merge operator-provided sampling defaults beneath explicitly supplied - /// request fields. HTTP and gRPC adapters use the same precedence policy. - pub(crate) fn apply_preferred(&mut self, preferred: &serde_json::Value) -> Result<(), String> { - let mut merged = preferred - .as_object() - .ok_or_else(|| "preferred_sampling_params must be a JSON object".to_string())? - .clone(); - let request_value = serde_json::to_value(&*self).map_err(|e| e.to_string())?; - let request = request_value - .as_object() - .ok_or_else(|| "SamplingParams did not serialize as an object".to_string())?; - for field in &self.explicit_fields { - if let Some(value) = request.get(field) { - merged.insert(field.clone(), value.clone()); - } - } - *self = sampling_params_from_value(serde_json::Value::Object(merged))?; - Ok(()) - } - /// `__post_init__` → `normalize` → `verify`, the order /// `TokenizerManager._create_tokenized_object` runs them in. `Err` is a /// request-local 400. `skip_tokenizer_init` stands in for Python's @@ -1220,35 +1105,4 @@ mod tests { let err = norm_err(&json).to_string(); assert!(err.contains("at most"), "{err}"); } - - #[test] - fn preferred_params_fill_only_omitted_request_fields() { - let preferred = serde_json::json!({ - "temperature": 0.25, - "top_p": 0.75, - "max_new_tokens": 4096 - }); - let mut input: SamplingParamsInput = - serde_json::from_str(r#"{"temperature": 1.0, "top_p": null}"#).unwrap(); - input.apply_preferred(&preferred).unwrap(); - let SamplingParamsInput::One(params) = input else { - panic!("expected scalar params") - }; - assert_eq!(params.temperature, 1.0, "explicit default wins"); - assert_eq!(params.top_p, 1.0, "explicit null keeps the type default"); - assert_eq!(params.max_new_tokens, Some(4096), "omitted uses preferred"); - } - - #[test] - fn preferred_params_apply_to_every_batched_object() { - let preferred = serde_json::json!({"temperature": 0.25, "top_p": 0.75}); - let mut input: SamplingParamsInput = - serde_json::from_str(r#"[{"temperature": 0.5}, {"top_p": 0.9}]"#).unwrap(); - input.apply_preferred(&preferred).unwrap(); - let SamplingParamsInput::Many(params) = input else { - panic!("expected batched params") - }; - assert_eq!((params[0].temperature, params[0].top_p), (0.5, 0.75)); - assert_eq!((params[1].temperature, params[1].top_p), (0.25, 0.9)); - } } diff --git a/rust/sglang-server/src/message/wire.rs b/rust/sglang-server/src/message/wire.rs new file mode 100644 index 0000000000..bfef9e4ab8 --- /dev/null +++ b/rust/sglang-server/src/message/wire.rs @@ -0,0 +1,410 @@ +//! The `/generate` wire contract is the generated `sglang_api_types` request +//! (the proto schema is the single source of truth for both servers). This +//! module is the one place its carriers become the internal request types: +//! scalar-or-list columns into [`OneOrMany`], media containers into +//! [`MmDataInput`], and the wire `SamplingParams` into the internal one that +//! carries normalization state. Nothing here validates batch shape; that is +//! [`into_requests`](super::request::into_requests). + +use std::collections::BTreeMap; + +use sglang_api_types::api::v1 as api; + +use super::multimodal::{MmDataInput, MmItem}; +use super::sampling::{CustomParamValue, SamplingParams}; +use super::types::{OneOrMany, TokenIds}; +use crate::utils::error::Error; + +pub fn string_or_list(v: api::StringOrList) -> Option> { + use api::string_or_list::Value; + v.value.map(|v| match v { + Value::One(s) => OneOrMany::One(s), + Value::Many(l) => OneOrMany::Many(l.items), + }) +} + +pub fn int64_or_list(v: api::Int64OrList) -> Option> { + use api::int64_or_list::Value; + v.value.map(|v| match v { + Value::One(x) => OneOrMany::One(x), + Value::Many(l) => OneOrMany::Many(l.items), + }) +} + +pub fn bool_or_list(v: api::BoolOrList) -> Option> { + use api::bool_or_list::Value; + v.value.map(|v| match v { + Value::One(x) => OneOrMany::One(x), + Value::Many(l) => OneOrMany::Many(l.items), + }) +} + +fn token_ids(v: api::TokenIds) -> TokenIds { + v.ids +} + +pub fn token_ids_or_list(v: api::TokenIdsOrList) -> Option> { + use api::token_ids_or_list::Value; + v.value.map(|v| match v { + Value::One(ids) => OneOrMany::One(token_ids(ids)), + Value::Many(l) => OneOrMany::Many(l.items.into_iter().map(token_ids).collect()), + }) +} + +pub fn optional_string_or_list(v: api::OptionalStringOrList) -> Option>> { + use api::optional_string_or_list::Value; + v.value.map(|v| match v { + Value::One(s) => OneOrMany::One(s.value), + Value::Many(l) => OneOrMany::Many(l.items.into_iter().map(|s| s.value).collect()), + }) +} + +pub fn optional_int64_or_list(v: api::OptionalInt64OrList) -> Option>> { + use api::optional_int64_or_list::Value; + v.value.map(|v| match v { + Value::One(x) => OneOrMany::One(x.value), + Value::Many(l) => OneOrMany::Many(l.items.into_iter().map(|x| x.value).collect()), + }) +} + +pub fn string_list_or_list(v: api::StringListOrList) -> Option>> { + use api::string_list_or_list::Value; + v.value.map(|v| match v { + Value::One(l) => OneOrMany::One(l.items), + Value::Many(ll) => OneOrMany::Many(ll.items.into_iter().map(|l| l.items).collect()), + }) +} + +/// A media item: a bare source, or a ref whose `url` must be present. proto3 +/// strings carry no presence, so an object without `url` decodes as an empty +/// one and is rejected here (Python's `load_image` fails on it the same way). +pub fn media_item(item: api::MediaItem) -> Result { + use api::media_item::Value; + match item.value { + // Only `url` reaches the Rust MM pipeline: the hints are read by model + // families it does not run (Python's `load_image` reduces an item to + // `.url` the same way). + Some(Value::Ref(r)) if !r.url.is_empty() => Ok(MmItem::Ref { url: r.url }), + Some(Value::Ref(_)) => Err(Error::Validation( + "a multimodal item object needs a non-empty `url`".into(), + )), + Some(Value::Source(s)) => Ok(MmItem::Source(s)), + // Unreachable from JSON (the untagged union always sets an arm). + None => Err(Error::Validation("empty multimodal item".into())), + } +} + +pub fn media_input(input: api::MediaInput) -> Result { + use api::media_input::Value; + let item = |i: api::OptionalMediaItem| i.item.map(media_item).transpose(); + Ok(match input.value { + Some(Value::Item(i)) => MmDataInput::One(media_item(i)?), + Some(Value::Many(l)) => { + MmDataInput::Many(l.items.into_iter().map(item).collect::>()?) + } + Some(Value::Nested(ll)) => MmDataInput::Nested( + ll.items + .into_iter() + .map(|entry| { + entry + .items + .map(|l| l.items.into_iter().map(item).collect::>()) + .transpose() + }) + .collect::>()?, + ), + None => MmDataInput::Many(Vec::new()), + }) +} + +/// The wire params, with the schema defaults already applied by the generated +/// decoder (an absent key reads its default), become the internal params. Only +/// `custom_params` needs parsing: it crosses the wire as JSON text. +impl TryFrom for SamplingParams { + type Error = String; + + fn try_from(p: api::SamplingParams) -> Result { + let defaults = SamplingParams::default(); + let custom_params: Option> = match p.custom_params { + None => None, + Some(text) => { + Some(serde_json::from_str(&text).map_err(|e| format!("custom_params: {e}"))?) + } + }; + Ok(SamplingParams { + max_new_tokens: p.max_new_tokens, + stop: p.stop.and_then(string_or_list), + stop_token_ids: p.stop_token_ids.map(|l| l.items), + stop_regex: p.stop_regex.and_then(string_or_list), + temperature: p.temperature.unwrap_or(defaults.temperature), + top_p: p.top_p.unwrap_or(defaults.top_p), + top_k: p.top_k.unwrap_or(defaults.top_k), + min_p: p.min_p.unwrap_or(defaults.min_p), + frequency_penalty: p.frequency_penalty.unwrap_or(defaults.frequency_penalty), + presence_penalty: p.presence_penalty.unwrap_or(defaults.presence_penalty), + repetition_penalty: p.repetition_penalty.unwrap_or(defaults.repetition_penalty), + min_new_tokens: p.min_new_tokens.unwrap_or(defaults.min_new_tokens), + n: p.n.unwrap_or(defaults.n), + beam_width: p.beam_width, + json_schema: p.json_schema, + regex: p.regex, + ebnf: p.ebnf, + structural_tag: p.structural_tag, + ignore_eos: p.ignore_eos.unwrap_or(defaults.ignore_eos), + skip_special_tokens: p + .skip_special_tokens + .unwrap_or(defaults.skip_special_tokens), + spaces_between_special_tokens: p + .spaces_between_special_tokens + .unwrap_or(defaults.spaces_between_special_tokens), + no_stop_trim: p.no_stop_trim.unwrap_or(defaults.no_stop_trim), + stream_interval: p.stream_interval, + logit_bias: (!p.logit_bias.is_empty()).then(|| p.logit_bias.into_iter().collect()), + sampling_seed: p.sampling_seed, + custom_params, + ..defaults + }) + } +} + +/// `sampling_params` as sent: one object (a broadcast in a batch) or one per +/// prompt. Not a [`OneOrMany`]: that carrier is sealed to wire-shaped items, +/// and the internal params are not one. +pub enum SamplingInput { + /// Boxed: the params are ~450 bytes, so an inline variant would make + /// every input that big regardless of which form arrived. + One(Box), + Many(Vec), +} + +pub fn sampling_params_or_list( + v: api::SamplingParamsOrList, +) -> Result, String> { + use api::sampling_params_or_list::Value; + Ok(match v.value { + None => None, + Some(Value::One(p)) => Some(SamplingInput::One(Box::new(p.try_into()?))), + Some(Value::Many(l)) => Some(SamplingInput::Many( + l.items + .into_iter() + .map(SamplingParams::try_from) + .collect::>()?, + )), + }) +} + +/// Launch-time preferred sampling params fill the keys a request did not +/// send, at the JSON level, before the body is decoded: `{**preferred, +/// **request}` per object, which is Python TokenizerManager's precedence. A +/// request key wins even when it carries the type's default or null, which +/// a decoded struct can no longer tell apart from an absent key. +pub fn merge_preferred_sampling( + body: &mut serde_json::Value, + preferred: &serde_json::Value, +) -> Result<(), String> { + let preferred = preferred + .as_object() + .ok_or_else(|| "preferred_sampling_params must be a JSON object".to_string())?; + let Some(body) = body.as_object_mut() else { + return Ok(()); // not an object: the decoder reports it + }; + let merge_into = |request: &mut serde_json::Value| { + if let Some(obj) = request.as_object_mut() { + for (k, v) in preferred { + obj.entry(k.clone()).or_insert_with(|| v.clone()); + } + } + }; + match body.get_mut("sampling_params") { + None | Some(serde_json::Value::Null) => { + body.insert( + "sampling_params".into(), + serde_json::Value::Object(preferred.clone()), + ); + } + Some(serde_json::Value::Array(items)) => items.iter_mut().for_each(merge_into), + Some(one) => merge_into(one), + } + Ok(()) +} + +/// The same fill for a request that arrived already decoded (gRPC): the +/// carrier round-trips through its JSON form so protobuf and HTTP share one +/// precedence rule. Protobuf has no null, so an unset field is an absent key +/// (the schema's null-emitting fields are stripped before the merge), never +/// the explicit null a JSON client can send. +pub fn fill_preferred_sampling( + params: Option, + preferred: &serde_json::Value, +) -> Result, String> { + let mut body = serde_json::json!({ "sampling_params": params }); + let strip_nulls = |object: &mut serde_json::Value| { + if let Some(map) = object.as_object_mut() { + map.retain(|_, value| !value.is_null()); + } + }; + match &mut body["sampling_params"] { + serde_json::Value::Array(items) => items.iter_mut().for_each(strip_nulls), + one => strip_nulls(one), + } + merge_preferred_sampling(&mut body, preferred)?; + serde_json::from_value(body["sampling_params"].take()).map_err(|e| e.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn decode(body: &str, preferred: serde_json::Value) -> Vec { + let mut value: serde_json::Value = serde_json::from_str(body).unwrap(); + merge_preferred_sampling(&mut value, &preferred).unwrap(); + let req: api::GenerateRequest = serde_json::from_value(value).unwrap(); + match sampling_params_or_list(req.sampling_params.unwrap()) + .unwrap() + .unwrap() + { + SamplingInput::One(p) => vec![*p], + SamplingInput::Many(v) => v, + } + } + + /// A request key wins over the preferred value, even as an explicit + /// default or null; only omitted keys take the preferred value. + #[test] + fn preferred_params_fill_only_omitted_request_fields() { + let preferred = + serde_json::json!({"temperature": 0.25, "top_p": 0.75, "max_new_tokens": 4096}); + let p = &decode( + r#"{"sampling_params": {"temperature": 1.0, "top_p": null}}"#, + preferred.clone(), + )[0]; + assert_eq!(p.temperature, 1.0, "explicit default wins"); + assert_eq!(p.top_p, 1.0, "explicit null keeps the type default"); + assert_eq!(p.max_new_tokens, Some(4096), "omitted uses preferred"); + let p = &decode(r#"{"text": "hi"}"#, preferred)[0]; + assert_eq!( + (p.temperature, p.max_new_tokens), + (0.25, Some(4096)), + "absent object takes all" + ); + } + + #[test] + fn preferred_params_apply_to_every_batched_object() { + let preferred = serde_json::json!({"temperature": 0.25, "top_p": 0.75}); + let ps = decode( + r#"{"sampling_params": [{"temperature": 0.5}, {"top_p": 0.9}]}"#, + preferred, + ); + assert_eq!((ps[0].temperature, ps[0].top_p), (0.5, 0.75)); + assert_eq!((ps[1].temperature, ps[1].top_p), (0.25, 0.9)); + } + + /// The generated `TokenIdsOrList` and the crate's own `OneOrMany` + /// accept and reject the same shapes (the first-element rule), so the + /// swap to the wire type changed no input_ids contract. + #[test] + fn token_ids_decoding_matches_one_or_many() { + for value in [ + "null", + "[]", + "[0]", + "[-2147483648,2147483647]", + "[[]]", + "[[],[1,-2]]", + "[[1,2],[3]]", + "1", + "true", + "{}", + r#""1""#, + "[null]", + "[true]", + r#"["1"]"#, + "[1.0]", + "[-0]", + "[2147483648]", + "[0,2147483648]", + "[1,[2]]", + "[[1],2]", + "[[[1]]]", + "[[1.0]]", + ] { + let expected = serde_json::from_str::>>(value); + let actual = serde_json::from_str::>(value) + .map(|v| v.and_then(token_ids_or_list)); + match (expected, actual) { + (Ok(e), Ok(a)) => assert_eq!(a, e, "{value}"), + (Err(_), Err(_)) => {} + (e, a) => panic!("{value}: expected {e:?}, got {a:?}"), + } + } + } + + #[test] + fn media_input_converts_every_container_form() { + let conv = |json: &str| -> MmDataInput { + media_input(serde_json::from_str::(json).unwrap()).unwrap() + }; + let src = |s: &str| MmItem::Source(s.to_owned()); + assert_eq!(conv(r#""u""#), MmDataInput::One(src("u"))); + assert_eq!( + conv(r#"{"url": "u", "detail": "high"}"#), + MmDataInput::One(MmItem::Ref { url: "u".into() }) + ); + assert_eq!( + conv(r#"["a", null, "b"]"#), + MmDataInput::Many(vec![Some(src("a")), None, Some(src("b"))]) + ); + assert_eq!(conv("[]"), MmDataInput::Many(vec![])); + assert_eq!( + conv(r#"[["a", null], null, []]"#), + MmDataInput::Nested(vec![Some(vec![Some(src("a")), None]), None, Some(vec![])]) + ); + } + + #[test] + fn custom_params_cross_as_json_text() { + let p: api::SamplingParams = + serde_json::from_str(r#"{"custom_params": {"k": [1, "x"]}, "logit_bias": {"5": 1.5}}"#) + .unwrap(); + let p = SamplingParams::try_from(p).unwrap(); + assert!( + p.custom_params + .as_ref() + .is_some_and(|m| m.contains_key("k")) + ); + assert_eq!( + p.logit_bias.as_ref().and_then(|m| m.get("5")).copied(), + Some(1.5) + ); + assert_eq!(p.temperature, 1.0, "schema default applied by the decoder"); + } + + /// The protobuf entry fills the same keys the JSON entry does: a set field + /// wins, an unset one takes the preferred value, and an absent carrier + /// takes them all. + #[test] + fn preferred_params_fill_unset_protobuf_fields() { + use api::sampling_params_or_list::Value; + let preferred = serde_json::json!({"temperature": 0.25, "max_new_tokens": 4096}); + let sent = api::SamplingParamsOrList { + value: Some(Value::One(api::SamplingParams { + temperature: Some(1.0), + ..Default::default() + })), + }; + let filled = fill_preferred_sampling(Some(sent), &preferred) + .unwrap() + .unwrap(); + let Some(Value::One(p)) = filled.value else { + panic!("one object in, one object out"); + }; + assert_eq!((p.temperature, p.max_new_tokens), (Some(1.0), Some(4096))); + + let filled = fill_preferred_sampling(None, &preferred).unwrap().unwrap(); + let Some(Value::One(p)) = filled.value else { + panic!("an absent carrier becomes the preferred object"); + }; + assert_eq!((p.temperature, p.max_new_tokens), (Some(0.25), Some(4096))); + } +} diff --git a/rust/sglang-server/src/multi_modality/payload.rs b/rust/sglang-server/src/multi_modality/payload.rs index 7f89d17266..cae540d238 100644 --- a/rust/sglang-server/src/multi_modality/payload.rs +++ b/rust/sglang-server/src/multi_modality/payload.rs @@ -10,7 +10,7 @@ use sglang_mm::common::fetch::{ByteBudget, fetch_bytes_budgeted}; use sglang_mm::driver::{ImageSource, MmInput}; use crate::message::multimodal::MmItem; -use crate::message::request::{MmData, ProcessorExtensions}; +use crate::message::request::MmData; use crate::message::types::TokenIds; /// Fully resolved media for a multimodal processor. I/O sources were @@ -23,9 +23,6 @@ pub struct ResolvedMediaWork { pub images: Vec, pub videos: Vec, pub audios: Vec, - /// Request fields owned by the selected processor rather than this shared - /// payload layer. - pub processor_extensions: ProcessorExtensions, } /// Resolve all modality fields in the fixed image/video/audio prefetch order. @@ -42,7 +39,6 @@ fn resolve_media_work_with_budget( image_data, video_data, audio_data, - processor_extensions, prefetched, mm_hashes: _, } = mm; @@ -59,7 +55,6 @@ fn resolve_media_work_with_budget( images, videos, audios, - processor_extensions, }) } @@ -83,16 +78,13 @@ fn collect_media( fetch_bytes_budgeted(&source, budget).map(Bytes::from) } } - MmItem::Preprocessed { format } => Err(format!( - "unsupported {field} item: preprocessed `{format}` input" - )), }) .collect() } /// True for sources the API layer must resolve before MM dispatch: I/O — network /// *or* disk, since a network mount can hang past any HTTP timeout — never runs -/// on the fixed MM worker pool (see `frontend::prefetch`). `data:` and bare +/// on the fixed MM worker pool (see `core::prefetch`). `data:` and bare /// base64 are pure CPU and stay on the worker. Lives next to [`image_source`] /// so the prefetch walk and the parse walk cannot drift. pub fn is_io_source(src: &str) -> bool { @@ -120,16 +112,12 @@ pub fn to_mm_input(input_ids: TokenIds, mm: MmData) -> Result { image_data, video_data, audio_data, - processor_extensions, prefetched, mm_hashes: _, } = mm; if !video_data.is_empty() || !audio_data.is_empty() { return Err("unsupported modality: video/audio input".into()); } - if !processor_extensions.is_empty() { - return Err("unsupported generate extensions for this processor".into()); - } let mut prefetched = prefetched.iter(); let images = image_data .into_iter() @@ -155,9 +143,6 @@ fn image_source( .map(|bytes| ImageSource::Bytes(bytes.to_vec())) .ok_or_else(|| "I/O-backed image source was not prefetched".to_string()) } - MmItem::Preprocessed { format } => Err(format!( - "unsupported image_data item: preprocessed `{format}` input" - )), } } @@ -207,31 +192,6 @@ mod tests { .unwrap() .contains("video/audio") ); - - let err = to_mm_input( - IDS.to_vec(), - image_work(vec![MmItem::Preprocessed { - format: "processor_output".into(), - }]), - ) - .err() - .unwrap(); - assert!(err.contains("preprocessed `processor_output`"), "{err}"); - - let extension = MmData { - processor_extensions: std::iter::once(( - "multimodal_custom".to_owned(), - rmpv::Value::Boolean(true), - )) - .collect(), - ..Default::default() - }; - assert!( - to_mm_input(IDS.to_vec(), extension) - .err() - .unwrap() - .contains("unsupported generate extensions") - ); } /// I/O-backed sources (URLs, file paths) take their prefetched bytes in walk diff --git a/rust/sglang-server/src/native_generation.rs b/rust/sglang-server/src/native_generation.rs deleted file mode 100644 index 2366bfb7f2..0000000000 --- a/rust/sglang-server/src/native_generation.rs +++ /dev/null @@ -1,160 +0,0 @@ -//! Shared wire semantics for SGLang-native generation responses. -//! -//! Native HTTP places these values in a JSON `meta_info` object, while -//! `runtime.v1` gRPC JSON-encodes each value into `map`. Keeping -//! the logical shape here prevents the sibling adapters from drifting without -//! making either adapter depend on the other. - -use crate::frontend::FrontendOutput; - -/// The text slot of a `[logprob, token_id, text]` tuple: the decoded token when -/// `return_text_in_logprobs` supplied a text buffer, else `null`. -pub(crate) fn text_slot(texts: Option<&[String]>, j: usize) -> serde_json::Value { - texts - .and_then(|texts| texts.get(j)) - .map(|text| serde_json::json!(text)) - .unwrap_or(serde_json::Value::Null) -} - -/// A decoded-text column becomes the tuples' text source only when populated. -pub(crate) fn opt_texts(texts: &[String]) -> Option<&[String]> { - (!texts.is_empty()).then_some(texts) -} - -/// The logprob slot of a tuple: a finite value, or `null` for the `NaN` sentinel. -pub(crate) fn lp_value(value: f32) -> serde_json::Value { - if value.is_nan() { - serde_json::Value::Null - } else { - serde_json::json!(value) - } -} - -/// SGLang logprob shape: a list of `[logprob, token_id, text]` tuples. -pub(crate) fn logprob_tuples( - values: &[f32], - token_ids: &[i32], - texts: Option<&[String]>, -) -> serde_json::Value { - let tuples = values - .iter() - .zip(token_ids.iter()) - .enumerate() - .map(|(index, (&value, &token_id))| { - serde_json::json!([lp_value(value), token_id, text_slot(texts, index)]) - }) - .collect(); - serde_json::Value::Array(tuples) -} - -/// Ragged top-k / requested-token shape: one entry per position, or `null` -/// where that position's length is zero. -pub(crate) fn ragged_logprob_tuples( - values: &[f32], - token_ids: &[i32], - lengths: &[u32], - texts: Option<&[String]>, -) -> serde_json::Value { - let mut positions = Vec::with_capacity(lengths.len()); - let mut offset = 0usize; - for &length in lengths { - let length = length as usize; - if length == 0 { - positions.push(serde_json::Value::Null); - } else { - // Bounds checks turn malformed parallel columns into a shortened - // row instead of panicking a transport worker. - let tuples = (offset..offset + length) - .filter_map(|index| { - Some(serde_json::json!([ - lp_value(*values.get(index)?), - *token_ids.get(index)?, - text_slot(texts, index) - ])) - }) - .collect(); - positions.push(serde_json::Value::Array(tuples)); - } - offset += length; - } - serde_json::Value::Array(positions) -} - -/// Reshape flat hidden-state values and per-row lengths into nested rows. -pub(crate) fn hidden_states_rows(values: &[f32], lengths: &[u32]) -> serde_json::Value { - let mut rows = Vec::with_capacity(lengths.len()); - let mut offset = 0usize; - for &length in lengths { - let length = length as usize; - rows.push(serde_json::json!( - values.get(offset..offset + length).unwrap_or(&[]) - )); - offset += length; - } - serde_json::Value::Array(rows) -} - -/// Build the heterogeneous native `meta_info` object before either adapter -/// chooses its wire encoding. -pub(crate) fn meta_info_value(out: &FrontendOutput, rid: &str) -> serde_json::Value { - let mut meta = serde_json::json!({ - "id": rid, - "prompt_tokens": out.prompt_tokens, - "completion_tokens": out.completion_tokens, - "finish_reason": out.finish_reason, - }); - let Some(extras) = out.extras.as_deref() else { - return meta; - }; - - // Python always sets input and output token logprobs together, including - // the empty input list emitted by a disaggregated decode node. - if !extras.out_lp_val.is_empty() || !extras.in_lp_val.is_empty() { - meta["output_token_logprobs"] = logprob_tuples( - &extras.out_lp_val, - &extras.out_lp_idx, - opt_texts(&extras.out_lp_txt), - ); - meta["input_token_logprobs"] = logprob_tuples( - &extras.in_lp_val, - &extras.in_lp_idx, - opt_texts(&extras.in_lp_txt), - ); - } - if !extras.out_top_lens.is_empty() { - meta["output_top_logprobs"] = ragged_logprob_tuples( - &extras.out_top_val, - &extras.out_top_idx, - &extras.out_top_lens, - opt_texts(&extras.out_top_txt), - ); - } - if !extras.in_top_lens.is_empty() { - meta["input_top_logprobs"] = ragged_logprob_tuples( - &extras.in_top_val, - &extras.in_top_idx, - &extras.in_top_lens, - opt_texts(&extras.in_top_txt), - ); - } - if !extras.out_tid_lens.is_empty() { - meta["output_token_ids_logprobs"] = ragged_logprob_tuples( - &extras.out_tid_val, - &extras.out_tid_idx, - &extras.out_tid_lens, - opt_texts(&extras.out_tid_txt), - ); - } - if !extras.in_tid_lens.is_empty() { - meta["input_token_ids_logprobs"] = ragged_logprob_tuples( - &extras.in_tid_val, - &extras.in_tid_idx, - &extras.in_tid_lens, - opt_texts(&extras.in_tid_txt), - ); - } - if !extras.hidden_lens.is_empty() { - meta["hidden_states"] = hidden_states_rows(&extras.hidden_val, &extras.hidden_lens); - } - meta -} diff --git a/rust/sglang-server/src/tokenizer_manager/detokenizer.rs b/rust/sglang-server/src/tokenizer_manager/detokenizer.rs index 323848b406..c72485fae6 100644 --- a/rust/sglang-server/src/tokenizer_manager/detokenizer.rs +++ b/rust/sglang-server/src/tokenizer_manager/detokenizer.rs @@ -184,7 +184,7 @@ impl Runnable for DetokenizerWorker { // Plain `recv`: exits when the `DetokMsg` channel closes (every `Senders` // clone gone). On shutdown that happens once to-scheduler/from-scheduler - // exit — frontend calls own only their abort sender, not detok channels. + // exit — core calls own only their abort sender, not detok channels. while let Ok(msg) = self.rx.recv() { match msg { DetokMsg::Register { @@ -395,7 +395,7 @@ fn handle_chunk( } let _ = st.fsm.apply(Event::Disconnect); // Abort ONLY when the sink is full. `Closed` means the handler future is - // already gone, so its `FrontendCall` has queued the abort; do not emit + // already gone, so its `CoreCall` has queued the abort; do not emit // the same lifecycle event a second time. A full sink still has a live // handler, so the detokenizer must initiate cleanup itself. if matches!(e, SinkError::Full) { diff --git a/rust/sglang-server/src/tokenizer_manager/to_scheduler_tests.rs b/rust/sglang-server/src/tokenizer_manager/to_scheduler_tests.rs index 1c4e9734f2..452a25cf90 100644 --- a/rust/sglang-server/src/tokenizer_manager/to_scheduler_tests.rs +++ b/rust/sglang-server/src/tokenizer_manager/to_scheduler_tests.rs @@ -102,7 +102,7 @@ fn ids_len(req: &SchedulerRequest) -> usize { /// further chunk can be delivered, and tell the scheduler to stop generating. /// /// Neither releases anything, and nothing needs them to. Release ordering used -/// to be the delicate part here — `FrontendCall::drop` releasing a rid right +/// to be the delicate part here — `CoreCall::drop` releasing a rid right /// after enqueuing the abort ordered the SEND, not the EFFECT, so a retry of /// the same rid could `Register` ahead of the stale abort and be torn down by /// it. `Rid::from_client` removes the premise: a retry carries a different diff --git a/rust/sglang-server/src/tokenizer_manager/wiring.rs b/rust/sglang-server/src/tokenizer_manager/wiring.rs index 1e0633ec2d..353b29c6cc 100644 --- a/rust/sglang-server/src/tokenizer_manager/wiring.rs +++ b/rust/sglang-server/src/tokenizer_manager/wiring.rs @@ -45,12 +45,12 @@ const ADMISSION_PENDING: u8 = 0; const ADMISSION_ACCEPTED: u8 = 1; const ADMISSION_CANCELLED: u8 = 2; -/// Cancellation-safe ownership handoff between a frontend call and Intake. +/// Cancellation-safe ownership handoff between a core call and Intake. /// /// A bounded `send_async` can enqueue a request and wake its sender without the /// sender being polled again. If that task is then cancelled, this token decides /// exactly one outcome: either Intake observes the prior cancellation and drops -/// the request, or the frontend observes prior admission and emits an abort. +/// the request, or the core observes prior admission and emits an abort. #[derive(Clone)] pub struct RequestAdmission { state: Arc, @@ -91,7 +91,7 @@ impl RequestAdmission { /// cannot be tangled up with an abort still in flight for the original. #[derive(Clone, Debug)] pub enum AbortSource { - /// From an in-flight frontend call drop. Owns the release. + /// From an in-flight core call drop. Owns the release. Guard(Rid), /// From a detokenizer terminal path. Aborts the scheduler work. Detok(Rid), diff --git a/rust/sglang-server/src/utils/runtime.rs b/rust/sglang-server/src/utils/runtime.rs index 378c2554b9..811362abfe 100644 --- a/rust/sglang-server/src/utils/runtime.rs +++ b/rust/sglang-server/src/utils/runtime.rs @@ -301,16 +301,16 @@ pub fn start(cfg: RuntimeConfig) -> Result { // One transport-neutral entrance to the runtime. Each configured listener // receives a clone; no listener owns or reconstructs scheduler wiring. - let frontend = crate::frontend::FrontendHandle::new( + let core = crate::api_server::core::CoreHandle::new( senders.tok_manager_tx.clone(), senders.abort_tx.clone(), - crate::frontend::FrontendConfig { + crate::api_server::core::CoreConfig { response_capacity: cfg.rust_server_args.stage_channel_cap, response_activity: response_activity.clone(), startup_ready: cfg.server_args.skip_server_warmup, is_disaggregation: cfg.server_args.is_disaggregation(), mm_limits: cfg.server_args.limit_mm_data_per_request.clone(), - metadata: crate::frontend::FrontendMetadata::from(cfg.server_args.as_ref()), + metadata: crate::api_server::core::CoreMetadata::from(cfg.server_args.as_ref()), }, ); @@ -320,7 +320,8 @@ pub fn start(cfg: RuntimeConfig) -> Result { let api_cores = plan.as_ref().map(|p| p.api.clone()); let shutdown_rx = shutdown_rx.clone(); let grpc_server = grpc_listener.map(|listener| { - let service = crate::grpc::GrpcService::new(frontend.clone(), &cfg.server_args); + let service = + api_server::grpc::service::GrpcService::new(core.clone(), &cfg.server_args); (listener, service) }); let handle = std::thread::Builder::new() @@ -341,14 +342,17 @@ pub fn start(cfg: RuntimeConfig) -> Result { } let rt = builder.build().expect("build api runtime"); rt.block_on(async move { - let http = api_server::app::serve( + let http = api_server::http::app::serve( http_listener, - frontend, + core, cfg.server_args.clone(), shutdown_rx.clone(), ); if let Some((listener, service)) = grpc_server { - tokio::join!(http, crate::grpc::serve(listener, service, shutdown_rx)); + tokio::join!( + http, + api_server::grpc::app::serve(listener, service, shutdown_rx) + ); } else { http.await; } @@ -375,8 +379,8 @@ mod tests { use super::*; use crate::message::config::{RuntimeConfig, RustServerServerArgs, ServerArgs}; use crate::message::response::{BatchHeader, frame_decode_batch_cols}; - use sglang_grpc_types::sglang::runtime::v1 as proto; - use sglang_grpc_types::sglang::runtime::v1::sglang_service_client::SglangServiceClient; + use sglang_api_types::api::v1 as proto; + use sglang_api_types::api::v1::sglang_service_client::SglangServiceClient; fn free_loopback_addrs() -> (std::net::SocketAddr, std::net::SocketAddr) { // Hold both probes at once so the OS cannot return the same ephemeral @@ -412,6 +416,21 @@ mod tests { SglangServiceClient::new(endpoint.connect().await.expect("connect gRPC client")) } + fn grpc_ids_request(ids: &[i64], rid: &str) -> proto::GenerateRequest { + proto::GenerateRequest { + input_ids: Some(proto::TokenIdsOrList { + value: Some(proto::token_ids_or_list::Value::One(proto::TokenIds { + ids: ids.to_vec(), + })), + }), + rid: Some(proto::StringOrList { + value: Some(proto::string_or_list::Value::One(rid.into())), + }), + stream: Some(true), + ..Default::default() + } + } + fn scheduler_rid(header: &[u8]) -> String { let value = rmpv::decode::read_value(&mut std::io::Cursor::new(header)).unwrap(); value @@ -643,9 +662,9 @@ mod tests { drop(rebound); } - /// End to end over the configured socket: the generated runtime.v1 client + /// End to end over the configured socket: the generated api.v1 client /// reaches the existing scheduler ring, and an existing scheduler frame is - /// translated back into a protobuf response. Shutdown then closes both + /// translated back into a protobuf frame. Shutdown then closes both /// transport listeners. #[test] fn configured_grpc_serves_generation_and_closes_with_http() { @@ -663,12 +682,7 @@ mod tests { let mut client = connect_grpc(grpc_addr).await; tokio::time::timeout( Duration::from_secs(5), - client.generate(proto::GenerateRequest { - input_ids: vec![1, 2, 3], - stream: Some(true), - rid: Some("network-generation".into()), - ..Default::default() - }), + client.generate(grpc_ids_request(&[1, 2, 3], "network-generation")), ) .await .expect("timed out starting Generate RPC") @@ -694,25 +708,29 @@ mod tests { .push(terminal_token_frame(rid, &[9, 10])) ); - let response = client_runtime + let item = client_runtime .block_on(async { tokio::time::timeout(Duration::from_secs(2), stream.message()).await }) .expect("timed out waiting for gRPC response") .expect("gRPC stream error") .expect("gRPC stream ended without a response"); - assert_eq!(response.output_ids, vec![9, 10]); - assert!(response.finished); - assert_eq!(response.meta_info["id"], r#""network-generation""#); + let Some(proto::generate_stream_item::Item::Frame(response)) = item.item else { + panic!("expected a frame, got {item:?}"); + }; + assert_eq!(response.output_ids.map(|ids| ids.ids), Some(vec![9, 10])); + let meta_info = response.meta_info.expect("frame meta_info"); + assert_eq!(meta_info.id, "network-generation"); + assert!(meta_info.finish_reason.is_some()); rt.request_shutdown(); assert!(std::net::TcpStream::connect(http_addr).is_err()); assert!(std::net::TcpStream::connect(grpc_addr).is_err()); } - /// The existing runtime.v1 listener accepts bodies above Tonic's 4 MiB - /// default. Keep that transport policy when mounting the same protocol on - /// the Rust frontend; reaching adapter validation proves the body decoded. + /// The Python-backed gRPC listener accepts bodies above Tonic's 4 MiB + /// default. Keep that transport policy on the Rust frontend; reaching + /// adapter validation proves the body decoded. #[test] fn grpc_listener_preserves_existing_large_message_limit() { use tonic::Code; @@ -729,19 +747,23 @@ mod tests { tokio::time::timeout( std::time::Duration::from_secs(10), client.generate(proto::GenerateRequest { - input_ids: vec![1], - // This field is intentionally unsupported. An - // UNIMPLEMENTED status proves the >4 MiB protobuf reached - // the adapter instead of Tonic rejecting it on size. - routing_key: Some("x".repeat(5 * 1024 * 1024)), - ..Default::default() + // `text` and `input_ids` together is a normalization + // error: an INVALID_ARGUMENT status proves the >4 MiB + // protobuf reached the adapter instead of Tonic rejecting + // it on size. + text: Some(proto::StringOrList { + value: Some(proto::string_or_list::Value::One( + "x".repeat(5 * 1024 * 1024), + )), + }), + ..grpc_ids_request(&[1], "large") }), ) .await .expect("timed out sending large Generate RPC") - .expect_err("unsupported routing_key must fail") + .expect_err("text and input_ids together must fail") }); - assert_eq!(error.code(), Code::Unimplemented); + assert_eq!(error.code(), Code::InvalidArgument); rt.request_shutdown(); assert!(std::net::TcpStream::connect(grpc_addr).is_err()); @@ -765,12 +787,7 @@ mod tests { let mut client = connect_grpc(grpc_addr).await; let stream = tokio::time::timeout( Duration::from_secs(5), - client.generate(proto::GenerateRequest { - input_ids: vec![1, 2, 3], - stream: Some(true), - rid: Some("in-flight".into()), - ..Default::default() - }), + client.generate(grpc_ids_request(&[1, 2, 3], "in-flight")), ) .await .expect("timed out starting Generate RPC") diff --git a/test/registered/rust/test_run_rust_tests.py b/test/registered/rust/test_run_rust_tests.py index 09345d6df5..e410ca1c99 100644 --- a/test/registered/rust/test_run_rust_tests.py +++ b/test/registered/rust/test_run_rust_tests.py @@ -11,7 +11,7 @@ from sglang.test.test_utils import CustomTestCase BUILD_AND_RUN_TIMEOUT_S = 900 RUST_WORKSPACE = Path(__file__).resolve().parents[3] / "rust" -GRPC_TYPES_PACKAGE = "sglang-grpc-types" +GRPC_TYPES_PACKAGE = "sglang-api-types" register_cpu_ci(est_time=170, suite="base-a-test-cpu") diff --git a/test/registered/rust/test_rust_extension.py b/test/registered/rust/test_rust_extension.py index 2873e6c8ea..0c5d851363 100644 --- a/test/registered/rust/test_rust_extension.py +++ b/test/registered/rust/test_rust_extension.py @@ -590,21 +590,23 @@ crate-type = ["cdylib"] self.assertNotIn(module_name, sys.modules) def test_checked_in_crates_are_discovered_from_wheel_metadata(self): - grpc_proto = (rust_extension._RUST_WORKSPACE.parent / "proto").resolve() + # No crate reads `proto/` at build time: the generated bindings are + # checked in under rust/sglang-api-types (`cargo gen-api`), so the + # workspace is every extension's only source input. for python_module, package, library, features, source_inputs in ( ( "sglang.srt.rust_extensions._server", "sglang-server", "sglang_server", (), - (grpc_proto,), + (), ), ( "sglang.srt.rust_extensions._grpc", "sglang-grpc", "sglang_grpc_core", (), - (grpc_proto,), + (), ), ( "sglang.srt.rust_extensions._multimodal", diff --git a/test/registered/unit/api/v1/test_json_contract.py b/test/registered/unit/api/v1/test_json_contract.py new file mode 100644 index 0000000000..1f8c8c2496 --- /dev/null +++ b/test/registered/unit/api/v1/test_json_contract.py @@ -0,0 +1,325 @@ +"""JSON-contract tests for the generated Python API types (sglang.api.v1). + +Mirrors rust/sglang-api-types/tests/json_contract.rs case for case (minus the +protobuf-arrival accessors, which have no Python counterpart), so the Rust and +Python emitters of sglang-api-codegen are held to one contract. +""" + +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +import json +import unittest + +from sglang.api.v1.api_types import ( + FinishAbort, + FinishLength, + FinishStop, + GenerateMetaInfo, + GenerateRequest, + GenerateResponse, + JsonContractError, + LogprobEntry, + MediaRef, + SamplingParams, + decode_FinishReason, + decode_MediaInput, + decode_OptionalInt64OrList, + decode_OptionalStringOrList, + decode_SamplingParamsOrList, + decode_StringListOrList, + decode_StringOrList, + decode_TokenIdsOrList, + encode_FinishReason, + encode_MediaInput, + encode_OptionalStringOrList, + encode_StringOrList, +) +from sglang.test.test_utils import CustomTestCase + + +def compact(value) -> str: + """serde_json::to_string parity: no whitespace.""" + return json.dumps(value, separators=(",", ":")) + + +def sp(text: str) -> SamplingParams: + return SamplingParams.from_json_value(json.loads(text)) + + +class TestSamplingParams(CustomTestCase): + def test_defaults_match_hand_written(self): + """Absent keys yield the schema defaults.""" + p = sp("{}") + self.assertEqual(p.max_new_tokens, 128) + self.assertEqual(p.temperature, 1.0) + self.assertEqual(p.top_p, 1.0) + self.assertEqual(p.top_k, 1 << 30) + self.assertEqual(p.min_p, 0.0) + self.assertEqual(p.repetition_penalty, 1.0) + self.assertEqual(p.min_new_tokens, 0) + self.assertEqual(p.n, 1) + self.assertIs(p.ignore_eos, False) + self.assertIs(p.skip_special_tokens, True) + self.assertIs(p.spaces_between_special_tokens, True) + self.assertIs(p.no_stop_trim, False) + + def test_constructor_defaults_match_absent_keys(self): + """A dataclass built by hand carries the same schema defaults.""" + self.assertEqual(SamplingParams(), sp("{}")) + + def test_null_resets_default(self): + """null_resets_default: an explicit null is the default, not an error.""" + p = sp('{"temperature": null, "top_k": null, "ignore_eos": null}') + self.assertEqual(p.temperature, 1.0) + self.assertEqual(p.top_k, 1 << 30) + self.assertIs(p.ignore_eos, False) + + def test_max_new_tokens_absent_vs_null(self): + """null_is_none: absent = 128, null = unbounded.""" + self.assertEqual(sp("{}").max_new_tokens, 128) + self.assertIsNone(sp('{"max_new_tokens": null}').max_new_tokens) + self.assertEqual(sp('{"max_new_tokens": 7}').max_new_tokens, 7) + + def test_denies_unknown_keys(self): + """DENY policy with serde's own error text.""" + with self.assertRaises(JsonContractError) as ctx: + sp('{"temperatur": 0.5}') # codespell:ignore temperatur + self.assertTrue( + str(ctx.exception).startswith( + "unknown field `temperatur`, expected one of" # codespell:ignore temperatur + ), + str(ctx.exception), + ) + + def test_type_mismatch_uses_serde_text(self): + with self.assertRaises(JsonContractError) as ctx: + sp('{"temperature": "hot"}') + self.assertEqual(str(ctx.exception), 'invalid type: string "hot", expected f64') + + def test_or_list_keeps_field_errors(self): + """one_or_many over SamplingParams keeps field-level error texts.""" + with self.assertRaises(JsonContractError) as ctx: + decode_SamplingParamsOrList( + {"temperatur": 1} # codespell:ignore temperatur + ) + self.assertTrue( + str(ctx.exception).startswith( + "unknown field `temperatur`" # codespell:ignore temperatur + ), + str(ctx.exception), + ) + many = decode_SamplingParamsOrList([{"n": 2}, {}]) + self.assertEqual([p.n for p in many], [2, 1]) + + def test_raw_json_passthrough(self): + """raw_json: custom_params crosses untouched, integers included.""" + p = sp('{"custom_params": {"k": 3}}') + self.assertEqual(p.custom_params, {"k": 3}) + self.assertEqual(p.to_json_value()["custom_params"], {"k": 3}) + + def test_int_key_map(self): + p = sp('{"logit_bias": {"42": -1}}') + self.assertEqual(p.logit_bias, {"42": -1.0}) + + +class TestGenerateRequest(CustomTestCase): + def test_ignores_unknown_keys(self): + """IGNORE policy: unported Python fields must not 400.""" + req = GenerateRequest.from_json_value( + { + "text": "hi", + "priority": 3, + "session_id": "s", + "custom_logit_processor": "x", + } + ) + self.assertEqual(req.text, "hi") + + def test_media_input_shapes_dispatch(self): + """Typed media: the value's shape picks the container form, nulls are + kept per element, and a ref object decodes to a MediaRef.""" + req = GenerateRequest.from_json_value( + {"image_data": [["data:image/png;base64,xx"], None, []]} + ) + self.assertEqual(req.image_data, [["data:image/png;base64,xx"], None, []]) + flat = decode_MediaInput(["a", None, {"url": "b", "detail": "high"}]) + self.assertEqual(flat[:2], ["a", None]) + self.assertIsInstance(flat[2], MediaRef) + self.assertEqual((flat[2].url, flat[2].detail), ("b", "high")) + self.assertEqual(decode_MediaInput("u"), "u") + self.assertEqual(decode_MediaInput([]), []) + for value in ( + ["a", None, {"url": "b", "detail": "high"}], + [["a", None], None, []], + "u", + ): + self.assertEqual(encode_MediaInput(decode_MediaInput(value)), value) + + def test_media_ref_is_typed_and_strict(self): + with self.assertRaises(JsonContractError) as ctx: + decode_MediaInput( + {"url": "u", "max_dynamic_patc": 6} + ) # codespell:ignore patc + self.assertIn("unknown field", str(ctx.exception)) + with self.assertRaises(JsonContractError): + decode_MediaInput([{"format": "processor_output"}]) + with self.assertRaises(JsonContractError): + decode_MediaInput(["a", ["b"]]) + + def test_string_list_or_list_shape_dispatch(self): + self.assertEqual(decode_StringListOrList(["a", "b"]), ["a", "b"]) + self.assertEqual( + decode_StringListOrList([["a"], ["b", "c"]]), [["a"], ["b", "c"]] + ) + self.assertEqual(decode_StringListOrList([]), []) + + def test_stream_null_resets_default(self): + self.assertIs(GenerateRequest.from_json_value({"stream": None}).stream, False) + self.assertIs(GenerateRequest.from_json_value({"stream": True}).stream, True) + + def test_round_trip_batch(self): + body = { + "text": ["a", "b"], + "sampling_params": [{"n": 2}, {"temperature": 0.5}], + "return_logprob": True, + "bootstrap_host": [None, "h"], + } + req = GenerateRequest.from_json_value(body) + back = req.to_json_value() + self.assertEqual(back["text"], ["a", "b"]) + self.assertEqual(back["return_logprob"], True) + self.assertEqual(back["bootstrap_host"], [None, "h"]) + self.assertEqual(back["sampling_params"][1]["temperature"], 0.5) + + +class TestCarriers(CustomTestCase): + def test_one_or_many_flattens(self): + self.assertEqual(decode_StringOrList("a"), "a") + self.assertEqual(compact(encode_StringOrList("a")), '"a"') + self.assertEqual(decode_StringOrList(["a", "b"]), ["a", "b"]) + self.assertEqual(compact(encode_StringOrList(["a", "b"])), '["a","b"]') + with self.assertRaises(JsonContractError) as ctx: + decode_StringOrList({"x": 1}) + self.assertEqual( + str(ctx.exception), + "invalid type: map, expected a value or an array of values for StringOrList", + ) + + def test_token_ids_or_list_shape_dispatch(self): + """[1,2] is one id list, [[1],[2,3]] is a batch, [] is one empty list.""" + self.assertEqual(decode_TokenIdsOrList([1, 2]), [1, 2]) + self.assertEqual(decode_TokenIdsOrList([[1], [2, 3]]), [[1], [2, 3]]) + self.assertEqual(decode_TokenIdsOrList([]), []) + + def test_optional_carriers_keep_element_nulls(self): + """PD bootstrap columns: [null, "h"] keeps per-item presence.""" + v = decode_OptionalStringOrList([None, "h"]) + self.assertEqual(v, [None, "h"]) + self.assertEqual(compact(encode_OptionalStringOrList(v)), '[null,"h"]') + self.assertEqual(decode_OptionalInt64OrList(17000), 17000) + + +class TestFinishReason(CustomTestCase): + def test_round_trips(self): + """Tagged by "type"; matched keeps its wire shape (str / int / int list).""" + for text in [ + '{"type":"stop","matched":""}', + '{"type":"stop","matched":7}', + '{"type":"stop","matched":[1,2]}', + '{"type":"length","length":2}', + ]: + want = json.loads(text) + parsed = decode_FinishReason(want) + self.assertEqual(encode_FinishReason(parsed), want, text) + + def test_variant_classes(self): + self.assertIsInstance(decode_FinishReason({"type": "stop"}), FinishStop) + self.assertIsInstance( + decode_FinishReason({"type": "length", "length": 2}), FinishLength + ) + self.assertIsInstance(decode_FinishReason({"type": "abort"}), FinishAbort) + + def test_abort_emits_null_keys(self): + """An abort carries all three keys, nulls included.""" + parsed = decode_FinishReason({"type": "abort", "message": "m"}) + back = encode_FinishReason(parsed) + self.assertEqual( + back, + {"type": "abort", "message": "m", "status_code": None, "err_type": None}, + ) + + def test_unknown_type_passes_through(self): + """The forward-compat escape hatch: unknown tags round-trip unchanged.""" + raw = {"type": "paused", "step": 3} + parsed = decode_FinishReason(raw) + self.assertIsInstance(parsed, dict) + self.assertEqual(encode_FinishReason(parsed), raw) + + def test_malformed_known_type_passes_through(self): + parsed = decode_FinishReason({"type": "stop", "matched": {"bad": "shape"}}) + self.assertIsInstance(parsed, dict) + + +class TestFrames(CustomTestCase): + def test_logprob_tuple_shape(self): + """[logprob, token_id, text|null]; the logprob slot is nullable.""" + e = LogprobEntry(logprob=-0.25, token_id=7, text=None) + self.assertEqual(compact(e.to_json_value()), "[-0.25,7,null]") + parsed = LogprobEntry.from_json_value([-0.5, 9, "x"]) + self.assertEqual(parsed.token_id, 9) + self.assertEqual(parsed.text, "x") + first_prefill = LogprobEntry.from_json_value([None, 10, None]) + self.assertIsNone(first_prefill.logprob) + self.assertEqual(compact(first_prefill.to_json_value()), "[null,10,null]") + with self.assertRaises(JsonContractError) as ctx: + LogprobEntry.from_json_value([1.0, 2]) + self.assertEqual( + str(ctx.exception), "invalid length 2, expected a 3-element tuple" + ) + + def test_generate_response_matches_frame_value_shape(self): + """Key order (text, meta_info, output_ids), conditional keys, and + finish_reason always present (null pre-terminal).""" + frame = GenerateResponse( + text="ok", + output_ids=[7, 8], + meta_info=GenerateMetaInfo( + id="client-rid", + prompt_tokens=5, + completion_tokens=2, + finish_reason=None, + ), + index=None, + ) + self.assertEqual( + compact(frame.to_json_value()), + '{"text":"ok","meta_info":{"id":"client-rid","prompt_tokens":5,"completion_tokens":2,"finish_reason":null},"output_ids":[7,8]}', + ) + + def test_frame_round_trip_with_logprobs(self): + wire = { + "text": "ok", + "meta_info": { + "id": "r", + "prompt_tokens": 1, + "completion_tokens": 1, + "finish_reason": {"type": "stop", "matched": 2}, + "output_token_logprobs": [[-0.1, 5, "x"]], + "output_top_logprobs": [None, [[-0.2, 6, None]]], + "hidden_states": [[0.5, 0.25]], + "e2e_latency": 0.01, + }, + "output_ids": [5], + "index": 0, + } + frame = GenerateResponse.from_json_value(wire) + self.assertIsInstance(frame.meta_info.finish_reason, FinishStop) + self.assertEqual(frame.meta_info.output_top_logprobs[0], None) + self.assertEqual(frame.meta_info.output_top_logprobs[1][0].token_id, 6) + self.assertEqual(frame.to_json_value(), wire) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/api/v1/test_parity.py b/test/registered/unit/api/v1/test_parity.py new file mode 100644 index 0000000000..7d90aed06c --- /dev/null +++ b/test/registered/unit/api/v1/test_parity.py @@ -0,0 +1,250 @@ +"""Parity of the three `/generate` vocabularies: the hand-written Python +request structs (`io_struct.GenerateReqInput`, `SamplingParams`), the +generated Python types (`sglang.api.v1.api_types`), and the generated Rust +types (`rust/sglang-api-types`). + +The cross-language half runs through `rust/sglang-api-types/testdata/ +json_parity.json`: both generators must turn each fixture input into the same +canonical bytes (`tests/json_parity.rs` checks the Rust side). The Python half +pins the field sets against the hand-written structs and proves a schema +request still normalizes through the dataclass the Python server runs. +""" + +import dataclasses +import json +import unittest +from pathlib import Path + +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel + +maybe_stub_sgl_kernel() # must precede any import that pulls in sgl_kernel + +from sglang.api.v1 import api_types +from sglang.api.v1.api_types import ( + GenerateMetaInfo, + GenerateRequest, + GenerateResponse, + SamplingParams, +) +from sglang.srt.managers.io_struct import GenerateReqInput +from sglang.srt.sampling.sampling_params import SamplingParams as PySamplingParams + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +REPO_ROOT = Path(__file__).resolve().parents[5] +FIXTURES = REPO_ROOT / "rust" / "sglang-api-types" / "testdata" / "json_parity.json" + +# `GenerateReqInput` fields with no schema counterpart. Each is either +# Engine-only (tensors, callbacks), set by the server after admission, or a +# feature the HTTP contract has not adopted. A new dataclass field must be +# added to the proto or listed here, so a client-facing field cannot slip +# into the Python server without the Rust server and the contract seeing it. +PYTHON_ONLY_REQUEST_FIELDS = frozenset( + { + "background", + "cache_salt", + "conversation_id", + "custom_labels", + "custom_logit_processor", + "data_parallel_rank", + "encoder_urls", + "external_trace_header", + "extra_key", + "http_worker_ipc", + "image_max_dynamic_patch", + "images_config", + "input_embeds", + "kv_hints", + "log_metrics", + "lora_id", + "lora_path", + "max_dynamic_patch", + "max_thinking_tokens", + "min_dynamic_patch", + "mm_content_hashes", + "modalities", + "multi_item_delimiter_indices", + "need_wait_for_mm_inputs", + "no_logs", + "num_items_assigned", + "positional_embed_overrides", + "priority", + "received_time", + "require_reasoning", + "return_bytes", + "return_entropy", + "return_flat_raw_top_logprobs", + "return_flat_raw_top_logprobs_b64", + "return_indexer_topk", + "return_prompt_token_ids", + "return_routed_experts", + "return_sampling_mask", + "routed_experts_start_len", + "routing_key", + "sampling_logprobs_mode", + "session_id", + "session_params", + "token_indices_to_pool", + "use_audio_in_video", + "video_config", + "video_max_dynamic_patch", + } +) + +# `SamplingParams` members that `normalize()` derives; never on the wire. +DERIVED_SAMPLING_FIELDS = frozenset( + { + "stop_strs", + "stop_regex_strs", + "stop_str_max_len", + "stop_regex_max_len", + "is_normalized", + "ebnf_full_assistant", + } +) + + +def compact(value) -> str: + """serde_json::to_string parity: no whitespace.""" + return json.dumps(value, separators=(",", ":")) + + +def load_fixtures(): + with open(FIXTURES) as f: + return json.load(f)["cases"] + + +class TestCrossLanguageFixtures(CustomTestCase): + def test_fixtures_decode_to_their_canonical_bytes(self): + """The Python emitter agrees with the Rust emitter on every fixture: + same keys, same order, same presence and null rules, and the + canonical form is a fixed point.""" + cases = load_fixtures() + self.assertTrue(cases) + for case in cases: + with self.subTest(case=f"{case['type']}: {case['name']}"): + cls = getattr(api_types, case["type"]) + decoded = cls.from_json_value(case["input"]) + self.assertEqual(compact(decoded.to_json_value()), case["canonical"]) + again = cls.from_json_value(json.loads(case["canonical"])) + self.assertEqual(compact(again.to_json_value()), case["canonical"]) + + def test_every_fixture_type_is_a_generated_struct(self): + """A fixture naming a type the Python module lacks would be skipped + silently by a lookup-by-name; make it a failure.""" + for case in load_fixtures(): + self.assertTrue(hasattr(api_types, case["type"]), case["type"]) + + +class TestRequestFieldParity(CustomTestCase): + def test_schema_request_fields_are_a_subset_of_the_dataclass(self): + """Every schema field is a `GenerateReqInput` field, so a schema body + constructs the dataclass as-is, and the dataclass-only fields are + exactly the pinned list.""" + schema = set(GenerateRequest.__struct_fields__) + dataclass_fields = {f.name for f in dataclasses.fields(GenerateReqInput)} + self.assertEqual(schema - dataclass_fields, set()) + self.assertEqual(dataclass_fields - schema, PYTHON_ONLY_REQUEST_FIELDS) + + def test_schema_sampling_fields_match_the_python_api(self): + """The schema's SamplingParams is the Python struct minus the fields + `normalize()` derives, in the same order.""" + python_api = [ + name + for name in PySamplingParams.__struct_fields__ + if name not in DERIVED_SAMPLING_FIELDS + ] + self.assertEqual(set(python_api), set(SamplingParams.__struct_fields__)) + self.assertEqual( + set(PySamplingParams.__struct_fields__) - set(python_api), + DERIVED_SAMPLING_FIELDS, + ) + + def test_request_fixtures_normalize_through_the_dataclass(self): + """A schema-decoded body, re-encoded, is a valid `GenerateReqInput` + and survives `normalize_batch_and_arguments` with its prompt, batch + shape, stream flag, and sampling values intact.""" + for case in load_fixtures(): + if case["type"] != "GenerateRequest": + continue + with self.subTest(case=case["name"]): + req = GenerateRequest.from_json_value(case["input"]) + wire = req.to_json_value() + dataclass_req = GenerateReqInput(**wire) + dataclass_req.normalize_batch_and_arguments() + self.assertEqual(dataclass_req.stream, wire["stream"]) + if "text" in wire: + self.assertEqual(dataclass_req.text, wire["text"]) + batch = isinstance(wire["text"], list) + else: + self.assertEqual(dataclass_req.input_ids, wire["input_ids"]) + batch = isinstance(wire["input_ids"][0], list) + self.assertEqual(dataclass_req.is_single, not batch) + if "sampling_params" in wire: + sent = wire["sampling_params"] + got = dataclass_req.sampling_params + if isinstance(sent, list): + self.assertEqual(len(got), len(sent)) + for sent_item, got_item in zip(sent, got): + self.assertEqual({**got_item, **sent_item}, got_item) + else: + self.assertEqual({**got, **sent}, got) + if "rid" in wire: + self.assertEqual(dataclass_req.rid, wire["rid"]) + + +class TestResponseParity(CustomTestCase): + def test_python_server_meta_info_decodes_to_the_schema_subset(self): + """The Python server's `meta_info` carries keys the schema does not + model (weight_version, num_retractions, spec stats, ...). The + generated types ignore them and re-emit the schema subset, so a + Rust-server client and a Python-server client see one shape.""" + python_frame = { + "text": "ok", + "meta_info": { + "id": "rid", + "finish_reason": {"type": "stop", "matched": 2}, + "prompt_tokens": 3, + "weight_version": "default", + "num_retractions": 0, + "completion_tokens": 2, + "cached_tokens": 1, + "spec_accept_rate": 0.5, + "e2e_latency": 0.25, + }, + } + frame = GenerateResponse.from_json_value(python_frame) + back = frame.to_json_value() + self.assertEqual( + list(back["meta_info"]), + [ + "id", + "prompt_tokens", + "completion_tokens", + "finish_reason", + "e2e_latency", + ], + ) + self.assertEqual( + back["meta_info"]["finish_reason"], {"type": "stop", "matched": 2} + ) + self.assertNotIn("output_ids", back) + + def test_response_key_order_is_the_proto_field_order(self): + """The Rust HTTP server's memoized cumulative frame writes these keys + by hand in this order (`Accumulated::frame_json`); the generated + serializers must keep matching it.""" + self.assertEqual( + list(GenerateResponse.__struct_fields__), + ["text", "meta_info", "output_ids", "index"], + ) + self.assertEqual( + list(GenerateMetaInfo.__struct_fields__)[:4], + ["id", "prompt_tokens", "completion_tokens", "finish_reason"], + ) + self.assertEqual(list(GenerateMetaInfo.__struct_fields__)[-1], "e2e_latency") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/entrypoints/test_generate_api_contract.py b/test/registered/unit/entrypoints/test_generate_api_contract.py new file mode 100644 index 0000000000..8ab3b42c1d --- /dev/null +++ b/test/registered/unit/entrypoints/test_generate_api_contract.py @@ -0,0 +1,73 @@ +"""The `/generate` schema gate (`entrypoints/api_contract.py`): the typed media +contract that replaced the raw_json passthrough, checked without a server.""" + +import unittest + +from sglang.srt.entrypoints.api_contract import generate_contract_error +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=3, suite="base-a-test-cpu") + + +class TestGenerateContract(CustomTestCase): + def test_every_media_container_form_is_accepted(self): + for image_data in ( + "u", + {"url": "u", "detail": "high"}, + ["u", None, {"url": "v"}], + [], + [["a", "b"], None, []], + ): + with self.subTest(image_data=image_data): + self.assertIsNone( + generate_contract_error({"text": "hi", "image_data": image_data}) + ) + self.assertIsNone( + generate_contract_error({"text": "hi", "mm_hashes": ["a1b2", "0xff"]}) + ) + self.assertIsNone( + generate_contract_error( + { + "text": "hi", + "video_data": {"url": "v", "fps": 2.0, "use_audio": True}, + } + ) + ) + + def test_unknown_media_hint_is_rejected(self): + """A hint the schema does not name used to be silently dropped.""" + error = generate_contract_error( + { + "text": "hi", + "image_data": {"url": "u", "max_dynamic_patc": 4}, + } # codespell:ignore patc + ) + self.assertIsNotNone(error) + self.assertIn("unknown field", error) + + def test_preprocessed_inputs_are_not_an_http_shape(self): + """processor_output / precomputed_embedding carry tensors: Engine-only.""" + error = generate_contract_error( + {"input_ids": [1, 2], "image_data": [{"format": "processor_output"}]} + ) + self.assertIsNotNone(error) + + def test_mixed_list_shapes_are_rejected(self): + error = generate_contract_error({"text": "hi", "image_data": ["a", ["b"]]}) + self.assertIsNotNone(error) + + def test_sampling_params_follow_the_schema(self): + self.assertIsNone( + generate_contract_error( + {"text": "hi", "sampling_params": {"beam_width": 2, "n": 2}} + ) + ) + bad = {"temperatur": 1} # codespell:ignore temperatur + error = generate_contract_error({"text": "hi", "sampling_params": bad}) + self.assertIsNotNone(error) + self.assertIn("unknown field", error) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/sampling/test_sampling_params.py b/test/registered/unit/sampling/test_sampling_params.py index af53964607..fe40af2e2c 100644 --- a/test/registered/unit/sampling/test_sampling_params.py +++ b/test/registered/unit/sampling/test_sampling_params.py @@ -484,7 +484,8 @@ class TestSamplingParamsMsgspecStruct(CustomTestCase): ) source = rust_path.read_text() start = source.index("pub struct SamplingParams {") - end = source.index("\n}\n\n/// The `/generate`", start) + # The struct's own closing brace: the first one back at column 0. + end = source.index("\n}\n", start) rust_fields = tuple( re.findall( r"^\s*pub ([a-z][a-z0-9_]*):",