diff --git a/proto/sglang/api/v1/generate.proto b/proto/sglang/api/v1/generate.proto index 5a3734e102..75cf98de90 100644 --- a/proto/sglang/api/v1/generate.proto +++ b/proto/sglang/api/v1/generate.proto @@ -66,12 +66,51 @@ message GenerateRequest { // Apply grammar constraints only after the reasoning section, for every // batch item. optional bool require_reasoning = 26; - // Fields this server cannot honour. Unknown keys are ignored, but - // dropping these would silently change the result (base-model output for a - // LoRA request, a stateless answer for a session request), so a non-null - // value is rejected instead. - optional string lora_path = 27 [(sglang.json.v1.field_json).raw_json = true]; + // A field this server cannot honour. Unknown keys are ignored, but dropping + // this one would silently return a stateless answer for a session request, + // so a non-null value is rejected instead. optional string session_params = 28 [(sglang.json.v1.field_json).raw_json = true]; + + // Orchestrator controls used by native gRPC clients such as Dynamo. + // `lora_path` and `kv_hints` follow Python's `GenerateReqInput`: a scalar + // applies to every prompt, a list carries one (nullable) value per prompt. + OptionalStringOrList lora_path = 27; + optional string routing_key = 31; + optional uint64 max_thinking_tokens = 32; + KvHintsEnvelopeOrList kv_hints = 29; +} + +message KvHintsEnvelopeOrList { + oneof value { + option (sglang.json.v1.oneof_json).shape = ONEOF_SHAPE_ONE_OR_MANY; + KvHintsEnvelope one = 1; + KvHintsEnvelopeList many = 2; + } +} + +message KvHintsEnvelopeList { + option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_BARE_LIST; + repeated OptionalKvHintsEnvelope items = 1; +} + +message OptionalKvHintsEnvelope { + option (sglang.json.v1.message_json).shape = MESSAGE_SHAPE_TRANSPARENT; + optional KvHintsEnvelope envelope = 1; +} + +// Versioned KV-hint envelope forwarded from a trusted orchestrator. Action +// payloads are JSON because each action type owns its own schema. +message KvHintsEnvelope { + string protocol_version = 1; + string message_id = 2; + repeated KvHintAction actions = 3; +} + +message KvHintAction { + string action_id = 1; + string action_type = 2; + string action_version = 3; + optional string payload = 4 [(sglang.json.v1.field_json).raw_json = true]; } message SamplingParamsOrList { diff --git a/proto/sglang/api/v1/sampling.proto b/proto/sglang/api/v1/sampling.proto index 0b4fd78898..9703f9d8a6 100644 --- a/proto/sglang/api/v1/sampling.proto +++ b/proto/sglang/api/v1/sampling.proto @@ -98,6 +98,9 @@ message SamplingParams { // the number of returned sequences (n <= beam_width). optional int64 beam_width = 26; WatermarkRequestConfig watermark = 27; + // Literal alternatives. The server lowers these to an escaped regex before + // grammar compilation, matching the legacy native gRPC contract. + repeated string choice = 28; } message WatermarkRequestConfig { diff --git a/proto/sglang/api/v1/service.proto b/proto/sglang/api/v1/service.proto index 11adf88796..33bed674a8 100644 --- a/proto/sglang/api/v1/service.proto +++ b/proto/sglang/api/v1/service.proto @@ -27,6 +27,57 @@ service SglangService { // `/server_info`: one control round-trip, shaped through the runtime-metric // allowlist (never the raw server-args dump, which embeds keys). rpc GetServerInfo(GetServerInfoRequest) returns (GetServerInfoResponse); + + // The same static model cards as HTTP `/v1/models`. + rpc ListModels(ListModelsRequest) returns (ListModelsResponse); + + // Decode token IDs locally, without running model inference. + rpc Detokenize(DetokenizeRequest) returns (DetokenizeResponse); + + // Cancel by the native rid used by HTTP `/generate` or gRPC Generate, or + // cancel all active frontend generations with abort_all. HTTP chat/completion + // response IDs are not supported as rid selectors. + // Duplicate native IDs cancel all matching generations. A later reuse of the + // ID is unaffected. Internal control/detokenization requests are not targeted. + rpc Abort(AbortRequest) returns (AbortResponse); +} + +message ListModelsRequest {} + +message ListModelsResponse { + repeated ModelCard models = 1; +} + +message ModelCard { + string id = 1; + string root = 2; + optional string parent = 3; + uint64 max_model_len = 4; +} + +message DetokenizeRequest { + repeated int64 tokens = 1; +} + +message DetokenizeResponse { + string text = 1; +} + +message AbortRequest { + // Native generation rid (HTTP `/generate` or gRPC Generate), not an HTTP + // chat/completion response ID. + // Required and nonempty unless abort_all is true. + string rid = 1; + // When true, rid must be empty. Targets all frontend generations active at + // this call, including HTTP chat/completions. + bool abort_all = 2; +} + +message AbortResponse { + // True if at least one active generation matched and cancellation was + // requested. Not an acknowledgement that the scheduler has stopped yet. + // False for an unknown/already-finished ID; repeated aborts are harmless. + bool success = 1; } message HealthCheckRequest {} @@ -69,4 +120,49 @@ message GetServerInfoResponse { // step_time_dict), untyped because the scheduler owns their shapes; JSON // passes them through verbatim. repeated string internal_states = 7 [(sglang.json.v1.field_json).raw_json = true]; + + // Public serving endpoint configuration. + string host = 8; + uint32 port = 9; + bool incremental_streaming_output = 10; + string disaggregation_mode = 11; + + // Distributed deployment metadata required by out-of-process orchestrators. + DeploymentInfo deployment_info = 12; +} + +message DeploymentInfo { + // Distributed process topology. + optional uint32 disaggregation_bootstrap_port = 1; + uint32 node_rank = 2; + uint32 nnodes = 3; + uint32 dp_size = 4; + optional uint32 dp_rank = 5; + optional string dist_init_addr = 6; + uint32 base_gpu_id = 7; + optional uint32 page_size = 8; + uint32 dcp_size = 9; + + // Request scheduling and execution limits. + optional uint64 max_running_requests = 10; + uint64 max_prefill_tokens = 11; + bool enable_dp_attention = 12; + optional string speculative_algorithm = 13; + + // Hierarchical KV-cache configuration. + bool enable_hierarchical_cache = 14; + optional double hicache_ratio = 15; + uint64 hicache_size = 16; + optional string hicache_write_policy = 17; + optional uint64 hicache_host_total_tokens = 18; + + // KV-cache event streams published by this deployment. + repeated KvEventSource kv_event_sources = 19; +} + +message KvEventSource { + uint32 dp_rank = 1; + string endpoint = 2; + string topic = 3; + uint32 block_size = 4; } diff --git a/python/sglang/api/v1/api_types.py b/python/sglang/api/v1/api_types.py index 562b79b323..661a1f3882 100644 --- a/python/sglang/api/v1/api_types.py +++ b/python/sglang/api/v1/api_types.py @@ -204,6 +204,7 @@ class SamplingParams(msgspec.Struct, kw_only=True, frozen=True): custom_params: Optional[Any] = None beam_width: Optional[int] = None watermark: Optional[WatermarkRequestConfig] = None + choice: List[str] = [] @classmethod def from_json_value(cls, value: Any) -> "SamplingParams": @@ -261,8 +262,11 @@ class GenerateRequest(msgspec.Struct, kw_only=True, frozen=True): priority: Optional[int] = None cache_salt: Optional[StringOrList] = None require_reasoning: Optional[bool] = None - lora_path: Optional[Any] = None + lora_path: Optional[OptionalStringOrList] = None session_params: Optional[Any] = None + kv_hints: Optional[KvHintsEnvelopeOrList] = None + routing_key: Optional[str] = None + max_thinking_tokens: Optional[int] = None @classmethod def from_json_value(cls, value: Any) -> "GenerateRequest": @@ -274,6 +278,41 @@ class GenerateRequest(msgspec.Struct, kw_only=True, frozen=True): return encode_GenerateRequest(self) +class KvHintsEnvelope(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.KvHintsEnvelope: JSON object (unknown keys ignored).""" + + protocol_version: str = "" + message_id: str = "" + actions: List[KvHintAction] = [] + + @classmethod + def from_json_value(cls, value: Any) -> "KvHintsEnvelope": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_KvHintsEnvelope(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_KvHintsEnvelope(self) + + +class KvHintAction(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.KvHintAction: JSON object (unknown keys ignored).""" + + action_id: str = "" + action_type: str = "" + action_version: str = "" + payload: Optional[Any] = None + + @classmethod + def from_json_value(cls, value: Any) -> "KvHintAction": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_KvHintAction(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_KvHintAction(self) + + class LogprobEntry(msgspec.Struct, kw_only=True, frozen=True): """sglang.api.v1.LogprobEntry: JSON positional array [logprob, token_id, text].""" @@ -382,6 +421,113 @@ class MediaRef(msgspec.Struct, kw_only=True, frozen=True): return encode_MediaRef(self) +class ListModelsRequest(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.ListModelsRequest: JSON object (unknown keys ignored).""" + + @classmethod + def from_json_value(cls, value: Any) -> "ListModelsRequest": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_ListModelsRequest(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_ListModelsRequest(self) + + +class ListModelsResponse(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.ListModelsResponse: JSON object (unknown keys ignored).""" + + models: List[ModelCard] = [] + + @classmethod + def from_json_value(cls, value: Any) -> "ListModelsResponse": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_ListModelsResponse(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_ListModelsResponse(self) + + +class ModelCard(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.ModelCard: JSON object (unknown keys ignored).""" + + id: str = "" + root: str = "" + parent: Optional[str] = None + max_model_len: int = 0 + + @classmethod + def from_json_value(cls, value: Any) -> "ModelCard": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_ModelCard(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_ModelCard(self) + + +class DetokenizeRequest(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.DetokenizeRequest: JSON object (unknown keys ignored).""" + + tokens: List[int] = [] + + @classmethod + def from_json_value(cls, value: Any) -> "DetokenizeRequest": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_DetokenizeRequest(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_DetokenizeRequest(self) + + +class DetokenizeResponse(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.DetokenizeResponse: JSON object (unknown keys ignored).""" + + text: str = "" + + @classmethod + def from_json_value(cls, value: Any) -> "DetokenizeResponse": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_DetokenizeResponse(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_DetokenizeResponse(self) + + +class AbortRequest(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.AbortRequest: JSON object (unknown keys ignored).""" + + rid: str = "" + abort_all: bool = False + + @classmethod + def from_json_value(cls, value: Any) -> "AbortRequest": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_AbortRequest(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_AbortRequest(self) + + +class AbortResponse(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.AbortResponse: JSON object (unknown keys ignored).""" + + success: bool = False + + @classmethod + def from_json_value(cls, value: Any) -> "AbortResponse": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_AbortResponse(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_AbortResponse(self) + + class HealthCheckRequest(msgspec.Struct, kw_only=True, frozen=True): """sglang.api.v1.HealthCheckRequest: JSON object (unknown keys ignored).""" @@ -469,6 +615,11 @@ class GetServerInfoResponse(msgspec.Struct, kw_only=True, frozen=True): max_total_num_tokens: Optional[int] = None version: str = "" internal_states: List[Any] = [] + host: str = "" + port: int = 0 + incremental_streaming_output: bool = False + disaggregation_mode: str = "" + deployment_info: Optional[DeploymentInfo] = None @classmethod def from_json_value(cls, value: Any) -> "GetServerInfoResponse": @@ -480,6 +631,57 @@ class GetServerInfoResponse(msgspec.Struct, kw_only=True, frozen=True): return encode_GetServerInfoResponse(self) +class DeploymentInfo(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.DeploymentInfo: JSON object (unknown keys ignored).""" + + disaggregation_bootstrap_port: Optional[int] = None + node_rank: int = 0 + nnodes: int = 0 + dp_size: int = 0 + dp_rank: Optional[int] = None + dist_init_addr: Optional[str] = None + base_gpu_id: int = 0 + page_size: Optional[int] = None + dcp_size: int = 0 + max_running_requests: Optional[int] = None + max_prefill_tokens: int = 0 + enable_dp_attention: bool = False + speculative_algorithm: Optional[str] = None + enable_hierarchical_cache: bool = False + hicache_ratio: Optional[float] = None + hicache_size: int = 0 + hicache_write_policy: Optional[str] = None + hicache_host_total_tokens: Optional[int] = None + kv_event_sources: List[KvEventSource] = [] + + @classmethod + def from_json_value(cls, value: Any) -> "DeploymentInfo": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_DeploymentInfo(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_DeploymentInfo(self) + + +class KvEventSource(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.KvEventSource: JSON object (unknown keys ignored).""" + + dp_rank: int = 0 + endpoint: str = "" + topic: str = "" + block_size: int = 0 + + @classmethod + def from_json_value(cls, value: Any) -> "KvEventSource": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_KvEventSource(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_KvEventSource(self) + + # ---- type aliases: JSON shapes without an object of their own ---- # sglang.api.v1.TokenIds: JSON bare array. @@ -518,6 +720,9 @@ MatchedTokens = List[int] # sglang.api.v1.FinishReason: JSON object tagged by its type key; unknown tags pass through as a dict. FinishReason = Union[FinishStop, FinishLength, FinishAbort, Dict[str, Any]] +# sglang.api.v1.OptionalKvHintsEnvelope: JSON: the wrapped value itself, null when unset. +OptionalKvHintsEnvelope = Optional[KvHintsEnvelope] + # sglang.api.v1.SamplingParamsList: JSON bare array. SamplingParamsList = List[SamplingParams] @@ -578,9 +783,15 @@ OptionalInt64OrList = Union[OptionalInt64, OptionalInt64List] # sglang.api.v1.Matched: JSON: the value's shape picks the arm. Matched = Union[int, str, MatchedTokens] +# sglang.api.v1.KvHintsEnvelopeList: JSON bare array. +KvHintsEnvelopeList = List[OptionalKvHintsEnvelope] + # sglang.api.v1.SamplingParamsOrList: JSON: one value or an array of values. SamplingParamsOrList = Union[SamplingParams, SamplingParamsList] +# sglang.api.v1.KvHintsEnvelopeOrList: JSON: one value or an array of values. +KvHintsEnvelopeOrList = Union[KvHintsEnvelope, KvHintsEnvelopeList] + # ---- codecs: parsed JSON value <-> Python value ---- @@ -903,7 +1114,7 @@ def encode_FinishReason(v: FinishReason) -> Any: raise TypeError(f"FinishReason: unsupported value {type(v).__name__}") -_FIELDS_SamplingParams = ("max_new_tokens", "stop", "stop_token_ids", "stop_regex", "temperature", "top_p", "top_k", "min_p", "frequency_penalty", "presence_penalty", "repetition_penalty", "min_new_tokens", "n", "json_schema", "regex", "ebnf", "structural_tag", "ignore_eos", "skip_special_tokens", "spaces_between_special_tokens", "no_stop_trim", "stream_interval", "logit_bias", "sampling_seed", "custom_params", "beam_width", "watermark",) +_FIELDS_SamplingParams = ("max_new_tokens", "stop", "stop_token_ids", "stop_regex", "temperature", "top_p", "top_k", "min_p", "frequency_penalty", "presence_penalty", "repetition_penalty", "min_new_tokens", "n", "json_schema", "regex", "ebnf", "structural_tag", "ignore_eos", "skip_special_tokens", "spaces_between_special_tokens", "no_stop_trim", "stream_interval", "logit_bias", "sampling_seed", "custom_params", "beam_width", "watermark", "choice",) def decode_SamplingParams(value: Any) -> SamplingParams: @@ -964,6 +1175,8 @@ def decode_SamplingParams(value: Any) -> SamplingParams: kw["beam_width"] = None if x is None else _expect_int(x, expected="i64") elif key == "watermark": kw["watermark"] = None if x is None else decode_WatermarkRequestConfig(x) + elif key == "choice": + kw["choice"] = [] if x is None else [_expect_str(e, expected="a string") for e in _expect_seq(x, expected="a sequence")] else: raise _unknown_field(key, fields=_FIELDS_SamplingParams) return SamplingParams(**kw) @@ -1024,6 +1237,8 @@ def encode_SamplingParams(v: SamplingParams) -> Dict[str, Any]: d["beam_width"] = v.beam_width if v.watermark is not None: d["watermark"] = encode_WatermarkRequestConfig(v.watermark) + if v.choice: + d["choice"] = list(v.choice) return d @@ -1113,9 +1328,15 @@ def decode_GenerateRequest(value: Any) -> GenerateRequest: elif key == "require_reasoning": kw["require_reasoning"] = None if x is None else _expect_bool(x, expected="a boolean") elif key == "lora_path": - kw["lora_path"] = x + kw["lora_path"] = None if x is None else decode_OptionalStringOrList(x) elif key == "session_params": kw["session_params"] = x + elif key == "kv_hints": + kw["kv_hints"] = None if x is None else decode_KvHintsEnvelopeOrList(x) + elif key == "routing_key": + kw["routing_key"] = None if x is None else _expect_str(x, expected="a string") + elif key == "max_thinking_tokens": + kw["max_thinking_tokens"] = None if x is None else _expect_uint(x, expected="u64") return GenerateRequest(**kw) @@ -1174,9 +1395,90 @@ def encode_GenerateRequest(v: GenerateRequest) -> Dict[str, Any]: if v.require_reasoning is not None: d["require_reasoning"] = v.require_reasoning if v.lora_path is not None: - d["lora_path"] = v.lora_path + d["lora_path"] = encode_OptionalStringOrList(v.lora_path) if v.session_params is not None: d["session_params"] = v.session_params + if v.kv_hints is not None: + d["kv_hints"] = encode_KvHintsEnvelopeOrList(v.kv_hints) + if v.routing_key is not None: + d["routing_key"] = v.routing_key + if v.max_thinking_tokens is not None: + d["max_thinking_tokens"] = v.max_thinking_tokens + return d + + +def decode_KvHintsEnvelopeOrList(value: Any) -> KvHintsEnvelopeOrList: + if isinstance(value, list): + return decode_KvHintsEnvelopeList(value) + return decode_KvHintsEnvelope(_expect_map(value, expected="a value or an array of values for KvHintsEnvelopeOrList")) + + +def encode_KvHintsEnvelopeOrList(v: KvHintsEnvelopeOrList) -> Any: + if isinstance(v, list): + return encode_KvHintsEnvelopeList(v) + return encode_KvHintsEnvelope(v) + + +def decode_KvHintsEnvelopeList(value: Any) -> KvHintsEnvelopeList: + return [decode_OptionalKvHintsEnvelope(e) for e in _expect_seq(value, expected="a sequence")] + + +def encode_KvHintsEnvelopeList(v: KvHintsEnvelopeList) -> List[Any]: + return [encode_OptionalKvHintsEnvelope(e) for e in v] + + +def decode_OptionalKvHintsEnvelope(value: Any) -> OptionalKvHintsEnvelope: + return None if value is None else decode_KvHintsEnvelope(value) + + +def encode_OptionalKvHintsEnvelope(v: OptionalKvHintsEnvelope) -> Any: + return None if v is None else encode_KvHintsEnvelope(v) + + +def decode_KvHintsEnvelope(value: Any) -> KvHintsEnvelope: + m = _expect_map(value, expected="struct KvHintsEnvelope") + kw: Dict[str, Any] = {} + for key, x in m.items(): + if key == "protocol_version": + kw["protocol_version"] = _expect_str(x, expected="a string") + elif key == "message_id": + kw["message_id"] = _expect_str(x, expected="a string") + elif key == "actions": + kw["actions"] = [] if x is None else [decode_KvHintAction(e) for e in _expect_seq(x, expected="a sequence")] + return KvHintsEnvelope(**kw) + + +def encode_KvHintsEnvelope(v: KvHintsEnvelope) -> Dict[str, Any]: + d: Dict[str, Any] = {} + d["protocol_version"] = v.protocol_version + d["message_id"] = v.message_id + if v.actions: + d["actions"] = [encode_KvHintAction(e) for e in v.actions] + return d + + +def decode_KvHintAction(value: Any) -> KvHintAction: + m = _expect_map(value, expected="struct KvHintAction") + kw: Dict[str, Any] = {} + for key, x in m.items(): + if key == "action_id": + kw["action_id"] = _expect_str(x, expected="a string") + elif key == "action_type": + kw["action_type"] = _expect_str(x, expected="a string") + elif key == "action_version": + kw["action_version"] = _expect_str(x, expected="a string") + elif key == "payload": + kw["payload"] = x + return KvHintAction(**kw) + + +def encode_KvHintAction(v: KvHintAction) -> Dict[str, Any]: + d: Dict[str, Any] = {} + d["action_id"] = v.action_id + d["action_type"] = v.action_type + d["action_version"] = v.action_version + if v.payload is not None: + d["payload"] = v.payload return d @@ -1514,6 +1816,121 @@ def encode_MediaInput(v: MediaInput) -> Any: return v +def decode_ListModelsRequest(value: Any) -> ListModelsRequest: + m = _expect_map(value, expected="struct ListModelsRequest") + return ListModelsRequest() + + +def encode_ListModelsRequest(v: ListModelsRequest) -> Dict[str, Any]: + d: Dict[str, Any] = {} + return d + + +def decode_ListModelsResponse(value: Any) -> ListModelsResponse: + m = _expect_map(value, expected="struct ListModelsResponse") + kw: Dict[str, Any] = {} + for key, x in m.items(): + if key == "models": + kw["models"] = [] if x is None else [decode_ModelCard(e) for e in _expect_seq(x, expected="a sequence")] + return ListModelsResponse(**kw) + + +def encode_ListModelsResponse(v: ListModelsResponse) -> Dict[str, Any]: + d: Dict[str, Any] = {} + if v.models: + d["models"] = [encode_ModelCard(e) for e in v.models] + return d + + +def decode_ModelCard(value: Any) -> ModelCard: + m = _expect_map(value, expected="struct ModelCard") + kw: Dict[str, Any] = {} + for key, x in m.items(): + if key == "id": + kw["id"] = _expect_str(x, expected="a string") + elif key == "root": + kw["root"] = _expect_str(x, expected="a string") + elif key == "parent": + kw["parent"] = None if x is None else _expect_str(x, expected="a string") + elif key == "max_model_len": + kw["max_model_len"] = _expect_uint(x, expected="u64") + return ModelCard(**kw) + + +def encode_ModelCard(v: ModelCard) -> Dict[str, Any]: + d: Dict[str, Any] = {} + d["id"] = v.id + d["root"] = v.root + if v.parent is not None: + d["parent"] = v.parent + d["max_model_len"] = v.max_model_len + return d + + +def decode_DetokenizeRequest(value: Any) -> DetokenizeRequest: + m = _expect_map(value, expected="struct DetokenizeRequest") + kw: Dict[str, Any] = {} + for key, x in m.items(): + if key == "tokens": + kw["tokens"] = [] if x is None else [_expect_int(e, expected="i64") for e in _expect_seq(x, expected="a sequence")] + return DetokenizeRequest(**kw) + + +def encode_DetokenizeRequest(v: DetokenizeRequest) -> Dict[str, Any]: + d: Dict[str, Any] = {} + if v.tokens: + d["tokens"] = list(v.tokens) + return d + + +def decode_DetokenizeResponse(value: Any) -> DetokenizeResponse: + m = _expect_map(value, expected="struct DetokenizeResponse") + kw: Dict[str, Any] = {} + for key, x in m.items(): + if key == "text": + kw["text"] = _expect_str(x, expected="a string") + return DetokenizeResponse(**kw) + + +def encode_DetokenizeResponse(v: DetokenizeResponse) -> Dict[str, Any]: + d: Dict[str, Any] = {} + d["text"] = v.text + return d + + +def decode_AbortRequest(value: Any) -> AbortRequest: + m = _expect_map(value, expected="struct AbortRequest") + kw: Dict[str, Any] = {} + for key, x in m.items(): + if key == "rid": + kw["rid"] = _expect_str(x, expected="a string") + elif key == "abort_all": + kw["abort_all"] = _expect_bool(x, expected="a boolean") + return AbortRequest(**kw) + + +def encode_AbortRequest(v: AbortRequest) -> Dict[str, Any]: + d: Dict[str, Any] = {} + d["rid"] = v.rid + d["abort_all"] = v.abort_all + return d + + +def decode_AbortResponse(value: Any) -> AbortResponse: + m = _expect_map(value, expected="struct AbortResponse") + kw: Dict[str, Any] = {} + for key, x in m.items(): + if key == "success": + kw["success"] = _expect_bool(x, expected="a boolean") + return AbortResponse(**kw) + + +def encode_AbortResponse(v: AbortResponse) -> Dict[str, Any]: + d: Dict[str, Any] = {} + d["success"] = v.success + return d + + def decode_HealthCheckRequest(value: Any) -> HealthCheckRequest: m = _expect_map(value, expected="struct HealthCheckRequest") return HealthCheckRequest() @@ -1616,6 +2033,16 @@ def decode_GetServerInfoResponse(value: Any) -> GetServerInfoResponse: kw["version"] = _expect_str(x, expected="a string") elif key == "internal_states": kw["internal_states"] = [] if x is None else list(_expect_seq(x, expected="a sequence")) + elif key == "host": + kw["host"] = _expect_str(x, expected="a string") + elif key == "port": + kw["port"] = _expect_uint(x, expected="u32") + elif key == "incremental_streaming_output": + kw["incremental_streaming_output"] = _expect_bool(x, expected="a boolean") + elif key == "disaggregation_mode": + kw["disaggregation_mode"] = _expect_str(x, expected="a string") + elif key == "deployment_info": + kw["deployment_info"] = None if x is None else decode_DeploymentInfo(x) return GetServerInfoResponse(**kw) @@ -1630,4 +2057,113 @@ def encode_GetServerInfoResponse(v: GetServerInfoResponse) -> Dict[str, Any]: d["version"] = v.version if v.internal_states: d["internal_states"] = list(v.internal_states) + d["host"] = v.host + d["port"] = v.port + d["incremental_streaming_output"] = v.incremental_streaming_output + d["disaggregation_mode"] = v.disaggregation_mode + if v.deployment_info is not None: + d["deployment_info"] = encode_DeploymentInfo(v.deployment_info) + return d + + +def decode_DeploymentInfo(value: Any) -> DeploymentInfo: + m = _expect_map(value, expected="struct DeploymentInfo") + kw: Dict[str, Any] = {} + for key, x in m.items(): + if key == "disaggregation_bootstrap_port": + kw["disaggregation_bootstrap_port"] = None if x is None else _expect_uint(x, expected="u32") + elif key == "node_rank": + kw["node_rank"] = _expect_uint(x, expected="u32") + elif key == "nnodes": + kw["nnodes"] = _expect_uint(x, expected="u32") + elif key == "dp_size": + kw["dp_size"] = _expect_uint(x, expected="u32") + elif key == "dp_rank": + kw["dp_rank"] = None if x is None else _expect_uint(x, expected="u32") + elif key == "dist_init_addr": + kw["dist_init_addr"] = None if x is None else _expect_str(x, expected="a string") + elif key == "base_gpu_id": + kw["base_gpu_id"] = _expect_uint(x, expected="u32") + elif key == "page_size": + kw["page_size"] = None if x is None else _expect_uint(x, expected="u32") + elif key == "dcp_size": + kw["dcp_size"] = _expect_uint(x, expected="u32") + elif key == "max_running_requests": + kw["max_running_requests"] = None if x is None else _expect_uint(x, expected="u64") + elif key == "max_prefill_tokens": + kw["max_prefill_tokens"] = _expect_uint(x, expected="u64") + elif key == "enable_dp_attention": + kw["enable_dp_attention"] = _expect_bool(x, expected="a boolean") + elif key == "speculative_algorithm": + kw["speculative_algorithm"] = None if x is None else _expect_str(x, expected="a string") + elif key == "enable_hierarchical_cache": + kw["enable_hierarchical_cache"] = _expect_bool(x, expected="a boolean") + elif key == "hicache_ratio": + kw["hicache_ratio"] = None if x is None else _expect_float(x, expected="f64") + elif key == "hicache_size": + kw["hicache_size"] = _expect_uint(x, expected="u64") + elif key == "hicache_write_policy": + kw["hicache_write_policy"] = None if x is None else _expect_str(x, expected="a string") + elif key == "hicache_host_total_tokens": + kw["hicache_host_total_tokens"] = None if x is None else _expect_uint(x, expected="u64") + elif key == "kv_event_sources": + kw["kv_event_sources"] = [] if x is None else [decode_KvEventSource(e) for e in _expect_seq(x, expected="a sequence")] + return DeploymentInfo(**kw) + + +def encode_DeploymentInfo(v: DeploymentInfo) -> Dict[str, Any]: + d: Dict[str, Any] = {} + if v.disaggregation_bootstrap_port is not None: + d["disaggregation_bootstrap_port"] = v.disaggregation_bootstrap_port + d["node_rank"] = v.node_rank + d["nnodes"] = v.nnodes + d["dp_size"] = v.dp_size + if v.dp_rank is not None: + d["dp_rank"] = v.dp_rank + if v.dist_init_addr is not None: + d["dist_init_addr"] = v.dist_init_addr + d["base_gpu_id"] = v.base_gpu_id + if v.page_size is not None: + d["page_size"] = v.page_size + d["dcp_size"] = v.dcp_size + if v.max_running_requests is not None: + d["max_running_requests"] = v.max_running_requests + d["max_prefill_tokens"] = v.max_prefill_tokens + d["enable_dp_attention"] = v.enable_dp_attention + if v.speculative_algorithm is not None: + d["speculative_algorithm"] = v.speculative_algorithm + d["enable_hierarchical_cache"] = v.enable_hierarchical_cache + if v.hicache_ratio is not None: + d["hicache_ratio"] = float(v.hicache_ratio) + d["hicache_size"] = v.hicache_size + if v.hicache_write_policy is not None: + d["hicache_write_policy"] = v.hicache_write_policy + if v.hicache_host_total_tokens is not None: + d["hicache_host_total_tokens"] = v.hicache_host_total_tokens + if v.kv_event_sources: + d["kv_event_sources"] = [encode_KvEventSource(e) for e in v.kv_event_sources] + return d + + +def decode_KvEventSource(value: Any) -> KvEventSource: + m = _expect_map(value, expected="struct KvEventSource") + kw: Dict[str, Any] = {} + for key, x in m.items(): + if key == "dp_rank": + kw["dp_rank"] = _expect_uint(x, expected="u32") + elif key == "endpoint": + kw["endpoint"] = _expect_str(x, expected="a string") + elif key == "topic": + kw["topic"] = _expect_str(x, expected="a string") + elif key == "block_size": + kw["block_size"] = _expect_uint(x, expected="u32") + return KvEventSource(**kw) + + +def encode_KvEventSource(v: KvEventSource) -> Dict[str, Any]: + d: Dict[str, Any] = {} + d["dp_rank"] = v.dp_rank + d["endpoint"] = v.endpoint + d["topic"] = v.topic + d["block_size"] = v.block_size return d diff --git a/python/sglang/srt/arg_groups/parallel_hook.py b/python/sglang/srt/arg_groups/parallel_hook.py index fbe3d95de0..f1dcda0802 100644 --- a/python/sglang/srt/arg_groups/parallel_hook.py +++ b/python/sglang/srt/arg_groups/parallel_hook.py @@ -317,6 +317,12 @@ def handle_data_parallelism(server_args: Any): "_handle_data_parallelism", schedule_conservativeness=cfg.schedule_conservativeness * 0.3, ) + if envs.SGLANG_RUST_SERVER.get(): + declare_resolution( + server_args, + "_handle_data_parallelism", + enable_dp_attention_local_control_broadcast=True, + ) assert cfg.tp_size % cfg.attn_dp_size == 0 original_chunked_prefill_size = cfg.chunked_prefill_size declare_resolution( diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 644c3aac6e..30d0506c1b 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -657,6 +657,11 @@ class Scheduler( self.init_metrics_reporter() self.scheduler_stage_metrics = self.metrics_reporter.scheduler_stage_metrics + # The Rust frontend snapshots local KV-event discovery metadata while it + # builds its typed server info, so the publisher must exist before the + # frontend starts. + self.init_kv_events_publisher() + # Init schedule policy and new token estimation self.init_schedule_policy() @@ -713,8 +718,6 @@ class Scheduler( self.init_invariant_checker() - self.init_kv_events_publisher() - self.init_load_publisher() self.init_load_inquirer() diff --git a/python/sglang/srt/rust_server/config.py b/python/sglang/srt/rust_server/config.py index 17ea4e7270..6b9a97f1cd 100644 --- a/python/sglang/srt/rust_server/config.py +++ b/python/sglang/srt/rust_server/config.py @@ -9,6 +9,7 @@ from types import ModuleType from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple from sglang.srt.arg_groups.overrides import resolving_view +from sglang.srt.disaggregation.kv_events import select_kv_publisher_dp_rank from sglang.srt.entrypoints.v1_loads import ( _accelerator_name, _num_accelerators_per_dp_rank, @@ -17,12 +18,17 @@ from sglang.srt.environ import envs from sglang.srt.managers.load_snapshot import ShmLoadSnapshotWriter from sglang.srt.managers.utils import compute_num_reserved_tokens from sglang.srt.runtime_context import ( + get_device, get_disagg, + get_lora, + get_memory, get_mm, get_model, get_observability, + get_parallel, get_schedule, get_serving, + get_spec, ) from sglang.srt.sampling.watermarking.core import redact_watermark_secrets from sglang.srt.utils.common import build_server_info @@ -65,6 +71,38 @@ def _build_server_args( if get_serving().smg_grpc_mode or get_serving().grpc_mode else get_serving().grpc_port ) + deployment_info = { + "disaggregation_bootstrap_port": get_disagg().disaggregation_bootstrap_port, + "node_rank": get_parallel().node_rank, + "nnodes": get_parallel().nnodes, + "dp_size": get_parallel().num_dp_ranks, + "dp_rank": select_kv_publisher_dp_rank( + get_parallel().attn_dp_size, + get_parallel().attn_dp_rank, + get_parallel().dp_rank, + ), + "dist_init_addr": get_parallel().dist_init_addr, + "base_gpu_id": get_device().base_gpu_id, + "page_size": get_schedule().page_size, + "dcp_size": get_parallel().dcp_size, + "max_running_requests": get_schedule().max_running_requests, + "max_prefill_tokens": get_schedule().max_prefill_tokens, + "enable_dp_attention": get_parallel().attn_dp_size > 1, + "speculative_algorithm": get_spec().speculative_algorithm, + "enable_hierarchical_cache": get_memory().enable_hierarchical_cache, + "hicache_ratio": get_memory().hicache_ratio, + "hicache_size": get_memory().hicache_size, + "hicache_write_policy": get_memory().hicache_write_policy, + # Python's scheduler handshake only computes startup_time after all + # components have initialized, but the Rust frontend must start earlier + # to serve the disaggregation bootstrap registry. Read the already- + # initialized KV publisher directly instead of calling get_init_info() + # on a partially initialized scheduler. + "hicache_host_total_tokens": None, + "kv_event_sources": scheduler.kv_events_publisher.local_kv_event_sources( + scheduler.page_size * get_parallel().dcp_size + ), + } # `/v1/loads` reads this rank's slot of the scheduler's shared-memory # snapshots directly; the ZMQ transport (multi-node DP) has no file to read. shm_writer = scheduler.load_snapshot_writer @@ -120,6 +158,12 @@ def _build_server_args( if get_serving().preferred_sampling_params is not None else None ), + lora_ids={ + ref.lora_name: ref.lora_id + for ref in (get_lora().lora_paths or []) + if ref.lora_name is not None + }, + enable_strict_thinking=get_serving().enable_strict_thinking, limit_mm_data_per_request=get_mm().limit_mm_data_per_request or {}, allow_auto_truncate=get_serving().allow_auto_truncate, enable_return_hidden_states=sa.enable_return_hidden_states, @@ -134,6 +178,7 @@ def _build_server_args( # can serve them statically (no scheduler round-trip). version=__version__, max_total_num_tokens=scheduler.max_total_num_tokens, + deployment_info=json.dumps(deployment_info), launch_server_info=_launch_server_info(scheduler), load_snapshot_path=shm_writer.path if shm_writer else None, load_snapshot_dp_size=shm_writer.dp_size if shm_writer else 0, diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index 0e60c57001..7a471f337a 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -3680,6 +3680,8 @@ def start_follower_grpc_server(server_args, scheduler_info: Dict[str, Any]): serving = get_serving() if serving.grpc_port is None or serving.smg_grpc_mode or serving.grpc_mode: return None + if envs.SGLANG_RUST_SERVER.get(): + return None from sglang.srt.rust_extensions import load_rust_extension from sglang.srt.sampling.watermarking.core import redact_watermark_secrets diff --git a/rust/sglang-api-types/src/generated/sglang.api.v1.rs b/rust/sglang-api-types/src/generated/sglang.api.v1.rs index a926654030..8baa991b0d 100644 --- a/rust/sglang-api-types/src/generated/sglang.api.v1.rs +++ b/rust/sglang-api-types/src/generated/sglang.api.v1.rs @@ -306,6 +306,10 @@ pub struct SamplingParams { pub beam_width: ::core::option::Option, #[prost(message, optional, tag = "27")] pub watermark: ::core::option::Option, + /// Literal alternatives. The server lowers these to an escaped regex before + /// grammar compilation, matching the legacy native gRPC contract. + #[prost(string, repeated, tag = "28")] + pub choice: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct WatermarkRequestConfig { @@ -391,14 +395,69 @@ pub struct GenerateRequest { /// batch item. #[prost(bool, optional, tag = "26")] pub require_reasoning: ::core::option::Option, - /// Fields this server cannot honour. Unknown keys are ignored, but - /// dropping these would silently change the result (base-model output for a - /// LoRA request, a stateless answer for a session request), so a non-null - /// value is rejected instead. - #[prost(string, optional, tag = "27")] - pub lora_path: ::core::option::Option<::prost::alloc::string::String>, + /// A field this server cannot honour. Unknown keys are ignored, but dropping + /// this one would silently return a stateless answer for a session request, + /// so a non-null value is rejected instead. #[prost(string, optional, tag = "28")] pub session_params: ::core::option::Option<::prost::alloc::string::String>, + /// Orchestrator controls used by native gRPC clients such as Dynamo. + /// `lora_path` and `kv_hints` follow Python's `GenerateReqInput`: a scalar + /// applies to every prompt, a list carries one (nullable) value per prompt. + #[prost(message, optional, tag = "27")] + pub lora_path: ::core::option::Option, + #[prost(string, optional, tag = "31")] + pub routing_key: ::core::option::Option<::prost::alloc::string::String>, + #[prost(uint64, optional, tag = "32")] + pub max_thinking_tokens: ::core::option::Option, + #[prost(message, optional, tag = "29")] + pub kv_hints: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct KvHintsEnvelopeOrList { + #[prost(oneof = "kv_hints_envelope_or_list::Value", tags = "1, 2")] + pub value: ::core::option::Option, +} +/// Nested message and enum types in `KvHintsEnvelopeOrList`. +pub mod kv_hints_envelope_or_list { + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Value { + #[prost(message, tag = "1")] + One(super::KvHintsEnvelope), + #[prost(message, tag = "2")] + Many(super::KvHintsEnvelopeList), + } +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct KvHintsEnvelopeList { + #[prost(message, repeated, tag = "1")] + pub items: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct OptionalKvHintsEnvelope { + #[prost(message, optional, tag = "1")] + pub envelope: ::core::option::Option, +} +/// Versioned KV-hint envelope forwarded from a trusted orchestrator. Action +/// payloads are JSON because each action type owns its own schema. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct KvHintsEnvelope { + #[prost(string, tag = "1")] + pub protocol_version: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub message_id: ::prost::alloc::string::String, + #[prost(message, repeated, tag = "3")] + pub actions: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct KvHintAction { + #[prost(string, tag = "1")] + pub action_id: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub action_type: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub action_version: ::prost::alloc::string::String, + #[prost(string, optional, tag = "4")] + pub payload: ::core::option::Option<::prost::alloc::string::String>, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct SamplingParamsOrList { @@ -651,6 +710,54 @@ pub mod media_input { } } #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ListModelsRequest {} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ListModelsResponse { + #[prost(message, repeated, tag = "1")] + pub models: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ModelCard { + #[prost(string, tag = "1")] + pub id: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub root: ::prost::alloc::string::String, + #[prost(string, optional, tag = "3")] + pub parent: ::core::option::Option<::prost::alloc::string::String>, + #[prost(uint64, tag = "4")] + pub max_model_len: u64, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct DetokenizeRequest { + #[prost(int64, repeated, tag = "1")] + pub tokens: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct DetokenizeResponse { + #[prost(string, tag = "1")] + pub text: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct AbortRequest { + /// Native generation rid (HTTP `/generate` or gRPC Generate), not an HTTP + /// chat/completion response ID. + /// Required and nonempty unless abort_all is true. + #[prost(string, tag = "1")] + pub rid: ::prost::alloc::string::String, + /// When true, rid must be empty. Targets all frontend generations active at + /// this call, including HTTP chat/completions. + #[prost(bool, tag = "2")] + pub abort_all: bool, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct AbortResponse { + /// True if at least one active generation matched and cancellation was + /// requested. Not an acknowledgement that the scheduler has stopped yet. + /// False for an unknown/already-finished ID; repeated aborts are harmless. + #[prost(bool, tag = "1")] + pub success: bool, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct HealthCheckRequest {} #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct HealthCheckResponse { @@ -685,7 +792,7 @@ pub struct GetModelInfoResponse { } #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct GetServerInfoRequest {} -#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct GetServerInfoResponse { #[prost(string, tag = "1")] pub model_path: ::prost::alloc::string::String, @@ -705,4 +812,72 @@ pub struct GetServerInfoResponse { /// passes them through verbatim. #[prost(string, repeated, tag = "7")] pub internal_states: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + /// Public serving endpoint configuration. + #[prost(string, tag = "8")] + pub host: ::prost::alloc::string::String, + #[prost(uint32, tag = "9")] + pub port: u32, + #[prost(bool, tag = "10")] + pub incremental_streaming_output: bool, + #[prost(string, tag = "11")] + pub disaggregation_mode: ::prost::alloc::string::String, + /// Distributed deployment metadata required by out-of-process orchestrators. + #[prost(message, optional, tag = "12")] + pub deployment_info: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeploymentInfo { + /// Distributed process topology. + #[prost(uint32, optional, tag = "1")] + pub disaggregation_bootstrap_port: ::core::option::Option, + #[prost(uint32, tag = "2")] + pub node_rank: u32, + #[prost(uint32, tag = "3")] + pub nnodes: u32, + #[prost(uint32, tag = "4")] + pub dp_size: u32, + #[prost(uint32, optional, tag = "5")] + pub dp_rank: ::core::option::Option, + #[prost(string, optional, tag = "6")] + pub dist_init_addr: ::core::option::Option<::prost::alloc::string::String>, + #[prost(uint32, tag = "7")] + pub base_gpu_id: u32, + #[prost(uint32, optional, tag = "8")] + pub page_size: ::core::option::Option, + #[prost(uint32, tag = "9")] + pub dcp_size: u32, + /// Request scheduling and execution limits. + #[prost(uint64, optional, tag = "10")] + pub max_running_requests: ::core::option::Option, + #[prost(uint64, tag = "11")] + pub max_prefill_tokens: u64, + #[prost(bool, tag = "12")] + pub enable_dp_attention: bool, + #[prost(string, optional, tag = "13")] + pub speculative_algorithm: ::core::option::Option<::prost::alloc::string::String>, + /// Hierarchical KV-cache configuration. + #[prost(bool, tag = "14")] + pub enable_hierarchical_cache: bool, + #[prost(double, optional, tag = "15")] + pub hicache_ratio: ::core::option::Option, + #[prost(uint64, tag = "16")] + pub hicache_size: u64, + #[prost(string, optional, tag = "17")] + pub hicache_write_policy: ::core::option::Option<::prost::alloc::string::String>, + #[prost(uint64, optional, tag = "18")] + pub hicache_host_total_tokens: ::core::option::Option, + /// KV-cache event streams published by this deployment. + #[prost(message, repeated, tag = "19")] + pub kv_event_sources: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct KvEventSource { + #[prost(uint32, tag = "1")] + pub dp_rank: u32, + #[prost(string, tag = "2")] + pub endpoint: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub topic: ::prost::alloc::string::String, + #[prost(uint32, tag = "4")] + pub block_size: u32, } diff --git a/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs b/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs index 1b804b54b9..2f6e955e6d 100644 --- a/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs +++ b/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs @@ -1110,6 +1110,9 @@ impl ::serde::Serialize for SamplingParams { if let Some(v) = &self.watermark { ::serde::ser::SerializeMap::serialize_entry(&mut map, "watermark", v)?; } + if !self.choice.is_empty() { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "choice", &self.choice)?; + } ::serde::ser::SerializeMap::end(map) } } @@ -1144,6 +1147,7 @@ impl<'de> ::serde::Deserialize<'de> for SamplingParams { "custom_params", "beam_width", "watermark", + "choice", ]; struct V; impl<'de> ::serde::de::Visitor<'de> for V { @@ -1182,6 +1186,7 @@ impl<'de> ::serde::Deserialize<'de> for SamplingParams { let mut f_custom_params = None; let mut f_beam_width = None; let mut f_watermark = None; + let mut f_choice = None; while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { match key.as_ref() { "max_new_tokens" => { @@ -1394,6 +1399,15 @@ impl<'de> ::serde::Deserialize<'de> for SamplingParams { } f_watermark = Some(map.next_value::>()?); } + "choice" => { + if f_choice.is_some() { + return Err(::serde::de::Error::duplicate_field("choice")); + } + f_choice = Some( + map.next_value::>>()? + .unwrap_or_default(), + ); + } other => { return Err(::serde::de::Error::unknown_field(other, FIELDS)); } @@ -1428,6 +1442,7 @@ impl<'de> ::serde::Deserialize<'de> for SamplingParams { custom_params: f_custom_params.flatten(), beam_width: f_beam_width.flatten(), watermark: f_watermark.flatten(), + choice: f_choice.unwrap_or_default(), }) } } @@ -1603,12 +1618,7 @@ impl ::serde::Serialize for GenerateRequest { ::serde::ser::SerializeMap::serialize_entry(&mut map, "require_reasoning", v)?; } if let Some(v) = &self.lora_path { - ::serde::ser::SerializeMap::serialize_entry( - &mut map, - "lora_path", - &::serde_json::from_str::<::serde_json::Value>(v) - .unwrap_or(::serde_json::Value::Null), - )?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "lora_path", v)?; } if let Some(v) = &self.session_params { ::serde::ser::SerializeMap::serialize_entry( @@ -1618,6 +1628,15 @@ impl ::serde::Serialize for GenerateRequest { .unwrap_or(::serde_json::Value::Null), )?; } + if let Some(v) = &self.kv_hints { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "kv_hints", v)?; + } + if let Some(v) = &self.routing_key { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "routing_key", v)?; + } + if let Some(v) = &self.max_thinking_tokens { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "max_thinking_tokens", v)?; + } ::serde::ser::SerializeMap::end(map) } } @@ -1662,6 +1681,9 @@ impl<'de> ::serde::Deserialize<'de> for GenerateRequest { let mut f_require_reasoning = None; let mut f_lora_path = None; let mut f_session_params = None; + let mut f_kv_hints = None; + let mut f_routing_key = None; + let mut f_max_thinking_tokens = None; while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { match key.as_ref() { "rid" => { @@ -1848,10 +1870,7 @@ impl<'de> ::serde::Deserialize<'de> for GenerateRequest { if f_lora_path.is_some() { return Err(::serde::de::Error::duplicate_field("lora_path")); } - f_lora_path = Some( - map.next_value::>()? - .map(|v| v.to_string()), - ); + f_lora_path = Some(map.next_value::>()?); } "session_params" => { if f_session_params.is_some() { @@ -1862,6 +1881,27 @@ impl<'de> ::serde::Deserialize<'de> for GenerateRequest { .map(|v| v.to_string()), ); } + "kv_hints" => { + if f_kv_hints.is_some() { + return Err(::serde::de::Error::duplicate_field("kv_hints")); + } + f_kv_hints = Some(map.next_value::>()?); + } + "routing_key" => { + if f_routing_key.is_some() { + return Err(::serde::de::Error::duplicate_field("routing_key")); + } + f_routing_key = + Some(map.next_value::>()?); + } + "max_thinking_tokens" => { + if f_max_thinking_tokens.is_some() { + return Err(::serde::de::Error::duplicate_field( + "max_thinking_tokens", + )); + } + f_max_thinking_tokens = Some(map.next_value::>()?); + } _ => { map.next_value::<::serde::de::IgnoredAny>()?; } @@ -1896,6 +1936,242 @@ impl<'de> ::serde::Deserialize<'de> for GenerateRequest { require_reasoning: f_require_reasoning.flatten(), lora_path: f_lora_path.flatten(), session_params: f_session_params.flatten(), + kv_hints: f_kv_hints.flatten(), + routing_key: f_routing_key.flatten(), + max_thinking_tokens: f_max_thinking_tokens.flatten(), + }) + } + } + deserializer.deserialize_map(V) + } +} + +impl ::serde::Serialize for KvHintsEnvelopeOrList { + fn serialize(&self, serializer: S) -> Result { + match &self.value { + Some(kv_hints_envelope_or_list::Value::One(v)) => v.serialize(serializer), + Some(kv_hints_envelope_or_list::Value::Many(v)) => v.serialize(serializer), + None => serializer.serialize_none(), + } + } +} + +impl<'de> ::serde::Deserialize<'de> for KvHintsEnvelopeOrList { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = KvHintsEnvelopeOrList; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("a value or an array of values for KvHintsEnvelopeOrList") + } + fn visit_map>( + self, + map: A, + ) -> Result { + let one = ::deserialize( + ::serde::de::value::MapAccessDeserializer::new(map), + )?; + Ok(KvHintsEnvelopeOrList { + value: Some(kv_hints_envelope_or_list::Value::One(one)), + }) + } + fn visit_seq>( + self, + seq: A, + ) -> Result { + let many = ::deserialize( + ::serde::de::value::SeqAccessDeserializer::new(seq), + )?; + Ok(KvHintsEnvelopeOrList { + value: Some(kv_hints_envelope_or_list::Value::Many(many)), + }) + } + } + deserializer.deserialize_any(V) + } +} + +impl ::serde::Serialize for KvHintsEnvelopeList { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_seq(self.items.iter()) + } +} + +impl<'de> ::serde::Deserialize<'de> for KvHintsEnvelopeList { + fn deserialize>(deserializer: D) -> Result { + Ok(KvHintsEnvelopeList { + items: <::prost::alloc::vec::Vec as ::serde::Deserialize>::deserialize(deserializer)?, + }) + } +} + +impl ::serde::Serialize for OptionalKvHintsEnvelope { + fn serialize(&self, serializer: S) -> Result { + self.envelope.serialize(serializer) + } +} + +impl<'de> ::serde::Deserialize<'de> for OptionalKvHintsEnvelope { + fn deserialize>(deserializer: D) -> Result { + Ok(OptionalKvHintsEnvelope { + envelope: as ::serde::Deserialize>::deserialize(deserializer)?, + }) + } +} + +impl ::serde::Serialize for KvHintsEnvelope { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "protocol_version", + &self.protocol_version, + )?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "message_id", &self.message_id)?; + if !self.actions.is_empty() { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "actions", &self.actions)?; + } + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for KvHintsEnvelope { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = KvHintsEnvelope; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct KvHintsEnvelope") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut f_protocol_version = None; + let mut f_message_id = None; + let mut f_actions = None; + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + "protocol_version" => { + if f_protocol_version.is_some() { + return Err(::serde::de::Error::duplicate_field( + "protocol_version", + )); + } + f_protocol_version = + Some(map.next_value::<::prost::alloc::string::String>()?); + } + "message_id" => { + if f_message_id.is_some() { + return Err(::serde::de::Error::duplicate_field("message_id")); + } + f_message_id = + Some(map.next_value::<::prost::alloc::string::String>()?); + } + "actions" => { + if f_actions.is_some() { + return Err(::serde::de::Error::duplicate_field("actions")); + } + f_actions = Some( + map.next_value::>>()? + .unwrap_or_default(), + ); + } + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(KvHintsEnvelope { + protocol_version: f_protocol_version.unwrap_or_default(), + message_id: f_message_id.unwrap_or_default(), + actions: f_actions.unwrap_or_default(), + }) + } + } + deserializer.deserialize_map(V) + } +} + +impl ::serde::Serialize for KvHintAction { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "action_id", &self.action_id)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "action_type", &self.action_type)?; + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "action_version", + &self.action_version, + )?; + if let Some(v) = &self.payload { + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "payload", + &::serde_json::from_str::<::serde_json::Value>(v) + .unwrap_or(::serde_json::Value::Null), + )?; + } + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for KvHintAction { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = KvHintAction; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct KvHintAction") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut f_action_id = None; + let mut f_action_type = None; + let mut f_action_version = None; + let mut f_payload = None; + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + "action_id" => { + if f_action_id.is_some() { + return Err(::serde::de::Error::duplicate_field("action_id")); + } + f_action_id = Some(map.next_value::<::prost::alloc::string::String>()?); + } + "action_type" => { + if f_action_type.is_some() { + return Err(::serde::de::Error::duplicate_field("action_type")); + } + f_action_type = + Some(map.next_value::<::prost::alloc::string::String>()?); + } + "action_version" => { + if f_action_version.is_some() { + return Err(::serde::de::Error::duplicate_field("action_version")); + } + f_action_version = + Some(map.next_value::<::prost::alloc::string::String>()?); + } + "payload" => { + if f_payload.is_some() { + return Err(::serde::de::Error::duplicate_field("payload")); + } + f_payload = Some( + map.next_value::>()? + .map(|v| v.to_string()), + ); + } + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(KvHintAction { + action_id: f_action_id.unwrap_or_default(), + action_type: f_action_type.unwrap_or_default(), + action_version: f_action_version.unwrap_or_default(), + payload: f_payload.flatten(), }) } } @@ -2887,6 +3163,348 @@ impl<'de> ::serde::Deserialize<'de> for MediaInput { } } +impl ::serde::Serialize for ListModelsRequest { + fn serialize(&self, serializer: S) -> Result { + let map = serializer.serialize_map(None)?; + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for ListModelsRequest { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = ListModelsRequest; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct ListModelsRequest") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(ListModelsRequest {}) + } + } + deserializer.deserialize_map(V) + } +} + +impl ::serde::Serialize for ListModelsResponse { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + if !self.models.is_empty() { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "models", &self.models)?; + } + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for ListModelsResponse { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = ListModelsResponse; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct ListModelsResponse") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut f_models = None; + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + "models" => { + if f_models.is_some() { + return Err(::serde::de::Error::duplicate_field("models")); + } + f_models = Some( + map.next_value::>>()? + .unwrap_or_default(), + ); + } + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(ListModelsResponse { + models: f_models.unwrap_or_default(), + }) + } + } + deserializer.deserialize_map(V) + } +} + +impl ::serde::Serialize for ModelCard { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "id", &self.id)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "root", &self.root)?; + if let Some(v) = &self.parent { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "parent", v)?; + } + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "max_model_len", + &self.max_model_len, + )?; + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for ModelCard { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = ModelCard; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct ModelCard") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut f_id = None; + let mut f_root = None; + let mut f_parent = None; + let mut f_max_model_len = None; + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + "id" => { + if f_id.is_some() { + return Err(::serde::de::Error::duplicate_field("id")); + } + f_id = Some(map.next_value::<::prost::alloc::string::String>()?); + } + "root" => { + if f_root.is_some() { + return Err(::serde::de::Error::duplicate_field("root")); + } + f_root = Some(map.next_value::<::prost::alloc::string::String>()?); + } + "parent" => { + if f_parent.is_some() { + return Err(::serde::de::Error::duplicate_field("parent")); + } + f_parent = + Some(map.next_value::>()?); + } + "max_model_len" => { + if f_max_model_len.is_some() { + return Err(::serde::de::Error::duplicate_field("max_model_len")); + } + f_max_model_len = Some(map.next_value::()?); + } + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(ModelCard { + id: f_id.unwrap_or_default(), + root: f_root.unwrap_or_default(), + parent: f_parent.flatten(), + max_model_len: f_max_model_len.unwrap_or_default(), + }) + } + } + deserializer.deserialize_map(V) + } +} + +impl ::serde::Serialize for DetokenizeRequest { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + if !self.tokens.is_empty() { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "tokens", &self.tokens)?; + } + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for DetokenizeRequest { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = DetokenizeRequest; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct DetokenizeRequest") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut f_tokens = None; + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + "tokens" => { + if f_tokens.is_some() { + return Err(::serde::de::Error::duplicate_field("tokens")); + } + f_tokens = + Some(map.next_value::>>()?.unwrap_or_default()); + } + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(DetokenizeRequest { + tokens: f_tokens.unwrap_or_default(), + }) + } + } + deserializer.deserialize_map(V) + } +} + +impl ::serde::Serialize for DetokenizeResponse { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "text", &self.text)?; + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for DetokenizeResponse { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = DetokenizeResponse; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct DetokenizeResponse") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut f_text = None; + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + "text" => { + if f_text.is_some() { + return Err(::serde::de::Error::duplicate_field("text")); + } + f_text = Some(map.next_value::<::prost::alloc::string::String>()?); + } + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(DetokenizeResponse { + text: f_text.unwrap_or_default(), + }) + } + } + deserializer.deserialize_map(V) + } +} + +impl ::serde::Serialize for AbortRequest { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "rid", &self.rid)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "abort_all", &self.abort_all)?; + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for AbortRequest { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = AbortRequest; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct AbortRequest") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut f_rid = None; + let mut f_abort_all = None; + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + "rid" => { + if f_rid.is_some() { + return Err(::serde::de::Error::duplicate_field("rid")); + } + f_rid = Some(map.next_value::<::prost::alloc::string::String>()?); + } + "abort_all" => { + if f_abort_all.is_some() { + return Err(::serde::de::Error::duplicate_field("abort_all")); + } + f_abort_all = Some(map.next_value::()?); + } + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(AbortRequest { + rid: f_rid.unwrap_or_default(), + abort_all: f_abort_all.unwrap_or_default(), + }) + } + } + deserializer.deserialize_map(V) + } +} + +impl ::serde::Serialize for AbortResponse { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "success", &self.success)?; + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for AbortResponse { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = AbortResponse; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct AbortResponse") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut f_success = None; + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + "success" => { + if f_success.is_some() { + return Err(::serde::de::Error::duplicate_field("success")); + } + f_success = Some(map.next_value::()?); + } + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(AbortResponse { + success: f_success.unwrap_or_default(), + }) + } + } + deserializer.deserialize_map(V) + } +} + impl ::serde::Serialize for HealthCheckRequest { fn serialize(&self, serializer: S) -> Result { let map = serializer.serialize_map(None)?; @@ -3233,6 +3851,21 @@ impl ::serde::Serialize for GetServerInfoResponse { .collect(); ::serde::ser::SerializeMap::serialize_entry(&mut map, "internal_states", &rows)?; } + ::serde::ser::SerializeMap::serialize_entry(&mut map, "host", &self.host)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "port", &self.port)?; + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "incremental_streaming_output", + &self.incremental_streaming_output, + )?; + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "disaggregation_mode", + &self.disaggregation_mode, + )?; + if let Some(v) = &self.deployment_info { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "deployment_info", v)?; + } ::serde::ser::SerializeMap::end(map) } } @@ -3256,6 +3889,11 @@ impl<'de> ::serde::Deserialize<'de> for GetServerInfoResponse { let mut f_max_total_num_tokens = None; let mut f_version = None; let mut f_internal_states = None; + let mut f_host = None; + let mut f_port = None; + let mut f_incremental_streaming_output = None; + let mut f_disaggregation_mode = None; + let mut f_deployment_info = None; while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { match key.as_ref() { "model_path" => { @@ -3315,6 +3953,41 @@ impl<'de> ::serde::Deserialize<'de> for GetServerInfoResponse { .collect(), ); } + "host" => { + if f_host.is_some() { + return Err(::serde::de::Error::duplicate_field("host")); + } + f_host = Some(map.next_value::<::prost::alloc::string::String>()?); + } + "port" => { + if f_port.is_some() { + return Err(::serde::de::Error::duplicate_field("port")); + } + f_port = Some(map.next_value::()?); + } + "incremental_streaming_output" => { + if f_incremental_streaming_output.is_some() { + return Err(::serde::de::Error::duplicate_field( + "incremental_streaming_output", + )); + } + f_incremental_streaming_output = Some(map.next_value::()?); + } + "disaggregation_mode" => { + if f_disaggregation_mode.is_some() { + return Err(::serde::de::Error::duplicate_field( + "disaggregation_mode", + )); + } + f_disaggregation_mode = + Some(map.next_value::<::prost::alloc::string::String>()?); + } + "deployment_info" => { + if f_deployment_info.is_some() { + return Err(::serde::de::Error::duplicate_field("deployment_info")); + } + f_deployment_info = Some(map.next_value::>()?); + } _ => { map.next_value::<::serde::de::IgnoredAny>()?; } @@ -3328,6 +4001,352 @@ impl<'de> ::serde::Deserialize<'de> for GetServerInfoResponse { max_total_num_tokens: f_max_total_num_tokens.flatten(), version: f_version.unwrap_or_default(), internal_states: f_internal_states.unwrap_or_default(), + host: f_host.unwrap_or_default(), + port: f_port.unwrap_or_default(), + incremental_streaming_output: f_incremental_streaming_output + .unwrap_or_default(), + disaggregation_mode: f_disaggregation_mode.unwrap_or_default(), + deployment_info: f_deployment_info.flatten(), + }) + } + } + deserializer.deserialize_map(V) + } +} + +impl ::serde::Serialize for DeploymentInfo { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + if let Some(v) = &self.disaggregation_bootstrap_port { + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "disaggregation_bootstrap_port", + v, + )?; + } + ::serde::ser::SerializeMap::serialize_entry(&mut map, "node_rank", &self.node_rank)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "nnodes", &self.nnodes)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "dp_size", &self.dp_size)?; + if let Some(v) = &self.dp_rank { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "dp_rank", v)?; + } + if let Some(v) = &self.dist_init_addr { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "dist_init_addr", v)?; + } + ::serde::ser::SerializeMap::serialize_entry(&mut map, "base_gpu_id", &self.base_gpu_id)?; + if let Some(v) = &self.page_size { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "page_size", v)?; + } + ::serde::ser::SerializeMap::serialize_entry(&mut map, "dcp_size", &self.dcp_size)?; + if let Some(v) = &self.max_running_requests { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "max_running_requests", v)?; + } + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "max_prefill_tokens", + &self.max_prefill_tokens, + )?; + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "enable_dp_attention", + &self.enable_dp_attention, + )?; + if let Some(v) = &self.speculative_algorithm { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "speculative_algorithm", v)?; + } + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "enable_hierarchical_cache", + &self.enable_hierarchical_cache, + )?; + if let Some(v) = &self.hicache_ratio { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "hicache_ratio", v)?; + } + ::serde::ser::SerializeMap::serialize_entry(&mut map, "hicache_size", &self.hicache_size)?; + if let Some(v) = &self.hicache_write_policy { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "hicache_write_policy", v)?; + } + if let Some(v) = &self.hicache_host_total_tokens { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "hicache_host_total_tokens", v)?; + } + if !self.kv_event_sources.is_empty() { + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "kv_event_sources", + &self.kv_event_sources, + )?; + } + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for DeploymentInfo { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = DeploymentInfo; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct DeploymentInfo") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut f_disaggregation_bootstrap_port = None; + let mut f_node_rank = None; + let mut f_nnodes = None; + let mut f_dp_size = None; + let mut f_dp_rank = None; + let mut f_dist_init_addr = None; + let mut f_base_gpu_id = None; + let mut f_page_size = None; + let mut f_dcp_size = None; + let mut f_max_running_requests = None; + let mut f_max_prefill_tokens = None; + let mut f_enable_dp_attention = None; + let mut f_speculative_algorithm = None; + let mut f_enable_hierarchical_cache = None; + let mut f_hicache_ratio = None; + let mut f_hicache_size = None; + let mut f_hicache_write_policy = None; + let mut f_hicache_host_total_tokens = None; + let mut f_kv_event_sources = None; + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + "disaggregation_bootstrap_port" => { + if f_disaggregation_bootstrap_port.is_some() { + return Err(::serde::de::Error::duplicate_field( + "disaggregation_bootstrap_port", + )); + } + f_disaggregation_bootstrap_port = + Some(map.next_value::>()?); + } + "node_rank" => { + if f_node_rank.is_some() { + return Err(::serde::de::Error::duplicate_field("node_rank")); + } + f_node_rank = Some(map.next_value::()?); + } + "nnodes" => { + if f_nnodes.is_some() { + return Err(::serde::de::Error::duplicate_field("nnodes")); + } + f_nnodes = Some(map.next_value::()?); + } + "dp_size" => { + if f_dp_size.is_some() { + return Err(::serde::de::Error::duplicate_field("dp_size")); + } + f_dp_size = Some(map.next_value::()?); + } + "dp_rank" => { + if f_dp_rank.is_some() { + return Err(::serde::de::Error::duplicate_field("dp_rank")); + } + f_dp_rank = Some(map.next_value::>()?); + } + "dist_init_addr" => { + if f_dist_init_addr.is_some() { + return Err(::serde::de::Error::duplicate_field("dist_init_addr")); + } + f_dist_init_addr = + Some(map.next_value::>()?); + } + "base_gpu_id" => { + if f_base_gpu_id.is_some() { + return Err(::serde::de::Error::duplicate_field("base_gpu_id")); + } + f_base_gpu_id = Some(map.next_value::()?); + } + "page_size" => { + if f_page_size.is_some() { + return Err(::serde::de::Error::duplicate_field("page_size")); + } + f_page_size = Some(map.next_value::>()?); + } + "dcp_size" => { + if f_dcp_size.is_some() { + return Err(::serde::de::Error::duplicate_field("dcp_size")); + } + f_dcp_size = Some(map.next_value::()?); + } + "max_running_requests" => { + if f_max_running_requests.is_some() { + return Err(::serde::de::Error::duplicate_field( + "max_running_requests", + )); + } + f_max_running_requests = Some(map.next_value::>()?); + } + "max_prefill_tokens" => { + if f_max_prefill_tokens.is_some() { + return Err(::serde::de::Error::duplicate_field( + "max_prefill_tokens", + )); + } + f_max_prefill_tokens = Some(map.next_value::()?); + } + "enable_dp_attention" => { + if f_enable_dp_attention.is_some() { + return Err(::serde::de::Error::duplicate_field( + "enable_dp_attention", + )); + } + f_enable_dp_attention = Some(map.next_value::()?); + } + "speculative_algorithm" => { + if f_speculative_algorithm.is_some() { + return Err(::serde::de::Error::duplicate_field( + "speculative_algorithm", + )); + } + f_speculative_algorithm = + Some(map.next_value::>()?); + } + "enable_hierarchical_cache" => { + if f_enable_hierarchical_cache.is_some() { + return Err(::serde::de::Error::duplicate_field( + "enable_hierarchical_cache", + )); + } + f_enable_hierarchical_cache = Some(map.next_value::()?); + } + "hicache_ratio" => { + if f_hicache_ratio.is_some() { + return Err(::serde::de::Error::duplicate_field("hicache_ratio")); + } + f_hicache_ratio = Some(map.next_value::>()?); + } + "hicache_size" => { + if f_hicache_size.is_some() { + return Err(::serde::de::Error::duplicate_field("hicache_size")); + } + f_hicache_size = Some(map.next_value::()?); + } + "hicache_write_policy" => { + if f_hicache_write_policy.is_some() { + return Err(::serde::de::Error::duplicate_field( + "hicache_write_policy", + )); + } + f_hicache_write_policy = + Some(map.next_value::>()?); + } + "hicache_host_total_tokens" => { + if f_hicache_host_total_tokens.is_some() { + return Err(::serde::de::Error::duplicate_field( + "hicache_host_total_tokens", + )); + } + f_hicache_host_total_tokens = Some(map.next_value::>()?); + } + "kv_event_sources" => { + if f_kv_event_sources.is_some() { + return Err(::serde::de::Error::duplicate_field( + "kv_event_sources", + )); + } + f_kv_event_sources = Some( + map.next_value::>>()? + .unwrap_or_default(), + ); + } + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(DeploymentInfo { + disaggregation_bootstrap_port: f_disaggregation_bootstrap_port.flatten(), + node_rank: f_node_rank.unwrap_or_default(), + nnodes: f_nnodes.unwrap_or_default(), + dp_size: f_dp_size.unwrap_or_default(), + dp_rank: f_dp_rank.flatten(), + dist_init_addr: f_dist_init_addr.flatten(), + base_gpu_id: f_base_gpu_id.unwrap_or_default(), + page_size: f_page_size.flatten(), + dcp_size: f_dcp_size.unwrap_or_default(), + max_running_requests: f_max_running_requests.flatten(), + max_prefill_tokens: f_max_prefill_tokens.unwrap_or_default(), + enable_dp_attention: f_enable_dp_attention.unwrap_or_default(), + speculative_algorithm: f_speculative_algorithm.flatten(), + enable_hierarchical_cache: f_enable_hierarchical_cache.unwrap_or_default(), + hicache_ratio: f_hicache_ratio.flatten(), + hicache_size: f_hicache_size.unwrap_or_default(), + hicache_write_policy: f_hicache_write_policy.flatten(), + hicache_host_total_tokens: f_hicache_host_total_tokens.flatten(), + kv_event_sources: f_kv_event_sources.unwrap_or_default(), + }) + } + } + deserializer.deserialize_map(V) + } +} + +impl ::serde::Serialize for KvEventSource { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "dp_rank", &self.dp_rank)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "endpoint", &self.endpoint)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "topic", &self.topic)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "block_size", &self.block_size)?; + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for KvEventSource { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = KvEventSource; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct KvEventSource") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut f_dp_rank = None; + let mut f_endpoint = None; + let mut f_topic = None; + let mut f_block_size = None; + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + "dp_rank" => { + if f_dp_rank.is_some() { + return Err(::serde::de::Error::duplicate_field("dp_rank")); + } + f_dp_rank = Some(map.next_value::()?); + } + "endpoint" => { + if f_endpoint.is_some() { + return Err(::serde::de::Error::duplicate_field("endpoint")); + } + f_endpoint = Some(map.next_value::<::prost::alloc::string::String>()?); + } + "topic" => { + if f_topic.is_some() { + return Err(::serde::de::Error::duplicate_field("topic")); + } + f_topic = Some(map.next_value::<::prost::alloc::string::String>()?); + } + "block_size" => { + if f_block_size.is_some() { + return Err(::serde::de::Error::duplicate_field("block_size")); + } + f_block_size = Some(map.next_value::()?); + } + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(KvEventSource { + dp_rank: f_dp_rank.unwrap_or_default(), + endpoint: f_endpoint.unwrap_or_default(), + topic: f_topic.unwrap_or_default(), + block_size: f_block_size.unwrap_or_default(), }) } } diff --git a/rust/sglang-api-types/src/generated/sglang.api.v1.tonic.rs b/rust/sglang-api-types/src/generated/sglang.api.v1.tonic.rs index 0c7a0e53eb..e7f8f3a55c 100644 --- a/rust/sglang-api-types/src/generated/sglang.api.v1.tonic.rs +++ b/rust/sglang-api-types/src/generated/sglang.api.v1.tonic.rs @@ -168,6 +168,59 @@ pub mod sglang_service_client { )); self.inner.unary(req, path, codec).await } + /// The same static model cards as HTTP `/v1/models`. + pub async fn list_models( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = + http::uri::PathAndQuery::from_static("/sglang.api.v1.SglangService/ListModels"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("sglang.api.v1.SglangService", "ListModels")); + self.inner.unary(req, path, codec).await + } + /// Decode token IDs locally, without running model inference. + pub async fn detokenize( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = + http::uri::PathAndQuery::from_static("/sglang.api.v1.SglangService/Detokenize"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("sglang.api.v1.SglangService", "Detokenize")); + self.inner.unary(req, path, codec).await + } + /// Cancel by the native rid used by HTTP `/generate` or gRPC Generate, or + /// cancel all active frontend generations with abort_all. HTTP chat/completion + /// response IDs are not supported as rid selectors. + /// Duplicate native IDs cancel all matching generations. A later reuse of the + /// ID is unaffected. Internal control/detokenization requests are not targeted. + pub async fn abort( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/sglang.api.v1.SglangService/Abort"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("sglang.api.v1.SglangService", "Abort")); + self.inner.unary(req, path, codec).await + } } } /// Generated server implementations. @@ -212,6 +265,25 @@ pub mod sglang_service_server { &self, request: tonic::Request, ) -> std::result::Result, tonic::Status>; + /// The same static model cards as HTTP `/v1/models`. + async fn list_models( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + /// Decode token IDs locally, without running model inference. + async fn detokenize( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + /// Cancel by the native rid used by HTTP `/generate` or gRPC Generate, or + /// cancel all active frontend generations with abort_all. HTTP chat/completion + /// response IDs are not supported as rid selectors. + /// Duplicate native IDs cancel all matching generations. A later reuse of the + /// ID is unaffected. Internal control/detokenization requests are not targeted. + async fn abort( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; } #[derive(Debug)] pub struct SglangServiceServer { @@ -453,6 +525,122 @@ pub mod sglang_service_server { }; Box::pin(fut) } + "/sglang.api.v1.SglangService/ListModels" => { + #[allow(non_camel_case_types)] + struct ListModelsSvc(pub Arc); + impl tonic::server::UnaryService for ListModelsSvc { + type Response = super::ListModelsResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::list_models(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = ListModelsSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.api.v1.SglangService/Detokenize" => { + #[allow(non_camel_case_types)] + struct DetokenizeSvc(pub Arc); + impl tonic::server::UnaryService for DetokenizeSvc { + type Response = super::DetokenizeResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::detokenize(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = DetokenizeSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.api.v1.SglangService/Abort" => { + #[allow(non_camel_case_types)] + struct AbortSvc(pub Arc); + impl tonic::server::UnaryService for AbortSvc { + type Response = super::AbortResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = + async move { ::abort(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = AbortSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } _ => Box::pin(async move { let mut response = http::Response::new(tonic::body::Body::default()); let headers = response.headers_mut(); diff --git a/rust/sglang-api-types/testdata/json_parity.json b/rust/sglang-api-types/testdata/json_parity.json index 9de2711db9..e90af6f9d2 100644 --- a/rust/sglang-api-types/testdata/json_parity.json +++ b/rust/sglang-api-types/testdata/json_parity.json @@ -1,6 +1,42 @@ { "_": "Cross-language JSON parity fixtures for sglang.api.v1. Each case: decoding `input` and re-encoding yields `canonical` byte for byte, and `canonical` re-encodes to itself. Checked by rust/sglang-api-types/tests/json_parity.rs and test/registered/unit/api/v1/test_parity.py.", "cases": [ + { + "type": "ListModelsResponse", + "name": "model cards preserve a uint64 context limit and omit absent parent", + "input": {"models": [{"id": "model", "root": "model", "max_model_len": 4294967296}]}, + "canonical": "{\"models\":[{\"id\":\"model\",\"root\":\"model\",\"max_model_len\":4294967296}]}" + }, + { + "type": "DetokenizeRequest", + "name": "token IDs share the native generate int64 representation", + "input": {"tokens": [1, 2147483648]}, + "canonical": "{\"tokens\":[1,2147483648]}" + }, + { + "type": "DetokenizeResponse", + "name": "decoded text", + "input": {"text": "hello"}, + "canonical": "{\"text\":\"hello\"}" + }, + { + "type": "AbortRequest", + "name": "single public request ID", + "input": {"rid": "request", "abort_all": false}, + "canonical": "{\"rid\":\"request\",\"abort_all\":false}" + }, + { + "type": "AbortRequest", + "name": "all generations", + "input": {"rid": "", "abort_all": true}, + "canonical": "{\"rid\":\"\",\"abort_all\":true}" + }, + { + "type": "AbortResponse", + "name": "unknown ID is an explicit unsuccessful match", + "input": {"success": false}, + "canonical": "{\"success\":false}" + }, { "type": "GenerateRequest", "name": "single text prompt with a sampling object; stream absent", @@ -317,9 +353,41 @@ "weight": 2.0 } } - ] + ], + "host": "127.0.0.1", + "port": 30000, + "incremental_streaming_output": true, + "disaggregation_mode": "decode", + "deployment_info": { + "disaggregation_bootstrap_port": 8998, + "node_rank": 1, + "nnodes": 2, + "dp_size": 4, + "dp_rank": 1, + "dist_init_addr": "10.0.0.1:5000", + "base_gpu_id": 2, + "page_size": 64, + "dcp_size": 2, + "max_running_requests": 32, + "max_prefill_tokens": 8192, + "enable_dp_attention": true, + "speculative_algorithm": "EAGLE", + "enable_hierarchical_cache": true, + "hicache_ratio": 0.5, + "hicache_size": 4096, + "hicache_write_policy": "write_back", + "hicache_host_total_tokens": 16384, + "kv_event_sources": [ + { + "dp_rank": 1, + "endpoint": "tcp://127.0.0.1:5557", + "topic": "kv-events", + "block_size": 64 + } + ] + } }, - "canonical": "{\"model_path\":\"/m\",\"served_model_name\":\"m\",\"tokenizer_path\":\"/t\",\"max_context_length\":4096,\"max_total_num_tokens\":8192,\"version\":\"0.5.0\",\"internal_states\":[{\"last_gen_throughput\":1.5,\"memory_usage\":{\"weight\":2.0}}]}" + "canonical": "{\"model_path\":\"/m\",\"served_model_name\":\"m\",\"tokenizer_path\":\"/t\",\"max_context_length\":4096,\"max_total_num_tokens\":8192,\"version\":\"0.5.0\",\"internal_states\":[{\"last_gen_throughput\":1.5,\"memory_usage\":{\"weight\":2.0}}],\"host\":\"127.0.0.1\",\"port\":30000,\"incremental_streaming_output\":true,\"disaggregation_mode\":\"decode\",\"deployment_info\":{\"disaggregation_bootstrap_port\":8998,\"node_rank\":1,\"nnodes\":2,\"dp_size\":4,\"dp_rank\":1,\"dist_init_addr\":\"10.0.0.1:5000\",\"base_gpu_id\":2,\"page_size\":64,\"dcp_size\":2,\"max_running_requests\":32,\"max_prefill_tokens\":8192,\"enable_dp_attention\":true,\"speculative_algorithm\":\"EAGLE\",\"enable_hierarchical_cache\":true,\"hicache_ratio\":0.5,\"hicache_size\":4096,\"hicache_write_policy\":\"write_back\",\"hicache_host_total_tokens\":16384,\"kv_event_sources\":[{\"dp_rank\":1,\"endpoint\":\"tcp://127.0.0.1:5557\",\"topic\":\"kv-events\",\"block_size\":64}]}}" }, { "type": "SamplingParams", diff --git a/rust/sglang-api-types/tests/json_parity.rs b/rust/sglang-api-types/tests/json_parity.rs index 9a0ae0a700..23ea68656f 100644 --- a/rust/sglang-api-types/tests/json_parity.rs +++ b/rust/sglang-api-types/tests/json_parity.rs @@ -44,6 +44,11 @@ fn fixtures_decode_to_their_canonical_bytes() { "GenerateStreamError" => check::(&name, input, canonical), "GetModelInfoResponse" => check::(&name, input, canonical), "GetServerInfoResponse" => check::(&name, input, canonical), + "ListModelsResponse" => check::(&name, input, canonical), + "DetokenizeRequest" => check::(&name, input, canonical), + "DetokenizeResponse" => check::(&name, input, canonical), + "AbortRequest" => check::(&name, input, canonical), + "AbortResponse" => check::(&name, input, canonical), other => panic!("fixture type {other} has no Rust mapping; add it here"), } } diff --git a/rust/sglang-bench/tests/grpc.rs b/rust/sglang-bench/tests/grpc.rs index 66c41dc62e..6eaf920333 100644 --- a/rust/sglang-bench/tests/grpc.rs +++ b/rust/sglang-bench/tests/grpc.rs @@ -102,8 +102,30 @@ impl SglangService for Mock { max_total_num_tokens: Some(8192), version: "test".into(), internal_states: vec![r#"{"avg_spec_accept_length": 2.5}"#.into()], + ..Default::default() })) } + + async fn list_models( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("list_models")) + } + + async fn detokenize( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("detokenize")) + } + + async fn abort( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("abort")) + } } /// Bind on a free port and serve the mock on its own runtime thread, so the diff --git a/rust/sglang-server/src/api_server/core/handle.rs b/rust/sglang-server/src/api_server/core/handle.rs index 190bd0245f..21e049cf08 100644 --- a/rust/sglang-server/src/api_server/core/handle.rs +++ b/rust/sglang-server/src/api_server/core/handle.rs @@ -1,16 +1,17 @@ //! The shared handle: submission with admission and cancellation ownership, //! runtime-to-semantic event translation, and the typed control operations -//! (model info, server info, flush cache, weights check, tokenize, detokenize, -//! health probe). +//! (model info, model list, server info, flush cache, weights check, tokenize, +//! detokenize, health probe). -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; use std::sync::{ - Arc, + Arc, Mutex, atomic::{AtomicBool, Ordering}, }; use std::time::Duration; -use tokio::sync::mpsc; +use sglang_api_types::api::v1 as api; +use tokio::sync::{mpsc, oneshot}; use crate::message::config::{DisaggregationMode, ServerArgs}; use crate::message::ids::Rid; @@ -59,6 +60,10 @@ pub(crate) struct CoreMetadata { max_context_length: u64, max_total_num_tokens: u64, version: String, + host: String, + port: u16, + incremental_streaming_output: bool, + deployment_info: crate::message::config::DeploymentInfo, launch_server_info: Arc>, } @@ -77,6 +82,10 @@ impl From<&ServerArgs> for CoreMetadata { max_context_length: args.model_config.context_len, max_total_num_tokens: args.max_total_num_tokens, version: args.version.clone(), + host: args.host.clone(), + port: args.port, + incremental_streaming_output: args.incremental_streaming_output, + deployment_info: args.deployment_info.clone(), launch_server_info: args.launch_server_info.0.clone(), } } @@ -100,6 +109,7 @@ struct CoreInner { is_disaggregation: bool, mm_limits: BTreeMap, metadata: CoreMetadata, + generations: Generations, } impl CoreHandle { @@ -118,6 +128,7 @@ impl CoreHandle { is_disaggregation: config.is_disaggregation, mm_limits: config.mm_limits, metadata: config.metadata, + generations: Generations::default(), }), } } @@ -140,6 +151,23 @@ impl CoreHandle { &self, mut request: GenerateRequest, ) -> Result { + let mut generation = self + .inner + .generations + .register(request.rid.clone(), self.inner.abort_tx.clone()); + tokio::select! { + biased; + _ = generation.cancelled() => return Err(CoreError::Cancelled("request aborted".into())), + result = self.prepare_generations(std::slice::from_mut(&mut request)) => result?, + } + self.submit_generation(request, generation) + .await + .map(CoreCall::new) + } + + /// Submit core-owned work without exposing it through the public Abort + /// registry. Drop still owns cancellation through `RuntimeCall`. + async fn generate_internal(&self, mut request: GenerateRequest) -> Result { self.prepare_generations(std::slice::from_mut(&mut request)) .await?; self.submit(RequestKind::Generate(Box::new(request))) @@ -147,27 +175,72 @@ impl CoreHandle { .map(CoreCall::new) } - /// Preprocess and submit a group atomically from the adapter's perspective: - /// no request is admitted until shared multimodal preparation succeeds for - /// the entire group. If admission later fails part-way through, dropping - /// the already-created calls aborts those accepted requests. + /// Prepare the group before submitting any surviving request. Explicit + /// cancellation produces only that item's terminal result, preserving batch + /// order. Other preparation/admission failures still reject the group and + /// dropping already-created calls aborts any accepted requests. pub(crate) async fn generate_batch( &self, mut requests: Vec, ) -> Result, CoreError> { - self.prepare_generations(&mut requests).await?; + if requests.is_empty() { + return Ok(Vec::new()); + } + let mut generations: Vec<_> = requests + .iter() + .map(|request| { + self.inner + .generations + .register(request.rid.clone(), self.inner.abort_tx.clone()) + }) + .collect(); + // Fetch concurrently, but stop only the cancelled item's preparation. + // Per-item cancellation lets try_join_all use wake-driven polling for + // large batches, without rescanning every receiver on each media wake. + futures::future::try_join_all(requests.iter_mut().zip(&mut generations).map( + |(request, generation)| async move { + tokio::select! { + biased; + _ = generation.cancelled() => Ok(()), + result = self.prepare_generations(std::slice::from_mut(request)) => result, + } + }, + )) + .await?; let mut calls = Vec::with_capacity(requests.len()); - for request in requests { - calls.push( - self.submit(RequestKind::Generate(Box::new(request))) - .await - .map(CoreCall::new)?, - ); + for (request, generation) in requests.into_iter().zip(generations) { + let rid = request.rid.clone(); + let call = match self.submit_generation(request, generation).await { + Ok(call) => CoreCall::new(call), + // Cancellation may happen during preparation or while waiting + // for intake. Keep this slot without rolling back siblings. + Err(CoreError::Cancelled(_)) => { + CoreCall::cancelled(rid, self.inner.abort_tx.clone()) + } + Err(error) => return Err(error), + }; + calls.push(call); } Ok(calls) } + /// Cancel active generations by public ID, or all active generations. + /// The admission gate and internal IDs keep cancellation safe across + /// preprocessing, queueing, duplicate public IDs, and ID reuse. + pub(crate) fn abort(&self, rid: Option<&str>) -> bool { + self.inner.generations.abort(rid, &self.inner.abort_tx) + } + + async fn submit_generation( + &self, + request: GenerateRequest, + generation: Generation, + ) -> Result { + self.submit_with_generation(RequestKind::Generate(Box::new(request)), Some(generation)) + .await + } + async fn prepare_generations(&self, requests: &mut [GenerateRequest]) -> Result<(), CoreError> { prefetch::prefetch_all(requests, &self.inner.mm_limits) .await @@ -191,6 +264,18 @@ impl CoreHandle { } } + /// Return the served model cards shared by HTTP and gRPC discovery. + /// This is static metadata; no runtime request is submitted. + pub(crate) fn list_models(&self) -> Vec { + let metadata = &self.inner.metadata; + vec![api::ModelCard { + id: metadata.served_model_name.clone(), + root: metadata.served_model_name.clone(), + parent: None, + max_model_len: metadata.max_context_length, + }] + } + /// Return public server metadata with the current scheduler metrics. /// Raw control bytes and the scheduler's full launch-argument dump never /// cross the core boundary. @@ -208,6 +293,11 @@ impl CoreHandle { frontend: "rust", startup_time: internal_state.rust_server_startup_time.take(), internal_states: vec![internal_state], + host: metadata.host.clone(), + port: metadata.port, + incremental_streaming_output: metadata.incremental_streaming_output, + disaggregation_mode: metadata.disaggregation_mode, + deployment_info: metadata.deployment_info.clone(), }) } @@ -360,7 +450,7 @@ impl CoreHandle { // The probe is intentionally not drained. Its call stays alive while // the heartbeat is observed, then aborts on drop to clean up a probe // that a busy scheduler skipped without producing a terminal frame. - let _probe = self.generate(probe).await?; + let _probe = self.generate_internal(probe).await?; let deadline = tokio::time::Instant::now() + timeout; loop { @@ -383,7 +473,7 @@ impl CoreHandle { (Some("The capital city of France is".to_string()), None) }; let mut call = self - .generate(GenerateRequest { + .generate_internal(GenerateRequest { rid: Rid::new(), text, input_ids, @@ -417,10 +507,21 @@ impl CoreHandle { /// request is accepted, the returned call owns cancellation until it /// observes a terminal response. async fn submit(&self, kind: RequestKind) -> Result { + self.submit_with_generation(kind, None).await + } + + async fn submit_with_generation( + &self, + kind: RequestKind, + generation: Option, + ) -> Result { let rid = request_rid(&kind); let expected_response = ExpectedResponse::for_request(&kind); let (response_tx, response_rx) = mpsc::channel(self.inner.response_capacity); - let admission = RequestAdmission::pending(); + let admission = generation + .as_ref() + .map(|g| g.admission.clone()) + .unwrap_or_else(RequestAdmission::pending); // Arm ownership before awaiting the bounded handoff. If this future is // cancelled after the event is queued but before the await observes // success, dropping `call` still cancels the request. @@ -431,6 +532,7 @@ impl CoreHandle { admission: admission.clone(), expected_response, in_flight: true, + generation, }; let request = Request { rid: rid.clone(), @@ -439,13 +541,17 @@ impl CoreHandle { kind, }; - if self - .inner - .intake_tx - .send_async(TmEvent::Intake { request, admission }) - .await - .is_err() - { + let sent = tokio::select! { + biased; + _ = async { + match call.generation.as_mut() { + Some(generation) => generation.cancelled().await, + None => std::future::pending().await, + } + } => return Err(CoreError::Cancelled("request aborted".into())), + result = self.inner.intake_tx.send_async(TmEvent::Intake { request, admission }) => result, + }; + if sent.is_err() { // The channel returned the event, so Intake cannot own this request. // Disarm before returning the transport-neutral rejection. call.in_flight = false; @@ -468,7 +574,7 @@ fn translate_runtime_error(error: Error) -> CoreError { } } -/// One accepted generation request and its semantic response stream. +/// One generation's semantic response stream, including pre-admission cancellation. /// /// Runtime response variants are translated here so adapters cannot depend on /// scheduler/control encodings. Dropping an unfinished call retains the @@ -486,6 +592,23 @@ impl CoreCall { } } + /// Retain a cancelled batch item's ID and position without runtime work. + fn cancelled(rid: Rid, abort_tx: flume::Sender) -> Self { + let (response_tx, response_rx) = mpsc::channel(1); + response_tx + .try_send(ResponseItem::Error(Error::Disconnected)) + .expect("new response channel has room for its terminal error"); + Self::new(RuntimeCall { + rid, + response_rx, + abort_tx, + admission: RequestAdmission::pending(), + expected_response: ExpectedResponse::Generate, + in_flight: false, + generation: None, + }) + } + /// ID that adapters should return to their client. pub(crate) fn public_id(&self) -> &str { self.runtime.rid.client_facing() @@ -543,6 +666,7 @@ impl CoreCall { admission, expected_response: ExpectedResponse::Generate, in_flight: true, + generation: None, }) } } @@ -617,6 +741,123 @@ struct InternalStateEnvelope { internal_state: InternalState, } +type Entries = HashMap>; + +/// Public-ID lookup for explicit cancellation. Internal IDs remain unique; +/// this index never replaces the runtime's request identity or admission gate. +#[derive(Clone, Default)] +struct Generations(Arc>); + +struct Entry { + admission: RequestAdmission, + cancelled: oneshot::Sender<()>, +} + +impl Generations { + fn register(&self, rid: Rid, abort_tx: flume::Sender) -> Generation { + let admission = RequestAdmission::pending(); + let (cancelled, cancellation) = oneshot::channel(); + self.0 + .lock() + .unwrap() + .entry(rid.client_facing().into()) + .or_default() + .insert( + rid.clone(), + Entry { + admission: admission.clone(), + cancelled, + }, + ); + Generation { + registry: self.clone(), + rid, + admission, + cancellation, + abort_tx, + armed: true, + } + } + + fn abort(&self, rid: Option<&str>, abort_tx: &flume::Sender) -> bool { + // Detach a snapshot under the lock, then notify outside it. Completion + // and a new request reusing the same public ID cannot remove each other. + let selected = { + let mut entries = self.0.lock().unwrap(); + match rid { + Some(rid) => entries.remove(rid).into_iter().collect::>(), + None => std::mem::take(&mut *entries).into_values().collect(), + } + }; + let mut matched = false; + for entries in selected { + for (rid, entry) in entries { + matched = true; + // Stop backend work immediately, even if the client is not + // polling its response stream. The same admission gate also + // prevents a queued request from starting after cancellation. + if entry.admission.cancel_requires_abort() { + let _ = abort_tx.send(AbortSource::Guard(rid)); + } + let _ = entry.cancelled.send(()); + } + } + matched + } + + fn remove(&self, rid: &Rid) { + let mut entries = self.0.lock().unwrap(); + if let Some(group) = entries.get_mut(rid.client_facing()) { + group.remove(rid); + if group.is_empty() { + entries.remove(rid.client_facing()); + } + } + } +} + +/// Lives from preprocessing through the terminal response. Drop owns both +/// removal from the public-ID index and cancellation of unfinished work. +struct Generation { + registry: Generations, + rid: Rid, + admission: RequestAdmission, + cancellation: oneshot::Receiver<()>, + abort_tx: flume::Sender, + armed: bool, +} + +impl Generation { + async fn cancelled(&mut self) { + // Preparation and submission may both observe the same cancellation; + // do not poll a oneshot receiver again after consuming its notification. + if !self.is_cancelled() { + let _ = (&mut self.cancellation).await; + } + } + + fn is_cancelled(&self) -> bool { + self.admission.is_cancelled() + } + + fn complete(&mut self) { + self.admission.complete(); + self.registry.remove(&self.rid); + self.armed = false; + } +} + +impl Drop for Generation { + fn drop(&mut self) { + if self.armed { + self.registry.remove(&self.rid); + if self.admission.cancel_requires_abort() { + let _ = self.abort_tx.send(AbortSource::Guard(self.rid.clone())); + } + } + } +} + /// Runtime-facing request guard kept private behind [`CoreCall`]. /// /// This type is the only owner of the raw response channel and admission/abort @@ -629,17 +870,35 @@ struct RuntimeCall { admission: RequestAdmission, expected_response: ExpectedResponse, in_flight: bool, + generation: Option, } impl RuntimeCall { async fn recv(&mut self) -> Option { - let item = self.response_rx.recv().await; + let item = tokio::select! { + biased; + _ = async { + match self.generation.as_mut() { + Some(generation) => generation.cancelled().await, + None => std::future::pending().await, + } + } => Some(ResponseItem::Error(Error::Disconnected)), + item = self.response_rx.recv() => item, + }; self.observe(item.as_ref()); item } fn try_recv(&mut self) -> Result { - let item = self.response_rx.try_recv()?; + let item = if self + .generation + .as_ref() + .is_some_and(Generation::is_cancelled) + { + ResponseItem::Error(Error::Disconnected) + } else { + self.response_rx.try_recv()? + }; self.observe(Some(&item)); Ok(item) } @@ -647,13 +906,16 @@ impl RuntimeCall { fn observe(&mut self, item: Option<&ResponseItem>) { if item.is_some_and(|item| self.expected_response.is_terminal(item)) { self.in_flight = false; + if let Some(generation) = &mut self.generation { + generation.complete(); + } } } } impl Drop for RuntimeCall { fn drop(&mut self) { - if self.in_flight { + if self.in_flight && self.generation.is_none() { // Pending cancellation is consumed by Intake before it starts work. // Once Intake has accepted the request, use the normal abort lane. if self.admission.cancel_requires_abort() { @@ -798,6 +1060,40 @@ mod tests { request } + #[test] + fn list_models_returns_static_cards_without_runtime_work() { + let (intake_tx, intake_rx) = flume::unbounded(); + let (abort_tx, abort_rx) = flume::unbounded(); + let handle = CoreHandle::new( + intake_tx, + abort_tx, + CoreConfig { + response_capacity: 8, + response_activity: Default::default(), + startup_ready: false, + is_disaggregation: false, + mm_limits: BTreeMap::new(), + metadata: CoreMetadata { + served_model_name: "served-model".into(), + max_context_length: 1_u64 << 32, + ..Default::default() + }, + }, + ); + + assert_eq!( + handle.list_models(), + vec![api::ModelCard { + id: "served-model".into(), + root: "served-model".into(), + parent: None, + max_model_len: 1_u64 << 32, + }] + ); + assert!(intake_rx.try_recv().is_err()); + assert!(abort_rx.try_recv().is_err()); + } + #[tokio::test] async fn cancellation_after_intake_acceptance_aborts_unobserved_handoff() { let (intake_tx, intake_rx) = flume::bounded(0); @@ -1064,11 +1360,17 @@ mod tests { false, BTreeMap::new(), ); - let probe = tokio::spawn(async move { handle.probe_health(Duration::from_secs(1)).await }); + let probe_handle = handle.clone(); + let probe = + tokio::spawn(async move { probe_handle.probe_health(Duration::from_secs(1)).await }); let request = accept_intake(intake_rx.recv_async().await.unwrap()); let rid = request.rid.clone(); assert!(rid.as_str().starts_with("HEALTH_CHECK_")); + assert!( + !handle.abort(None), + "internal probes must not enter the public abort-all registry" + ); activity.fetch_add(1, Ordering::Relaxed); assert_eq!(probe.await.unwrap().unwrap(), HealthStatus::Healthy); @@ -1128,6 +1430,10 @@ mod tests { assert_eq!(warmup.input_ids, None); assert_eq!(warmup.sampling_params.max_new_tokens, Some(8)); assert_eq!(warmup.bootstrap_host, None); + assert!( + !harness.handle.abort(None), + "startup warmup must not enter the public abort-all registry" + ); request .sink .try_send(ResponseItem::Done(ChunkEvent::default())) @@ -1234,4 +1540,289 @@ mod tests { assert_eq!(task.await.unwrap().unwrap(), vec![1, 2]); assert!(harness.abort_rx.try_recv().is_err()); } + + fn public_request(rid: &str) -> GenerateRequest { + GenerateRequest { + rid: Rid::from_client(rid), + ..Default::default() + } + } + + #[tokio::test] + async fn explicit_abort_during_batch_preprocessing_preserves_other_items() { + for aborted in ["A", "B"] { + let harness = Harness::unbounded(8); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let requests = ["A", "B"] + .into_iter() + .map(|rid| { + let mut request = public_request(rid); + if rid == aborted { + request.mm = Some(Box::new(MmData { + image_data: vec![MmItem::Source(format!("http://{addr}/blocked.png"))], + ..Default::default() + })); + } + request + }) + .collect(); + let mut batch = Box::pin(harness.handle.generate_batch(requests)); + assert!(futures::poll!(batch.as_mut()).is_pending()); + // Hold the media response open so cancellation happens during + // preprocessing, not after the request reaches intake. + let (media, _) = tokio::time::timeout(Duration::from_secs(5), listener.accept()) + .await + .unwrap() + .unwrap(); + assert!(harness.intake_rx.try_recv().is_err()); + assert!(harness.handle.abort(Some(aborted))); + let mut calls = tokio::time::timeout(Duration::from_secs(5), batch) + .await + .expect("cancelled media must not hold up its sibling") + .unwrap(); + drop(media); + + assert_eq!( + calls.iter().map(CoreCall::public_id).collect::>(), + ["A", "B"] + ); + let request = accept_intake(harness.intake_rx.try_recv().unwrap()); + assert_ne!(request.rid.client_facing(), aborted); + assert!(harness.intake_rx.try_recv().is_err()); + request + .sink + .try_send(ResponseItem::Done(ChunkEvent::default())) + .unwrap(); + for call in &mut calls { + let was_aborted = call.public_id() == aborted; + let event = call.recv().await; + if was_aborted { + assert!(matches!( + event, + Some(CoreEvent::Failed(CoreError::Cancelled(_))) + )); + } else { + assert!(matches!(event, Some(CoreEvent::Finished(_)))); + } + assert!(call.recv().await.is_none()); + } + assert!(!harness.handle.abort(None)); + assert!(harness.abort_rx.try_recv().is_err()); + } + } + + #[tokio::test] + async fn explicit_abort_during_batch_admission_preserves_other_items() { + let (intake_tx, intake_rx) = flume::bounded(0); + let (abort_tx, abort_rx) = flume::unbounded(); + let handle = test_handle(intake_tx, abort_tx, 8); + let mut batch = Box::pin( + handle.generate_batch(["A", "B", "C"].into_iter().map(public_request).collect()), + ); + assert!(futures::poll!(batch.as_mut()).is_pending()); + let first = accept_intake(intake_rx.recv_async().await.unwrap()); + assert_eq!(first.rid.client_facing(), "A"); + + // A is accepted; B is blocked on intake; C has not been submitted yet. + assert!(futures::poll!(batch.as_mut()).is_pending()); + assert!(handle.abort(Some("B"))); + assert!(futures::poll!(batch.as_mut()).is_pending()); + let last = accept_intake(intake_rx.recv_async().await.unwrap()); + assert_eq!(last.rid.client_facing(), "C"); + let mut calls = batch.await.unwrap(); + assert_eq!( + calls.iter().map(CoreCall::public_id).collect::>(), + ["A", "B", "C"] + ); + assert!(intake_rx.try_recv().is_err()); + assert!(abort_rx.try_recv().is_err(), "A must remain active"); + + for request in [first, last] { + request + .sink + .try_send(ResponseItem::Done(ChunkEvent::default())) + .unwrap(); + } + for (index, call) in calls.iter_mut().enumerate() { + let event = call.recv().await; + if index == 1 { + assert!(matches!( + event, + Some(CoreEvent::Failed(CoreError::Cancelled(_))) + )); + } else { + assert!(matches!(event, Some(CoreEvent::Finished(_)))); + } + assert!(call.recv().await.is_none()); + } + assert!(!handle.abort(None)); + assert!(abort_rx.try_recv().is_err()); + } + + #[tokio::test] + async fn explicit_abort_cancels_duplicates_but_not_later_id_reuse() { + let harness = Harness::unbounded(8); + let mut first = harness + .handle + .generate(public_request("same")) + .await + .unwrap(); + let mut second = harness + .handle + .generate(public_request("same")) + .await + .unwrap(); + let old_a = accept_intake(harness.intake_rx.recv().unwrap()); + let old_b = accept_intake(harness.intake_rx.recv().unwrap()); + assert!(harness.handle.abort(Some("same"))); + assert!(!harness.handle.abort(Some("same"))); + // Backend cancellation does not depend on polling either response. + let aborted: std::collections::HashSet<_> = (0..2) + .map(|_| harness.abort_rx.try_recv().unwrap().rid().clone()) + .collect(); + assert_eq!(aborted, [old_a.rid, old_b.rid].into_iter().collect()); + + let new = harness + .handle + .generate(public_request("same")) + .await + .unwrap(); + let new_request = accept_intake(harness.intake_rx.recv().unwrap()); + for call in [&mut first, &mut second] { + assert!(matches!( + call.recv().await, + Some(CoreEvent::Failed(CoreError::Cancelled(_))) + )); + assert!(call.recv().await.is_none()); + } + drop((first, second)); + assert!(harness.abort_rx.try_recv().is_err()); + assert!(harness.handle.abort(Some("same"))); + assert_eq!(harness.abort_rx.try_recv().unwrap().rid(), &new_request.rid); + drop(new); + assert!(harness.abort_rx.try_recv().is_err()); + } + + #[tokio::test] + async fn explicit_abort_before_admission_prevents_intake_acceptance() { + let harness = Harness::unbounded(8); + let mut call = harness + .handle + .generate(public_request("queued")) + .await + .unwrap(); + assert!(harness.handle.abort(Some("queued"))); + let TmEvent::Intake { admission, .. } = harness.intake_rx.recv().unwrap() else { + panic!("expected intake"); + }; + assert!(!admission.try_accept()); + assert!(matches!( + call.recv().await, + Some(CoreEvent::Failed(CoreError::Cancelled(_))) + )); + drop(call); + assert!(harness.abort_rx.try_recv().is_err()); + } + + #[tokio::test] + async fn explicit_abort_wakes_a_submission_blocked_on_admission() { + let (intake_tx, intake_rx) = flume::bounded(0); + let (abort_tx, abort_rx) = flume::unbounded(); + let handle = test_handle(intake_tx, abort_tx, 8); + let mut submit = Box::pin(handle.generate(public_request("blocked"))); + assert!(futures::poll!(submit.as_mut()).is_pending()); + assert!(handle.abort(Some("blocked"))); + assert!(matches!(submit.await, Err(CoreError::Cancelled(_)))); + if let Ok(TmEvent::Intake { admission, .. }) = intake_rx.try_recv() { + assert!(!admission.try_accept()); + } + assert!(abort_rx.try_recv().is_err()); + } + + #[tokio::test] + async fn completed_and_dropped_calls_are_removed_from_abort_lookup() { + let harness = Harness::unbounded(8); + let mut call = harness + .handle + .generate(public_request("done")) + .await + .unwrap(); + let request = accept_intake(harness.intake_rx.recv().unwrap()); + request + .sink + .try_send(ResponseItem::Done(ChunkEvent::default())) + .unwrap(); + assert!(matches!(call.recv().await, Some(CoreEvent::Finished(_)))); + assert!(!harness.handle.abort(Some("done"))); + drop(call); + assert!(harness.abort_rx.try_recv().is_err()); + + let call = harness + .handle + .generate(public_request("dropped")) + .await + .unwrap(); + let request = accept_intake(harness.intake_rx.recv().unwrap()); + drop(call); + assert_eq!(harness.abort_rx.try_recv().unwrap().rid(), &request.rid); + assert!(!harness.handle.abort(Some("dropped"))); + assert!(!harness.handle.abort(None)); + } + + #[tokio::test] + async fn abort_all_excludes_detokenization_and_preserves_future_requests() { + let harness = Harness::unbounded(8); + let mut decode = Box::pin(harness.handle.detokenize(vec![1], true)); + assert!(futures::poll!(decode.as_mut()).is_pending()); + let decoder = accept_intake(harness.intake_rx.recv().unwrap()); + let a = harness.handle.generate(public_request("a")).await.unwrap(); + let b = harness.handle.generate(public_request("b")).await.unwrap(); + let req_a = accept_intake(harness.intake_rx.recv().unwrap()); + let req_b = accept_intake(harness.intake_rx.recv().unwrap()); + assert!(harness.handle.abort(None)); + let aborted: std::collections::HashSet<_> = (0..2) + .map(|_| harness.abort_rx.try_recv().unwrap().rid().clone()) + .collect(); + assert_eq!(aborted, [req_a.rid, req_b.rid].into_iter().collect()); + decoder + .sink + .try_send(ResponseItem::Data(Bytes::from_static(b"text"))) + .unwrap(); + assert_eq!(decode.await.unwrap(), "text"); + drop((a, b)); + assert!(harness.abort_rx.try_recv().is_err()); + assert!(!harness.handle.abort(None)); + let future = harness + .handle + .generate(public_request("future")) + .await + .unwrap(); + let request = accept_intake(harness.intake_rx.recv().unwrap()); + drop(future); + assert_eq!(harness.abort_rx.try_recv().unwrap().rid(), &request.rid); + } + + #[tokio::test] + async fn abort_is_observed_when_draining_buffered_frames() { + let harness = Harness::unbounded(8); + let mut call = harness + .handle + .generate(public_request("buffered")) + .await + .unwrap(); + let request = accept_intake(harness.intake_rx.recv().unwrap()); + request + .sink + .try_send(ResponseItem::Frame(ChunkEvent::default())) + .unwrap(); + assert!(harness.handle.abort(Some("buffered"))); + let mut events = Vec::new(); + call.drain_ready(&mut events); + assert!(matches!( + events.as_slice(), + [CoreEvent::Failed(CoreError::Cancelled(_))] + )); + assert!(call.recv().await.is_none()); + } } diff --git a/rust/sglang-server/src/api_server/core/prefetch.rs b/rust/sglang-server/src/api_server/core/prefetch.rs index 5875838761..c67472d9c7 100644 --- a/rust/sglang-server/src/api_server/core/prefetch.rs +++ b/rust/sglang-server/src/api_server/core/prefetch.rs @@ -10,7 +10,7 @@ //! [`crate::multi_modality::payload::to_mm_input`] swaps back in. use std::collections::BTreeMap; -use std::sync::Arc; +use std::sync::{Arc, LazyLock}; use bytes::Bytes; use sglang_mm::common::fetch::{ByteBudget, fetch_bytes_budgeted, fetch_local_file_budgeted}; @@ -22,7 +22,7 @@ use crate::multi_modality::payload::io_sources; /// Global bound on concurrent media fetches across all in-flight requests; /// excess acquisitions queue on the semaphore without holding a thread. -static PERMITS: Semaphore = Semaphore::const_new(32); +static PERMITS: LazyLock> = LazyLock::new(|| Arc::new(Semaphore::new(32))); /// Fill [`MmData::prefetched`] for every request, all fetches across the batch /// concurrent. Any failure rejects the call (a 400, as on the Python path). @@ -96,26 +96,43 @@ async fn fetch_ordered(sources: Vec, total_bytes: u64) -> Result(permits: Arc, work: F) -> Result +where + F: FnOnce() -> T + Send + 'static, + T: Send + 'static, +{ + let permit = permits + .acquire_owned() + .await + .expect("semaphore never closed"); + tokio::task::spawn_blocking(move || { + // Keep the permit with the blocking operation: dropping the async + // caller cannot make uncancellable work disappear from the bound. + let _permit = permit; + work() + }) + .await + .map_err(|e| format!("media prefetch: {e}")) +} + #[cfg(test)] mod tests { use super::*; @@ -310,4 +327,40 @@ mod tests { assert_eq!(fetched.len(), 2); assert!(error.contains("request media byte budget"), "{error}"); } + + #[tokio::test] + async fn cancelled_caller_does_not_release_blocking_permit_early() { + let permits = Arc::new(Semaphore::new(1)); + let (started_tx, started_rx) = std::sync::mpsc::sync_channel(0); + let (release_tx, release_rx) = std::sync::mpsc::sync_channel(0); + let task_permits = Arc::clone(&permits); + let task = tokio::spawn(async move { + run_blocking_bounded(task_permits, move || { + started_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + }) + .await + }); + + tokio::task::spawn_blocking(move || started_rx.recv().unwrap()) + .await + .unwrap(); + task.abort(); + tokio::task::yield_now().await; + assert_eq!( + permits.available_permits(), + 0, + "the blocking operation must retain its permit after caller cancellation" + ); + + release_tx.send(()).unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(1), async { + while permits.available_permits() == 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(permits.available_permits(), 1); + } } diff --git a/rust/sglang-server/src/api_server/grpc.rs b/rust/sglang-server/src/api_server/grpc.rs index a1e94e442e..4226d44313 100644 --- a/rust/sglang-server/src/api_server/grpc.rs +++ b/rust/sglang-server/src/api_server/grpc.rs @@ -1,11 +1,12 @@ //! Tonic adapter for the native `sglang.api.v1` service. //! //! The service is the gRPC rendering of the contract the native HTTP endpoints -//! serve: `Generate` is `/generate`, `HealthCheck` is `/health_generate`, and -//! `GetModelInfo` / `GetServerInfo` are their HTTP namesakes. Requests decode -//! through the shared `/generate` fan-out and responses stream through the -//! shared [`CoreHandle`], which owns preprocessing, admission, runtime -//! communication, and request cancellation. +//! serve: `Generate` is `/generate`, `HealthCheck` is `/health_generate`, +//! `ListModels` is `/v1/models`, and `GetModelInfo` / `GetServerInfo` are their +//! HTTP namesakes. The service also adds `Detokenize` and `Abort`. +//! Requests decode through the shared `/generate` fan-out and responses stream +//! through the shared [`CoreHandle`], which owns preprocessing, admission, +//! runtime communication, and request cancellation. //! //! The thin Tonic-service structure, streamed response approach. diff --git a/rust/sglang-server/src/api_server/grpc/info.rs b/rust/sglang-server/src/api_server/grpc/info.rs index d125b6d388..a5da39bc63 100644 --- a/rust/sglang-server/src/api_server/grpc/info.rs +++ b/rust/sglang-server/src/api_server/grpc/info.rs @@ -50,6 +50,11 @@ pub(super) fn server_info(info: ServerInfo) -> Result Result, + ) -> Result, Status> { + Ok(Response::new(api::ListModelsResponse { + models: self.core.list_models(), + })) + } + + async fn detokenize( + &self, + request: Request, + ) -> Result, Status> { + if !self.config.has_tokenizer { + return Err(Status::unimplemented("detokenization requires a tokenizer")); + } + let text = tokio::time::timeout( + self.config.response_timeout, + self.core.detokenize( + request.into_inner().tokens, + true, /* skip_special_tokens */ + ), + ) + .await + .map_err(|_| Status::deadline_exceeded("detokenization timed out"))? + .map_err(native_api::status)?; + Ok(Response::new(api::DetokenizeResponse { text })) + } + + async fn abort( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + if request.abort_all != request.rid.is_empty() { + return Err(Status::invalid_argument( + "provide either a nonempty rid or abort_all=true, not both", + )); + } + Ok(Response::new(api::AbortResponse { + success: self + .core + .abort((!request.abort_all).then_some(request.rid.as_str())), + })) + } } #[cfg(test)] @@ -168,7 +220,7 @@ mod tests { use super::GrpcService; use crate::api_server::core::{CoreConfig, CoreHandle, CoreMetadata}; - use crate::message::config::PreferredSamplingParams; + use crate::message::config::{ModelConfig, PreferredSamplingParams, ServerArgs}; use crate::message::finish_reason::FinishReason; use crate::message::ids::Rid; use crate::message::request::{GenerateRequest, Request as RuntimeRequest, RequestKind}; @@ -211,7 +263,14 @@ mod tests { startup_ready, is_disaggregation: false, mm_limits: Default::default(), - metadata: CoreMetadata::default(), + metadata: CoreMetadata::from(&ServerArgs { + served_model_name: "test-model".into(), + model_config: ModelConfig { + context_len: 4096, + ..Default::default() + }, + ..Default::default() + }), }, ); Self { @@ -719,4 +778,179 @@ mod tests { let probe = stalled.next_generation().await; assert_eq!(probe.request.input_ids.as_deref(), Some(&[0][..])); } + + #[tokio::test] + async fn list_models_uses_static_metadata_without_intake() { + let harness = Harness::new(2, false, Duration::from_secs(1)); + let models = harness + .service + .list_models(Request::new(api::ListModelsRequest {})) + .await + .unwrap() + .into_inner() + .models; + assert_eq!( + models, + vec![api::ModelCard { + id: "test-model".into(), + root: "test-model".into(), + parent: None, + max_model_len: 4096, + }] + ); + assert!(harness.intake_rx.try_recv().is_err()); + } + + #[tokio::test] + async fn detokenize_returns_text_and_maps_invalid_ids() { + let harness = Harness::new(2, false, Duration::from_secs(1)); + for invalid in [false, true] { + let tokens = if invalid { vec![-1] } else { vec![1, 2] }; + let decode = harness + .service + .detokenize(Request::new(api::DetokenizeRequest { + tokens: tokens.clone(), + })); + let runtime = async { + let TmEvent::Intake { request, admission } = + harness.intake_rx.recv_async().await.unwrap() + else { + panic!("expected intake"); + }; + assert!(admission.try_accept()); + assert!( + matches!(&request.kind, RequestKind::Detokenize { token_ids, skip_special_tokens: true } if *token_ids == tokens) + ); + let reply = if invalid { + ResponseItem::Error(crate::utils::error::Error::Validation( + "invalid token ID".into(), + )) + } else { + ResponseItem::Data(bytes::Bytes::from_static("héllo".as_bytes())) + }; + request.sink.try_send(reply).unwrap(); + }; + let (result, ()) = tokio::join!(decode, runtime); + if invalid { + assert_eq!(result.unwrap_err().code(), Code::InvalidArgument); + } else { + assert_eq!(result.unwrap().into_inner().text, "héllo"); + } + assert!(harness.abort_rx.try_recv().is_err()); + } + } + + #[tokio::test] + async fn detokenize_rejects_missing_tokenizer_without_submission() { + let mut harness = Harness::new(2, false, Duration::from_secs(1)); + harness.service.config.has_tokenizer = false; + let error = harness + .service + .detokenize(Request::new(api::DetokenizeRequest { tokens: vec![1] })) + .await + .unwrap_err(); + assert_eq!(error.code(), Code::Unimplemented); + assert!(harness.intake_rx.try_recv().is_err()); + } + + #[tokio::test] + async fn detokenize_timeout_releases_the_internal_call() { + let harness = Harness::new(2, false, Duration::from_millis(10)); + let decode = harness + .service + .detokenize(Request::new(api::DetokenizeRequest { tokens: vec![1] })); + let runtime = async { + let TmEvent::Intake { request, admission } = + harness.intake_rx.recv_async().await.unwrap() + else { + panic!("expected intake"); + }; + assert!(admission.try_accept()); + let aborted = harness.abort_rx.recv_async().await.unwrap(); + assert_eq!(aborted.rid(), &request.rid); + }; + let (result, ()) = tokio::join!(decode, runtime); + assert_eq!(result.unwrap_err().code(), Code::DeadlineExceeded); + assert!(harness.abort_rx.try_recv().is_err()); + } + + #[tokio::test] + async fn abort_rpc_cancels_generation_with_an_in_stream_error() { + let harness = Harness::new(2, false, Duration::from_secs(1)); + let mut stream = harness + .service + .generate(Request::new(ids_request(&[1], Some("abort-me")))) + .await + .unwrap() + .into_inner(); + let intake = harness.next_generation().await; + assert!(intake.admission.try_accept()); + let reply = harness + .service + .abort(Request::new(api::AbortRequest { + rid: "abort-me".into(), + abort_all: false, + })) + .await + .unwrap() + .into_inner(); + assert!(reply.success); + assert_eq!(harness.abort_rx.try_recv().unwrap().rid(), &intake.rid); + let error = stream_error(stream.next().await.unwrap().unwrap()); + assert_eq!(error.error.unwrap().code, 499); + assert!(stream.next().await.is_none()); + assert!( + !harness + .service + .abort(Request::new(api::AbortRequest { + rid: "abort-me".into(), + abort_all: false, + })) + .await + .unwrap() + .into_inner() + .success + ); + drop(stream); + assert!(harness.abort_rx.try_recv().is_err()); + } + + #[tokio::test] + async fn abort_validates_selector_and_supports_all() { + let harness = Harness::new(2, false, Duration::from_secs(1)); + for (rid, abort_all) in [("", false), ("ambiguous", true)] { + let error = harness + .service + .abort(Request::new(api::AbortRequest { + rid: rid.into(), + abort_all, + })) + .await + .unwrap_err(); + assert_eq!(error.code(), Code::InvalidArgument); + } + let stream = harness + .service + .generate(Request::new(ids_request(&[1], Some("all")))) + .await + .unwrap() + .into_inner(); + let intake = harness.next_generation().await; + assert!(intake.admission.try_accept()); + assert!( + harness + .service + .abort(Request::new(api::AbortRequest { + rid: String::new(), + abort_all: true, + })) + .await + .unwrap() + .into_inner() + .success + ); + assert_eq!(harness.abort_rx.try_recv().unwrap().rid(), &intake.rid); + drop(stream); + assert!(harness.abort_rx.try_recv().is_err()); + } } diff --git a/rust/sglang-server/src/api_server/http/app.rs b/rust/sglang-server/src/api_server/http/app.rs index 2ef6438197..c0f81e431f 100644 --- a/rust/sglang-server/src/api_server/http/app.rs +++ b/rust/sglang-server/src/api_server/http/app.rs @@ -13,6 +13,7 @@ use crate::api_server::core::CoreHandle; use crate::api_server::disaggregation::bootstrap as pd_bootstrap; use crate::api_server::log; use crate::message::config::ServerArgs; +use crate::message::request::GenerationPolicy; use crate::utils::environ; /// HTTP adapter state: the shared core capability, immutable server @@ -25,6 +26,7 @@ use crate::utils::environ; pub(super) struct AppState { pub(super) core: CoreHandle, pub(super) server_args: Arc, + pub(super) generation_policy: GenerationPolicy, pub(super) chat_formatter: Option, } @@ -69,9 +71,11 @@ pub async fn serve( shutdown: flume::Receiver<()>, ) { let chat_formatter = openai::load_chat_support(&server_args); + let generation_policy = GenerationPolicy::from(server_args.as_ref()); let state = Arc::new(AppState { core, server_args: server_args.clone(), + generation_policy, chat_formatter, }); // Cancelled with the api runtime on shutdown. diff --git a/rust/sglang-server/src/api_server/http/info.rs b/rust/sglang-server/src/api_server/http/info.rs index 33497f3716..fb7c0e43f3 100644 --- a/rust/sglang-server/src/api_server/http/info.rs +++ b/rust/sglang-server/src/api_server/http/info.rs @@ -308,6 +308,9 @@ mod tests { ( Arc::new(AppState { core, + generation_policy: crate::message::request::GenerationPolicy::from( + server_args.as_ref(), + ), server_args, chat_formatter: None, }), diff --git a/rust/sglang-server/src/api_server/http/loads.rs b/rust/sglang-server/src/api_server/http/loads.rs index 448730faf7..795927d330 100644 --- a/rust/sglang-server/src/api_server/http/loads.rs +++ b/rust/sglang-server/src/api_server/http/loads.rs @@ -476,6 +476,7 @@ mod tests { metadata: crate::api_server::core::CoreMetadata::default(), }, ), + generation_policy: crate::message::request::GenerationPolicy::from(&server_args), server_args: Arc::new(server_args), chat_formatter: None, }); diff --git a/rust/sglang-server/src/api_server/http/metrics.rs b/rust/sglang-server/src/api_server/http/metrics.rs index 01e810e4b5..0bd34a1158 100644 --- a/rust/sglang-server/src/api_server/http/metrics.rs +++ b/rust/sglang-server/src/api_server/http/metrics.rs @@ -81,6 +81,9 @@ mod tests { metadata: Default::default(), }, ), + generation_policy: crate::message::request::GenerationPolicy::from( + server_args.as_ref(), + ), server_args, chat_formatter: None, }); diff --git a/rust/sglang-server/src/api_server/http/native_api.rs b/rust/sglang-server/src/api_server/http/native_api.rs index e6e01e38f1..9710750e69 100644 --- a/rust/sglang-server/src/api_server/http/native_api.rs +++ b/rust/sglang-server/src/api_server/http/native_api.rs @@ -17,6 +17,8 @@ use crate::api_server::core::{ }; #[cfg(test)] use crate::message::ids::Rid; +#[cfg(test)] +use crate::message::request::GenerationPolicy; use crate::message::request::{GenerateRequest, into_requests}; use crate::message::wire::merge_preferred_sampling; use crate::utils::{ @@ -211,7 +213,7 @@ async fn generate( let stream = body.stream.unwrap_or(false); // Fan `text`/`input_ids`/`sampling_params` (scalar or list) into per-request // payloads. `is_batch` = list form -> the response is a JSON array. - let (payloads, is_batch) = match into_requests(body) { + let (payloads, is_batch) = match into_requests(body, &state.generation_policy) { Ok(v) => v, // Request normalization only returns validation failures. Err(e) => { @@ -571,6 +573,7 @@ mod tests { }, ), server_args: Arc::new(crate::message::config::ServerArgs::default()), + generation_policy: GenerationPolicy::default(), chat_formatter: None, }); @@ -609,6 +612,7 @@ mod tests { metadata: crate::api_server::core::CoreMetadata::default(), }, ), + generation_policy: crate::message::request::GenerationPolicy::from(&server_args), server_args: Arc::new(server_args), chat_formatter: None, }); @@ -648,6 +652,7 @@ mod tests { let state = Arc::new(AppState { core, server_args: Arc::new(crate::message::config::ServerArgs::default()), + generation_policy: GenerationPolicy::default(), chat_formatter: None, }); diff --git a/rust/sglang-server/src/api_server/http/openai/chat.rs b/rust/sglang-server/src/api_server/http/openai/chat.rs index 08e0abff68..e6410f09b0 100644 --- a/rust/sglang-server/src/api_server/http/openai/chat.rs +++ b/rust/sglang-server/src/api_server/http/openai/chat.rs @@ -1254,6 +1254,7 @@ mod tests { .unwrap(); let state = Arc::new(super::super::AppState { core, + generation_policy: crate::message::request::GenerationPolicy::from(args.as_ref()), server_args: args, chat_formatter: Some(super::super::ChatFormatter::HuggingFace(formatter)), }); diff --git a/rust/sglang-server/src/api_server/http/openai/models.rs b/rust/sglang-server/src/api_server/http/openai/models.rs index 124366c07d..d41d94db1a 100644 --- a/rust/sglang-server/src/api_server/http/openai/models.rs +++ b/rust/sglang-server/src/api_server/http/openai/models.rs @@ -9,6 +9,8 @@ use axum::{ }; use std::sync::Arc; +use sglang_api_types::api::v1 as api; + use super::{AppState, openai_error, unix_seconds_u32}; pub(super) fn routes() -> Router> { @@ -17,33 +19,42 @@ pub(super) fn routes() -> Router> { .route("/v1/models/{model}", get(retrieve_model)) } -/// `GET /v1/models` — OpenAI-compatible model list. Served from `server_args`; +/// `GET /v1/models` — OpenAI-compatible model list. Served from core metadata; /// no scheduler round-trip. async fn available_models(State(state): State>) -> Response { - let base = model_card(&state); - Json(serde_json::json!({ "object": "list", "data": [base] })).into_response() + let models: Vec<_> = state + .core + .list_models() + .into_iter() + .map(model_card) + .collect(); + Json(serde_json::json!({ "object": "list", "data": models })).into_response() } async fn retrieve_model(State(state): State>, Path(model): Path) -> Response { - if model != state.server_args.served_model_name { + let Some(card) = state + .core + .list_models() + .into_iter() + .find(|card| card.id == model) + else { return openai_error( StatusCode::NOT_FOUND, format!("The model `{model}` does not exist"), false, ); - } - Json(model_card(&state)).into_response() + }; + Json(model_card(card)).into_response() } -fn model_card(state: &AppState) -> serde_json::Value { - let name = &state.server_args.served_model_name; +fn model_card(card: api::ModelCard) -> serde_json::Value { serde_json::json!({ - "id": name, + "id": card.id, "object": "model", "created": unix_seconds_u32(), "owned_by": "sglang", - "root": name, - "parent": serde_json::Value::Null, - "max_model_len": state.server_args.model_config.context_len, + "root": card.root, + "parent": card.parent, + "max_model_len": card.max_model_len, }) } diff --git a/rust/sglang-server/src/api_server/http/openai/test_utils.rs b/rust/sglang-server/src/api_server/http/openai/test_utils.rs index f91abd6b5e..d0a9a375ae 100644 --- a/rust/sglang-server/src/api_server/http/openai/test_utils.rs +++ b/rust/sglang-server/src/api_server/http/openai/test_utils.rs @@ -106,9 +106,11 @@ pub(super) fn server_args() -> Arc { } pub(super) fn app_state(core: CoreHandle) -> Arc { + let server_args = server_args(); Arc::new(super::AppState { core, - server_args: server_args(), + generation_policy: crate::message::request::GenerationPolicy::from(server_args.as_ref()), + server_args, chat_formatter: None, }) } @@ -157,9 +159,11 @@ fn answering_app_state() -> Arc { metadata: crate::api_server::core::CoreMetadata::from(server_args().as_ref()), }, ); + let server_args = server_args(); Arc::new(super::AppState { core, - server_args: server_args(), + generation_policy: crate::message::request::GenerationPolicy::from(server_args.as_ref()), + server_args, chat_formatter: Some(super::ChatFormatter::Legacy(Box::new( super::template::LegacyFormatter { spec: super::template::builtin_template("chatml").unwrap(), diff --git a/rust/sglang-server/src/api_server/http/openai/tokenize.rs b/rust/sglang-server/src/api_server/http/openai/tokenize.rs index b7a6457087..810de5dd47 100644 --- a/rust/sglang-server/src/api_server/http/openai/tokenize.rs +++ b/rust/sglang-server/src/api_server/http/openai/tokenize.rs @@ -273,6 +273,9 @@ mod tests { metadata: CoreMetadata::from(server_args.as_ref()), }, ), + generation_policy: crate::message::request::GenerationPolicy::from( + server_args.as_ref(), + ), server_args, chat_formatter: Some( load_chat_formatter( diff --git a/rust/sglang-server/src/api_server/http/request_headers.rs b/rust/sglang-server/src/api_server/http/request_headers.rs index dd596abc42..efaed3dcd1 100644 --- a/rust/sglang-server/src/api_server/http/request_headers.rs +++ b/rust/sglang-server/src/api_server/http/request_headers.rs @@ -57,7 +57,7 @@ pub(super) fn apply_header_overrides(body: &mut Value, headers: &HeaderMap) -> R #[cfg(test)] mod tests { use super::*; - use crate::message::request::into_requests; + use crate::message::request::{GenerationPolicy, into_requests}; fn headers(pairs: &[(&'static str, &str)]) -> HeaderMap { pairs @@ -95,7 +95,11 @@ mod tests { "disagg_prefill_dp_rank": 1, "priority": -2, }) ); - let (requests, _) = into_requests(serde_json::from_value(body.clone()).unwrap()).unwrap(); + let (requests, _) = into_requests( + serde_json::from_value(body.clone()).unwrap(), + &GenerationPolicy::default(), + ) + .unwrap(); let req = &requests[0]; assert_eq!(req.rid.client_facing(), "router"); assert_eq!(req.bootstrap_host.as_deref(), Some("10.0.0.1")); diff --git a/rust/sglang-server/src/lib.rs b/rust/sglang-server/src/lib.rs index 254a952322..0f1246fa36 100644 --- a/rust/sglang-server/src/lib.rs +++ b/rust/sglang-server/src/lib.rs @@ -38,7 +38,7 @@ use pyo3::pybacked::PyBackedBytes; use pyo3::types::PyBytes; use crate::message::config::RuntimeConfig; -use crate::utils::startup::{grpc_listen_addr, listen_addr, value_error}; +use crate::utils::startup::{advertise_listener_ports, grpc_listen_addr, listen_addr, value_error}; use crate::utils::{logging, runtime, shm::ShmSegment}; /// One drained request handed to Python by [`Server::recv_requests`]: the @@ -304,6 +304,7 @@ impl Server { .map_err(|e| value_error("bad listen address", e))?; let grpc_addr = grpc_listen_addr(&server_args, port_offset) .map_err(|e| value_error("bad gRPC listen address", e))?; + let server_args = advertise_listener_ports(server_args, http_addr, grpc_addr); let cfg = RuntimeConfig { rust_server_args: RustServerServerArgs { diff --git a/rust/sglang-server/src/message/config.rs b/rust/sglang-server/src/message/config.rs index f6d6e2aec6..3e5e336dae 100644 --- a/rust/sglang-server/src/message/config.rs +++ b/rust/sglang-server/src/message/config.rs @@ -22,7 +22,7 @@ use std::collections::BTreeMap; use std::net::SocketAddr; use std::sync::Arc; -use serde::Serialize; +use serde::{Deserialize, Serialize}; /// Boot knobs specific to the embedded rust server — none of these exist in /// the Python-built [`ServerArgs`]; they arrive as explicit @@ -156,6 +156,9 @@ pub struct ServerArgs { /// Launch-time sampling defaults merged beneath per-request values and /// advertised by `/get_model_info`. pub preferred_sampling_params: Option, + /// Native-generation launch policy normally applied by Python ingress. + pub lora_ids: BTreeMap, + pub enable_strict_thinking: bool, /// Per-modality media-count limits from `--limit-mm-data-per-request`. pub limit_mm_data_per_request: BTreeMap, /// Over-long inputs are truncated to fit the context instead of 400ing, and @@ -178,6 +181,8 @@ pub struct ServerArgs { /// and the scheduler-derived KV token capacity, reported by `/server_info`. pub version: String, pub max_total_num_tokens: u64, + /// Resolved deployment metadata used by out-of-process orchestrators. + pub deployment_info: DeploymentInfo, /// Python's `/server_info` fields that are fixed at launch, which /// `/server_info` reports beneath its runtime fields. pub launch_server_info: LaunchServerInfo, @@ -224,6 +229,8 @@ impl ServerArgs { disaggregation_mode, model_config, preferred_sampling_params, + lora_ids, + enable_strict_thinking, limit_mm_data_per_request, allow_auto_truncate, enable_return_hidden_states, @@ -231,6 +238,7 @@ impl ServerArgs { tokenizer_model_max_length, version, max_total_num_tokens, + deployment_info, launch_server_info, load_snapshot_path, load_snapshot_dp_size, @@ -268,6 +276,8 @@ impl ServerArgs { disaggregation_mode: DisaggregationMode, model_config: ModelConfig, preferred_sampling_params: Option, + lora_ids: BTreeMap, + enable_strict_thinking: bool, limit_mm_data_per_request: BTreeMap, allow_auto_truncate: bool, enable_return_hidden_states: bool, @@ -275,6 +285,7 @@ impl ServerArgs { tokenizer_model_max_length: i64, version: String, max_total_num_tokens: u64, + deployment_info: DeploymentInfo, launch_server_info: LaunchServerInfo, load_snapshot_path: Option, load_snapshot_dp_size: usize, @@ -310,6 +321,8 @@ impl ServerArgs { disaggregation_mode, model_config, preferred_sampling_params, + lora_ids, + enable_strict_thinking, limit_mm_data_per_request, allow_auto_truncate, enable_return_hidden_states, @@ -317,6 +330,7 @@ impl ServerArgs { tokenizer_model_max_length, version, max_total_num_tokens, + deployment_info, launch_server_info, load_snapshot_path, load_snapshot_dp_size, @@ -360,6 +374,8 @@ impl Default for ServerArgs { disaggregation_mode: DisaggregationMode::Null, model_config: ModelConfig::default(), preferred_sampling_params: None, + lora_ids: BTreeMap::new(), + enable_strict_thinking: false, limit_mm_data_per_request: BTreeMap::new(), allow_auto_truncate: false, enable_return_hidden_states: false, @@ -367,6 +383,7 @@ impl Default for ServerArgs { tokenizer_model_max_length: -1, version: String::new(), max_total_num_tokens: 0, + deployment_info: DeploymentInfo::default(), launch_server_info: LaunchServerInfo::default(), load_snapshot_path: None, load_snapshot_dp_size: 0, @@ -378,6 +395,51 @@ impl Default for ServerArgs { } } +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +#[serde(default)] +pub struct DeploymentInfo { + pub disaggregation_bootstrap_port: Option, + pub node_rank: u32, + pub nnodes: u32, + pub dp_size: u32, + pub dp_rank: Option, + pub dist_init_addr: Option, + pub base_gpu_id: u32, + pub page_size: Option, + pub dcp_size: u32, + pub max_running_requests: Option, + pub max_prefill_tokens: u64, + pub enable_dp_attention: bool, + pub speculative_algorithm: Option, + pub enable_hierarchical_cache: bool, + pub hicache_ratio: Option, + pub hicache_size: u64, + pub hicache_write_policy: Option, + pub hicache_host_total_tokens: Option, + pub kv_event_sources: Vec, +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +pub struct KvEventSource { + pub dp_rank: u32, + pub endpoint: String, + pub topic: String, + pub block_size: u32, +} + +impl<'py> pyo3::FromPyObject<'_, 'py> for DeploymentInfo { + type Error = pyo3::PyErr; + + fn extract(obj: pyo3::Borrowed<'_, 'py, pyo3::PyAny>) -> pyo3::PyResult { + let text = obj.extract::()?; + serde_json::from_str(&text).map_err(|e| { + pyo3::exceptions::PyValueError::new_err(format!( + "deployment_info is not valid JSON: {e}" + )) + }) + } +} + /// Prometheus metrics, as Python serves them with `--enable-metrics`. #[pyo3::pyclass(frozen, from_py_object, module = "sglang.srt.rust_extensions._server")] #[derive(Clone, Debug)] diff --git a/rust/sglang-server/src/message/ids.rs b/rust/sglang-server/src/message/ids.rs index 7d26846c10..7e9a92dd9d 100644 --- a/rust/sglang-server/src/message/ids.rs +++ b/rust/sglang-server/src/message/ids.rs @@ -74,8 +74,9 @@ impl Rid { /// so the second would evict the first's sink, 500 that client mid-generation /// and deliver its remaining chunks to the second's connection. Uniquifying /// here makes the collision unrepresentable rather than something a duplicate - /// check has to catch — no in-flight registry, no admission/release ordering - /// to get wrong, and both clients get served instead of the second being 400'd. + /// check has to catch. The core's explicit-cancellation index groups public + /// IDs but does not assign identity: both clients get served instead of the + /// second being 400'd, and an old internal abort cannot affect an ID reuse. /// /// The client never sees this: [`client_facing`](Self::client_facing) strips it /// back off for `meta_info.id`. Only the scheduler wire and its logs carry the diff --git a/rust/sglang-server/src/message/info.rs b/rust/sglang-server/src/message/info.rs index 1031989f86..36c9b10d6f 100644 --- a/rust/sglang-server/src/message/info.rs +++ b/rust/sglang-server/src/message/info.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use serde::{Deserialize, Serialize}; -use super::config::{DisaggregationMode, PreferredSamplingParams}; +use super::config::{DeploymentInfo, DisaggregationMode, PreferredSamplingParams}; /// Static model metadata shared by every public transport. #[derive(Clone, Debug, Serialize)] @@ -42,6 +42,11 @@ pub(crate) struct ServerInfo { #[serde(skip_serializing_if = "Option::is_none")] pub(crate) startup_time: Option, pub(crate) internal_states: Vec, + pub(crate) host: String, + pub(crate) port: u16, + pub(crate) incremental_streaming_output: bool, + pub(crate) disaggregation_mode: DisaggregationMode, + pub(crate) deployment_info: DeploymentInfo, } impl ServerInfo { diff --git a/rust/sglang-server/src/message/io_struct.rs b/rust/sglang-server/src/message/io_struct.rs index 969a516216..2cea897341 100644 --- a/rust/sglang-server/src/message/io_struct.rs +++ b/rust/sglang-server/src/message/io_struct.rs @@ -40,12 +40,11 @@ wire_struct! { return_indexer_topk: bool, session_id: (), session_params: (), - lora_id: (), + lora_id: Option<&'a str>, custom_logit_processor: (), positional_embed_overrides: (), - /// PD-disaggregation block. Everything after `cache_salt` (the last - /// field emitted) has a msgspec default in Python and is omitted - /// (short arrays decode with defaulted tails). + /// PD-disaggregation block. Everything after `kv_hints` (the last + /// field emitted) has a msgspec default in Python and is omitted. bootstrap_host: Option<&'a str>, bootstrap_port: Option, bootstrap_room: Option, @@ -53,16 +52,10 @@ wire_struct! { decode_tp_size: Option, routed_dp_rank: Option, disagg_prefill_dp_rank: Option, - /// The scheduler decodes this header by position (Python's - /// `TokenizedGenerateReqInput` is a msgspec `array_like` struct), so - /// `routing_key` is sent at its default for the fields after it to land - /// in their slots. - routing_key: (), + routing_key: Option<&'a str>, require_reasoning: bool, priority: Option, extra_key: Option<&'a str>, - /// Defaults, so `cache_salt`, the prefix cache's second key, lands in - /// its slot. no_logs: bool, return_bytes: bool, return_entropy: bool, @@ -73,6 +66,7 @@ wire_struct! { token_indices_to_pool: (), time_stats: (), cache_salt: Option<&'a str>, + kv_hints: Option<&'a super::request::KvHintsEnvelope>, } } @@ -139,7 +133,7 @@ impl<'a> From<&'a GenerateRequest> for TokenizedGenerateReqInput<'a> { return_indexer_topk: false, session_id: (), session_params: (), - lora_id: (), + lora_id: req.lora_id.as_deref(), custom_logit_processor: (), positional_embed_overrides: (), bootstrap_host: req.bootstrap_host.as_deref(), @@ -149,7 +143,7 @@ impl<'a> From<&'a GenerateRequest> for TokenizedGenerateReqInput<'a> { decode_tp_size: req.decode_tp_size, routed_dp_rank: req.routed_dp_rank, disagg_prefill_dp_rank: req.disagg_prefill_dp_rank, - routing_key: (), + routing_key: req.routing_key.as_deref(), require_reasoning: req.require_reasoning, priority: req.priority, extra_key: req.extra_key.as_deref(), @@ -163,6 +157,7 @@ impl<'a> From<&'a GenerateRequest> for TokenizedGenerateReqInput<'a> { token_indices_to_pool: (), time_stats: (), cache_salt: req.cache_salt.as_deref(), + kv_hints: req.kv_hints.as_ref(), } } } @@ -299,9 +294,8 @@ mod tests { let bytes = TokenizedGenerateReqInput::from(&req).encode().unwrap(); let val = rmpv::decode::read_value(&mut &bytes[..]).unwrap(); let arr = val.as_array().expect("array"); - // msgspec requires >= 14 (through `stream`); we emit 46 (through - // `cache_salt`). Trailing defaulted fields are omitted. - assert_eq!(arr.len(), 46, "header ends at cache_salt"); + // The orchestrator controls extend the array through `kv_hints`. + assert_eq!(arr.len(), 47, "header ends at kv_hints"); assert_eq!(arr[0].as_str(), Some("TokenizedGenerateReqInput")); assert_eq!(arr[1].as_str(), Some("r1")); assert!(arr[5].is_nil(), "idx 5 must be input_embeds (nil)"); @@ -331,6 +325,49 @@ mod tests { ); } + /// Python's `KvHintsEnvelope` / `KvHintAction` are dict-shaped msgspec + /// structs, so a positional array at the kv_hints slot fails to decode. + #[test] + fn header_kv_hints_are_map_encoded() { + use sglang_api_types::api::v1 as api; + let kv_hints = crate::message::request::KvHintsEnvelope::try_from(api::KvHintsEnvelope { + protocol_version: "1".into(), + message_id: "m".into(), + actions: vec![api::KvHintAction { + action_id: "a".into(), + action_type: "pin".into(), + action_version: "1".into(), + payload: Some(r#"{"blocks":[1,2]}"#.into()), + }], + }) + .unwrap(); + let req = GenerateRequest { + rid: "r1".into(), + input_ids: Some(vec![1]), + kv_hints: Some(kv_hints), + ..Default::default() + }; + let bytes = TokenizedGenerateReqInput::from(&req).encode().unwrap(); + let val = rmpv::decode::read_value(&mut &bytes[..]).unwrap(); + let envelope = val.as_array().expect("array")[46] + .as_map() + .expect("kv_hints must be a map"); + let get = |map: &[(rmpv::Value, rmpv::Value)], key: &str| { + map.iter() + .find(|(k, _)| k.as_str() == Some(key)) + .map(|(_, v)| v.clone()) + .unwrap_or_else(|| panic!("missing key {key}")) + }; + assert_eq!(get(envelope, "protocol_version").as_str(), Some("1")); + assert_eq!(get(envelope, "message_id").as_str(), Some("m")); + let actions = get(envelope, "actions"); + let action = actions.as_array().unwrap()[0] + .as_map() + .expect("kv_hints action must be a map"); + assert_eq!(get(action, "action_type").as_str(), Some("pin")); + assert!(get(action, "payload").is_map()); + } + /// The PD block must land on Python's wire indices 25–31, with the filler /// block (17–24) holding its defaults — a shift here silently routes KV /// transfers to the wrong host/room. The cache keys and `require_reasoning` diff --git a/rust/sglang-server/src/message/request.rs b/rust/sglang-server/src/message/request.rs index 24e25293a0..110a8548b5 100644 --- a/rust/sglang-server/src/message/request.rs +++ b/rust/sglang-server/src/message/request.rs @@ -1,7 +1,7 @@ //! The `/generate` request path: the HTTP body and its per-request fan-out //! ([`GenerateBody`] → [`GenerateRequest`]s). -use std::collections::HashSet; +use std::collections::{BTreeMap, HashSet}; use std::sync::LazyLock; use bytes::Bytes; @@ -14,6 +14,7 @@ use super::response::ResponseSink; use super::sampling::SamplingParams; use super::types::{OneOrMany, OneOrManyItem, TokenIds}; use super::wire; +use crate::message::config::ServerArgs; use crate::message::ids::Rid; use crate::utils::fsm::RequestState; use crate::utils::{environ::env_i64, error::Error}; @@ -55,7 +56,25 @@ const JSON_TO_HEAP_FACTOR: usize = 8; /// The wire type is the generated `sglang_api_types` request (the schema is /// the contract); its carriers become the internal ones through `message::wire` /// on the way in. -pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec, bool), Error> { +#[derive(Clone, Debug, Default)] +pub struct GenerationPolicy { + lora_ids: BTreeMap, + enable_strict_thinking: bool, +} + +impl From<&ServerArgs> for GenerationPolicy { + fn from(args: &ServerArgs) -> Self { + Self { + lora_ids: args.lora_ids.clone(), + enable_strict_thinking: args.enable_strict_thinking, + } + } +} + +pub fn into_requests( + req: api_v1::GenerateRequest, + policy: &GenerationPolicy, +) -> Result<(Vec, bool), Error> { let api_v1::GenerateRequest { rid, text, @@ -79,22 +98,20 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec Result<(Vec Result<(Vec, _>>()?; + let kv_hints = kv_hints_column(kv_hints, n)?; let mut requests: Vec = izip!( rids, texts, @@ -352,6 +385,8 @@ pub fn into_requests(req: api_v1::GenerateRequest) -> Result<(Vec Result<(Vec, /// The prefix cache's second namespace key. pub cache_salt: Option, - /// Scheduling priority, shared by every item of a batch. + /// Scheduler/orchestrator controls forwarded by the native API. `lora_id` + /// accepts the preloaded adapter identifier; dynamic LoRA loading remains + /// owned by the Python registry and is intentionally not implied here. + pub lora_id: Option, + pub routing_key: Option, pub priority: Option, - /// Grammar constraints start after the reasoning section. pub require_reasoning: bool, + pub kv_hints: Option, /// Multimodal inputs. Consumed by the Encoding stage, which ships them to /// the MM worker pool; never read by the tokenizer or serialized onto the /// scheduler header. Boxed so the common text-only request doesn't grow @@ -577,6 +633,81 @@ pub struct GenerateRequest { pub mm_buffers: Vec, } +#[derive(Debug, Clone)] +pub struct KvHintsEnvelope { + protocol_version: String, + message_id: String, + actions: Vec, +} + +#[derive(Debug, Clone)] +struct KvHintAction { + action_id: String, + action_type: String, + action_version: String, + payload: serde_json::Value, +} + +// Python's kv_hints structs are not `array_like`, so they must reach msgspec as +// maps; a derived Serialize goes through rmp_serde's struct-as-array encoding. +impl serde::Serialize for KvHintsEnvelope { + fn serialize(&self, serializer: S) -> Result { + use serde::ser::SerializeMap; // codespell:ignore ser + let mut map = serializer.serialize_map(Some(3))?; + map.serialize_entry("protocol_version", &self.protocol_version)?; + map.serialize_entry("message_id", &self.message_id)?; + map.serialize_entry("actions", &self.actions)?; + map.end() + } +} + +impl serde::Serialize for KvHintAction { + fn serialize(&self, serializer: S) -> Result { + use serde::ser::SerializeMap; // codespell:ignore ser + let mut map = serializer.serialize_map(Some(4))?; + map.serialize_entry("action_id", &self.action_id)?; + map.serialize_entry("action_type", &self.action_type)?; + map.serialize_entry("action_version", &self.action_version)?; + map.serialize_entry("payload", &self.payload)?; + map.end() + } +} + +impl TryFrom for KvHintsEnvelope { + type Error = Error; + + fn try_from(value: api_v1::KvHintsEnvelope) -> Result { + let actions = value + .actions + .into_iter() + .map(|action| { + let payload: serde_json::Value = serde_json::from_str( + action.payload.as_deref().unwrap_or("{}"), + ) + .map_err(|error| { + Error::Validation(format!("kv_hints action payload is not JSON: {error}")) + })?; + if !payload.is_object() { + return Err(Error::Validation( + "kv_hints action payload must be a JSON object".into(), + )); + } + Ok(KvHintAction { + action_id: action.action_id, + action_type: action.action_type, + action_version: action.action_version, + payload, + }) + }) + .collect::, Error>>()?; + Ok(Self { + protocol_version: value.protocol_version, + message_id: value.message_id, + actions, + }) + } +} + /// The multimodal fields of one request (see [`GenerateRequest::mm`]), each /// modality already fanned out to this request's own item list. Also the MM /// processor's input: the MM worker moves it out of the request whole, and @@ -690,6 +821,56 @@ pub(crate) fn check_broadcast_budget(per_clone: usize, n: usize, name: &str) -> Ok(()) } +/// Map a prompt's adapter name to its launch-time LoRA id. A missing or empty +/// name runs the base model, as Python treats a falsy `lora_path`. +fn resolve_lora_id( + name: Option, + policy: &GenerationPolicy, +) -> Result, Error> { + match name.as_deref() { + None | Some("") => Ok(None), + Some(name) => policy.lora_ids.get(name).cloned().map(Some).ok_or_else(|| { + Error::Validation(format!( + "LoRA adapter '{name}' is not loaded; loaded adapters: {:?}", + policy.lora_ids.keys().collect::>() + )) + }), + } +} + +/// Per-prompt `kv_hints`, following Python `_normalize_kv_hints`: one envelope +/// applies to every prompt, a list carries one nullable envelope per prompt. +fn kv_hints_column( + value: Option, + n: usize, +) -> Result>, Error> { + use api_v1::kv_hints_envelope_or_list::Value; + match value.and_then(|v| v.value) { + None => Ok(vec![None; n]), + Some(Value::One(envelope)) => { + let envelope = KvHintsEnvelope::try_from(envelope)?; + let per_clone = serde_json::to_vec(&envelope) + .expect("KvHintsEnvelope contains only JSON-serializable values") + .len() + .saturating_mul(JSON_TO_HEAP_FACTOR); + check_broadcast_budget(per_clone, n, "kv_hints")?; + Ok(vec![Some(envelope); n]) + } + Some(Value::Many(list)) => { + if list.items.len() != n { + return Err(Error::Validation(format!( + "kv_hints list length {} does not match batch size {n}", + list.items.len() + ))); + } + list.items + .into_iter() + .map(|item| item.envelope.map(KvHintsEnvelope::try_from).transpose()) + .collect() + } + } +} + fn fan_out( value: Option>, n: usize, @@ -760,7 +941,80 @@ mod tests { const TEST_VOCAB: u64 = 1000; fn requests(body: &str) -> Result<(Vec, bool), Error> { - into_requests(serde_json::from_str::(body).unwrap()) + requests_with_policy(body, &GenerationPolicy::default()) + } + + fn requests_with_policy( + body: &str, + policy: &GenerationPolicy, + ) -> Result<(Vec, bool), Error> { + into_requests( + serde_json::from_str::(body).unwrap(), + policy, + ) + } + + #[test] + fn launch_policy_is_applied_before_scheduler_submission() { + let mut lora_ids = BTreeMap::new(); + lora_ids.insert("adapter".into(), "stable-id".into()); + let policy = GenerationPolicy { + lora_ids, + enable_strict_thinking: true, + }; + let (requests, _) = requests_with_policy( + r#"{"text":"hi","lora_path":"adapter","max_thinking_tokens":23}"#, + &policy, + ) + .unwrap(); + assert_eq!(requests[0].lora_id.as_deref(), Some("stable-id")); + assert!(matches!( + requests[0] + .sampling_params + .custom_params + .as_ref() + .and_then(|params| params.get("thinking_budget")), + Some(super::super::sampling::CustomParamValue::Unsigned(23)) + )); + + let error = + requests_with_policy(r#"{"text":"hi","lora_path":"missing"}"#, &policy).unwrap_err(); + assert!(error.to_string().contains("not loaded"), "{error}"); + } + + #[test] + fn max_thinking_tokens_requires_strict_thinking() { + let error = requests(r#"{"text":"hi","max_thinking_tokens":23}"#).unwrap_err(); + assert!( + error.to_string().contains("--enable-strict-thinking"), + "{error}" + ); + } + + #[test] + fn orchestrator_scalar_broadcasts_are_bounded() { + let prompts = vec!["x"; 4096]; + let routing_key = "k".repeat(MAX_BROADCAST_CLONE_BYTES / prompts.len() + 1); + let body = serde_json::json!({"text": prompts, "routing_key": routing_key}); + let error = requests(&body.to_string()).unwrap_err(); + assert!(error.to_string().contains("routing_key"), "{error}"); + + let payload = "v".repeat(MAX_BROADCAST_CLONE_BYTES / 8 / 4096 + 1); + let body = serde_json::json!({ + "text": vec!["x"; 4096], + "kv_hints": { + "protocol_version": "1", + "message_id": "m", + "actions": [{ + "action_id": "a", + "action_type": "t", + "action_version": "1", + "payload": serde_json::json!({"value": payload}).to_string() + }] + } + }); + let error = requests(&body.to_string()).unwrap_err(); + assert!(error.to_string().contains("kv_hints"), "{error}"); } /// Scalar `text` → one item, not a batch (response stays a single object). @@ -858,18 +1112,18 @@ mod tests { assert!(ps[0].sampling_params.normalize(false, TEST_VOCAB).is_err()); } - /// Unported `GenerateReqInput` fields are IGNORED, not rejected. + /// Remaining unported `GenerateReqInput` fields are IGNORED, not rejected. /// /// These are all real fields on Python's `GenerateReqInput` that this server /// has not ported. `deny_unknown_fields` turned every one of them into a 400, /// so a client that worked against the Python server broke here — and the - /// wire-compat fields (`lora_path`, `image_data`, `return_routed_experts`) had - /// to be declared and dropped by hand just to let `bench_serving` through. + /// wire-compat fields (`image_data`, `return_routed_experts`) had to be + /// declared and dropped by hand just to let `bench_serving` through. /// FastAPI's pydantic dataclass drops extras, so ignoring them is the parity /// behavior; a typo being silently ignored is the same trade Python makes. - /// The exceptions are `lora_path` and `session_params`: dropping them would - /// silently change the answer, so a non-null value is a 400 (as Python's is - /// on a server without LoRA or without that session), and `null` passes. + /// The exception is `session_params`: dropping it would silently change the + /// answer, so a non-null value is a 400 (as Python's is without that + /// session), and `null` passes. #[test] fn unported_generate_req_input_fields_are_ignored() { for field in [ @@ -880,7 +1134,6 @@ mod tests { r#""lora_path": null"#, r#""image_data": "base64""#, r#""return_routed_experts": true"#, - r#""bootstrap_host": "h""#, // Python has no top-level `n` either, and ignores it just the same. r#""n": 1"#, r#""totally_made_up": 1"#, @@ -891,14 +1144,10 @@ mod tests { assert_eq!(ps.len(), 1, "{field}"); assert_eq!(ps[0].text.as_deref(), Some("hi"), "{field}"); } - for field in [ - r#""lora_path": "adapter""#, - r#""session_params": {"id": "s"}"#, - ] { - let body = format!(r#"{{"text": "hi", {field}}}"#); - let err = requests(&body).unwrap_err().to_string(); - assert!(err.contains("not supported"), "{field}: {err}"); - } + let field = r#""session_params": {"id": "s"}"#; + let body = format!(r#"{{"text": "hi", {field}}}"#); + let err = requests(&body).unwrap_err().to_string(); + assert!(err.contains("not supported"), "{field}: {err}"); } /// Client-supplied rid semantics mirror Python's `_normalize_batch`: a @@ -938,9 +1187,9 @@ mod tests { } /// The native `bench_serving` payload (a `GenerateReqInput` superset) parses: - /// its `lora_path`/`return_routed_experts` are accepted-but-ignored and a - /// `null` `image_data` means "no multimodal input", so `split` succeeds - /// while the real fields survive. + /// its optional `lora_path` is null, `return_routed_experts` is + /// accepted-but-ignored, and a null `image_data` means "no multimodal + /// input", so `split` succeeds while the real fields survive. #[test] fn accepts_bench_serving_payload() { let (ps, is_batch) = requests( diff --git a/rust/sglang-server/src/message/wire.rs b/rust/sglang-server/src/message/wire.rs index 9d13df81a0..44c6022920 100644 --- a/rust/sglang-server/src/message/wire.rs +++ b/rust/sglang-server/src/message/wire.rs @@ -126,8 +126,28 @@ pub fn media_input(input: api::MediaInput) -> Result { impl TryFrom for SamplingParams { type Error = String; - fn try_from(p: api::SamplingParams) -> Result { + fn try_from(mut p: api::SamplingParams) -> Result { let defaults = SamplingParams::default(); + if !p.choice.is_empty() { + if p.regex.is_some() + || p.json_schema.is_some() + || p.ebnf.is_some() + || p.structural_tag.is_some() + { + return Err( + "choice cannot be combined with regex, json_schema, ebnf, or structural_tag" + .into(), + ); + } + p.regex = Some(format!( + "(?:{})", + p.choice + .iter() + .map(|choice| regex_escape(choice)) + .collect::>() + .join("|") + )); + } let custom_params: Option> = match p.custom_params { None => None, Some(text) => { @@ -175,6 +195,20 @@ impl TryFrom for SamplingParams { } } +fn regex_escape(value: &str) -> String { + let mut escaped = String::with_capacity(value.len()); + for ch in value.chars() { + if matches!( + ch, + '.' | '^' | '$' | '*' | '+' | '?' | '{' | '}' | '[' | ']' | '\\' | '|' | '(' | ')' + ) { + escaped.push('\\'); + } + escaped.push(ch); + } + escaped +} + /// `sampling_params` as sent: one object (a broadcast in a batch) or one per /// prompt. Not a [`OneOrMany`]: that carrier is sealed to wire-shaped items, /// and the internal params are not one. @@ -391,6 +425,13 @@ mod tests { assert_eq!(p.temperature, 1.0, "schema default applied by the decoder"); } + #[test] + fn choices_lower_to_an_escaped_regex() { + let p: api::SamplingParams = serde_json::from_str(r#"{"choice":["a.b","x+y"]}"#).unwrap(); + let p = SamplingParams::try_from(p).unwrap(); + assert_eq!(p.regex.as_deref(), Some(r"(?:a\.b|x\+y)")); + } + #[test] fn watermark_crosses_generated_schema() { let key = "0123456789abcdef"; diff --git a/rust/sglang-server/src/tokenizer_manager/channel.rs b/rust/sglang-server/src/tokenizer_manager/channel.rs index a7dba17d47..572ba81af9 100644 --- a/rust/sglang-server/src/tokenizer_manager/channel.rs +++ b/rust/sglang-server/src/tokenizer_manager/channel.rs @@ -19,10 +19,12 @@ use crate::message::request::SchedulerRequest; #[derive(Clone)] pub struct ToSchedulerTx { tx: flume::Sender, + priority_tx: flume::Sender, } pub struct ToSchedulerRx { rx: flume::Receiver, + priority_rx: flume::Receiver, /// One-slot buffer holding a message consumed by a blocking [`wait`] so the /// scheduler can park on idle without losing it — the next [`drain`] returns /// it first. Only ever touched by the single consumer (the Python thread), @@ -41,6 +43,14 @@ impl ToSchedulerTx { pub fn try_push(&self, msg: SchedulerRequest) -> bool { self.tx.try_send(msg).is_ok() } + + /// Reserved, unbounded lane for scheduler cancellation. Active-request + /// cardinality bounds this queue, and request-ring saturation must not make + /// cancellation lossy. + #[inline] + pub fn push_priority(&self, msg: SchedulerRequest) -> bool { + self.priority_tx.send(msg).is_ok() + } } impl ToSchedulerRx { @@ -58,6 +68,12 @@ impl ToSchedulerRx { if let Some(m) = self.stash.lock().unwrap().take() { batch.push(m); } + while batch.len() < max { + match self.priority_rx.try_recv() { + Ok(m) => batch.push(m), + Err(_) => break, + } + } while batch.len() < max { match self.rx.try_recv() { Ok(m) => batch.push(m), @@ -77,12 +93,18 @@ impl ToSchedulerRx { if self.stash.lock().unwrap().is_some() { return true; } - match self.rx.recv_timeout(timeout) { - Ok(m) => { + let selected = flume::Selector::new() + .recv(&self.priority_rx, |result| result.ok()) + .recv(&self.rx, |result| result.ok()) + .wait_timeout(timeout) + .ok() + .flatten(); + match selected { + Some(m) => { *self.stash.lock().unwrap() = Some(m); true } - Err(_) => false, // Timeout or Disconnected + None => false, // Timeout or disconnected } } } @@ -127,10 +149,12 @@ impl FromSchedulerRx { /// Build both halves of a bounded ring. pub fn to_scheduler(cap: usize) -> (ToSchedulerTx, ToSchedulerRx) { let (tx, rx) = flume::bounded(cap); + let (priority_tx, priority_rx) = flume::unbounded(); ( - ToSchedulerTx { tx }, + ToSchedulerTx { tx, priority_tx }, ToSchedulerRx { rx, + priority_rx, stash: Mutex::new(None), }, ) @@ -183,6 +207,30 @@ mod tests { assert_eq!(rx.drain(16).len(), 1); } + #[test] + fn priority_lane_bypasses_full_request_ring() { + let (tx, rx) = to_scheduler(1); + assert!(tx.try_push(msg(b"request"))); + assert!(!tx.try_push(msg(b"full"))); + assert!(tx.push_priority(msg(b"abort"))); + + let drained = rx.drain(2); + assert_eq!(drained.len(), 2); + assert_eq!(drained[0].header, Bytes::from_static(b"abort")); + assert_eq!(drained[1].header, Bytes::from_static(b"request")); + } + + #[test] + fn wait_wakes_on_priority_push() { + let (tx, rx) = to_scheduler(1); + std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(20)); + let _ = tx.push_priority(msg(b"abort")); + }); + assert!(rx.wait(Duration::from_secs(5))); + assert_eq!(rx.drain(1)[0].header, Bytes::from_static(b"abort")); + } + /// A full from_scheduler channel parks the producer until the consumer drains — the /// committed frame is delivered in order, never dropped. #[test] diff --git a/rust/sglang-server/src/tokenizer_manager/to_scheduler.rs b/rust/sglang-server/src/tokenizer_manager/to_scheduler.rs index ec59bc803d..a09664ac8c 100644 --- a/rust/sglang-server/src/tokenizer_manager/to_scheduler.rs +++ b/rust/sglang-server/src/tokenizer_manager/to_scheduler.rs @@ -452,11 +452,9 @@ impl Intake { /// Client disconnected (or a detok terminal): deregister the sink, then push an /// `AbortReq(rid)` so the scheduler stops generating for it. /// - /// A failed push is logged, not retried: the scheduler keeps generating and the - /// chunks arrive for a rid no longer in the detok table, where they are dropped. - /// That wastes GPU work until the request finishes on its own, but it cannot be - /// misdelivered — the rid is unique to this request for the process's lifetime - /// ([`Rid::from_client`]), so no later request can ever answer to it. + /// Aborts use the reserved scheduler-control lane, so request-ring + /// backpressure cannot discard cancellation after public selector state is + /// retired. /// /// A request out in a pool never reached the scheduler, so there is nothing to /// abort there yet: it is marked `Aborted` and dropped when it returns. @@ -475,19 +473,13 @@ impl Intake { return; } - // The channel is BOUNDED and drops pushes under exactly the load this matters - // for, so report the miss rather than assuming the scheduler was told. match ControlRequest::AbortReq(AbortReq::new(rid.as_str().to_string(), false)).encode() { Ok(header) => { - if !self.to_scheduler_tx.try_push(SchedulerRequest { + if !self.to_scheduler_tx.push_priority(SchedulerRequest { header, buffers: Vec::new(), }) { - tracing::error!( - rid = %rid, - "abort dropped: to_scheduler channel is full; the scheduler keeps generating \ - for this request until it finishes on its own" - ); + tracing::debug!(rid = %rid, "abort ignored during scheduler shutdown"); } } Err(e) => tracing::error!(rid = %rid, error = %e, "abort encode failed"), diff --git a/rust/sglang-server/src/tokenizer_manager/to_scheduler_tests.rs b/rust/sglang-server/src/tokenizer_manager/to_scheduler_tests.rs index d2d1b22c46..afd9b4e64d 100644 --- a/rust/sglang-server/src/tokenizer_manager/to_scheduler_tests.rs +++ b/rust/sglang-server/src/tokenizer_manager/to_scheduler_tests.rs @@ -575,9 +575,9 @@ fn tokenize_goes_to_the_pool_without_detok_registration() { /// The detok entry is dropped either way — that is the half that must not /// depend on the channel. /// -/// Channel capacity 1: the first abort pushes, the second finds it full. +/// Request-ring capacity does not limit the reserved abort lane. #[test] -fn abort_deregisters_even_when_the_channel_push_is_dropped() { +fn abort_deregisters_and_bypasses_request_ring_capacity() { let (tok_tx, _tok_rx) = flume::unbounded(); let (detok_tx, detok_rx) = flume::unbounded(); let (abort_tx, abort_rx) = flume::unbounded::(); @@ -587,7 +587,7 @@ fn abort_deregisters_even_when_the_channel_push_is_dropped() { tokenizer_tx: tok_tx, detokenizer_tx: vec![detok_tx], }; - let (producer, _consumer) = to_scheduler(1); + let (producer, consumer) = to_scheduler(1); let (_tm_tx, tm_rx) = flume::unbounded(); let (sd_tx, sd_rx) = flume::unbounded::<()>(); std::mem::forget(sd_tx); @@ -611,6 +611,7 @@ fn abort_deregisters_even_when_the_channel_push_is_dropped() { "{expected}: the detok entry must be dropped even when the to_scheduler channel is full", ); } + assert_eq!(consumer.drain(8).len(), 2, "both aborts must be delivered"); } /// The rid keys the detok table and rides on every chunk of every decode step, diff --git a/rust/sglang-server/src/tokenizer_manager/wiring.rs b/rust/sglang-server/src/tokenizer_manager/wiring.rs index bc7f1b9290..68e6c7270f 100644 --- a/rust/sglang-server/src/tokenizer_manager/wiring.rs +++ b/rust/sglang-server/src/tokenizer_manager/wiring.rs @@ -44,6 +44,7 @@ pub enum TmEvent { const ADMISSION_PENDING: u8 = 0; const ADMISSION_ACCEPTED: u8 = 1; const ADMISSION_CANCELLED: u8 = 2; +const ADMISSION_COMPLETED: u8 = 3; /// Cancellation-safe ownership handoff between a core call and Intake. /// @@ -79,19 +80,33 @@ impl RequestAdmission { pub fn cancel_requires_abort(&self) -> bool { self.state.swap(ADMISSION_CANCELLED, Ordering::AcqRel) == ADMISSION_ACCEPTED } + + pub fn is_cancelled(&self) -> bool { + self.state.load(Ordering::Acquire) == ADMISSION_CANCELLED + } + + /// Prevent an explicit abort racing with an observed terminal response + /// from sending a redundant scheduler abort. + pub fn complete(&self) { + let _ = self.state.compare_exchange( + ADMISSION_ACCEPTED, + ADMISSION_COMPLETED, + Ordering::AcqRel, + Ordering::Acquire, + ); + } } /// The source of the abort request. [`Intake::on_abort`] deregisters the /// detok entry for every source and tells the scheduler to stop for all but /// [`Self::Local`]; the source is otherwise kept for diagnostics. /// -/// There is no in-flight rid registry to keep consistent, and so no release -/// ordering to get wrong: [`Rid::from_client`] makes every client-supplied rid -/// internally unique, so a resubmit of the "same" rid is a different `Rid` and -/// cannot be tangled up with an abort still in flight for the original. +/// [`Rid::from_client`] makes every client-supplied rid internally unique, so +/// an abort already in flight cannot affect a later reuse of the public ID. +/// Explicit cancellation resolves public IDs in the core before using this lane. #[derive(Clone, Debug)] pub enum AbortSource { - /// From an in-flight core call drop. Owns the release. + /// From a core call drop or explicit core cancellation. Owns the release. Guard(Rid), /// From a detokenizer terminal path. Aborts the scheduler work. Detok(Rid), diff --git a/rust/sglang-server/src/utils/startup.rs b/rust/sglang-server/src/utils/startup.rs index 6cb94e21f0..40481ee8bf 100644 --- a/rust/sglang-server/src/utils/startup.rs +++ b/rust/sglang-server/src/utils/startup.rs @@ -5,7 +5,7 @@ use std::net::SocketAddr; use pyo3::PyErr; use pyo3::exceptions::PyValueError; -use crate::message::config::ServerArgs; +use crate::message::config::{DisaggregationMode, ServerArgs}; /// A `ValueError` for a boot-time failure, as `"{context}: {err}"`. pub(crate) fn value_error(context: &str, err: impl std::fmt::Display) -> PyErr { @@ -46,6 +46,23 @@ fn listen_addr_for_port( Ok(addr) } +/// Rewrite the advertised ports to the ones this listener binds, so a DP +/// listener's server info names its own endpoints rather than the base ports. +pub(crate) fn advertise_listener_ports( + mut server_args: ServerArgs, + http_addr: SocketAddr, + grpc_addr: Option, +) -> ServerArgs { + server_args.port = http_addr.port(); + server_args.grpc_port = grpc_addr.map(|addr| addr.port()); + // Rust-mode prefill serves the PD bootstrap registry on its API port; on + // decode the field names the prefill peer's port and must stay unchanged. + if server_args.disaggregation_mode == DisaggregationMode::Prefill { + server_args.deployment_info.disaggregation_bootstrap_port = Some(http_addr.port()); + } + server_args +} + #[cfg(test)] mod tests { use super::*; @@ -91,6 +108,41 @@ mod tests { ); } + /// A DP listener at a nonzero offset must not advertise rank 0's ports: + /// decode would rendezvous with the wrong prefill bootstrap registry. + #[test] + fn dp_listener_advertises_its_own_ports() { + let base = ServerArgs { + host: "::".into(), + port: 30_000, + grpc_port: Some(50_051), + ..Default::default() + }; + let http = listen_addr(&base, Some(2)).unwrap(); + let grpc = grpc_listen_addr(&base, Some(2)).unwrap(); + + let mut prefill = base.clone(); + prefill.disaggregation_mode = DisaggregationMode::Prefill; + prefill.deployment_info.disaggregation_bootstrap_port = Some(30_000); + let prefill = advertise_listener_ports(prefill, http, grpc); + assert_eq!(prefill.port, 30_002); + assert_eq!(prefill.grpc_port, Some(50_053)); + assert_eq!( + prefill.deployment_info.disaggregation_bootstrap_port, + Some(30_002) + ); + + let mut decode = base; + decode.disaggregation_mode = DisaggregationMode::Decode; + decode.deployment_info.disaggregation_bootstrap_port = Some(8_998); + let decode = advertise_listener_ports(decode, http, grpc); + assert_eq!(decode.port, 30_002); + assert_eq!( + decode.deployment_info.disaggregation_bootstrap_port, + Some(8_998) + ); + } + #[test] fn grpc_listen_addr_rejects_port_overflow() { let args = ServerArgs { diff --git a/test/registered/unit/api/v1/test_parity.py b/test/registered/unit/api/v1/test_parity.py index 30ae036d5b..72662c11df 100644 --- a/test/registered/unit/api/v1/test_parity.py +++ b/test/registered/unit/api/v1/test_parity.py @@ -53,11 +53,9 @@ PYTHON_ONLY_REQUEST_FIELDS = frozenset( "image_max_dynamic_patch", "images_config", "input_embeds", - "kv_hints", "log_metrics", "lora_id", "max_dynamic_patch", - "max_thinking_tokens", "min_dynamic_patch", "mm_content_hashes", "mm_processor_kwargs", @@ -78,7 +76,6 @@ PYTHON_ONLY_REQUEST_FIELDS = frozenset( "return_routed_experts", "return_sampling_mask", "routed_experts_start_len", - "routing_key", "sampling_logprobs_mode", "session_id", "token_indices_to_pool", @@ -112,6 +109,11 @@ def load_fixtures(): return json.load(f)["cases"] +# Schema-only sampling inputs the Rust server lowers before the scheduler +# (choice -> regex); Python's SamplingParams never receives them. +SCHEMA_ONLY_SAMPLING_FIELDS = frozenset({"choice"}) + + class TestCrossLanguageFixtures(CustomTestCase): def test_fixtures_decode_to_their_canonical_bytes(self): """The Python emitter agrees with the Rust emitter on every fixture: @@ -152,7 +154,10 @@ class TestRequestFieldParity(CustomTestCase): for name in PySamplingParams.__struct_fields__ if name not in DERIVED_SAMPLING_FIELDS ] - self.assertEqual(set(python_api), set(SamplingParams.__struct_fields__)) + self.assertEqual( + set(python_api), + set(SamplingParams.__struct_fields__) - SCHEMA_ONLY_SAMPLING_FIELDS, + ) self.assertEqual( set(PySamplingParams.__struct_fields__) - set(python_api), DERIVED_SAMPLING_FIELDS, diff --git a/test/registered/unit/entrypoints/test_rust_server_dp_ports.py b/test/registered/unit/entrypoints/test_rust_server_dp_ports.py index 85c47c8a03..1fd2630c24 100644 --- a/test/registered/unit/entrypoints/test_rust_server_dp_ports.py +++ b/test/registered/unit/entrypoints/test_rust_server_dp_ports.py @@ -1,3 +1,4 @@ +import json from types import SimpleNamespace from unittest.mock import MagicMock, patch @@ -23,6 +24,15 @@ def _scheduler_for_typed_config(): hf_config=SimpleNamespace(model_type=None), get_default_sampling_params=lambda: {}, ), + get_init_info=MagicMock( + side_effect=AssertionError("typed config must not read late init info") + ), + kv_events_publisher=SimpleNamespace( + local_kv_event_sources=MagicMock( + return_value=[{"endpoint": "tcp://127.0.0.1:5557"}] + ) + ), + page_size=16, rust_server_tokenizer_path=lambda: "tokenizer", tokenizer=None, max_total_num_tokens=1024, @@ -60,18 +70,58 @@ def test_typed_config_forwards_resolved_launch_fields(overrides, kwarg, expected DefaultSamplingParams=MagicMock(return_value="sampling-defaults"), ServerArgs=MagicMock(return_value="server-args"), ) + scheduler = _scheduler_for_typed_config() with ( get_context().override_server_args(grpc_port=50051, **overrides), + get_parallel().override(attn_dp_rank=0, dp_rank=None), patch.object(rust_extensions, "load_rust_extension", return_value=extension), patch.object(rust_config, "compute_num_reserved_tokens", return_value=0), patch.object(rust_config, "_launch_server_info", return_value="{}"), ): - assert ( - rust_config._build_server_args(_scheduler_for_typed_config()) - == "server-args" - ) + assert rust_config._build_server_args(scheduler) == "server-args" assert extension.ServerArgs.call_args.kwargs[kwarg] == expected + scheduler.get_init_info.assert_not_called() + scheduler.kv_events_publisher.local_kv_event_sources.assert_called_once_with(16) + deployment_info = json.loads( + extension.ServerArgs.call_args.kwargs["deployment_info"] + ) + assert deployment_info["kv_event_sources"] == [{"endpoint": "tcp://127.0.0.1:5557"}] + + +@pytest.mark.parametrize( + "server_args,ranks,expected", + [ + ({"tp_size": 4, "attn_dp_size": 4}, {"attn_dp_rank": 2, "dp_rank": None}, 2), + ({"dp_size": 2}, {"attn_dp_rank": 0, "dp_rank": 1}, 1), + ({}, {"attn_dp_rank": 0, "dp_rank": None}, 0), + ], + ids=["attention-dp", "replica-dp", "no-dp"], +) +def test_deployment_info_reports_listener_dp_rank(server_args, ranks, expected): + """Each DP listener must advertise its own rank, matching its KV-event + source; a per-listener orchestrator cannot otherwise tell listeners apart.""" + extension = SimpleNamespace( + DisaggregationMode=SimpleNamespace( + Null="null", Prefill="prefill", Decode="decode" + ), + ModelConfig=MagicMock(return_value="model-config"), + DefaultSamplingParams=MagicMock(return_value="sampling-defaults"), + ServerArgs=MagicMock(return_value="server-args"), + ) + with ( + get_context().override_server_args(**server_args), + get_parallel().override(**ranks), + patch.object(rust_extensions, "load_rust_extension", return_value=extension), + patch.object(rust_config, "compute_num_reserved_tokens", return_value=0), + patch.object(rust_config, "_launch_server_info", return_value="{}"), + ): + rust_config._build_server_args(_scheduler_for_typed_config()) + + deployment_info = json.loads( + extension.ServerArgs.call_args.kwargs["deployment_info"] + ) + assert deployment_info["dp_rank"] == expected @pytest.mark.parametrize("enable_metrics", [False, True]) @@ -79,6 +129,7 @@ def test_typed_config_points_metrics_at_the_multiproc_dir(enable_metrics): extension = MagicMock() with ( get_context().override_server_args(enable_metrics=enable_metrics), + get_parallel().override(attn_dp_rank=0, dp_rank=None), patch.dict("os.environ", {"PROMETHEUS_MULTIPROC_DIR": "/tmp/prom"}), patch.object(rust_extensions, "load_rust_extension", return_value=extension), patch.object(rust_config, "compute_num_reserved_tokens", return_value=0), diff --git a/test/registered/unit/entrypoints/test_server_info.py b/test/registered/unit/entrypoints/test_server_info.py index bd66bd223e..e64baa22b8 100644 --- a/test/registered/unit/entrypoints/test_server_info.py +++ b/test/registered/unit/entrypoints/test_server_info.py @@ -32,6 +32,7 @@ import msgspec.structs from sglang.srt.arg_groups.validation_hook import check_load_publish_args from sglang.srt.entrypoints import http_server from sglang.srt.entrypoints.grpc_bridge import RuntimeHandle +from sglang.srt.environ import envs from sglang.srt.lora.lora_registry import LoRARef from sglang.srt.managers.tokenizer_manager import TokenizerManager from sglang.srt.runtime_context import publish, reset_context @@ -207,6 +208,28 @@ class TestServerInfoTransportParity(CustomTestCase): host="127.0.0.1", port=50051, server_info_json=leader_json ) + def test_rust_server_followers_skip_runtime_metadata_server(self): + """A follower node hosting a Rust listener must not bind grpc_port a + second time; that node's KV publishers are served by the listener.""" + with ( + envs.SGLANG_RUST_SERVER.override(True), + patch.object( + common, + "get_serving", + return_value=SimpleNamespace( + host="127.0.0.1", + grpc_port=50051, + smg_grpc_mode=False, + grpc_mode=False, + ), + ), + patch("sglang.srt.rust_extensions.load_rust_extension") as load, + ): + self.assertIsNone( + common.start_follower_grpc_server(ServerArgs(model_path="dummy"), {}) + ) + load.return_value.start_metadata_server.assert_not_called() + class TestServerInfoKvEventsField(CustomTestCase): """The new `kv_events` field is wired correctly across the full diff --git a/test/registered/unit/managers/test_output_streamer_logprobs.py b/test/registered/unit/managers/test_output_streamer_logprobs.py index e85a7d5f19..c800db0a32 100644 --- a/test/registered/unit/managers/test_output_streamer_logprobs.py +++ b/test/registered/unit/managers/test_output_streamer_logprobs.py @@ -66,7 +66,9 @@ class _FakeReq: return self.output_ids_through_stop, 0 -def _make_accumulator() -> _GenerationStreamAccumulator: +def _make_accumulator( + *, rust_server_mode: bool = False +) -> _GenerationStreamAccumulator: return _GenerationStreamAccumulator( return_logprob=True, return_hidden_states=False, @@ -77,6 +79,7 @@ def _make_accumulator() -> _GenerationStreamAccumulator: default_stream_interval=1, default_force_stream_interval=1, get_cached_tokens_details=lambda req: None, + rust_server_mode=rust_server_mode, is_request_level_customized_info_key=lambda key: False, current_weight_version=None, ) @@ -103,6 +106,17 @@ class TestOutputStreamerLogprobs(unittest.TestCase): self.assertEqual(req.send_output_token_logprobs_offset, 1) self.assertEqual(accumulator.output_token_logprobs_val, [[-0.5]]) + def test_rust_server_mode_reports_cached_tokens_per_request(self): + """The Rust frontend rejects a batch whose cached_tokens column is shorter than rids.""" + req = _FakeReq(is_retracted=False, max_new_tokens=16) + req.cached_tokens = 7 + accumulator = _make_accumulator(rust_server_mode=True) + + accumulator.accept(req=req) + + self.assertEqual(accumulator.rids, ["req"]) + self.assertEqual(accumulator.cached_tokens, [7]) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index feda6feae5..80297d95aa 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -4360,6 +4360,31 @@ class TestNoneMeansUnset(CustomTestCase): self.assertIsNotNone(server_args.mamba_full_memory_ratio) +class TestRustServerDpAttentionControlBroadcast(unittest.TestCase): + """Under the Rust server every attention-DP leader hosts its own listener; + a control request (abort, server info) sent to a leader other than rank 0 + must reach its scheduler instead of being overwritten by rank 0's + broadcast.""" + + def test_rust_server_broadcasts_control_within_each_dp_group(self): + server_args = ServerArgs( + model_path="dummy", + attn_dp_size=2, + tp_size=2, + chunked_prefill_size=8192, + cuda_graph_config=CudaGraphConfig( + prefill=PhaseConfig(backend=Backend.DISABLED) + ), + ) + with envs.SGLANG_RUST_SERVER.override(True): + parallel_hook.handle_data_parallelism(server_args) + self.assertTrue( + resolution_result( + server_args, "enable_dp_attention_local_control_broadcast", False + ) + ) + + class TestTpLmHeadAllToAllNcclGraphRegister(unittest.TestCase): """The graph-captured TP LM-head all-to-all must not run with NCCL's graph buffer registration: registered graph-pool temporaries deadlock the