[Feat] rust/grpc: Add ListModels, Detokenize, and Abort to api.v1 (#42118)

Signed-off-by: jain-ria <riajain@NVIDIA.com>
Co-authored-by: ishandhanani <82981111+ishandhanani@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Ishan Dhanani <ishandhanani@gmail.com>
This commit is contained in:
jain-ria
2026-10-19 09:53:56 +08:00
committed by GitHub
co-authored by ishandhanani Claude Opus 5.5 Ishan Dhanani
parent 377cd867fd
commit fea48754de
46 changed files with 4011 additions and 219 deletions
+540 -4
View File
@@ -204,6 +204,7 @@ class SamplingParams(msgspec.Struct, kw_only=True, frozen=True):
custom_params: Optional[Any] = None
beam_width: Optional[int] = None
watermark: Optional[WatermarkRequestConfig] = None
choice: List[str] = []
@classmethod
def from_json_value(cls, value: Any) -> "SamplingParams":
@@ -261,8 +262,11 @@ class GenerateRequest(msgspec.Struct, kw_only=True, frozen=True):
priority: Optional[int] = None
cache_salt: Optional[StringOrList] = None
require_reasoning: Optional[bool] = None
lora_path: Optional[Any] = None
lora_path: Optional[OptionalStringOrList] = None
session_params: Optional[Any] = None
kv_hints: Optional[KvHintsEnvelopeOrList] = None
routing_key: Optional[str] = None
max_thinking_tokens: Optional[int] = None
@classmethod
def from_json_value(cls, value: Any) -> "GenerateRequest":
@@ -274,6 +278,41 @@ class GenerateRequest(msgspec.Struct, kw_only=True, frozen=True):
return encode_GenerateRequest(self)
class KvHintsEnvelope(msgspec.Struct, kw_only=True, frozen=True):
"""sglang.api.v1.KvHintsEnvelope: JSON object (unknown keys ignored)."""
protocol_version: str = ""
message_id: str = ""
actions: List[KvHintAction] = []
@classmethod
def from_json_value(cls, value: Any) -> "KvHintsEnvelope":
"""Decode a parsed JSON value; raises JsonContractError on a contract violation."""
return decode_KvHintsEnvelope(value)
def to_json_value(self) -> Dict[str, Any]:
"""Encode to a JSON-ready value (json.dumps-compatible)."""
return encode_KvHintsEnvelope(self)
class KvHintAction(msgspec.Struct, kw_only=True, frozen=True):
"""sglang.api.v1.KvHintAction: JSON object (unknown keys ignored)."""
action_id: str = ""
action_type: str = ""
action_version: str = ""
payload: Optional[Any] = None
@classmethod
def from_json_value(cls, value: Any) -> "KvHintAction":
"""Decode a parsed JSON value; raises JsonContractError on a contract violation."""
return decode_KvHintAction(value)
def to_json_value(self) -> Dict[str, Any]:
"""Encode to a JSON-ready value (json.dumps-compatible)."""
return encode_KvHintAction(self)
class LogprobEntry(msgspec.Struct, kw_only=True, frozen=True):
"""sglang.api.v1.LogprobEntry: JSON positional array [logprob, token_id, text]."""
@@ -382,6 +421,113 @@ class MediaRef(msgspec.Struct, kw_only=True, frozen=True):
return encode_MediaRef(self)
class ListModelsRequest(msgspec.Struct, kw_only=True, frozen=True):
"""sglang.api.v1.ListModelsRequest: JSON object (unknown keys ignored)."""
@classmethod
def from_json_value(cls, value: Any) -> "ListModelsRequest":
"""Decode a parsed JSON value; raises JsonContractError on a contract violation."""
return decode_ListModelsRequest(value)
def to_json_value(self) -> Dict[str, Any]:
"""Encode to a JSON-ready value (json.dumps-compatible)."""
return encode_ListModelsRequest(self)
class ListModelsResponse(msgspec.Struct, kw_only=True, frozen=True):
"""sglang.api.v1.ListModelsResponse: JSON object (unknown keys ignored)."""
models: List[ModelCard] = []
@classmethod
def from_json_value(cls, value: Any) -> "ListModelsResponse":
"""Decode a parsed JSON value; raises JsonContractError on a contract violation."""
return decode_ListModelsResponse(value)
def to_json_value(self) -> Dict[str, Any]:
"""Encode to a JSON-ready value (json.dumps-compatible)."""
return encode_ListModelsResponse(self)
class ModelCard(msgspec.Struct, kw_only=True, frozen=True):
"""sglang.api.v1.ModelCard: JSON object (unknown keys ignored)."""
id: str = ""
root: str = ""
parent: Optional[str] = None
max_model_len: int = 0
@classmethod
def from_json_value(cls, value: Any) -> "ModelCard":
"""Decode a parsed JSON value; raises JsonContractError on a contract violation."""
return decode_ModelCard(value)
def to_json_value(self) -> Dict[str, Any]:
"""Encode to a JSON-ready value (json.dumps-compatible)."""
return encode_ModelCard(self)
class DetokenizeRequest(msgspec.Struct, kw_only=True, frozen=True):
"""sglang.api.v1.DetokenizeRequest: JSON object (unknown keys ignored)."""
tokens: List[int] = []
@classmethod
def from_json_value(cls, value: Any) -> "DetokenizeRequest":
"""Decode a parsed JSON value; raises JsonContractError on a contract violation."""
return decode_DetokenizeRequest(value)
def to_json_value(self) -> Dict[str, Any]:
"""Encode to a JSON-ready value (json.dumps-compatible)."""
return encode_DetokenizeRequest(self)
class DetokenizeResponse(msgspec.Struct, kw_only=True, frozen=True):
"""sglang.api.v1.DetokenizeResponse: JSON object (unknown keys ignored)."""
text: str = ""
@classmethod
def from_json_value(cls, value: Any) -> "DetokenizeResponse":
"""Decode a parsed JSON value; raises JsonContractError on a contract violation."""
return decode_DetokenizeResponse(value)
def to_json_value(self) -> Dict[str, Any]:
"""Encode to a JSON-ready value (json.dumps-compatible)."""
return encode_DetokenizeResponse(self)
class AbortRequest(msgspec.Struct, kw_only=True, frozen=True):
"""sglang.api.v1.AbortRequest: JSON object (unknown keys ignored)."""
rid: str = ""
abort_all: bool = False
@classmethod
def from_json_value(cls, value: Any) -> "AbortRequest":
"""Decode a parsed JSON value; raises JsonContractError on a contract violation."""
return decode_AbortRequest(value)
def to_json_value(self) -> Dict[str, Any]:
"""Encode to a JSON-ready value (json.dumps-compatible)."""
return encode_AbortRequest(self)
class AbortResponse(msgspec.Struct, kw_only=True, frozen=True):
"""sglang.api.v1.AbortResponse: JSON object (unknown keys ignored)."""
success: bool = False
@classmethod
def from_json_value(cls, value: Any) -> "AbortResponse":
"""Decode a parsed JSON value; raises JsonContractError on a contract violation."""
return decode_AbortResponse(value)
def to_json_value(self) -> Dict[str, Any]:
"""Encode to a JSON-ready value (json.dumps-compatible)."""
return encode_AbortResponse(self)
class HealthCheckRequest(msgspec.Struct, kw_only=True, frozen=True):
"""sglang.api.v1.HealthCheckRequest: JSON object (unknown keys ignored)."""
@@ -469,6 +615,11 @@ class GetServerInfoResponse(msgspec.Struct, kw_only=True, frozen=True):
max_total_num_tokens: Optional[int] = None
version: str = ""
internal_states: List[Any] = []
host: str = ""
port: int = 0
incremental_streaming_output: bool = False
disaggregation_mode: str = ""
deployment_info: Optional[DeploymentInfo] = None
@classmethod
def from_json_value(cls, value: Any) -> "GetServerInfoResponse":
@@ -480,6 +631,57 @@ class GetServerInfoResponse(msgspec.Struct, kw_only=True, frozen=True):
return encode_GetServerInfoResponse(self)
class DeploymentInfo(msgspec.Struct, kw_only=True, frozen=True):
"""sglang.api.v1.DeploymentInfo: JSON object (unknown keys ignored)."""
disaggregation_bootstrap_port: Optional[int] = None
node_rank: int = 0
nnodes: int = 0
dp_size: int = 0
dp_rank: Optional[int] = None
dist_init_addr: Optional[str] = None
base_gpu_id: int = 0
page_size: Optional[int] = None
dcp_size: int = 0
max_running_requests: Optional[int] = None
max_prefill_tokens: int = 0
enable_dp_attention: bool = False
speculative_algorithm: Optional[str] = None
enable_hierarchical_cache: bool = False
hicache_ratio: Optional[float] = None
hicache_size: int = 0
hicache_write_policy: Optional[str] = None
hicache_host_total_tokens: Optional[int] = None
kv_event_sources: List[KvEventSource] = []
@classmethod
def from_json_value(cls, value: Any) -> "DeploymentInfo":
"""Decode a parsed JSON value; raises JsonContractError on a contract violation."""
return decode_DeploymentInfo(value)
def to_json_value(self) -> Dict[str, Any]:
"""Encode to a JSON-ready value (json.dumps-compatible)."""
return encode_DeploymentInfo(self)
class KvEventSource(msgspec.Struct, kw_only=True, frozen=True):
"""sglang.api.v1.KvEventSource: JSON object (unknown keys ignored)."""
dp_rank: int = 0
endpoint: str = ""
topic: str = ""
block_size: int = 0
@classmethod
def from_json_value(cls, value: Any) -> "KvEventSource":
"""Decode a parsed JSON value; raises JsonContractError on a contract violation."""
return decode_KvEventSource(value)
def to_json_value(self) -> Dict[str, Any]:
"""Encode to a JSON-ready value (json.dumps-compatible)."""
return encode_KvEventSource(self)
# ---- type aliases: JSON shapes without an object of their own ----
# sglang.api.v1.TokenIds: JSON bare array.
@@ -518,6 +720,9 @@ MatchedTokens = List[int]
# sglang.api.v1.FinishReason: JSON object tagged by its type key; unknown tags pass through as a dict.
FinishReason = Union[FinishStop, FinishLength, FinishAbort, Dict[str, Any]]
# sglang.api.v1.OptionalKvHintsEnvelope: JSON: the wrapped value itself, null when unset.
OptionalKvHintsEnvelope = Optional[KvHintsEnvelope]
# sglang.api.v1.SamplingParamsList: JSON bare array.
SamplingParamsList = List[SamplingParams]
@@ -578,9 +783,15 @@ OptionalInt64OrList = Union[OptionalInt64, OptionalInt64List]
# sglang.api.v1.Matched: JSON: the value's shape picks the arm.
Matched = Union[int, str, MatchedTokens]
# sglang.api.v1.KvHintsEnvelopeList: JSON bare array.
KvHintsEnvelopeList = List[OptionalKvHintsEnvelope]
# sglang.api.v1.SamplingParamsOrList: JSON: one value or an array of values.
SamplingParamsOrList = Union[SamplingParams, SamplingParamsList]
# sglang.api.v1.KvHintsEnvelopeOrList: JSON: one value or an array of values.
KvHintsEnvelopeOrList = Union[KvHintsEnvelope, KvHintsEnvelopeList]
# ---- codecs: parsed JSON value <-> Python value ----
@@ -903,7 +1114,7 @@ def encode_FinishReason(v: FinishReason) -> Any:
raise TypeError(f"FinishReason: unsupported value {type(v).__name__}")
_FIELDS_SamplingParams = ("max_new_tokens", "stop", "stop_token_ids", "stop_regex", "temperature", "top_p", "top_k", "min_p", "frequency_penalty", "presence_penalty", "repetition_penalty", "min_new_tokens", "n", "json_schema", "regex", "ebnf", "structural_tag", "ignore_eos", "skip_special_tokens", "spaces_between_special_tokens", "no_stop_trim", "stream_interval", "logit_bias", "sampling_seed", "custom_params", "beam_width", "watermark",)
_FIELDS_SamplingParams = ("max_new_tokens", "stop", "stop_token_ids", "stop_regex", "temperature", "top_p", "top_k", "min_p", "frequency_penalty", "presence_penalty", "repetition_penalty", "min_new_tokens", "n", "json_schema", "regex", "ebnf", "structural_tag", "ignore_eos", "skip_special_tokens", "spaces_between_special_tokens", "no_stop_trim", "stream_interval", "logit_bias", "sampling_seed", "custom_params", "beam_width", "watermark", "choice",)
def decode_SamplingParams(value: Any) -> SamplingParams:
@@ -964,6 +1175,8 @@ def decode_SamplingParams(value: Any) -> SamplingParams:
kw["beam_width"] = None if x is None else _expect_int(x, expected="i64")
elif key == "watermark":
kw["watermark"] = None if x is None else decode_WatermarkRequestConfig(x)
elif key == "choice":
kw["choice"] = [] if x is None else [_expect_str(e, expected="a string") for e in _expect_seq(x, expected="a sequence")]
else:
raise _unknown_field(key, fields=_FIELDS_SamplingParams)
return SamplingParams(**kw)
@@ -1024,6 +1237,8 @@ def encode_SamplingParams(v: SamplingParams) -> Dict[str, Any]:
d["beam_width"] = v.beam_width
if v.watermark is not None:
d["watermark"] = encode_WatermarkRequestConfig(v.watermark)
if v.choice:
d["choice"] = list(v.choice)
return d
@@ -1113,9 +1328,15 @@ def decode_GenerateRequest(value: Any) -> GenerateRequest:
elif key == "require_reasoning":
kw["require_reasoning"] = None if x is None else _expect_bool(x, expected="a boolean")
elif key == "lora_path":
kw["lora_path"] = x
kw["lora_path"] = None if x is None else decode_OptionalStringOrList(x)
elif key == "session_params":
kw["session_params"] = x
elif key == "kv_hints":
kw["kv_hints"] = None if x is None else decode_KvHintsEnvelopeOrList(x)
elif key == "routing_key":
kw["routing_key"] = None if x is None else _expect_str(x, expected="a string")
elif key == "max_thinking_tokens":
kw["max_thinking_tokens"] = None if x is None else _expect_uint(x, expected="u64")
return GenerateRequest(**kw)
@@ -1174,9 +1395,90 @@ def encode_GenerateRequest(v: GenerateRequest) -> Dict[str, Any]:
if v.require_reasoning is not None:
d["require_reasoning"] = v.require_reasoning
if v.lora_path is not None:
d["lora_path"] = v.lora_path
d["lora_path"] = encode_OptionalStringOrList(v.lora_path)
if v.session_params is not None:
d["session_params"] = v.session_params
if v.kv_hints is not None:
d["kv_hints"] = encode_KvHintsEnvelopeOrList(v.kv_hints)
if v.routing_key is not None:
d["routing_key"] = v.routing_key
if v.max_thinking_tokens is not None:
d["max_thinking_tokens"] = v.max_thinking_tokens
return d
def decode_KvHintsEnvelopeOrList(value: Any) -> KvHintsEnvelopeOrList:
if isinstance(value, list):
return decode_KvHintsEnvelopeList(value)
return decode_KvHintsEnvelope(_expect_map(value, expected="a value or an array of values for KvHintsEnvelopeOrList"))
def encode_KvHintsEnvelopeOrList(v: KvHintsEnvelopeOrList) -> Any:
if isinstance(v, list):
return encode_KvHintsEnvelopeList(v)
return encode_KvHintsEnvelope(v)
def decode_KvHintsEnvelopeList(value: Any) -> KvHintsEnvelopeList:
return [decode_OptionalKvHintsEnvelope(e) for e in _expect_seq(value, expected="a sequence")]
def encode_KvHintsEnvelopeList(v: KvHintsEnvelopeList) -> List[Any]:
return [encode_OptionalKvHintsEnvelope(e) for e in v]
def decode_OptionalKvHintsEnvelope(value: Any) -> OptionalKvHintsEnvelope:
return None if value is None else decode_KvHintsEnvelope(value)
def encode_OptionalKvHintsEnvelope(v: OptionalKvHintsEnvelope) -> Any:
return None if v is None else encode_KvHintsEnvelope(v)
def decode_KvHintsEnvelope(value: Any) -> KvHintsEnvelope:
m = _expect_map(value, expected="struct KvHintsEnvelope")
kw: Dict[str, Any] = {}
for key, x in m.items():
if key == "protocol_version":
kw["protocol_version"] = _expect_str(x, expected="a string")
elif key == "message_id":
kw["message_id"] = _expect_str(x, expected="a string")
elif key == "actions":
kw["actions"] = [] if x is None else [decode_KvHintAction(e) for e in _expect_seq(x, expected="a sequence")]
return KvHintsEnvelope(**kw)
def encode_KvHintsEnvelope(v: KvHintsEnvelope) -> Dict[str, Any]:
d: Dict[str, Any] = {}
d["protocol_version"] = v.protocol_version
d["message_id"] = v.message_id
if v.actions:
d["actions"] = [encode_KvHintAction(e) for e in v.actions]
return d
def decode_KvHintAction(value: Any) -> KvHintAction:
m = _expect_map(value, expected="struct KvHintAction")
kw: Dict[str, Any] = {}
for key, x in m.items():
if key == "action_id":
kw["action_id"] = _expect_str(x, expected="a string")
elif key == "action_type":
kw["action_type"] = _expect_str(x, expected="a string")
elif key == "action_version":
kw["action_version"] = _expect_str(x, expected="a string")
elif key == "payload":
kw["payload"] = x
return KvHintAction(**kw)
def encode_KvHintAction(v: KvHintAction) -> Dict[str, Any]:
d: Dict[str, Any] = {}
d["action_id"] = v.action_id
d["action_type"] = v.action_type
d["action_version"] = v.action_version
if v.payload is not None:
d["payload"] = v.payload
return d
@@ -1514,6 +1816,121 @@ def encode_MediaInput(v: MediaInput) -> Any:
return v
def decode_ListModelsRequest(value: Any) -> ListModelsRequest:
m = _expect_map(value, expected="struct ListModelsRequest")
return ListModelsRequest()
def encode_ListModelsRequest(v: ListModelsRequest) -> Dict[str, Any]:
d: Dict[str, Any] = {}
return d
def decode_ListModelsResponse(value: Any) -> ListModelsResponse:
m = _expect_map(value, expected="struct ListModelsResponse")
kw: Dict[str, Any] = {}
for key, x in m.items():
if key == "models":
kw["models"] = [] if x is None else [decode_ModelCard(e) for e in _expect_seq(x, expected="a sequence")]
return ListModelsResponse(**kw)
def encode_ListModelsResponse(v: ListModelsResponse) -> Dict[str, Any]:
d: Dict[str, Any] = {}
if v.models:
d["models"] = [encode_ModelCard(e) for e in v.models]
return d
def decode_ModelCard(value: Any) -> ModelCard:
m = _expect_map(value, expected="struct ModelCard")
kw: Dict[str, Any] = {}
for key, x in m.items():
if key == "id":
kw["id"] = _expect_str(x, expected="a string")
elif key == "root":
kw["root"] = _expect_str(x, expected="a string")
elif key == "parent":
kw["parent"] = None if x is None else _expect_str(x, expected="a string")
elif key == "max_model_len":
kw["max_model_len"] = _expect_uint(x, expected="u64")
return ModelCard(**kw)
def encode_ModelCard(v: ModelCard) -> Dict[str, Any]:
d: Dict[str, Any] = {}
d["id"] = v.id
d["root"] = v.root
if v.parent is not None:
d["parent"] = v.parent
d["max_model_len"] = v.max_model_len
return d
def decode_DetokenizeRequest(value: Any) -> DetokenizeRequest:
m = _expect_map(value, expected="struct DetokenizeRequest")
kw: Dict[str, Any] = {}
for key, x in m.items():
if key == "tokens":
kw["tokens"] = [] if x is None else [_expect_int(e, expected="i64") for e in _expect_seq(x, expected="a sequence")]
return DetokenizeRequest(**kw)
def encode_DetokenizeRequest(v: DetokenizeRequest) -> Dict[str, Any]:
d: Dict[str, Any] = {}
if v.tokens:
d["tokens"] = list(v.tokens)
return d
def decode_DetokenizeResponse(value: Any) -> DetokenizeResponse:
m = _expect_map(value, expected="struct DetokenizeResponse")
kw: Dict[str, Any] = {}
for key, x in m.items():
if key == "text":
kw["text"] = _expect_str(x, expected="a string")
return DetokenizeResponse(**kw)
def encode_DetokenizeResponse(v: DetokenizeResponse) -> Dict[str, Any]:
d: Dict[str, Any] = {}
d["text"] = v.text
return d
def decode_AbortRequest(value: Any) -> AbortRequest:
m = _expect_map(value, expected="struct AbortRequest")
kw: Dict[str, Any] = {}
for key, x in m.items():
if key == "rid":
kw["rid"] = _expect_str(x, expected="a string")
elif key == "abort_all":
kw["abort_all"] = _expect_bool(x, expected="a boolean")
return AbortRequest(**kw)
def encode_AbortRequest(v: AbortRequest) -> Dict[str, Any]:
d: Dict[str, Any] = {}
d["rid"] = v.rid
d["abort_all"] = v.abort_all
return d
def decode_AbortResponse(value: Any) -> AbortResponse:
m = _expect_map(value, expected="struct AbortResponse")
kw: Dict[str, Any] = {}
for key, x in m.items():
if key == "success":
kw["success"] = _expect_bool(x, expected="a boolean")
return AbortResponse(**kw)
def encode_AbortResponse(v: AbortResponse) -> Dict[str, Any]:
d: Dict[str, Any] = {}
d["success"] = v.success
return d
def decode_HealthCheckRequest(value: Any) -> HealthCheckRequest:
m = _expect_map(value, expected="struct HealthCheckRequest")
return HealthCheckRequest()
@@ -1616,6 +2033,16 @@ def decode_GetServerInfoResponse(value: Any) -> GetServerInfoResponse:
kw["version"] = _expect_str(x, expected="a string")
elif key == "internal_states":
kw["internal_states"] = [] if x is None else list(_expect_seq(x, expected="a sequence"))
elif key == "host":
kw["host"] = _expect_str(x, expected="a string")
elif key == "port":
kw["port"] = _expect_uint(x, expected="u32")
elif key == "incremental_streaming_output":
kw["incremental_streaming_output"] = _expect_bool(x, expected="a boolean")
elif key == "disaggregation_mode":
kw["disaggregation_mode"] = _expect_str(x, expected="a string")
elif key == "deployment_info":
kw["deployment_info"] = None if x is None else decode_DeploymentInfo(x)
return GetServerInfoResponse(**kw)
@@ -1630,4 +2057,113 @@ def encode_GetServerInfoResponse(v: GetServerInfoResponse) -> Dict[str, Any]:
d["version"] = v.version
if v.internal_states:
d["internal_states"] = list(v.internal_states)
d["host"] = v.host
d["port"] = v.port
d["incremental_streaming_output"] = v.incremental_streaming_output
d["disaggregation_mode"] = v.disaggregation_mode
if v.deployment_info is not None:
d["deployment_info"] = encode_DeploymentInfo(v.deployment_info)
return d
def decode_DeploymentInfo(value: Any) -> DeploymentInfo:
m = _expect_map(value, expected="struct DeploymentInfo")
kw: Dict[str, Any] = {}
for key, x in m.items():
if key == "disaggregation_bootstrap_port":
kw["disaggregation_bootstrap_port"] = None if x is None else _expect_uint(x, expected="u32")
elif key == "node_rank":
kw["node_rank"] = _expect_uint(x, expected="u32")
elif key == "nnodes":
kw["nnodes"] = _expect_uint(x, expected="u32")
elif key == "dp_size":
kw["dp_size"] = _expect_uint(x, expected="u32")
elif key == "dp_rank":
kw["dp_rank"] = None if x is None else _expect_uint(x, expected="u32")
elif key == "dist_init_addr":
kw["dist_init_addr"] = None if x is None else _expect_str(x, expected="a string")
elif key == "base_gpu_id":
kw["base_gpu_id"] = _expect_uint(x, expected="u32")
elif key == "page_size":
kw["page_size"] = None if x is None else _expect_uint(x, expected="u32")
elif key == "dcp_size":
kw["dcp_size"] = _expect_uint(x, expected="u32")
elif key == "max_running_requests":
kw["max_running_requests"] = None if x is None else _expect_uint(x, expected="u64")
elif key == "max_prefill_tokens":
kw["max_prefill_tokens"] = _expect_uint(x, expected="u64")
elif key == "enable_dp_attention":
kw["enable_dp_attention"] = _expect_bool(x, expected="a boolean")
elif key == "speculative_algorithm":
kw["speculative_algorithm"] = None if x is None else _expect_str(x, expected="a string")
elif key == "enable_hierarchical_cache":
kw["enable_hierarchical_cache"] = _expect_bool(x, expected="a boolean")
elif key == "hicache_ratio":
kw["hicache_ratio"] = None if x is None else _expect_float(x, expected="f64")
elif key == "hicache_size":
kw["hicache_size"] = _expect_uint(x, expected="u64")
elif key == "hicache_write_policy":
kw["hicache_write_policy"] = None if x is None else _expect_str(x, expected="a string")
elif key == "hicache_host_total_tokens":
kw["hicache_host_total_tokens"] = None if x is None else _expect_uint(x, expected="u64")
elif key == "kv_event_sources":
kw["kv_event_sources"] = [] if x is None else [decode_KvEventSource(e) for e in _expect_seq(x, expected="a sequence")]
return DeploymentInfo(**kw)
def encode_DeploymentInfo(v: DeploymentInfo) -> Dict[str, Any]:
d: Dict[str, Any] = {}
if v.disaggregation_bootstrap_port is not None:
d["disaggregation_bootstrap_port"] = v.disaggregation_bootstrap_port
d["node_rank"] = v.node_rank
d["nnodes"] = v.nnodes
d["dp_size"] = v.dp_size
if v.dp_rank is not None:
d["dp_rank"] = v.dp_rank
if v.dist_init_addr is not None:
d["dist_init_addr"] = v.dist_init_addr
d["base_gpu_id"] = v.base_gpu_id
if v.page_size is not None:
d["page_size"] = v.page_size
d["dcp_size"] = v.dcp_size
if v.max_running_requests is not None:
d["max_running_requests"] = v.max_running_requests
d["max_prefill_tokens"] = v.max_prefill_tokens
d["enable_dp_attention"] = v.enable_dp_attention
if v.speculative_algorithm is not None:
d["speculative_algorithm"] = v.speculative_algorithm
d["enable_hierarchical_cache"] = v.enable_hierarchical_cache
if v.hicache_ratio is not None:
d["hicache_ratio"] = float(v.hicache_ratio)
d["hicache_size"] = v.hicache_size
if v.hicache_write_policy is not None:
d["hicache_write_policy"] = v.hicache_write_policy
if v.hicache_host_total_tokens is not None:
d["hicache_host_total_tokens"] = v.hicache_host_total_tokens
if v.kv_event_sources:
d["kv_event_sources"] = [encode_KvEventSource(e) for e in v.kv_event_sources]
return d
def decode_KvEventSource(value: Any) -> KvEventSource:
m = _expect_map(value, expected="struct KvEventSource")
kw: Dict[str, Any] = {}
for key, x in m.items():
if key == "dp_rank":
kw["dp_rank"] = _expect_uint(x, expected="u32")
elif key == "endpoint":
kw["endpoint"] = _expect_str(x, expected="a string")
elif key == "topic":
kw["topic"] = _expect_str(x, expected="a string")
elif key == "block_size":
kw["block_size"] = _expect_uint(x, expected="u32")
return KvEventSource(**kw)
def encode_KvEventSource(v: KvEventSource) -> Dict[str, Any]:
d: Dict[str, Any] = {}
d["dp_rank"] = v.dp_rank
d["endpoint"] = v.endpoint
d["topic"] = v.topic
d["block_size"] = v.block_size
return d
@@ -317,6 +317,12 @@ def handle_data_parallelism(server_args: Any):
"_handle_data_parallelism",
schedule_conservativeness=cfg.schedule_conservativeness * 0.3,
)
if envs.SGLANG_RUST_SERVER.get():
declare_resolution(
server_args,
"_handle_data_parallelism",
enable_dp_attention_local_control_broadcast=True,
)
assert cfg.tp_size % cfg.attn_dp_size == 0
original_chunked_prefill_size = cfg.chunked_prefill_size
declare_resolution(
+5 -2
View File
@@ -657,6 +657,11 @@ class Scheduler(
self.init_metrics_reporter()
self.scheduler_stage_metrics = self.metrics_reporter.scheduler_stage_metrics
# The Rust frontend snapshots local KV-event discovery metadata while it
# builds its typed server info, so the publisher must exist before the
# frontend starts.
self.init_kv_events_publisher()
# Init schedule policy and new token estimation
self.init_schedule_policy()
@@ -713,8 +718,6 @@ class Scheduler(
self.init_invariant_checker()
self.init_kv_events_publisher()
self.init_load_publisher()
self.init_load_inquirer()
+45
View File
@@ -9,6 +9,7 @@ from types import ModuleType
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple
from sglang.srt.arg_groups.overrides import resolving_view
from sglang.srt.disaggregation.kv_events import select_kv_publisher_dp_rank
from sglang.srt.entrypoints.v1_loads import (
_accelerator_name,
_num_accelerators_per_dp_rank,
@@ -17,12 +18,17 @@ from sglang.srt.environ import envs
from sglang.srt.managers.load_snapshot import ShmLoadSnapshotWriter
from sglang.srt.managers.utils import compute_num_reserved_tokens
from sglang.srt.runtime_context import (
get_device,
get_disagg,
get_lora,
get_memory,
get_mm,
get_model,
get_observability,
get_parallel,
get_schedule,
get_serving,
get_spec,
)
from sglang.srt.sampling.watermarking.core import redact_watermark_secrets
from sglang.srt.utils.common import build_server_info
@@ -65,6 +71,38 @@ def _build_server_args(
if get_serving().smg_grpc_mode or get_serving().grpc_mode
else get_serving().grpc_port
)
deployment_info = {
"disaggregation_bootstrap_port": get_disagg().disaggregation_bootstrap_port,
"node_rank": get_parallel().node_rank,
"nnodes": get_parallel().nnodes,
"dp_size": get_parallel().num_dp_ranks,
"dp_rank": select_kv_publisher_dp_rank(
get_parallel().attn_dp_size,
get_parallel().attn_dp_rank,
get_parallel().dp_rank,
),
"dist_init_addr": get_parallel().dist_init_addr,
"base_gpu_id": get_device().base_gpu_id,
"page_size": get_schedule().page_size,
"dcp_size": get_parallel().dcp_size,
"max_running_requests": get_schedule().max_running_requests,
"max_prefill_tokens": get_schedule().max_prefill_tokens,
"enable_dp_attention": get_parallel().attn_dp_size > 1,
"speculative_algorithm": get_spec().speculative_algorithm,
"enable_hierarchical_cache": get_memory().enable_hierarchical_cache,
"hicache_ratio": get_memory().hicache_ratio,
"hicache_size": get_memory().hicache_size,
"hicache_write_policy": get_memory().hicache_write_policy,
# Python's scheduler handshake only computes startup_time after all
# components have initialized, but the Rust frontend must start earlier
# to serve the disaggregation bootstrap registry. Read the already-
# initialized KV publisher directly instead of calling get_init_info()
# on a partially initialized scheduler.
"hicache_host_total_tokens": None,
"kv_event_sources": scheduler.kv_events_publisher.local_kv_event_sources(
scheduler.page_size * get_parallel().dcp_size
),
}
# `/v1/loads` reads this rank's slot of the scheduler's shared-memory
# snapshots directly; the ZMQ transport (multi-node DP) has no file to read.
shm_writer = scheduler.load_snapshot_writer
@@ -120,6 +158,12 @@ def _build_server_args(
if get_serving().preferred_sampling_params is not None
else None
),
lora_ids={
ref.lora_name: ref.lora_id
for ref in (get_lora().lora_paths or [])
if ref.lora_name is not None
},
enable_strict_thinking=get_serving().enable_strict_thinking,
limit_mm_data_per_request=get_mm().limit_mm_data_per_request or {},
allow_auto_truncate=get_serving().allow_auto_truncate,
enable_return_hidden_states=sa.enable_return_hidden_states,
@@ -134,6 +178,7 @@ def _build_server_args(
# can serve them statically (no scheduler round-trip).
version=__version__,
max_total_num_tokens=scheduler.max_total_num_tokens,
deployment_info=json.dumps(deployment_info),
launch_server_info=_launch_server_info(scheduler),
load_snapshot_path=shm_writer.path if shm_writer else None,
load_snapshot_dp_size=shm_writer.dp_size if shm_writer else 0,
+2
View File
@@ -3680,6 +3680,8 @@ def start_follower_grpc_server(server_args, scheduler_info: Dict[str, Any]):
serving = get_serving()
if serving.grpc_port is None or serving.smg_grpc_mode or serving.grpc_mode:
return None
if envs.SGLANG_RUST_SERVER.get():
return None
from sglang.srt.rust_extensions import load_rust_extension
from sglang.srt.sampling.watermarking.core import redact_watermark_secrets