From 8d7df07a56dbf53ec8db459f50353c7a45bf9b38 Mon Sep 17 00:00:00 2001 From: Apeiria <106635619+0130w@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:01:35 +0800 Subject: [PATCH] [Feat] rust-server: Add HiCache storage management endpoints (#44726) Signed-off-by: 0130w <1157374677@qq.com> Co-authored-by: Rain Jiang <96632942+rainj-me@users.noreply.github.com> --- proto/sglang/api/v1/hicache.proto | 42 ++ proto/sglang/api/v1/service.proto | 8 + python/sglang/api/v1/api_types.py | 191 ++++++ python/sglang/srt/rust_server/config.py | 4 + .../src/generated/sglang.api.v1.rs | 40 ++ .../src/generated/sglang.api.v1.serde.rs | 349 +++++++++++ .../src/generated/sglang.api.v1.tonic.rs | 263 +++++++++ .../testdata/json_parity.json | 91 +++ rust/sglang-api-types/tests/json_parity.rs | 16 + rust/sglang-server/src/api_server/core.rs | 2 + .../src/api_server/core/handle.rs | 2 +- .../src/api_server/core/hicache.rs | 213 +++++++ .../src/api_server/grpc/service.rs | 163 +++++- rust/sglang-server/src/api_server/http.rs | 1 + rust/sglang-server/src/api_server/http/app.rs | 10 +- .../src/api_server/http/hicache.rs | 549 ++++++++++++++++++ rust/sglang-server/src/message/config.rs | 24 + rust/sglang-server/src/message/io_struct.rs | 44 ++ rust/sglang-server/src/utils/runtime.rs | 82 ++- test/registered/unit/api/v1/test_parity.py | 40 +- .../entrypoints/test_rust_server_dp_ports.py | 16 + 21 files changed, 2137 insertions(+), 13 deletions(-) create mode 100644 proto/sglang/api/v1/hicache.proto create mode 100644 rust/sglang-server/src/api_server/core/hicache.rs create mode 100644 rust/sglang-server/src/api_server/http/hicache.rs diff --git a/proto/sglang/api/v1/hicache.proto b/proto/sglang/api/v1/hicache.proto new file mode 100644 index 0000000000..912883c35e --- /dev/null +++ b/proto/sglang/api/v1/hicache.proto @@ -0,0 +1,42 @@ +// HiCache storage management APIs shared by HTTP JSON and gRPC. +syntax = "proto3"; + +package sglang.api.v1; + +import "sglang/json/v1/options.proto"; + +message GetHiCacheStorageRequest {} + +message AttachHiCacheStorageRequest { + // Required and non-null in the Python HTTP API; handlers validate presence. + // An empty string is passed through for scheduler-side backend validation. + optional string hicache_storage_backend = 1; + optional string hicache_storage_backend_extra_config_json = 2; + optional string hicache_storage_prefetch_policy = 3; + optional string hicache_write_policy = 4; +} + +message DetachHiCacheStorageRequest {} + +message ClearHiCacheStorageRequest {} + +message HiCacheStatus { + optional string hicache_storage_backend = 1 [ + (sglang.json.v1.field_json).emit_null_when_absent = true + ]; + optional string hicache_storage_backend_extra_config = 2 [ + (sglang.json.v1.field_json).emit_null_when_absent = true + ]; + optional string hicache_storage_prefetch_policy = 3 [ + (sglang.json.v1.field_json).emit_null_when_absent = true + ]; + optional string hicache_write_policy = 4 [ + (sglang.json.v1.field_json).emit_null_when_absent = true + ]; +} + +// Result shared by the HTTP and gRPC management APIs. +message HiCacheStorageResponse { + bool success = 1; + string message = 2; +} diff --git a/proto/sglang/api/v1/service.proto b/proto/sglang/api/v1/service.proto index 33bed674a8..e2d02cf40d 100644 --- a/proto/sglang/api/v1/service.proto +++ b/proto/sglang/api/v1/service.proto @@ -9,6 +9,7 @@ syntax = "proto3"; package sglang.api.v1; import "sglang/api/v1/generate.proto"; +import "sglang/api/v1/hicache.proto"; import "sglang/json/v1/options.proto"; service SglangService { @@ -28,6 +29,13 @@ service SglangService { // allowlist (never the raw server-args dump, which embeds keys). rpc GetServerInfo(GetServerInfoRequest) returns (GetServerInfoResponse); + // `/hicache/storage-backend`: configuration snapshot and runtime attach/detach. + rpc GetHiCacheStorage(GetHiCacheStorageRequest) returns (HiCacheStatus); + rpc AttachHiCacheStorage(AttachHiCacheStorageRequest) returns (HiCacheStorageResponse); + rpc DetachHiCacheStorage(DetachHiCacheStorageRequest) returns (HiCacheStorageResponse); + + // `/hicache/storage-backend/clear`: clear the storage backend's cached entries. + rpc ClearHiCacheStorage(ClearHiCacheStorageRequest) returns (HiCacheStorageResponse); // The same static model cards as HTTP `/v1/models`. rpc ListModels(ListModelsRequest) returns (ListModelsResponse); diff --git a/python/sglang/api/v1/api_types.py b/python/sglang/api/v1/api_types.py index 661a1f3882..5e8791fab1 100644 --- a/python/sglang/api/v1/api_types.py +++ b/python/sglang/api/v1/api_types.py @@ -421,6 +421,97 @@ class MediaRef(msgspec.Struct, kw_only=True, frozen=True): return encode_MediaRef(self) +class GetHiCacheStorageRequest(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.GetHiCacheStorageRequest: JSON object (unknown keys ignored).""" + + @classmethod + def from_json_value(cls, value: Any) -> "GetHiCacheStorageRequest": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_GetHiCacheStorageRequest(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_GetHiCacheStorageRequest(self) + + +class AttachHiCacheStorageRequest(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.AttachHiCacheStorageRequest: JSON object (unknown keys ignored).""" + + hicache_storage_backend: Optional[str] = None + hicache_storage_backend_extra_config_json: Optional[str] = None + hicache_storage_prefetch_policy: Optional[str] = None + hicache_write_policy: Optional[str] = None + + @classmethod + def from_json_value(cls, value: Any) -> "AttachHiCacheStorageRequest": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_AttachHiCacheStorageRequest(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_AttachHiCacheStorageRequest(self) + + +class DetachHiCacheStorageRequest(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.DetachHiCacheStorageRequest: JSON object (unknown keys ignored).""" + + @classmethod + def from_json_value(cls, value: Any) -> "DetachHiCacheStorageRequest": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_DetachHiCacheStorageRequest(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_DetachHiCacheStorageRequest(self) + + +class ClearHiCacheStorageRequest(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.ClearHiCacheStorageRequest: JSON object (unknown keys ignored).""" + + @classmethod + def from_json_value(cls, value: Any) -> "ClearHiCacheStorageRequest": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_ClearHiCacheStorageRequest(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_ClearHiCacheStorageRequest(self) + + +class HiCacheStatus(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.HiCacheStatus: JSON object (unknown keys ignored).""" + + hicache_storage_backend: Optional[str] = None + hicache_storage_backend_extra_config: Optional[str] = None + hicache_storage_prefetch_policy: Optional[str] = None + hicache_write_policy: Optional[str] = None + + @classmethod + def from_json_value(cls, value: Any) -> "HiCacheStatus": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_HiCacheStatus(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_HiCacheStatus(self) + + +class HiCacheStorageResponse(msgspec.Struct, kw_only=True, frozen=True): + """sglang.api.v1.HiCacheStorageResponse: JSON object (unknown keys ignored).""" + + success: bool = False + message: str = "" + + @classmethod + def from_json_value(cls, value: Any) -> "HiCacheStorageResponse": + """Decode a parsed JSON value; raises JsonContractError on a contract violation.""" + return decode_HiCacheStorageResponse(value) + + def to_json_value(self) -> Dict[str, Any]: + """Encode to a JSON-ready value (json.dumps-compatible).""" + return encode_HiCacheStorageResponse(self) + + class ListModelsRequest(msgspec.Struct, kw_only=True, frozen=True): """sglang.api.v1.ListModelsRequest: JSON object (unknown keys ignored).""" @@ -1816,6 +1907,106 @@ def encode_MediaInput(v: MediaInput) -> Any: return v +def decode_GetHiCacheStorageRequest(value: Any) -> GetHiCacheStorageRequest: + m = _expect_map(value, expected="struct GetHiCacheStorageRequest") + return GetHiCacheStorageRequest() + + +def encode_GetHiCacheStorageRequest(v: GetHiCacheStorageRequest) -> Dict[str, Any]: + d: Dict[str, Any] = {} + return d + + +def decode_AttachHiCacheStorageRequest(value: Any) -> AttachHiCacheStorageRequest: + m = _expect_map(value, expected="struct AttachHiCacheStorageRequest") + kw: Dict[str, Any] = {} + for key, x in m.items(): + if key == "hicache_storage_backend": + kw["hicache_storage_backend"] = None if x is None else _expect_str(x, expected="a string") + elif key == "hicache_storage_backend_extra_config_json": + kw["hicache_storage_backend_extra_config_json"] = None if x is None else _expect_str(x, expected="a string") + elif key == "hicache_storage_prefetch_policy": + kw["hicache_storage_prefetch_policy"] = None if x is None else _expect_str(x, expected="a string") + elif key == "hicache_write_policy": + kw["hicache_write_policy"] = None if x is None else _expect_str(x, expected="a string") + return AttachHiCacheStorageRequest(**kw) + + +def encode_AttachHiCacheStorageRequest(v: AttachHiCacheStorageRequest) -> Dict[str, Any]: + d: Dict[str, Any] = {} + if v.hicache_storage_backend is not None: + d["hicache_storage_backend"] = v.hicache_storage_backend + if v.hicache_storage_backend_extra_config_json is not None: + d["hicache_storage_backend_extra_config_json"] = v.hicache_storage_backend_extra_config_json + if v.hicache_storage_prefetch_policy is not None: + d["hicache_storage_prefetch_policy"] = v.hicache_storage_prefetch_policy + if v.hicache_write_policy is not None: + d["hicache_write_policy"] = v.hicache_write_policy + return d + + +def decode_DetachHiCacheStorageRequest(value: Any) -> DetachHiCacheStorageRequest: + m = _expect_map(value, expected="struct DetachHiCacheStorageRequest") + return DetachHiCacheStorageRequest() + + +def encode_DetachHiCacheStorageRequest(v: DetachHiCacheStorageRequest) -> Dict[str, Any]: + d: Dict[str, Any] = {} + return d + + +def decode_ClearHiCacheStorageRequest(value: Any) -> ClearHiCacheStorageRequest: + m = _expect_map(value, expected="struct ClearHiCacheStorageRequest") + return ClearHiCacheStorageRequest() + + +def encode_ClearHiCacheStorageRequest(v: ClearHiCacheStorageRequest) -> Dict[str, Any]: + d: Dict[str, Any] = {} + return d + + +def decode_HiCacheStatus(value: Any) -> HiCacheStatus: + m = _expect_map(value, expected="struct HiCacheStatus") + kw: Dict[str, Any] = {} + for key, x in m.items(): + if key == "hicache_storage_backend": + kw["hicache_storage_backend"] = None if x is None else _expect_str(x, expected="a string") + elif key == "hicache_storage_backend_extra_config": + kw["hicache_storage_backend_extra_config"] = None if x is None else _expect_str(x, expected="a string") + elif key == "hicache_storage_prefetch_policy": + kw["hicache_storage_prefetch_policy"] = None if x is None else _expect_str(x, expected="a string") + elif key == "hicache_write_policy": + kw["hicache_write_policy"] = None if x is None else _expect_str(x, expected="a string") + return HiCacheStatus(**kw) + + +def encode_HiCacheStatus(v: HiCacheStatus) -> Dict[str, Any]: + d: Dict[str, Any] = {} + d["hicache_storage_backend"] = v.hicache_storage_backend + d["hicache_storage_backend_extra_config"] = v.hicache_storage_backend_extra_config + d["hicache_storage_prefetch_policy"] = v.hicache_storage_prefetch_policy + d["hicache_write_policy"] = v.hicache_write_policy + return d + + +def decode_HiCacheStorageResponse(value: Any) -> HiCacheStorageResponse: + m = _expect_map(value, expected="struct HiCacheStorageResponse") + kw: Dict[str, Any] = {} + for key, x in m.items(): + if key == "success": + kw["success"] = _expect_bool(x, expected="a boolean") + elif key == "message": + kw["message"] = _expect_str(x, expected="a string") + return HiCacheStorageResponse(**kw) + + +def encode_HiCacheStorageResponse(v: HiCacheStorageResponse) -> Dict[str, Any]: + d: Dict[str, Any] = {} + d["success"] = v.success + d["message"] = v.message + return d + + def decode_ListModelsRequest(value: Any) -> ListModelsRequest: m = _expect_map(value, expected="struct ListModelsRequest") return ListModelsRequest() diff --git a/python/sglang/srt/rust_server/config.py b/python/sglang/srt/rust_server/config.py index c636c33631..db6f383813 100644 --- a/python/sglang/srt/rust_server/config.py +++ b/python/sglang/srt/rust_server/config.py @@ -117,6 +117,10 @@ def _build_server_args( revision=get_model().revision, load_format=get_model().load_format, weight_version=get_serving().weight_version, + hicache_storage_backend=get_memory().hicache_storage_backend, + hicache_storage_backend_extra_config=get_memory().hicache_storage_backend_extra_config, + hicache_storage_prefetch_policy=get_memory().hicache_storage_prefetch_policy, + hicache_write_policy=get_memory().hicache_write_policy, host=get_serving().host, port=get_serving().port, # The shared field is also populated for legacy SMG mode; only forward diff --git a/rust/sglang-api-types/src/generated/sglang.api.v1.rs b/rust/sglang-api-types/src/generated/sglang.api.v1.rs index 8baa991b0d..e301fed478 100644 --- a/rust/sglang-api-types/src/generated/sglang.api.v1.rs +++ b/rust/sglang-api-types/src/generated/sglang.api.v1.rs @@ -710,6 +710,46 @@ pub mod media_input { } } #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetHiCacheStorageRequest {} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct AttachHiCacheStorageRequest { + /// Required and non-null in the Python HTTP API; handlers validate presence. + /// An empty string is passed through for scheduler-side backend validation. + #[prost(string, optional, tag = "1")] + pub hicache_storage_backend: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "2")] + pub hicache_storage_backend_extra_config_json: + ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "3")] + pub hicache_storage_prefetch_policy: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "4")] + pub hicache_write_policy: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct DetachHiCacheStorageRequest {} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ClearHiCacheStorageRequest {} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct HiCacheStatus { + #[prost(string, optional, tag = "1")] + pub hicache_storage_backend: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "2")] + pub hicache_storage_backend_extra_config: + ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "3")] + pub hicache_storage_prefetch_policy: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "4")] + pub hicache_write_policy: ::core::option::Option<::prost::alloc::string::String>, +} +/// Result shared by the HTTP and gRPC management APIs. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct HiCacheStorageResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, tag = "2")] + pub message: ::prost::alloc::string::String, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct ListModelsRequest {} #[derive(Clone, PartialEq, ::prost::Message)] pub struct ListModelsResponse { diff --git a/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs b/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs index 2f6e955e6d..762c9da8d5 100644 --- a/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs +++ b/rust/sglang-api-types/src/generated/sglang.api.v1.serde.rs @@ -3163,6 +3163,355 @@ impl<'de> ::serde::Deserialize<'de> for MediaInput { } } +impl ::serde::Serialize for GetHiCacheStorageRequest { + fn serialize(&self, serializer: S) -> Result { + let map = serializer.serialize_map(None)?; + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for GetHiCacheStorageRequest { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = GetHiCacheStorageRequest; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct GetHiCacheStorageRequest") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(GetHiCacheStorageRequest {}) + } + } + deserializer.deserialize_map(V) + } +} + +impl ::serde::Serialize for AttachHiCacheStorageRequest { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + if let Some(v) = &self.hicache_storage_backend { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "hicache_storage_backend", v)?; + } + if let Some(v) = &self.hicache_storage_backend_extra_config_json { + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "hicache_storage_backend_extra_config_json", + v, + )?; + } + if let Some(v) = &self.hicache_storage_prefetch_policy { + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "hicache_storage_prefetch_policy", + v, + )?; + } + if let Some(v) = &self.hicache_write_policy { + ::serde::ser::SerializeMap::serialize_entry(&mut map, "hicache_write_policy", v)?; + } + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for AttachHiCacheStorageRequest { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = AttachHiCacheStorageRequest; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct AttachHiCacheStorageRequest") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut f_hicache_storage_backend = None; + let mut f_hicache_storage_backend_extra_config_json = None; + let mut f_hicache_storage_prefetch_policy = None; + let mut f_hicache_write_policy = None; + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + "hicache_storage_backend" => { + if f_hicache_storage_backend.is_some() { + return Err(::serde::de::Error::duplicate_field( + "hicache_storage_backend", + )); + } + f_hicache_storage_backend = + Some(map.next_value::>()?); + } + "hicache_storage_backend_extra_config_json" => { + if f_hicache_storage_backend_extra_config_json.is_some() { + return Err(::serde::de::Error::duplicate_field( + "hicache_storage_backend_extra_config_json", + )); + } + f_hicache_storage_backend_extra_config_json = + Some(map.next_value::>()?); + } + "hicache_storage_prefetch_policy" => { + if f_hicache_storage_prefetch_policy.is_some() { + return Err(::serde::de::Error::duplicate_field( + "hicache_storage_prefetch_policy", + )); + } + f_hicache_storage_prefetch_policy = + Some(map.next_value::>()?); + } + "hicache_write_policy" => { + if f_hicache_write_policy.is_some() { + return Err(::serde::de::Error::duplicate_field( + "hicache_write_policy", + )); + } + f_hicache_write_policy = + Some(map.next_value::>()?); + } + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(AttachHiCacheStorageRequest { + hicache_storage_backend: f_hicache_storage_backend.flatten(), + hicache_storage_backend_extra_config_json: + f_hicache_storage_backend_extra_config_json.flatten(), + hicache_storage_prefetch_policy: f_hicache_storage_prefetch_policy.flatten(), + hicache_write_policy: f_hicache_write_policy.flatten(), + }) + } + } + deserializer.deserialize_map(V) + } +} + +impl ::serde::Serialize for DetachHiCacheStorageRequest { + fn serialize(&self, serializer: S) -> Result { + let map = serializer.serialize_map(None)?; + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for DetachHiCacheStorageRequest { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = DetachHiCacheStorageRequest; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct DetachHiCacheStorageRequest") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(DetachHiCacheStorageRequest {}) + } + } + deserializer.deserialize_map(V) + } +} + +impl ::serde::Serialize for ClearHiCacheStorageRequest { + fn serialize(&self, serializer: S) -> Result { + let map = serializer.serialize_map(None)?; + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for ClearHiCacheStorageRequest { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = ClearHiCacheStorageRequest; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct ClearHiCacheStorageRequest") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(ClearHiCacheStorageRequest {}) + } + } + deserializer.deserialize_map(V) + } +} + +impl ::serde::Serialize for HiCacheStatus { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "hicache_storage_backend", + &self.hicache_storage_backend, + )?; + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "hicache_storage_backend_extra_config", + &self.hicache_storage_backend_extra_config, + )?; + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "hicache_storage_prefetch_policy", + &self.hicache_storage_prefetch_policy, + )?; + ::serde::ser::SerializeMap::serialize_entry( + &mut map, + "hicache_write_policy", + &self.hicache_write_policy, + )?; + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for HiCacheStatus { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = HiCacheStatus; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct HiCacheStatus") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut f_hicache_storage_backend = None; + let mut f_hicache_storage_backend_extra_config = None; + let mut f_hicache_storage_prefetch_policy = None; + let mut f_hicache_write_policy = None; + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + "hicache_storage_backend" => { + if f_hicache_storage_backend.is_some() { + return Err(::serde::de::Error::duplicate_field( + "hicache_storage_backend", + )); + } + f_hicache_storage_backend = + Some(map.next_value::>()?); + } + "hicache_storage_backend_extra_config" => { + if f_hicache_storage_backend_extra_config.is_some() { + return Err(::serde::de::Error::duplicate_field( + "hicache_storage_backend_extra_config", + )); + } + f_hicache_storage_backend_extra_config = + Some(map.next_value::>()?); + } + "hicache_storage_prefetch_policy" => { + if f_hicache_storage_prefetch_policy.is_some() { + return Err(::serde::de::Error::duplicate_field( + "hicache_storage_prefetch_policy", + )); + } + f_hicache_storage_prefetch_policy = + Some(map.next_value::>()?); + } + "hicache_write_policy" => { + if f_hicache_write_policy.is_some() { + return Err(::serde::de::Error::duplicate_field( + "hicache_write_policy", + )); + } + f_hicache_write_policy = + Some(map.next_value::>()?); + } + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(HiCacheStatus { + hicache_storage_backend: f_hicache_storage_backend.flatten(), + hicache_storage_backend_extra_config: f_hicache_storage_backend_extra_config + .flatten(), + hicache_storage_prefetch_policy: f_hicache_storage_prefetch_policy.flatten(), + hicache_write_policy: f_hicache_write_policy.flatten(), + }) + } + } + deserializer.deserialize_map(V) + } +} + +impl ::serde::Serialize for HiCacheStorageResponse { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(None)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "success", &self.success)?; + ::serde::ser::SerializeMap::serialize_entry(&mut map, "message", &self.message)?; + ::serde::ser::SerializeMap::end(map) + } +} + +impl<'de> ::serde::Deserialize<'de> for HiCacheStorageResponse { + fn deserialize>(deserializer: D) -> Result { + struct V; + impl<'de> ::serde::de::Visitor<'de> for V { + type Value = HiCacheStorageResponse; + fn expecting(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result { + f.write_str("struct HiCacheStorageResponse") + } + fn visit_map>( + self, + mut map: A, + ) -> Result { + let mut f_success = None; + let mut f_message = None; + while let Some(key) = map.next_key::<::std::borrow::Cow<'_, str>>()? { + match key.as_ref() { + "success" => { + if f_success.is_some() { + return Err(::serde::de::Error::duplicate_field("success")); + } + f_success = Some(map.next_value::()?); + } + "message" => { + if f_message.is_some() { + return Err(::serde::de::Error::duplicate_field("message")); + } + f_message = Some(map.next_value::<::prost::alloc::string::String>()?); + } + _ => { + map.next_value::<::serde::de::IgnoredAny>()?; + } + } + } + Ok(HiCacheStorageResponse { + success: f_success.unwrap_or_default(), + message: f_message.unwrap_or_default(), + }) + } + } + deserializer.deserialize_map(V) + } +} + impl ::serde::Serialize for ListModelsRequest { fn serialize(&self, serializer: S) -> Result { let map = serializer.serialize_map(None)?; diff --git a/rust/sglang-api-types/src/generated/sglang.api.v1.tonic.rs b/rust/sglang-api-types/src/generated/sglang.api.v1.tonic.rs index e7f8f3a55c..d743054d18 100644 --- a/rust/sglang-api-types/src/generated/sglang.api.v1.tonic.rs +++ b/rust/sglang-api-types/src/generated/sglang.api.v1.tonic.rs @@ -168,6 +168,83 @@ pub mod sglang_service_client { )); self.inner.unary(req, path, codec).await } + /// `/hicache/storage-backend`: configuration snapshot and runtime attach/detach. + pub async fn get_hi_cache_storage( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/sglang.api.v1.SglangService/GetHiCacheStorage", + ); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "sglang.api.v1.SglangService", + "GetHiCacheStorage", + )); + self.inner.unary(req, path, codec).await + } + pub async fn attach_hi_cache_storage( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/sglang.api.v1.SglangService/AttachHiCacheStorage", + ); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "sglang.api.v1.SglangService", + "AttachHiCacheStorage", + )); + self.inner.unary(req, path, codec).await + } + pub async fn detach_hi_cache_storage( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/sglang.api.v1.SglangService/DetachHiCacheStorage", + ); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "sglang.api.v1.SglangService", + "DetachHiCacheStorage", + )); + self.inner.unary(req, path, codec).await + } + /// `/hicache/storage-backend/clear`: clear the storage backend's cached entries. + pub async fn clear_hi_cache_storage( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/sglang.api.v1.SglangService/ClearHiCacheStorage", + ); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "sglang.api.v1.SglangService", + "ClearHiCacheStorage", + )); + self.inner.unary(req, path, codec).await + } /// The same static model cards as HTTP `/v1/models`. pub async fn list_models( &mut self, @@ -265,6 +342,24 @@ pub mod sglang_service_server { &self, request: tonic::Request, ) -> std::result::Result, tonic::Status>; + /// `/hicache/storage-backend`: configuration snapshot and runtime attach/detach. + async fn get_hi_cache_storage( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + async fn attach_hi_cache_storage( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + async fn detach_hi_cache_storage( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + /// `/hicache/storage-backend/clear`: clear the storage backend's cached entries. + async fn clear_hi_cache_storage( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; /// The same static model cards as HTTP `/v1/models`. async fn list_models( &self, @@ -525,6 +620,174 @@ pub mod sglang_service_server { }; Box::pin(fut) } + "/sglang.api.v1.SglangService/GetHiCacheStorage" => { + #[allow(non_camel_case_types)] + struct GetHiCacheStorageSvc(pub Arc); + impl + tonic::server::UnaryService + for GetHiCacheStorageSvc + { + type Response = super::HiCacheStatus; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::get_hi_cache_storage(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = GetHiCacheStorageSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.api.v1.SglangService/AttachHiCacheStorage" => { + #[allow(non_camel_case_types)] + struct AttachHiCacheStorageSvc(pub Arc); + impl + tonic::server::UnaryService + for AttachHiCacheStorageSvc + { + type Response = super::HiCacheStorageResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::attach_hi_cache_storage(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = AttachHiCacheStorageSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.api.v1.SglangService/DetachHiCacheStorage" => { + #[allow(non_camel_case_types)] + struct DetachHiCacheStorageSvc(pub Arc); + impl + tonic::server::UnaryService + for DetachHiCacheStorageSvc + { + type Response = super::HiCacheStorageResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::detach_hi_cache_storage(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = DetachHiCacheStorageSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/sglang.api.v1.SglangService/ClearHiCacheStorage" => { + #[allow(non_camel_case_types)] + struct ClearHiCacheStorageSvc(pub Arc); + impl + tonic::server::UnaryService + for ClearHiCacheStorageSvc + { + type Response = super::HiCacheStorageResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::clear_hi_cache_storage(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = ClearHiCacheStorageSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } "/sglang.api.v1.SglangService/ListModels" => { #[allow(non_camel_case_types)] struct ListModelsSvc(pub Arc); diff --git a/rust/sglang-api-types/testdata/json_parity.json b/rust/sglang-api-types/testdata/json_parity.json index e90af6f9d2..078fdaa738 100644 --- a/rust/sglang-api-types/testdata/json_parity.json +++ b/rust/sglang-api-types/testdata/json_parity.json @@ -437,6 +437,97 @@ } }, "canonical": "{\"text\":\"\",\"meta_info\":{\"id\":\"r\",\"prompt_tokens\":1,\"completion_tokens\":0,\"finish_reason\":{\"type\":\"novel\",\"detail\":3,\"again\":[1,{\"z\":1,\"a\":2}]},\"cached_tokens\":0,\"reasoning_tokens\":0}}" + }, + { + "type": "AttachHiCacheStorageRequest", + "name": "backend with absent optional policies", + "input": { + "hicache_storage_backend": "file" + }, + "canonical": "{\"hicache_storage_backend\":\"file\"}" + }, + { + "type": "HiCacheStatus", + "name": "unset configuration is emitted as null", + "input": {}, + "canonical": "{\"hicache_storage_backend\":null,\"hicache_storage_backend_extra_config\":null,\"hicache_storage_prefetch_policy\":null,\"hicache_write_policy\":null}" + }, + { + "type": "HiCacheStorageResponse", + "name": "scheduler rejection with message", + "input": { + "success": false, + "message": "scheduler is not idle" + }, + "canonical": "{\"success\":false,\"message\":\"scheduler is not idle\"}" + }, + { + "type": "ClearHiCacheStorageRequest", + "name": "empty clear request", + "input": {}, + "canonical": "{}" + }, + { + "type": "DetachHiCacheStorageRequest", + "name": "empty detach request", + "input": {}, + "canonical": "{}" + }, + { + "type": "GetHiCacheStorageRequest", + "name": "empty status request", + "input": {}, + "canonical": "{}" + }, + { + "type": "AttachHiCacheStorageRequest", + "name": "all attach options", + "input": { + "hicache_storage_backend": "file", + "hicache_storage_backend_extra_config_json": "{}", + "hicache_storage_prefetch_policy": "timeout", + "hicache_write_policy": "write_through" + }, + "canonical": "{\"hicache_storage_backend\":\"file\",\"hicache_storage_backend_extra_config_json\":\"{}\",\"hicache_storage_prefetch_policy\":\"timeout\",\"hicache_write_policy\":\"write_through\"}" + }, + { + "type": "AttachHiCacheStorageRequest", + "name": "empty backend is validated by the scheduler", + "input": { + "hicache_storage_backend": "" + }, + "canonical": "{\"hicache_storage_backend\":\"\"}" + }, + { + "type": "AttachHiCacheStorageRequest", + "name": "all optional fields explicitly null", + "input": { + "hicache_storage_backend": "file", + "hicache_storage_backend_extra_config_json": null, + "hicache_storage_prefetch_policy": null, + "hicache_write_policy": null + }, + "canonical": "{\"hicache_storage_backend\":\"file\"}" + }, + { + "type": "AttachHiCacheStorageRequest", + "name": "unknown fields ignored", + "input": { + "hicache_storage_backend": "file", + "unknown_option": true + }, + "canonical": "{\"hicache_storage_backend\":\"file\"}" + }, + { + "type": "AttachHiCacheStorageRequest", + "name": "empty optional strings preserved", + "input": { + "hicache_storage_backend": "file", + "hicache_storage_backend_extra_config_json": "", + "hicache_storage_prefetch_policy": "", + "hicache_write_policy": "" + }, + "canonical": "{\"hicache_storage_backend\":\"file\",\"hicache_storage_backend_extra_config_json\":\"\",\"hicache_storage_prefetch_policy\":\"\",\"hicache_write_policy\":\"\"}" } ] } diff --git a/rust/sglang-api-types/tests/json_parity.rs b/rust/sglang-api-types/tests/json_parity.rs index 23ea68656f..51623ced13 100644 --- a/rust/sglang-api-types/tests/json_parity.rs +++ b/rust/sglang-api-types/tests/json_parity.rs @@ -44,6 +44,22 @@ fn fixtures_decode_to_their_canonical_bytes() { "GenerateStreamError" => check::(&name, input, canonical), "GetModelInfoResponse" => check::(&name, input, canonical), "GetServerInfoResponse" => check::(&name, input, canonical), + "AttachHiCacheStorageRequest" => { + check::(&name, input, canonical) + } + "HiCacheStatus" => check::(&name, input, canonical), + "HiCacheStorageResponse" => { + check::(&name, input, canonical) + } + "ClearHiCacheStorageRequest" => { + check::(&name, input, canonical) + } + "DetachHiCacheStorageRequest" => { + check::(&name, input, canonical) + } + "GetHiCacheStorageRequest" => { + check::(&name, input, canonical) + } "ListModelsResponse" => check::(&name, input, canonical), "DetokenizeRequest" => check::(&name, input, canonical), "DetokenizeResponse" => check::(&name, input, canonical), diff --git a/rust/sglang-server/src/api_server/core.rs b/rust/sglang-server/src/api_server/core.rs index ff63f3be47..31724fdeb7 100644 --- a/rust/sglang-server/src/api_server/core.rs +++ b/rust/sglang-server/src/api_server/core.rs @@ -11,8 +11,10 @@ mod error; mod frame; mod handle; +mod hicache; mod prefetch; pub(crate) use error::{CoreError, CoreErrorKind}; pub(crate) use frame::{CoreEvent, CoreOutput, HealthStatus}; pub(crate) use handle::{CoreCall, CoreConfig, CoreHandle, CoreMetadata, recv_indexed}; +pub(crate) use hicache::HiCacheManager; diff --git a/rust/sglang-server/src/api_server/core/handle.rs b/rust/sglang-server/src/api_server/core/handle.rs index ca2ba04e09..3befe482a2 100644 --- a/rust/sglang-server/src/api_server/core/handle.rs +++ b/rust/sglang-server/src/api_server/core/handle.rs @@ -354,7 +354,7 @@ impl CoreHandle { }) } - async fn control(&self, request: ControlRequest) -> Result { + pub(super) async fn control(&self, request: ControlRequest) -> Result { let mut call = self.submit(RequestKind::Control(Box::new(request))).await?; match call.recv().await { Some(ResponseItem::Control(bytes)) => Ok(bytes), diff --git a/rust/sglang-server/src/api_server/core/hicache.rs b/rust/sglang-server/src/api_server/core/hicache.rs new file mode 100644 index 0000000000..b1a2759722 --- /dev/null +++ b/rust/sglang-server/src/api_server/core/hicache.rs @@ -0,0 +1,213 @@ +//! HiCache management shared by the HTTP and gRPC adapters. + +use sglang_api_types::api::v1 as api; +use tokio::sync::RwLock; + +use super::{CoreError, CoreHandle}; +use crate::message::config::ServerArgs; +use crate::message::ids::Rid; +use crate::message::io_struct::{ + AttachHiCacheStorageReqInput, ClearHiCacheReqInput, ControlRequest, + DetachHiCacheStorageReqInput, +}; + +pub(crate) struct HiCacheManager { + core: CoreHandle, + config: RwLock, +} + +impl HiCacheManager { + pub(crate) fn new(core: CoreHandle, args: &ServerArgs) -> Self { + Self { + core, + config: RwLock::new(api::HiCacheStatus { + hicache_storage_backend: args.hicache_storage_backend.clone(), + hicache_storage_backend_extra_config: args + .hicache_storage_backend_extra_config + .clone(), + hicache_storage_prefetch_policy: args.hicache_storage_prefetch_policy.clone(), + hicache_write_policy: args.hicache_write_policy.clone(), + }), + } + } + + pub(crate) async fn status(&self) -> api::HiCacheStatus { + self.config.read().await.clone() + } + + pub(crate) async fn clear(&self) -> Result { + let bytes = self + .core + .control(ControlRequest::ClearHiCacheReqInput( + ClearHiCacheReqInput::new(Rid::new().to_string()), + )) + .await?; + rmp_serde::from_slice::(&bytes) + .map(|output| api::HiCacheStorageResponse { + success: output.success, + message: String::new(), + }) + .map_err(|error| { + CoreError::InvalidResponse(format!("invalid HiCache clear response: {error}")) + }) + } + + pub(crate) async fn detach(&self) -> Result { + let mut config = self.config.write().await; + let bytes = self + .core + .control(ControlRequest::DetachHiCacheStorageReqInput( + DetachHiCacheStorageReqInput::new(Rid::new().to_string()), + )) + .await?; + let output: DetachHiCacheOutput = rmp_serde::from_slice(&bytes).map_err(|error| { + CoreError::InvalidResponse(format!("invalid HiCache detach response: {error}")) + })?; + if output.success { + config.hicache_storage_backend = None; + config.hicache_storage_backend_extra_config = None; + } + Ok(api::HiCacheStorageResponse { + success: output.success, + message: output.message, + }) + } + + pub(crate) async fn attach( + &self, + body: api::AttachHiCacheStorageRequest, + ) -> Result { + let backend = body.hicache_storage_backend.as_ref().ok_or_else(|| { + CoreError::InvalidArgument("missing field `hicache_storage_backend`".into()) + })?; + let mut config = self.config.write().await; + let bytes = self + .core + .control(ControlRequest::AttachHiCacheStorageReqInput( + AttachHiCacheStorageReqInput::new( + Rid::new().to_string(), + backend.clone(), + body.hicache_storage_backend_extra_config_json.clone(), + body.hicache_storage_prefetch_policy.clone(), + body.hicache_write_policy.clone(), + ), + )) + .await?; + let output: AttachHiCacheOutput = rmp_serde::from_slice(&bytes).map_err(|error| { + CoreError::InvalidResponse(format!("invalid HiCache attach response: {error}")) + })?; + if output.success { + config.hicache_storage_backend = body.hicache_storage_backend; + if let Some(value) = body.hicache_storage_backend_extra_config_json { + config.hicache_storage_backend_extra_config = Some(value); + } + if let Some(value) = body.hicache_storage_prefetch_policy { + config.hicache_storage_prefetch_policy = Some(value); + } + if let Some(value) = body.hicache_write_policy { + config.hicache_write_policy = Some(value); + } + } + Ok(api::HiCacheStorageResponse { + success: output.success, + message: output.message, + }) + } +} + +/// Scheduler response for clearing HiCache storage. +#[derive(serde::Deserialize)] +struct ClearHiCacheOutput { + success: bool, +} + +/// Scheduler response for detaching HiCache storage. +#[derive(serde::Deserialize)] +struct DetachHiCacheOutput { + success: bool, + #[serde(default)] + message: String, +} + +/// Scheduler response for attaching HiCache storage. +#[derive(serde::Deserialize)] +struct AttachHiCacheOutput { + success: bool, + #[serde(default)] + message: String, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::api_server::core::{CoreConfig, CoreMetadata}; + use crate::message::response::ResponseItem; + use crate::tokenizer_manager::wiring::TmEvent; + use bytes::Bytes; + use std::sync::Arc; + use std::time::Duration; + + /// GET snapshots share the read lock, while an in-flight attach excludes readers. + #[tokio::test] + async fn hicache_readers_wait_for_successful_configuration_update() { + let (intake_tx, intake_rx) = flume::unbounded(); + let (abort_tx, _) = flume::unbounded(); + let core = CoreHandle::new( + intake_tx, + abort_tx, + CoreConfig { + response_capacity: 8, + response_activity: Default::default(), + startup_ready: true, + is_disaggregation: false, + mm_limits: Default::default(), + metadata: CoreMetadata::default(), + }, + ); + let manager = Arc::new(HiCacheManager::new(core, &ServerArgs::default())); + // Readers can share the lock, but a writer must wait for them to finish. + let reader = manager.config.read().await; + assert!(manager.config.try_read().is_ok()); + assert!(manager.config.try_write().is_err()); + drop(reader); + + // Run attach separately so this test can act as the scheduler. + let handle = manager.clone(); + let attach = tokio::spawn(async move { + handle + .attach(api::AttachHiCacheStorageRequest { + hicache_storage_backend: Some("file".into()), + ..Default::default() + }) + .await + }); + let TmEvent::Intake { request, admission } = + tokio::time::timeout(Duration::from_secs(5), intake_rx.recv_async()) + .await + .unwrap() + .unwrap() + else { + panic!("expected control intake"); + }; + assert!(admission.try_accept()); + // Attach holds the write lock while waiting for the scheduler response. + assert!(manager.config.try_read().is_err()); + // Reply successfully so attach can update the configuration and release the lock. + request + .sink + .try_send(ResponseItem::Control(Bytes::from( + rmp_serde::to_vec_named(&serde_json::json!({"success": true})).unwrap(), + ))) + .unwrap(); + tokio::time::timeout(Duration::from_secs(5), attach) + .await + .unwrap() + .unwrap() + .unwrap(); + // Once attach completes, GET can read the updated configuration. + assert_eq!( + manager.status().await.hicache_storage_backend.as_deref(), + Some("file") + ); + } +} diff --git a/rust/sglang-server/src/api_server/grpc/service.rs b/rust/sglang-server/src/api_server/grpc/service.rs index 03ebe2966b..b38d93b4c2 100644 --- a/rust/sglang-server/src/api_server/grpc/service.rs +++ b/rust/sglang-server/src/api_server/grpc/service.rs @@ -2,6 +2,7 @@ //! `api.v1` messages and the shared core. use std::pin::Pin; +use std::sync::Arc; use std::time::{Duration, Instant}; use futures::Stream; @@ -10,7 +11,7 @@ use sglang_api_types::api::v1::sglang_service_server::SglangService; use tonic::{Request, Response, Status}; use super::{info, native_api}; -use crate::api_server::core::{CoreHandle, HealthStatus}; +use crate::api_server::core::{CoreHandle, HealthStatus, HiCacheManager}; use crate::message::config::{PreferredSamplingParams, ServerArgs}; use crate::message::request::{GenerationPolicy, into_requests}; use crate::message::wire::{ClientMaxNewTokens, fill_preferred_sampling}; @@ -40,10 +41,15 @@ struct AdapterConfig { pub(crate) struct GrpcService { core: CoreHandle, config: AdapterConfig, + hicache_manager: Arc, } impl GrpcService { - pub(crate) fn new(core: CoreHandle, server_args: &ServerArgs) -> Self { + pub(crate) fn new( + core: CoreHandle, + server_args: &ServerArgs, + hicache_manager: Arc, + ) -> Self { Self { core, config: AdapterConfig { @@ -56,6 +62,7 @@ impl GrpcService { has_tokenizer: !server_args.skip_tokenizer_init, generation_policy: server_args.into(), }, + hicache_manager, } } @@ -67,6 +74,7 @@ impl GrpcService { response_timeout: Duration, ) -> Self { Self { + hicache_manager: Arc::new(HiCacheManager::new(core.clone(), &ServerArgs::default())), core, config: AdapterConfig { preferred_sampling_params, @@ -165,6 +173,49 @@ impl SglangService for GrpcService { .map_err(Status::internal) } + async fn get_hi_cache_storage( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(self.hicache_manager.status().await)) + } + + async fn attach_hi_cache_storage( + &self, + request: Request, + ) -> Result, Status> { + let output = self + .hicache_manager + .attach(request.into_inner()) + .await + .map_err(native_api::status)?; + Ok(Response::new(output)) + } + + async fn detach_hi_cache_storage( + &self, + _request: Request, + ) -> Result, Status> { + let output = self + .hicache_manager + .detach() + .await + .map_err(native_api::status)?; + Ok(Response::new(output)) + } + + async fn clear_hi_cache_storage( + &self, + _request: Request, + ) -> Result, Status> { + let output = self + .hicache_manager + .clear() + .await + .map_err(native_api::status)?; + Ok(Response::new(output)) + } + async fn list_models( &self, _request: Request, @@ -794,6 +845,114 @@ mod tests { assert_eq!(probe.request.input_ids.as_deref(), Some(&[0][..])); } + /// Check gRPC validation, scheduler results, and the configuration returned by GET. + #[tokio::test] + async fn hicache_controls_preserve_results_and_configuration() { + let h = Harness::new(8, false, Duration::from_secs(5)); + let service = &h.service; + let initial = service + .get_hi_cache_storage(Request::new(api::GetHiCacheStorageRequest {})) + .await + .unwrap() + .into_inner(); + assert_eq!(initial, api::HiCacheStatus::default()); + let error = service + .attach_hi_cache_storage(Request::new(api::AttachHiCacheStorageRequest::default())) + .await + .unwrap_err(); + assert_eq!(error.code(), Code::InvalidArgument); + assert!(h.intake_rx.is_empty()); + + for success in [false, true] { + let response = hicache_reply( + &h, + "AttachHiCacheStorageReqInput", + success, + service.attach_hi_cache_storage(Request::new(api::AttachHiCacheStorageRequest { + hicache_storage_backend: Some("file".into()), + hicache_write_policy: Some("write_through".into()), + ..Default::default() + })), + ) + .await; + assert_eq!(response.success, success); + assert_eq!(response.message, "backend detail"); + } + for success in [false, true] { + let response = hicache_reply( + &h, + "ClearHiCacheReqInput", + success, + service.clear_hi_cache_storage(Request::new(api::ClearHiCacheStorageRequest {})), + ) + .await; + assert_eq!(response.success, success); + assert!(response.message.is_empty()); + } + for success in [false, true] { + let response = hicache_reply( + &h, + "DetachHiCacheStorageReqInput", + success, + service.detach_hi_cache_storage(Request::new(api::DetachHiCacheStorageRequest {})), + ) + .await; + assert_eq!(response.success, success); + assert_eq!(response.message, "backend detail"); + } + let status = service + .get_hi_cache_storage(Request::new(api::GetHiCacheStorageRequest {})) + .await + .unwrap() + .into_inner(); + assert_eq!( + status, + api::HiCacheStatus { + hicache_write_policy: Some("write_through".into()), + ..Default::default() + } + ); + } + + // Drive the RPC and a simulated scheduler reply together, checking the request tag. + async fn hicache_reply( + h: &Harness, + tag: &str, + success: bool, + call: impl std::future::Future< + Output = Result, tonic::Status>, + >, + ) -> api::HiCacheStorageResponse { + let scheduler = async { + let TmEvent::Intake { request, admission } = h.intake_rx.recv_async().await.unwrap() + else { + panic!("expected control intake"); + }; + assert!(admission.try_accept()); + let RequestKind::Control(control) = &request.kind else { + panic!("expected control request"); + }; + let wire: serde_json::Value = + rmp_serde::from_slice(&control.encode().unwrap()).unwrap(); + assert_eq!(wire[0], tag); + request + .sink + .try_send(ResponseItem::Control(bytes::Bytes::from( + rmp_serde::to_vec_named( + &serde_json::json!({"success": success, "message": "backend detail"}), + ) + .unwrap(), + ))) + .unwrap(); + }; + tokio::time::timeout(Duration::from_secs(5), async { + let (response, ()) = tokio::join!(call, scheduler); + response.unwrap().into_inner() + }) + .await + .expect("HiCache RPC did not finish") + } + #[tokio::test] async fn list_models_uses_static_metadata_without_intake() { let harness = Harness::new(2, false, Duration::from_secs(1)); diff --git a/rust/sglang-server/src/api_server/http.rs b/rust/sglang-server/src/api_server/http.rs index bcd48fcb4c..df61b7b801 100644 --- a/rust/sglang-server/src/api_server/http.rs +++ b/rust/sglang-server/src/api_server/http.rs @@ -1,4 +1,5 @@ pub(crate) mod app; +mod hicache; mod info; mod loads; mod metrics; diff --git a/rust/sglang-server/src/api_server/http/app.rs b/rust/sglang-server/src/api_server/http/app.rs index c0f81e431f..d3d747ad1b 100644 --- a/rust/sglang-server/src/api_server/http/app.rs +++ b/rust/sglang-server/src/api_server/http/app.rs @@ -7,9 +7,9 @@ use std::time::Duration; use axum::Router; -use super::{info, loads, metrics, native_api, openai}; +use super::{hicache, info, loads, metrics, native_api, openai}; use crate::HttpLayer; -use crate::api_server::core::CoreHandle; +use crate::api_server::core::{CoreHandle, HiCacheManager}; use crate::api_server::disaggregation::bootstrap as pd_bootstrap; use crate::api_server::log; use crate::message::config::ServerArgs; @@ -69,6 +69,7 @@ pub async fn serve( // which point `serve` stops accepting and its in-flight handlers are // aborted with the api runtime. shutdown: flume::Receiver<()>, + hicache_manager: Arc, ) { let chat_formatter = openai::load_chat_support(&server_args); let generation_policy = GenerationPolicy::from(server_args.as_ref()); @@ -97,8 +98,9 @@ pub async fn serve( // // No body limit, matching the Python server. let mut app = router - .layer(axum::extract::DefaultBodyLimit::disable()) - .with_state(state); + .with_state(state) + .merge(hicache::routes(hicache_manager)) + .layer(axum::extract::DefaultBodyLimit::disable()); if let Some(http_layer) = http_layer { app = http_layer(app); } diff --git a/rust/sglang-server/src/api_server/http/hicache.rs b/rust/sglang-server/src/api_server/http/hicache.rs new file mode 100644 index 0000000000..044b89cd5b --- /dev/null +++ b/rust/sglang-server/src/api_server/http/hicache.rs @@ -0,0 +1,549 @@ +//! HiCache routes with dedicated state shared with the gRPC adapter. + +use std::sync::Arc; + +use axum::{ + Json, Router, + extract::{State, rejection::JsonRejection}, + http::StatusCode, + response::{IntoResponse, Response}, + routing::{get, post}, +}; +use sglang_api_types::api::v1 as api; + +use crate::api_server::core::{CoreError, HiCacheManager}; + +pub(super) fn routes(manager: Arc) -> Router { + Router::new() + .route( + "/hicache/storage-backend", + get(status).put(attach).delete(detach), + ) + .route("/hicache/storage-backend/clear", post(clear)) + .with_state(manager) +} + +async fn status(State(manager): State>) -> Json { + Json(manager.status().await) +} + +async fn attach( + State(manager): State>, + body: Result, JsonRejection>, +) -> Response { + let body = match body { + Ok(Json(body)) => body, + Err(error) => return (StatusCode::UNPROCESSABLE_ENTITY, error.body_text()).into_response(), + }; + if body.hicache_storage_backend.is_none() { + return ( + StatusCode::UNPROCESSABLE_ENTITY, + "missing field `hicache_storage_backend`", + ) + .into_response(); + } + storage_response(manager.attach(body).await) +} + +async fn detach(State(manager): State>) -> Response { + storage_response(manager.detach().await) +} + +async fn clear(State(manager): State>) -> Response { + storage_response(manager.clear().await) +} + +fn storage_response(result: Result) -> Response { + match result { + Ok(output) => { + let status = if output.success { + StatusCode::OK + } else { + StatusCode::BAD_REQUEST + }; + (status, Json(output)).into_response() + } + Err(error) => (error.http_status(), error.to_string()).into_response(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::{ + body::{Body, to_bytes}, + http::Request, + }; + use bytes::Bytes; + use serde_json::{Value, json}; + use tower::ServiceExt; + + use crate::api_server::core::{CoreConfig, CoreHandle}; + use crate::message::{config::ServerArgs, request::RequestKind, response::ResponseItem}; + use crate::tokenizer_manager::wiring::TmEvent; + + fn app() -> (Router, flume::Receiver) { + app_with_args(ServerArgs::default()) + } + + fn app_with_args(args: ServerArgs) -> (Router, flume::Receiver) { + let (intake_tx, intake_rx) = flume::unbounded(); + let (abort_tx, _) = flume::unbounded(); + let core = CoreHandle::new( + intake_tx, + abort_tx, + CoreConfig { + response_capacity: 8, + response_activity: Default::default(), + startup_ready: true, + is_disaggregation: false, + mm_limits: Default::default(), + metadata: (&args).into(), + }, + ); + ( + routes(Arc::new(HiCacheManager::new(core, &args))), + intake_rx, + ) + } + + // Exercise the real router/core boundary, replacing only the scheduler reply. + async fn exchange( + method: &str, + uri: &str, + body: Value, + tag: &str, + fields: Value, + reply: ResponseItem, + ) -> (StatusCode, String) { + exchange_on(&app(), method, uri, body, tag, fields, reply).await + } + + async fn exchange_on( + app: &(Router, flume::Receiver), + method: &str, + uri: &str, + body: Value, + tag: &str, + fields: Value, + reply: ResponseItem, + ) -> (StatusCode, String) { + let (router, intake) = app; + let response = tokio::spawn( + router.clone().oneshot( + Request::builder() + .method(method) + .uri(uri) + .header("content-type", "application/json") + .body(Body::from(body.to_string())) + .unwrap(), + ), + ); + let TmEvent::Intake { request, admission } = + tokio::time::timeout(std::time::Duration::from_secs(5), intake.recv_async()) + .await + .expect("handler did not submit a control request") + .unwrap() + else { + panic!("expected control intake"); + }; + assert!(admission.try_accept()); + let RequestKind::Control(control) = &request.kind else { + panic!("expected control request") + }; + let header: Value = rmp_serde::from_slice(&control.encode().unwrap()).unwrap(); + let mut expected = vec![json!(tag), json!(request.rid.as_str()), Value::Null]; + expected.extend(fields.as_array().unwrap().iter().cloned()); + assert_eq!(header, Value::Array(expected)); + request.sink.try_send(reply).unwrap(); + let response = tokio::time::timeout(std::time::Duration::from_secs(5), response) + .await + .expect("handler did not finish") + .unwrap() + .unwrap(); + let status = response.status(); + if matches!(status, StatusCode::OK | StatusCode::BAD_REQUEST) { + assert_eq!(response.headers()["content-type"], "application/json"); + } + let body = to_bytes(response.into_body(), 4096).await.unwrap(); + let body = String::from_utf8(body.to_vec()).unwrap(); + // Normalize JSON key order for comparisons, preserving plain-text errors. + let body = + serde_json::from_str::(&body).map_or(body.clone(), |value| value.to_string()); + (status, body) + } + + fn reply(value: Value) -> ResponseItem { + ResponseItem::Control(Bytes::from(rmp_serde::to_vec_named(&value).unwrap())) + } + + /// Clear maps scheduler success to HTTP 200 and failure to HTTP 400. + #[tokio::test] + async fn clear_relays_success_and_failure() { + for (success, status) in [(true, StatusCode::OK), (false, StatusCode::BAD_REQUEST)] { + assert_eq!( + exchange( + "POST", + "/hicache/storage-backend/clear", + Value::Null, + "ClearHiCacheReqInput", + json!([]), + reply(json!({"success": success})) + ) + .await, + ( + status, + json!({"success": success, "message": ""}).to_string() + ) + ); + } + } + + /// Detach returns the generated response, defaulting absent messages to empty. + #[tokio::test] + async fn detach_preserves_message_and_handles_empty_or_missing_message() { + for payload in [ + json!({"success": true, "message": "Already detached."}), + json!({"success": false, "message": "scheduler is not idle"}), + json!({"success": true, "message": ""}), + json!({"success": false}), + ] { + let (status, body) = exchange( + "DELETE", + "/hicache/storage-backend", + Value::Null, + "DetachHiCacheStorageReqInput", + json!([]), + reply(payload.clone()), + ) + .await; + let success = payload["success"].as_bool().unwrap(); + assert_eq!( + status, + if success { + StatusCode::OK + } else { + StatusCode::BAD_REQUEST + } + ); + assert_eq!( + serde_json::from_str::(&body).unwrap(), + json!({ + "success": success, + "message": payload.get("message").cloned().unwrap_or(json!("")), + }) + ); + } + } + + /// Malformed replies and scheduler errors produce HTTP 500 for all three operations. + #[tokio::test] + async fn control_operations_report_scheduler_errors() { + for (method, uri, tag, operation, input, fields) in [ + ( + "POST", + "/hicache/storage-backend/clear", + "ClearHiCacheReqInput", + "clear", + Value::Null, + json!([]), + ), + ( + "DELETE", + "/hicache/storage-backend", + "DetachHiCacheStorageReqInput", + "detach", + Value::Null, + json!([]), + ), + ( + "PUT", + "/hicache/storage-backend", + "AttachHiCacheStorageReqInput", + "attach", + json!({"hicache_storage_backend": "file"}), + json!(["file", null, null, null]), + ), + ] { + for (response, expected_error) in [ + ( + reply(json!({"message": "missing success"})), + format!("invalid HiCache {operation} response"), + ), + ( + reply(json!({"success": "false"})), + format!("invalid HiCache {operation} response"), + ), + ( + ResponseItem::Control(Bytes::from_static(&[0xc1])), + format!("invalid HiCache {operation} response"), + ), + ( + ResponseItem::Error(crate::utils::error::Error::Internal( + "backend unavailable".into(), + )), + "backend unavailable".into(), + ), + ] { + let (status, body) = + exchange(method, uri, input.clone(), tag, fields.clone(), response).await; + assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR); + assert!(body.contains(&expected_error), "{body}"); + } + } + } + + /// Attach forwards all options and returns the scheduler result and message. + #[tokio::test] + async fn attach_http_forwards_options_and_relays_result() { + let body = json!({ + "hicache_storage_backend": "file", + "hicache_storage_backend_extra_config_json": "{}", + "hicache_storage_prefetch_policy": "timeout", + "hicache_write_policy": "write_through", + }); + for (success, status) in [(true, StatusCode::OK), (false, StatusCode::BAD_REQUEST)] { + assert_eq!( + exchange( + "PUT", + "/hicache/storage-backend", + body.clone(), + "AttachHiCacheStorageReqInput", + json!(["file", "{}", "timeout", "write_through"]), + reply(json!({"success": success, "message": "backend detail"})) + ) + .await, + ( + status, + json!({"success": success, "message": "backend detail"}).to_string() + ) + ); + } + } + + /// Python accepts empty names and null options; backend validation belongs to the scheduler. + #[tokio::test] + async fn attach_preserves_empty_strings_and_accepts_null_options() { + for (body, fields) in [ + ( + json!({"hicache_storage_backend": ""}), + json!(["", null, null, null]), + ), + ( + json!({ + "hicache_storage_backend": "file", + "hicache_storage_backend_extra_config_json": null, + "hicache_storage_prefetch_policy": null, + "hicache_write_policy": null, + }), + json!(["file", null, null, null]), + ), + ( + json!({ + "hicache_storage_backend": "file", + "hicache_storage_backend_extra_config_json": "", + "hicache_storage_prefetch_policy": "", + "hicache_write_policy": "", + }), + json!(["file", "", "", ""]), + ), + ] { + assert_eq!( + exchange( + "PUT", + "/hicache/storage-backend", + body, + "AttachHiCacheStorageReqInput", + fields, + reply(json!({"success": true})) + ) + .await + .0, + StatusCode::OK + ); + } + } + + /// Empty bodies, invalid JSON, and missing backend fields fail before scheduler submission. + #[tokio::test] + async fn attach_rejects_invalid_body_before_scheduler_submission() { + for (body, expected_error) in [ + ("", "EOF while parsing"), + ("{", "EOF while parsing"), + ("{}", "missing field `hicache_storage_backend`"), + ( + r#"{"hicache_storage_backend":null}"#, + "missing field `hicache_storage_backend`", + ), + (r#"{"hicache_storage_backend":1}"#, "invalid type"), + ( + r#"{"hicache_storage_backend":"file","hicache_write_policy":{}}"#, + "invalid type", + ), + ] { + let (router, intake) = app(); + let response = tokio::time::timeout( + std::time::Duration::from_secs(5), + router.oneshot( + Request::builder() + .method("PUT") + .uri("/hicache/storage-backend") + .header("content-type", "application/json") + .body(Body::from(body)) + .unwrap(), + ), + ) + .await + .expect("invalid attach request must not wait for scheduler") + .unwrap(); + assert_eq!( + response.status(), + StatusCode::UNPROCESSABLE_ENTITY, + "{body:?}" + ); + let response_body = to_bytes(response.into_body(), 4096).await.unwrap(); + let response_body = std::str::from_utf8(&response_body).unwrap(); + assert!(response_body.contains(expected_error), "{response_body}"); + assert!( + intake.is_empty(), + "invalid request reached scheduler: {body:?}" + ); + } + } + + /// GET returns a JSON object with the same four configuration fields as Python. + #[tokio::test] + async fn get_status_returns_python_configuration_fields() { + let (router, _intake) = app(); + let body = read_status(&router).await; + assert_eq!( + body, + json!({ + "hicache_storage_backend": null, + "hicache_storage_backend_extra_config": null, + "hicache_storage_prefetch_policy": null, + "hicache_write_policy": null, + }) + ); + } + + /// Only successful attach/detach operations update GET. Omitted options and + /// clear preserve the current configuration. + #[tokio::test] + async fn get_status_tracks_only_successful_control_operations() { + let app = app_with_args(ServerArgs { + hicache_storage_backend: Some("file".into()), + hicache_storage_backend_extra_config: Some("{}".into()), + hicache_storage_prefetch_policy: Some("timeout".into()), + hicache_write_policy: Some("write_through".into()), + ..ServerArgs::default() + }); + let initial = json!({ + "hicache_storage_backend": "file", + "hicache_storage_backend_extra_config": "{}", + "hicache_storage_prefetch_policy": "timeout", + "hicache_write_policy": "write_through", + }); + assert_eq!(read_status(&app.0).await, initial); + assert!(app.1.is_empty(), "GET must not submit scheduler work"); + + let body = json!({ + "hicache_storage_backend": "file", + "hicache_storage_backend_extra_config_json": "{\"path\":\"/cache\"}", + "hicache_storage_prefetch_policy": "wait_complete", + "hicache_write_policy": "write_back", + }); + let fields = json!([ + "file", + "{\"path\":\"/cache\"}", + "wait_complete", + "write_back" + ]); + for success in [false, true] { + exchange_on( + &app, + "PUT", + "/hicache/storage-backend", + body.clone(), + "AttachHiCacheStorageReqInput", + fields.clone(), + reply(json!({"success": success})), + ) + .await; + if !success { + assert_eq!(read_status(&app.0).await, initial); + } + } + let attached = json!({ + "hicache_storage_backend": "file", + "hicache_storage_backend_extra_config": "{\"path\":\"/cache\"}", + "hicache_storage_prefetch_policy": "wait_complete", + "hicache_write_policy": "write_back", + }); + assert_eq!(read_status(&app.0).await, attached); + // Omitted options retain their current values, like Python's config updates. + exchange_on( + &app, + "PUT", + "/hicache/storage-backend", + json!({"hicache_storage_backend": "file"}), + "AttachHiCacheStorageReqInput", + json!(["file", null, null, null]), + reply(json!({"success": true})), + ) + .await; + assert_eq!(read_status(&app.0).await, attached); + exchange_on( + &app, + "POST", + "/hicache/storage-backend/clear", + Value::Null, + "ClearHiCacheReqInput", + json!([]), + reply(json!({"success": true})), + ) + .await; + assert_eq!(read_status(&app.0).await, attached); + for success in [false, true] { + exchange_on( + &app, + "DELETE", + "/hicache/storage-backend", + Value::Null, + "DetachHiCacheStorageReqInput", + json!([]), + reply(json!({"success": success})), + ) + .await; + let mut expected = attached.clone(); + if success { + expected["hicache_storage_backend"] = Value::Null; + expected["hicache_storage_backend_extra_config"] = Value::Null; + } + assert_eq!(read_status(&app.0).await, expected); + } + } + + async fn read_status(router: &Router) -> Value { + let response = tokio::time::timeout( + std::time::Duration::from_secs(5), + router.clone().oneshot( + Request::builder() + .uri("/hicache/storage-backend") + .body(Body::empty()) + .unwrap(), + ), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert!( + response.headers()["content-type"] + .to_str() + .unwrap() + .starts_with("application/json") + ); + serde_json::from_slice(&to_bytes(response.into_body(), 4096).await.unwrap()).unwrap() + } +} diff --git a/rust/sglang-server/src/message/config.rs b/rust/sglang-server/src/message/config.rs index f1b4dd4daf..18f1391faa 100644 --- a/rust/sglang-server/src/message/config.rs +++ b/rust/sglang-server/src/message/config.rs @@ -106,6 +106,14 @@ pub struct ServerArgs { /// `"default"` on the Python side, so it is present in every blob; `None` /// only when the blob omits the key. pub weight_version: Option, + /// HiCache storage backend configured at startup. + pub hicache_storage_backend: Option, + /// Backend-specific configuration as a JSON string. + pub hicache_storage_backend_extra_config: Option, + /// Policy for prefetching cached entries from HiCache storage. + pub hicache_storage_prefetch_policy: Option, + /// Write policy for the hierarchical cache. + pub hicache_write_policy: Option, /// HTTP bind address (see [`Self::bind`]). pub host: String, pub port: u16, @@ -216,6 +224,10 @@ impl ServerArgs { revision, load_format, weight_version, + hicache_storage_backend, + hicache_storage_backend_extra_config, + hicache_storage_prefetch_policy, + hicache_write_policy, host, port, grpc_port, @@ -264,6 +276,10 @@ impl ServerArgs { revision: Option, load_format: Option, weight_version: Option, + hicache_storage_backend: Option, + hicache_storage_backend_extra_config: Option, + hicache_storage_prefetch_policy: Option, + hicache_write_policy: Option, host: String, port: u16, grpc_port: Option, @@ -310,6 +326,10 @@ impl ServerArgs { revision, load_format, weight_version, + hicache_storage_backend, + hicache_storage_backend_extra_config, + hicache_storage_prefetch_policy, + hicache_write_policy, host, port, grpc_port, @@ -364,6 +384,10 @@ impl Default for ServerArgs { revision: None, load_format: None, weight_version: None, + hicache_storage_backend: None, + hicache_storage_backend_extra_config: None, + hicache_storage_prefetch_policy: None, + hicache_write_policy: None, host: "127.0.0.1".into(), port: 30000, grpc_port: None, diff --git a/rust/sglang-server/src/message/io_struct.rs b/rust/sglang-server/src/message/io_struct.rs index 2cea897341..e7ed65d252 100644 --- a/rust/sglang-server/src/message/io_struct.rs +++ b/rust/sglang-server/src/message/io_struct.rs @@ -100,6 +100,20 @@ control_messages! { skip_tensor_list: Option>, selector: String, } + + /// Clear the HiCache storage backend. + ClearHiCacheReqInput {} + + /// Detach the HiCache storage backend. + DetachHiCacheStorageReqInput {} + + /// Attach a HiCache storage backend with optional configuration and policies. + AttachHiCacheStorageReqInput { + hicache_storage_backend: String, + hicache_storage_backend_extra_config_json: Option, + hicache_storage_prefetch_policy: Option, + hicache_write_policy: Option, + } } /// Borrow a request as its wire struct, resolving `Option` scalars to the wire @@ -162,6 +176,36 @@ impl<'a> From<&'a GenerateRequest> for TokenizedGenerateReqInput<'a> { } } +impl ClearHiCacheReqInput { + pub fn new(rid: String) -> Self { + Self { rid } + } +} + +impl DetachHiCacheStorageReqInput { + pub fn new(rid: String) -> Self { + Self { rid } + } +} + +impl AttachHiCacheStorageReqInput { + pub fn new( + rid: String, + hicache_storage_backend: String, + hicache_storage_backend_extra_config_json: Option, + hicache_storage_prefetch_policy: Option, + hicache_write_policy: Option, + ) -> Self { + Self { + rid, + hicache_storage_backend, + hicache_storage_backend_extra_config_json, + hicache_storage_prefetch_policy, + hicache_write_policy, + } + } +} + impl GetInternalStateReq { pub fn new(rid: String) -> Self { Self { rid } diff --git a/rust/sglang-server/src/utils/runtime.rs b/rust/sglang-server/src/utils/runtime.rs index 181c9f47cb..c74254e3a7 100644 --- a/rust/sglang-server/src/utils/runtime.rs +++ b/rust/sglang-server/src/utils/runtime.rs @@ -321,9 +321,16 @@ pub fn start(cfg: RuntimeConfig, http_layer: Option) -> Result) -> Result(body).unwrap(), + serde_json::json!({"success": true, "message": "attached"}) + ); + + let client_runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let status = client_runtime.block_on(async { + let mut client = connect_grpc(grpc_addr).await; + tokio::time::timeout( + Duration::from_secs(5), + client.get_hi_cache_storage(proto::GetHiCacheStorageRequest {}), + ) + .await + .unwrap() + .unwrap() + .into_inner() + }); + assert_eq!( + status, + proto::HiCacheStatus { + hicache_storage_backend: Some("file".into()), + hicache_write_policy: Some("write_through".into()), + ..Default::default() + } + ); + rt.request_shutdown(); + } + /// The Python-backed gRPC listener accepts bodies above Tonic's 4 MiB /// default. Keep that transport policy on the Rust frontend; reaching /// adapter validation proves the body decoded. diff --git a/test/registered/unit/api/v1/test_parity.py b/test/registered/unit/api/v1/test_parity.py index 72662c11df..6aa4acbe8c 100644 --- a/test/registered/unit/api/v1/test_parity.py +++ b/test/registered/unit/api/v1/test_parity.py @@ -1,4 +1,4 @@ -"""Parity of the three `/generate` vocabularies: the hand-written Python +"""Parity of the `/generate` and HiCache API vocabularies: the hand-written Python request structs (`io_struct.GenerateReqInput`, `SamplingParams`), the generated Python types (`sglang.api.v1.api_types`), and the generated Rust types (`rust/sglang-api-types`). @@ -6,8 +6,9 @@ types (`rust/sglang-api-types`). The cross-language half runs through `rust/sglang-api-types/testdata/ json_parity.json`: both generators must turn each fixture input into the same canonical bytes (`tests/json_parity.rs` checks the Rust side). The Python half -pins the field sets against the hand-written structs and proves a schema -request still normalizes through the dataclass the Python server runs. +pins the field sets against the hand-written structs, checks HiCache request +validation, and proves a generation request still normalizes through the +dataclass the Python server runs. """ import dataclasses @@ -15,6 +16,8 @@ import json import unittest from pathlib import Path +from pydantic import TypeAdapter + from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel @@ -27,7 +30,11 @@ from sglang.api.v1.api_types import ( GenerateResponse, SamplingParams, ) -from sglang.srt.managers.io_struct import GenerateReqInput +from sglang.srt.managers.io_struct import ( + AttachHiCacheStorageReqInput, + BaseReq, + GenerateReqInput, +) from sglang.srt.sampling.sampling_params import SamplingParams as PySamplingParams register_cpu_ci(est_time=5, suite="base-a-test-cpu") @@ -136,6 +143,31 @@ class TestCrossLanguageFixtures(CustomTestCase): self.assertTrue(hasattr(api_types, case["type"]), case["type"]) +class TestHiCacheRequestParity(CustomTestCase): + def test_attach_fields_match_python(self): + """Only scheduler transport fields are excluded from the public request.""" + self.assertEqual( + set(api_types.AttachHiCacheStorageRequest.__struct_fields__), + set(AttachHiCacheStorageReqInput.__struct_fields__) + - set(BaseReq.__struct_fields__), + ) + + def test_attach_fixtures_match_python_validation_and_defaults(self): + """Decode using the same Pydantic adapter as the Python HTTP endpoint.""" + adapter = TypeAdapter(AttachHiCacheStorageReqInput) + for case in load_fixtures(): + if case["type"] != "AttachHiCacheStorageRequest": + continue + with self.subTest(case=case["name"]): + python = adapter.validate_python(case["input"]) + generated = api_types.AttachHiCacheStorageRequest.from_json_value( + case["input"] + ) + self.assertIsNotNone(generated.hicache_storage_backend) + for name in generated.__struct_fields__: + self.assertEqual(getattr(generated, name), getattr(python, name)) + + class TestRequestFieldParity(CustomTestCase): def test_schema_request_fields_are_a_subset_of_the_dataclass(self): """Every schema field is a `GenerateReqInput` field, so a schema body diff --git a/test/registered/unit/entrypoints/test_rust_server_dp_ports.py b/test/registered/unit/entrypoints/test_rust_server_dp_ports.py index d778ecec51..aaedf42504 100644 --- a/test/registered/unit/entrypoints/test_rust_server_dp_ports.py +++ b/test/registered/unit/entrypoints/test_rust_server_dp_ports.py @@ -44,6 +44,18 @@ def _scheduler_for_typed_config(): "overrides,kwarg,expected", [ ({}, "grpc_port", 50051), + ({"hicache_storage_backend": "file"}, "hicache_storage_backend", "file"), + ( + {"hicache_storage_backend_extra_config": "{}"}, + "hicache_storage_backend_extra_config", + "{}", + ), + ( + {"hicache_storage_prefetch_policy": "timeout"}, + "hicache_storage_prefetch_policy", + "timeout", + ), + ({"hicache_write_policy": "write_back"}, "hicache_write_policy", "write_back"), ({"smg_grpc_mode": True}, "grpc_port", None), ({"grpc_mode": True}, "grpc_port", None), ({"default_priority_value": 5}, "default_priority_value", None), @@ -61,6 +73,10 @@ def _scheduler_for_typed_config(): ], ids=[ "native-grpc", + "hicache_storage_backend", + "hicache_storage_backend_extra_config", + "hicache_storage_prefetch_policy", + "hicache_write_policy", "legacy-smg", "deprecated-legacy-smg", "priority-off",