[Feat] sglang-server: serve /v1/loads from the Rust frontend (#44532)

Co-authored-by: metamergebot <metamergebot@users.noreply.github.com>
Co-authored-by: shodoco <shodoco@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-16 10:35:37 -07:00
committed by GitHub
co-authored by metamergebot shodoco
parent a038a0dca0
commit 26a2f18c02
10 changed files with 881 additions and 1 deletions
+15
View File
@@ -9,7 +9,12 @@ from types import ModuleType
from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple
from sglang.srt.arg_groups.overrides import resolving_view
from sglang.srt.entrypoints.v1_loads import (
_accelerator_name,
_num_accelerators_per_dp_rank,
)
from sglang.srt.environ import envs
from sglang.srt.managers.load_snapshot import ShmLoadSnapshotWriter
from sglang.srt.managers.utils import compute_num_reserved_tokens
from sglang.srt.runtime_context import (
get_disagg,
@@ -57,6 +62,11 @@ def _build_server_args(
if get_serving().smg_grpc_mode or get_serving().grpc_mode
else get_serving().grpc_port
)
# `/v1/loads` reads this rank's slot of the scheduler's shared-memory
# snapshots directly; the ZMQ transport (multi-node DP) has no file to read.
shm_writer = scheduler.load_snapshot_writer
if not isinstance(shm_writer, ShmLoadSnapshotWriter):
shm_writer = None
return ext.ServerArgs(
model_path=get_model().model_path,
served_model_name=get_serving().served_model_name,
@@ -118,6 +128,11 @@ def _build_server_args(
# can serve them statically (no scheduler round-trip).
version=__version__,
max_total_num_tokens=scheduler.max_total_num_tokens,
load_snapshot_path=shm_writer.path if shm_writer else None,
load_snapshot_dp_size=shm_writer.dp_size if shm_writer else 0,
load_snapshot_dp_rank=shm_writer.dp_rank if shm_writer else 0,
accelerator=_accelerator_name(),
num_accelerators=_num_accelerators_per_dp_rank(),
# Every process writes its prometheus_client samples under this
# directory (set before the schedulers start); `/metrics` merges them.
metrics=(
+1
View File
@@ -3767,6 +3767,7 @@ dependencies = [
"axum",
"bytemuck",
"bytes",
"chrono",
"core_affinity",
"dynamo-parsers",
"dynamo-protocols",
+2
View File
@@ -34,6 +34,8 @@ uuid = { workspace = true }
arc-swap = "1"
axum = { version = "0.8.9", features = ["json", "tokio"] }
# Already in the tree via dynamo-renderer; `/v1/loads` timestamps.
chrono = { version = "0.4", default-features = false, features = ["clock"] }
# Safe POD slice casts (feature buffers viewed as bytes for the shm copy).
bytemuck = "1"
core_affinity = "0.8"
@@ -1,5 +1,6 @@
pub(crate) mod app;
mod info;
mod loads;
mod metrics;
mod native_api;
mod openai;
@@ -7,7 +7,7 @@ use std::time::Duration;
use axum::Router;
use super::{info, metrics, native_api, openai};
use super::{info, loads, metrics, native_api, openai};
use crate::HttpLayer;
use crate::api_server::core::CoreHandle;
use crate::api_server::disaggregation::bootstrap as pd_bootstrap;
@@ -80,6 +80,7 @@ pub async fn serve(
// Each endpoint module registers its own routes and merges here.
let mut router = Router::new()
.merge(info::routes())
.merge(loads::routes())
.merge(native_api::routes())
.merge(openai::routes());
if server_args.metrics.is_some() && metrics_listener {
@@ -0,0 +1,563 @@
//! `GET /v1/loads`: the port of Python `entrypoints/v1_loads.py`. Each
//! scheduler publishes its per-DP-rank load snapshot into a shared-memory file
//! (`managers/load_snapshot.py`, `ShmLoadSnapshotWriter`); this reads that file
//! and renders Python's `LoadSnapshot.to_dict`, as JSON or Prometheus text.
//!
//! Unlike the Python server, which fronts every DP rank, a Rust listener serves
//! only its own scheduler's rank, so it reports itself as a single-rank engine:
//! its own slot, as rank 0.
use std::collections::BTreeSet;
use std::fs::File;
use std::os::unix::fs::FileExt;
use std::path::Path;
use std::sync::Arc;
use axum::extract::{Query, State};
use axum::http::{StatusCode, header};
use axum::response::{IntoResponse, Response};
use axum::routing::get;
use axum::{Json, Router};
use rustix::fs::{FlockOperation, flock};
use serde::Deserialize;
use serde_json::{Map, Value};
use super::app::AppState;
use super::native_api::native_error;
/// The file header, Python's `HEADER_STRUCT` (`<4sHHI`): magic, version,
/// dp_size, slot_size. Each rank's slot follows at `HEADER_LEN + rank *
/// slot_size`: a little-endian `u32` length, then the msgpack snapshot.
const MAGIC: &[u8; 4] = b"SLNS";
const VERSION: u16 = 2;
const HEADER_LEN: u64 = 12;
const SLOT_LEN_PREFIX: u64 = 4;
/// The zero value a field takes when the snapshot omits it.
#[derive(Clone, Copy)]
enum Zero {
Int,
Float,
Null,
}
impl Zero {
fn value(self) -> Value {
match self {
Zero::Int => Value::from(0),
Zero::Float => Value::from(0.0),
Zero::Null => Value::Null,
}
}
}
/// `LoadSnapshot`'s flat fields in declaration order. The snapshot is encoded
/// with `omit_defaults`, so an absent key takes its zero.
const CORE: &[(&str, Zero)] = &[
("timestamp", Zero::Float),
("dp_rank", Zero::Int),
("num_running_reqs", Zero::Int),
("num_waiting_reqs", Zero::Int),
("num_waiting_uncached_tokens", Zero::Int),
("num_used_tokens", Zero::Int),
("num_total_tokens", Zero::Int),
("num_active_tokens", Zero::Int),
("num_prealloc_ready_tokens", Zero::Int),
("max_total_num_tokens", Zero::Int),
("max_running_requests", Zero::Int),
("token_usage", Zero::Float),
("gen_throughput", Zero::Float),
("cache_hit_rate", Zero::Float),
("utilization", Zero::Float),
("total_prefill_uncached_tokens", Zero::Int),
("total_prefill_busy_us", Zero::Int),
("decode_moments", Zero::Null),
];
/// One optional sub-struct of `LoadSnapshot`, encoded as a positional array.
/// `None` marks a field without a default.
struct Section {
field: &'static str,
include: &'static str,
prometheus_prefix: &'static str,
fields: &'static [(&'static str, Option<Zero>)],
}
const SECTIONS: &[Section] = &[
Section {
field: "memory",
include: "memory",
prometheus_prefix: "memory",
fields: &[
("weight_gb", None),
("kv_cache_gb", None),
("graph_gb", None),
("token_capacity", None),
],
},
Section {
field: "speculative",
include: "spec",
prometheus_prefix: "spec",
fields: &[("accept_length", None), ("accept_rate", None)],
},
Section {
field: "lora",
include: "lora",
prometheus_prefix: "lora",
fields: &[
("slots_used", None),
("slots_total", None),
("utilization", None),
],
},
Section {
field: "disaggregation",
include: "disagg",
prometheus_prefix: "disagg",
fields: &[
("mode", None),
("prefill_bootstrap_queue_reqs", Some(Zero::Int)),
("prefill_inflight_queue_reqs", Some(Zero::Int)),
("decode_prealloc_queue_reqs", Some(Zero::Int)),
("decode_transfer_queue_reqs", Some(Zero::Int)),
("decode_retracted_queue_reqs", Some(Zero::Int)),
("kv_transfer_speed_gb_s", Some(Zero::Float)),
("kv_transfer_latency_ms", Some(Zero::Float)),
],
},
Section {
field: "queues",
include: "queues",
prometheus_prefix: "queues",
fields: &[
("waiting", None),
("grammar", None),
("paused", None),
("retracted", None),
("prealloc_ready", None),
],
},
];
const VALID_INCLUDES: &[&str] = &["all", "core", "disagg", "lora", "memory", "queues", "spec"];
pub(super) fn routes() -> Router<Arc<AppState>> {
Router::new().route("/v1/loads", get(loads))
}
#[derive(Deserialize)]
struct LoadsQuery {
dp_rank: Option<i64>,
include: Option<String>,
format: Option<String>,
}
async fn loads(State(state): State<Arc<AppState>>, Query(query): Query<LoadsQuery>) -> Response {
let args = &state.server_args;
let Some(path) = args.load_snapshot_path.as_deref() else {
return native_error(
StatusCode::SERVICE_UNAVAILABLE,
"load snapshots are not published to shared memory in this deployment",
false,
);
};
let include = match parse_include(query.include.as_deref()) {
Ok(include) => include,
Err(message) => return native_error(StatusCode::BAD_REQUEST, &message, false),
};
let ranks = match query.dp_rank {
None | Some(0) => vec![args.load_snapshot_dp_rank],
Some(_) => Vec::new(),
};
let loads: Vec<Map<String, Value>> =
read_snapshots(Path::new(path), args.load_snapshot_dp_size, &ranks)
.iter()
.filter_map(|snapshot| to_dict(snapshot, include.as_ref()))
.map(|mut load| {
load.insert("dp_rank".into(), Value::from(0));
load
})
.collect();
if query.format.as_deref() == Some("prometheus") {
return (
[(
header::CONTENT_TYPE,
"text/plain; version=0.0.4; charset=utf-8",
)],
prometheus(&loads),
)
.into_response();
}
let mut body = Map::new();
body.insert("timestamp".into(), Value::from(utc_now_isoformat()));
body.insert("version".into(), Value::from(args.version.as_str()));
body.insert("accelerator".into(), args.accelerator.clone().into());
body.insert(
"num_accelerators".into(),
Value::from(args.num_accelerators),
);
body.insert(
"loads".into(),
Value::Array(loads.into_iter().map(Value::Object).collect()),
);
Json(Value::Object(body)).into_response()
}
/// Python `to_dict`'s include rules: absent or `all` means every section.
/// Returns the selected section names, `None` for all of them.
fn parse_include(include: Option<&str>) -> Result<Option<BTreeSet<String>>, String> {
let Some(include) = include.filter(|include| !include.is_empty()) else {
return Ok(None);
};
let names: BTreeSet<String> = include
.split(',')
.map(|name| name.trim().to_owned())
.collect();
if names.contains("all") {
return Ok(None);
}
let invalid: Vec<&String> = names
.iter()
.filter(|name| !VALID_INCLUDES.contains(&name.as_str()))
.collect();
if !invalid.is_empty() {
return Err(format!(
"Invalid include sections: {invalid:?}. Valid options: {VALID_INCLUDES:?}"
));
}
Ok(Some(names))
}
/// The decoded snapshots of `ranks`, read under the shared lock Python's
/// reader takes. A missing or mismatched file, or an empty or malformed slot,
/// yields nothing for that rank, as Python's reader does.
fn read_snapshots(path: &Path, dp_size: usize, ranks: &[usize]) -> Vec<rmpv::Value> {
let Ok(file) = File::open(path) else {
return Vec::new();
};
if flock(&file, FlockOperation::LockShared).is_err() {
return Vec::new();
}
let mut header = [0u8; HEADER_LEN as usize];
if file.read_exact_at(&mut header, 0).is_err() {
return Vec::new();
}
let version = u16::from_le_bytes([header[4], header[5]]);
let file_dp_size = usize::from(u16::from_le_bytes([header[6], header[7]]));
let slot_size = u64::from(u32::from_le_bytes([
header[8], header[9], header[10], header[11],
]));
let file_len = file.metadata().map(|meta| meta.len()).unwrap_or(0);
if &header[..4] != MAGIC
|| version != VERSION
|| file_dp_size != dp_size
|| slot_size < SLOT_LEN_PREFIX
|| file_len < HEADER_LEN + dp_size as u64 * slot_size
{
tracing::warn!(path = %path.display(), "load snapshot header mismatch");
return Vec::new();
}
ranks
.iter()
.filter_map(|&rank| {
let offset = HEADER_LEN + rank as u64 * slot_size;
let mut len = [0u8; SLOT_LEN_PREFIX as usize];
file.read_exact_at(&mut len, offset).ok()?;
let len = u64::from(u32::from_le_bytes(len));
if len == 0 || len > slot_size - SLOT_LEN_PREFIX {
return None;
}
let mut payload = vec![0u8; len as usize];
file.read_exact_at(&mut payload, offset + SLOT_LEN_PREFIX)
.ok()?;
rmpv::decode::read_value(&mut payload.as_slice()).ok()
})
.collect()
}
/// Python `LoadSnapshot.to_dict`: every flat field, then each present section
/// the include set selects (`None` selects all). `None` for a payload Python's
/// decoder would reject.
fn to_dict(
snapshot: &rmpv::Value,
include: Option<&BTreeSet<String>>,
) -> Option<Map<String, Value>> {
let fields = snapshot.as_map()?;
let get = |name: &str| {
fields
.iter()
.find(|(key, _)| key.as_str() == Some(name))
.map(|(_, value)| value)
};
let mut load = Map::new();
for &(name, zero) in CORE {
let value = match get(name) {
Some(value) => json(value)?,
None => zero.value(),
};
load.insert(name.to_owned(), value);
}
for section in SECTIONS {
let selected = include.is_none_or(|include| include.contains(section.include));
let Some(values) = get(section.field).filter(|value| !value.is_nil()) else {
continue;
};
let values = values.as_array()?;
let mut object = Map::new();
for (i, &(name, zero)) in section.fields.iter().enumerate() {
let value = match (values.get(i), zero) {
(Some(value), _) => json(value)?,
(None, Some(zero)) => zero.value(),
(None, None) => return None,
};
object.insert(name.to_owned(), value);
}
if selected {
load.insert(section.field.to_owned(), Value::Object(object));
}
}
Some(load)
}
fn json(value: &rmpv::Value) -> Option<Value> {
Some(match value {
rmpv::Value::Nil => Value::Null,
rmpv::Value::Boolean(b) => Value::from(*b),
rmpv::Value::Integer(n) => n
.as_i64()
.map(Value::from)
.or_else(|| n.as_u64().map(Value::from))?,
rmpv::Value::F32(x) => Value::from(f64::from(*x)),
rmpv::Value::F64(x) => Value::from(*x),
rmpv::Value::String(s) => Value::from(s.as_str()?),
rmpv::Value::Array(items) => Value::Array(items.iter().map(json).collect::<Option<_>>()?),
_ => return None,
})
}
/// Python `_format_loads_prometheus`: one gauge per numeric field, samples
/// labelled by `dp_rank`, metrics in first-seen order.
fn prometheus(loads: &[Map<String, Value>]) -> String {
let mut metrics: Vec<(String, Vec<String>)> = Vec::new();
let mut add = |name: String, rank: &str, value: &Value| {
let sample = format!("{name}{{dp_rank=\"{rank}\"}} {}", prometheus_value(value));
match metrics.iter_mut().find(|(seen, _)| *seen == name) {
Some((_, samples)) => samples.push(sample),
None => metrics.push((name, vec![sample])),
}
};
for load in loads {
let rank = load
.get("dp_rank")
.map(Value::to_string)
.unwrap_or_default();
for (key, value) in load.iter().filter(|(key, _)| *key != "dp_rank") {
match value {
Value::Object(section) => {
let prefix = SECTIONS
.iter()
.find(|s| s.field == key)
.map_or(key.as_str(), |s| s.prometheus_prefix);
for (sub_key, sub_value) in section.iter().filter(|(_, v)| v.is_number()) {
add(format!("sglang_{prefix}_{sub_key}"), &rank, sub_value);
}
}
Value::Number(_) => add(format!("sglang_{key}"), &rank, value),
_ => {}
}
}
}
let mut lines = Vec::new();
for (name, samples) in metrics {
lines.push(format!("# TYPE {name} gauge"));
lines.extend(samples);
}
lines.join("\n") + "\n"
}
/// Python's `str()` of a number: integers bare, floats with a decimal point.
fn prometheus_value(value: &Value) -> String {
match value.as_f64() {
Some(x) if !value.is_i64() && !value.is_u64() => format!("{x:?}"),
_ => value.to_string(),
}
}
/// `datetime.now(timezone.utc).isoformat()`.
fn utc_now_isoformat() -> String {
chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Micros, false)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::message::config::ServerArgs;
use serde_json::json;
fn fixture() -> Value {
serde_json::from_str(include_str!("../../../testdata/load_snapshot.json")).unwrap()
}
/// The fixture's payloads in Python's file format, one slot per rank.
struct SnapshotFile(std::path::PathBuf);
impl SnapshotFile {
fn new(dp_size: usize) -> Self {
let slot_size = 16 * 1024u32;
let mut bytes = Vec::from(&MAGIC[..]);
bytes.extend(VERSION.to_le_bytes());
bytes.extend((dp_size as u16).to_le_bytes());
bytes.extend(slot_size.to_le_bytes());
for payload in fixture()["payloads"].as_array().unwrap() {
let hex = payload.as_str().unwrap();
let payload: Vec<u8> = (0..hex.len())
.step_by(2)
.map(|i| u8::from_str_radix(&hex[i..i + 2], 16).unwrap())
.collect();
let mut slot = (payload.len() as u32).to_le_bytes().to_vec();
slot.extend(payload);
slot.resize(slot_size as usize, 0);
bytes.extend(slot);
}
let path = std::env::temp_dir().join(format!("sglang-loads-{}", uuid::Uuid::new_v4()));
std::fs::write(&path, bytes).unwrap();
Self(path)
}
}
impl Drop for SnapshotFile {
fn drop(&mut self) {
let _ = std::fs::remove_file(&self.0);
}
}
/// The JSON and Prometheus renderings match what Python produces from the
/// same bytes, key order and float formatting included.
#[test]
fn renders_snapshots_as_python_does() {
let file = SnapshotFile::new(2);
let snapshots = read_snapshots(&file.0, 2, &[0, 1]);
assert_eq!(snapshots.len(), 2);
let expected = fixture();
for name in ["all", "core", "core,queues"] {
let include = parse_include(Some(name)).unwrap();
let loads: Vec<_> = snapshots
.iter()
.map(|s| to_dict(s, include.as_ref()).unwrap())
.collect();
assert_eq!(
serde_json::to_string(&loads).unwrap(),
serde_json::to_string(&expected["to_dict"][name]).unwrap(),
"to_dict({name})"
);
assert_eq!(
prometheus(&loads),
expected["prometheus"][name],
"prometheus({name})"
);
}
}
async fn get(server_args: ServerArgs, query: &str) -> (StatusCode, String) {
let (intake_tx, _intake_rx) = flume::unbounded();
let (abort_tx, _abort_rx) = flume::unbounded();
let state = Arc::new(AppState {
core: crate::api_server::core::CoreHandle::new(
intake_tx,
abort_tx,
crate::api_server::core::CoreConfig {
response_capacity: 8,
response_activity: Default::default(),
startup_ready: true,
is_disaggregation: false,
mm_limits: Default::default(),
metadata: crate::api_server::core::CoreMetadata::default(),
},
),
server_args: Arc::new(server_args),
chat_formatter: None,
});
let query = Query::try_from_uri(&format!("/v1/loads?{query}").parse().unwrap()).unwrap();
let response = loads(State(state), query).await;
let status = response.status();
let body = axum::body::to_bytes(response.into_body(), 64 * 1024)
.await
.unwrap();
(status, String::from_utf8(body.to_vec()).unwrap())
}
/// The endpoint wraps the listener's own slot, as rank 0, in Python's
/// envelope, and degrades as Python's reader does; it rejects unknown
/// sections and answers 503 when snapshots are not in shared memory.
#[tokio::test]
async fn serves_own_rank_in_python_envelope() {
let file = SnapshotFile::new(2);
let args = |path: &Path, dp_size, dp_rank| ServerArgs {
load_snapshot_path: Some(path.display().to_string()),
load_snapshot_dp_size: dp_size,
load_snapshot_dp_rank: dp_rank,
version: "1.2.3".into(),
accelerator: Some("GPU".into()),
num_accelerators: 4,
..Default::default()
};
let json = |body: &str| serde_json::from_str::<Value>(body).unwrap();
let expected = fixture();
let (status, body) = get(args(&file.0, 2, 0), "include=core").await;
assert_eq!(status, StatusCode::OK);
let body = json(&body);
let keys: Vec<_> = body.as_object().unwrap().keys().collect();
assert_eq!(
keys,
[
"timestamp",
"version",
"accelerator",
"num_accelerators",
"loads"
]
);
let timestamp = body["timestamp"].as_str().unwrap();
assert!(
timestamp.len() == 32 && timestamp.ends_with("+00:00"),
"{timestamp}"
);
assert_eq!(
(&body["version"], &body["accelerator"]),
(&json!("1.2.3"), &json!("GPU"))
);
assert_eq!(body["num_accelerators"], 4);
assert_eq!(body["loads"], json!([expected["to_dict"]["core"][0]]));
// The rank-1 listener reports the rank-1 slot, renumbered to 0.
let (_, body) = get(args(&file.0, 2, 1), "dp_rank=0").await;
let mut rank1 = expected["to_dict"]["all"][1].clone();
rank1["dp_rank"] = json!(0);
assert_eq!(json(&body)["loads"], json!([rank1]));
let ranks = |body: &str| json(body)["loads"].as_array().unwrap().len();
assert_eq!(ranks(&get(args(&file.0, 2, 1), "dp_rank=1").await.1), 0);
assert_eq!(
ranks(&get(args(&file.0, 3, 0), "").await.1),
0,
"dp_size mismatch"
);
let missing = Path::new("/nonexistent/sglang_loads.shm");
assert_eq!(ranks(&get(args(missing, 2, 0), "").await.1), 0);
let (status, body) = get(args(&file.0, 2, 0), "format=prometheus").await;
assert_eq!(status, StatusCode::OK);
let rank0 = expected["to_dict"]["all"][0].as_object().unwrap().clone();
assert_eq!(body, prometheus(&[rank0]));
let (status, body) = get(args(&file.0, 2, 0), "include=core,bogus").await;
assert_eq!(status, StatusCode::BAD_REQUEST);
assert!(body.contains("bogus"), "{body}");
let (status, _) = get(ServerArgs::default(), "").await;
assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE);
}
}
+29
View File
@@ -173,6 +173,15 @@ pub struct ServerArgs {
/// and the scheduler-derived KV token capacity, reported by `/server_info`.
pub version: String,
pub max_total_num_tokens: u64,
/// The scheduler's shared-memory load snapshot file, its DP rank count and
/// this listener's own slot, which `/v1/loads` reads (`None` when snapshots
/// go over ZMQ instead), plus the accelerator name and per-rank count that
/// endpoint reports.
pub load_snapshot_path: Option<String>,
pub load_snapshot_dp_size: usize,
pub load_snapshot_dp_rank: usize,
pub accelerator: Option<String>,
pub num_accelerators: u32,
/// Prometheus metrics (`--enable-metrics`); `None` serves no `/metrics`.
pub metrics: Option<MetricsConfig>,
}
@@ -213,6 +222,11 @@ impl ServerArgs {
num_reserved_tokens,
version,
max_total_num_tokens,
load_snapshot_path,
load_snapshot_dp_size,
load_snapshot_dp_rank,
accelerator,
num_accelerators,
metrics,
))]
// The parameter list IS the schema; one keyword per field, all required.
@@ -250,6 +264,11 @@ impl ServerArgs {
num_reserved_tokens: u64,
version: String,
max_total_num_tokens: u64,
load_snapshot_path: Option<String>,
load_snapshot_dp_size: usize,
load_snapshot_dp_rank: usize,
accelerator: Option<String>,
num_accelerators: u32,
metrics: Option<MetricsConfig>,
) -> Self {
Self {
@@ -285,6 +304,11 @@ impl ServerArgs {
num_reserved_tokens,
version,
max_total_num_tokens,
load_snapshot_path,
load_snapshot_dp_size,
load_snapshot_dp_rank,
accelerator,
num_accelerators,
metrics,
}
}
@@ -328,6 +352,11 @@ impl Default for ServerArgs {
num_reserved_tokens: 0,
version: String::new(),
max_total_num_tokens: 0,
load_snapshot_path: None,
load_snapshot_dp_size: 0,
load_snapshot_dp_rank: 0,
accelerator: None,
num_accelerators: 0,
metrics: None,
}
}
+190
View File
@@ -0,0 +1,190 @@
{
"payloads": [
"de0016a974696d657374616d70cb3ff4000000000000b06e756d5f72756e6e696e675f7265717307b06e756d5f77616974696e675f7265717303bb6e756d5f77616974696e675f756e6361636865645f746f6b656e73cd1000af6e756d5f757365645f746f6b656e73ce00014000b06e756d5f746f74616c5f746f6b656e73ce00016000b16e756d5f6163746976655f746f6b656e73ce000157c0b96e756d5f707265616c6c6f635f72656164795f746f6b656e73cd0200b46d61785f746f74616c5f6e756d5f746f6b656e73ce00100000b46d61785f72756e6e696e675f7265717565737473cd0100ab746f6b656e5f7573616765cb3fb4000000000000ae67656e5f7468726f756768707574cb40934a0000000000ae63616368655f6869745f72617465cb3fe4000000000000ab7574696c697a6174696f6ecb3fe0000000000000bd746f74616c5f70726566696c6c5f756e6361636865645f746f6b656e73ce3ade68b1b5746f74616c5f70726566696c6c5f627573795f7573ce075bcd15ae6465636f64655f6d6f6d656e747393cb4059000000000000cb4004000000000000cb3fc0000000000000a66d656d6f727994cb404e400000000000cb405e100000000000cb4000000000000000ce00100000ab73706563756c617469766592cb400c000000000000cb3fe8000000000000a46c6f7261930108cb3fc0000000000000ae6469736167677265676174696f6e98a66465636f64650000020000cb4029000000000000cb0000000000000000a6717565756573950301000204",
"de0001a764705f72616e6b01"
],
"to_dict": {
"all": [
{
"timestamp": 1.25,
"dp_rank": 0,
"num_running_reqs": 7,
"num_waiting_reqs": 3,
"num_waiting_uncached_tokens": 4096,
"num_used_tokens": 81920,
"num_total_tokens": 90112,
"num_active_tokens": 88000,
"num_prealloc_ready_tokens": 512,
"max_total_num_tokens": 1048576,
"max_running_requests": 256,
"token_usage": 0.078125,
"gen_throughput": 1234.5,
"cache_hit_rate": 0.625,
"utilization": 0.5,
"total_prefill_uncached_tokens": 987654321,
"total_prefill_busy_us": 123456789,
"decode_moments": [
100.0,
2.5,
0.125
],
"memory": {
"weight_gb": 60.5,
"kv_cache_gb": 120.25,
"graph_gb": 2.0,
"token_capacity": 1048576
},
"speculative": {
"accept_length": 3.5,
"accept_rate": 0.75
},
"lora": {
"slots_used": 1,
"slots_total": 8,
"utilization": 0.125
},
"disaggregation": {
"mode": "decode",
"prefill_bootstrap_queue_reqs": 0,
"prefill_inflight_queue_reqs": 0,
"decode_prealloc_queue_reqs": 2,
"decode_transfer_queue_reqs": 0,
"decode_retracted_queue_reqs": 0,
"kv_transfer_speed_gb_s": 12.5,
"kv_transfer_latency_ms": 0.0
},
"queues": {
"waiting": 3,
"grammar": 1,
"paused": 0,
"retracted": 2,
"prealloc_ready": 4
}
},
{
"timestamp": 0.0,
"dp_rank": 1,
"num_running_reqs": 0,
"num_waiting_reqs": 0,
"num_waiting_uncached_tokens": 0,
"num_used_tokens": 0,
"num_total_tokens": 0,
"num_active_tokens": 0,
"num_prealloc_ready_tokens": 0,
"max_total_num_tokens": 0,
"max_running_requests": 0,
"token_usage": 0.0,
"gen_throughput": 0.0,
"cache_hit_rate": 0.0,
"utilization": 0.0,
"total_prefill_uncached_tokens": 0,
"total_prefill_busy_us": 0,
"decode_moments": null
}
],
"core": [
{
"timestamp": 1.25,
"dp_rank": 0,
"num_running_reqs": 7,
"num_waiting_reqs": 3,
"num_waiting_uncached_tokens": 4096,
"num_used_tokens": 81920,
"num_total_tokens": 90112,
"num_active_tokens": 88000,
"num_prealloc_ready_tokens": 512,
"max_total_num_tokens": 1048576,
"max_running_requests": 256,
"token_usage": 0.078125,
"gen_throughput": 1234.5,
"cache_hit_rate": 0.625,
"utilization": 0.5,
"total_prefill_uncached_tokens": 987654321,
"total_prefill_busy_us": 123456789,
"decode_moments": [
100.0,
2.5,
0.125
]
},
{
"timestamp": 0.0,
"dp_rank": 1,
"num_running_reqs": 0,
"num_waiting_reqs": 0,
"num_waiting_uncached_tokens": 0,
"num_used_tokens": 0,
"num_total_tokens": 0,
"num_active_tokens": 0,
"num_prealloc_ready_tokens": 0,
"max_total_num_tokens": 0,
"max_running_requests": 0,
"token_usage": 0.0,
"gen_throughput": 0.0,
"cache_hit_rate": 0.0,
"utilization": 0.0,
"total_prefill_uncached_tokens": 0,
"total_prefill_busy_us": 0,
"decode_moments": null
}
],
"core,queues": [
{
"timestamp": 1.25,
"dp_rank": 0,
"num_running_reqs": 7,
"num_waiting_reqs": 3,
"num_waiting_uncached_tokens": 4096,
"num_used_tokens": 81920,
"num_total_tokens": 90112,
"num_active_tokens": 88000,
"num_prealloc_ready_tokens": 512,
"max_total_num_tokens": 1048576,
"max_running_requests": 256,
"token_usage": 0.078125,
"gen_throughput": 1234.5,
"cache_hit_rate": 0.625,
"utilization": 0.5,
"total_prefill_uncached_tokens": 987654321,
"total_prefill_busy_us": 123456789,
"decode_moments": [
100.0,
2.5,
0.125
],
"queues": {
"waiting": 3,
"grammar": 1,
"paused": 0,
"retracted": 2,
"prealloc_ready": 4
}
},
{
"timestamp": 0.0,
"dp_rank": 1,
"num_running_reqs": 0,
"num_waiting_reqs": 0,
"num_waiting_uncached_tokens": 0,
"num_used_tokens": 0,
"num_total_tokens": 0,
"num_active_tokens": 0,
"num_prealloc_ready_tokens": 0,
"max_total_num_tokens": 0,
"max_running_requests": 0,
"token_usage": 0.0,
"gen_throughput": 0.0,
"cache_hit_rate": 0.0,
"utilization": 0.0,
"total_prefill_uncached_tokens": 0,
"total_prefill_busy_us": 0,
"decode_moments": null
}
]
},
"prometheus": {
"all": "# TYPE sglang_timestamp gauge\nsglang_timestamp{dp_rank=\"0\"} 1.25\nsglang_timestamp{dp_rank=\"1\"} 0.0\n# TYPE sglang_num_running_reqs gauge\nsglang_num_running_reqs{dp_rank=\"0\"} 7\nsglang_num_running_reqs{dp_rank=\"1\"} 0\n# TYPE sglang_num_waiting_reqs gauge\nsglang_num_waiting_reqs{dp_rank=\"0\"} 3\nsglang_num_waiting_reqs{dp_rank=\"1\"} 0\n# TYPE sglang_num_waiting_uncached_tokens gauge\nsglang_num_waiting_uncached_tokens{dp_rank=\"0\"} 4096\nsglang_num_waiting_uncached_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_num_used_tokens gauge\nsglang_num_used_tokens{dp_rank=\"0\"} 81920\nsglang_num_used_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_num_total_tokens gauge\nsglang_num_total_tokens{dp_rank=\"0\"} 90112\nsglang_num_total_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_num_active_tokens gauge\nsglang_num_active_tokens{dp_rank=\"0\"} 88000\nsglang_num_active_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_num_prealloc_ready_tokens gauge\nsglang_num_prealloc_ready_tokens{dp_rank=\"0\"} 512\nsglang_num_prealloc_ready_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_max_total_num_tokens gauge\nsglang_max_total_num_tokens{dp_rank=\"0\"} 1048576\nsglang_max_total_num_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_max_running_requests gauge\nsglang_max_running_requests{dp_rank=\"0\"} 256\nsglang_max_running_requests{dp_rank=\"1\"} 0\n# TYPE sglang_token_usage gauge\nsglang_token_usage{dp_rank=\"0\"} 0.078125\nsglang_token_usage{dp_rank=\"1\"} 0.0\n# TYPE sglang_gen_throughput gauge\nsglang_gen_throughput{dp_rank=\"0\"} 1234.5\nsglang_gen_throughput{dp_rank=\"1\"} 0.0\n# TYPE sglang_cache_hit_rate gauge\nsglang_cache_hit_rate{dp_rank=\"0\"} 0.625\nsglang_cache_hit_rate{dp_rank=\"1\"} 0.0\n# TYPE sglang_utilization gauge\nsglang_utilization{dp_rank=\"0\"} 0.5\nsglang_utilization{dp_rank=\"1\"} 0.0\n# TYPE sglang_total_prefill_uncached_tokens gauge\nsglang_total_prefill_uncached_tokens{dp_rank=\"0\"} 987654321\nsglang_total_prefill_uncached_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_total_prefill_busy_us gauge\nsglang_total_prefill_busy_us{dp_rank=\"0\"} 123456789\nsglang_total_prefill_busy_us{dp_rank=\"1\"} 0\n# TYPE sglang_memory_weight_gb gauge\nsglang_memory_weight_gb{dp_rank=\"0\"} 60.5\n# TYPE sglang_memory_kv_cache_gb gauge\nsglang_memory_kv_cache_gb{dp_rank=\"0\"} 120.25\n# TYPE sglang_memory_graph_gb gauge\nsglang_memory_graph_gb{dp_rank=\"0\"} 2.0\n# TYPE sglang_memory_token_capacity gauge\nsglang_memory_token_capacity{dp_rank=\"0\"} 1048576\n# TYPE sglang_spec_accept_length gauge\nsglang_spec_accept_length{dp_rank=\"0\"} 3.5\n# TYPE sglang_spec_accept_rate gauge\nsglang_spec_accept_rate{dp_rank=\"0\"} 0.75\n# TYPE sglang_lora_slots_used gauge\nsglang_lora_slots_used{dp_rank=\"0\"} 1\n# TYPE sglang_lora_slots_total gauge\nsglang_lora_slots_total{dp_rank=\"0\"} 8\n# TYPE sglang_lora_utilization gauge\nsglang_lora_utilization{dp_rank=\"0\"} 0.125\n# TYPE sglang_disagg_prefill_bootstrap_queue_reqs gauge\nsglang_disagg_prefill_bootstrap_queue_reqs{dp_rank=\"0\"} 0\n# TYPE sglang_disagg_prefill_inflight_queue_reqs gauge\nsglang_disagg_prefill_inflight_queue_reqs{dp_rank=\"0\"} 0\n# TYPE sglang_disagg_decode_prealloc_queue_reqs gauge\nsglang_disagg_decode_prealloc_queue_reqs{dp_rank=\"0\"} 2\n# TYPE sglang_disagg_decode_transfer_queue_reqs gauge\nsglang_disagg_decode_transfer_queue_reqs{dp_rank=\"0\"} 0\n# TYPE sglang_disagg_decode_retracted_queue_reqs gauge\nsglang_disagg_decode_retracted_queue_reqs{dp_rank=\"0\"} 0\n# TYPE sglang_disagg_kv_transfer_speed_gb_s gauge\nsglang_disagg_kv_transfer_speed_gb_s{dp_rank=\"0\"} 12.5\n# TYPE sglang_disagg_kv_transfer_latency_ms gauge\nsglang_disagg_kv_transfer_latency_ms{dp_rank=\"0\"} 0.0\n# TYPE sglang_queues_waiting gauge\nsglang_queues_waiting{dp_rank=\"0\"} 3\n# TYPE sglang_queues_grammar gauge\nsglang_queues_grammar{dp_rank=\"0\"} 1\n# TYPE sglang_queues_paused gauge\nsglang_queues_paused{dp_rank=\"0\"} 0\n# TYPE sglang_queues_retracted gauge\nsglang_queues_retracted{dp_rank=\"0\"} 2\n# TYPE sglang_queues_prealloc_ready gauge\nsglang_queues_prealloc_ready{dp_rank=\"0\"} 4\n",
"core": "# TYPE sglang_timestamp gauge\nsglang_timestamp{dp_rank=\"0\"} 1.25\nsglang_timestamp{dp_rank=\"1\"} 0.0\n# TYPE sglang_num_running_reqs gauge\nsglang_num_running_reqs{dp_rank=\"0\"} 7\nsglang_num_running_reqs{dp_rank=\"1\"} 0\n# TYPE sglang_num_waiting_reqs gauge\nsglang_num_waiting_reqs{dp_rank=\"0\"} 3\nsglang_num_waiting_reqs{dp_rank=\"1\"} 0\n# TYPE sglang_num_waiting_uncached_tokens gauge\nsglang_num_waiting_uncached_tokens{dp_rank=\"0\"} 4096\nsglang_num_waiting_uncached_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_num_used_tokens gauge\nsglang_num_used_tokens{dp_rank=\"0\"} 81920\nsglang_num_used_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_num_total_tokens gauge\nsglang_num_total_tokens{dp_rank=\"0\"} 90112\nsglang_num_total_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_num_active_tokens gauge\nsglang_num_active_tokens{dp_rank=\"0\"} 88000\nsglang_num_active_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_num_prealloc_ready_tokens gauge\nsglang_num_prealloc_ready_tokens{dp_rank=\"0\"} 512\nsglang_num_prealloc_ready_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_max_total_num_tokens gauge\nsglang_max_total_num_tokens{dp_rank=\"0\"} 1048576\nsglang_max_total_num_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_max_running_requests gauge\nsglang_max_running_requests{dp_rank=\"0\"} 256\nsglang_max_running_requests{dp_rank=\"1\"} 0\n# TYPE sglang_token_usage gauge\nsglang_token_usage{dp_rank=\"0\"} 0.078125\nsglang_token_usage{dp_rank=\"1\"} 0.0\n# TYPE sglang_gen_throughput gauge\nsglang_gen_throughput{dp_rank=\"0\"} 1234.5\nsglang_gen_throughput{dp_rank=\"1\"} 0.0\n# TYPE sglang_cache_hit_rate gauge\nsglang_cache_hit_rate{dp_rank=\"0\"} 0.625\nsglang_cache_hit_rate{dp_rank=\"1\"} 0.0\n# TYPE sglang_utilization gauge\nsglang_utilization{dp_rank=\"0\"} 0.5\nsglang_utilization{dp_rank=\"1\"} 0.0\n# TYPE sglang_total_prefill_uncached_tokens gauge\nsglang_total_prefill_uncached_tokens{dp_rank=\"0\"} 987654321\nsglang_total_prefill_uncached_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_total_prefill_busy_us gauge\nsglang_total_prefill_busy_us{dp_rank=\"0\"} 123456789\nsglang_total_prefill_busy_us{dp_rank=\"1\"} 0\n",
"core,queues": "# TYPE sglang_timestamp gauge\nsglang_timestamp{dp_rank=\"0\"} 1.25\nsglang_timestamp{dp_rank=\"1\"} 0.0\n# TYPE sglang_num_running_reqs gauge\nsglang_num_running_reqs{dp_rank=\"0\"} 7\nsglang_num_running_reqs{dp_rank=\"1\"} 0\n# TYPE sglang_num_waiting_reqs gauge\nsglang_num_waiting_reqs{dp_rank=\"0\"} 3\nsglang_num_waiting_reqs{dp_rank=\"1\"} 0\n# TYPE sglang_num_waiting_uncached_tokens gauge\nsglang_num_waiting_uncached_tokens{dp_rank=\"0\"} 4096\nsglang_num_waiting_uncached_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_num_used_tokens gauge\nsglang_num_used_tokens{dp_rank=\"0\"} 81920\nsglang_num_used_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_num_total_tokens gauge\nsglang_num_total_tokens{dp_rank=\"0\"} 90112\nsglang_num_total_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_num_active_tokens gauge\nsglang_num_active_tokens{dp_rank=\"0\"} 88000\nsglang_num_active_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_num_prealloc_ready_tokens gauge\nsglang_num_prealloc_ready_tokens{dp_rank=\"0\"} 512\nsglang_num_prealloc_ready_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_max_total_num_tokens gauge\nsglang_max_total_num_tokens{dp_rank=\"0\"} 1048576\nsglang_max_total_num_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_max_running_requests gauge\nsglang_max_running_requests{dp_rank=\"0\"} 256\nsglang_max_running_requests{dp_rank=\"1\"} 0\n# TYPE sglang_token_usage gauge\nsglang_token_usage{dp_rank=\"0\"} 0.078125\nsglang_token_usage{dp_rank=\"1\"} 0.0\n# TYPE sglang_gen_throughput gauge\nsglang_gen_throughput{dp_rank=\"0\"} 1234.5\nsglang_gen_throughput{dp_rank=\"1\"} 0.0\n# TYPE sglang_cache_hit_rate gauge\nsglang_cache_hit_rate{dp_rank=\"0\"} 0.625\nsglang_cache_hit_rate{dp_rank=\"1\"} 0.0\n# TYPE sglang_utilization gauge\nsglang_utilization{dp_rank=\"0\"} 0.5\nsglang_utilization{dp_rank=\"1\"} 0.0\n# TYPE sglang_total_prefill_uncached_tokens gauge\nsglang_total_prefill_uncached_tokens{dp_rank=\"0\"} 987654321\nsglang_total_prefill_uncached_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_total_prefill_busy_us gauge\nsglang_total_prefill_busy_us{dp_rank=\"0\"} 123456789\nsglang_total_prefill_busy_us{dp_rank=\"1\"} 0\n# TYPE sglang_queues_waiting gauge\nsglang_queues_waiting{dp_rank=\"0\"} 3\n# TYPE sglang_queues_grammar gauge\nsglang_queues_grammar{dp_rank=\"0\"} 1\n# TYPE sglang_queues_paused gauge\nsglang_queues_paused{dp_rank=\"0\"} 0\n# TYPE sglang_queues_retracted gauge\nsglang_queues_retracted{dp_rank=\"0\"} 2\n# TYPE sglang_queues_prealloc_ready gauge\nsglang_queues_prealloc_ready{dp_rank=\"0\"} 4\n"
}
}
@@ -25,6 +25,7 @@ def _scheduler_for_typed_config():
),
rust_server_tokenizer_path=lambda: "tokenizer",
max_total_num_tokens=1024,
load_snapshot_writer=None,
)
@@ -1,5 +1,6 @@
"""Unit tests for LoadSnapshot SHM and ZMQ backends."""
import json
import os
import tempfile
import time
@@ -9,15 +10,21 @@ from unittest import mock
from sglang.srt.environ import envs
from sglang.srt.managers.load_snapshot import (
DisaggregationMetrics,
LoadSnapshot,
LoRAMetrics,
MemoryMetrics,
QueueMetrics,
ShmLoadSnapshotReader,
ShmLoadSnapshotWriter,
SpeculativeMetrics,
ZmqLoadSnapshotWriter,
ZmqShmLoadSnapshotReader,
_zmq_addr_for,
create_load_snapshot_reader,
create_load_snapshot_writer,
should_use_zmq,
snapshot_encoder,
zmq_reader_owner,
)
from sglang.srt.runtime_context import get_context
@@ -492,5 +499,75 @@ class TestEndToEndZmqSimulation(CustomTestCase):
os.unlink(shm_path)
# The Rust frontend's `/v1/loads` (rust/sglang-server/src/api_server/http/loads.rs)
# renders these snapshots from the same SHM bytes; its fixture must match Python.
RUST_LOADS_FIXTURE = os.path.join(
os.path.dirname(__file__),
*[os.pardir] * 4,
"rust/sglang-server/testdata/load_snapshot.json",
)
def build_rust_loads_fixture() -> dict:
"""Wire payloads for a full and a minimal rank, and what Python renders from
them. The values are arbitrary and pairwise distinct, so a field rendered
under the wrong name or in the wrong slot shows. Regenerate the fixture
with ``json.dump(build_rust_loads_fixture(), f, indent=2)`` after a
``LoadSnapshot`` change, then update the Rust renderer."""
from sglang.srt.entrypoints.v1_loads import _format_loads_prometheus
full = LoadSnapshot(
timestamp=1.25,
dp_rank=0,
num_running_reqs=7,
num_waiting_reqs=3,
num_waiting_uncached_tokens=4096,
num_used_tokens=81920,
num_total_tokens=90112,
num_active_tokens=88000,
num_prealloc_ready_tokens=512,
max_total_num_tokens=1048576,
max_running_requests=256,
token_usage=0.078125,
gen_throughput=1234.5,
cache_hit_rate=0.625,
utilization=0.5,
total_prefill_uncached_tokens=987654321,
total_prefill_busy_us=123456789,
decode_moments=[100.0, 2.5, 0.125],
memory=MemoryMetrics(
weight_gb=60.5, kv_cache_gb=120.25, graph_gb=2.0, token_capacity=1048576
),
speculative=SpeculativeMetrics(accept_length=3.5, accept_rate=0.75),
lora=LoRAMetrics(slots_used=1, slots_total=8, utilization=0.125),
disaggregation=DisaggregationMetrics(
mode="decode", decode_prealloc_queue_reqs=2, kv_transfer_speed_gb_s=12.5
),
queues=QueueMetrics(
waiting=3, grammar=1, paused=0, retracted=2, prealloc_ready=4
),
)
snapshots = [full, LoadSnapshot(dp_rank=1)]
includes = {"all": None, "core": {"core"}, "core,queues": {"core", "queues"}}
return {
"payloads": [snapshot_encoder.encode(s).hex() for s in snapshots],
"to_dict": {
name: [s.to_dict(include) for s in snapshots]
for name, include in includes.items()
},
"prometheus": {
name: _format_loads_prometheus(snapshots, include).body.decode()
for name, include in includes.items()
},
}
class TestRustLoadsFixture(CustomTestCase):
def test_fixture_matches_python(self):
with open(RUST_LOADS_FIXTURE) as f:
fixture = json.load(f)
self.assertEqual(fixture, build_rust_loads_fixture())
if __name__ == "__main__":
unittest.main()