[Feat] sglang-server: serve /v1/loads from the Rust frontend (#44532)
Co-authored-by: metamergebot <metamergebot@users.noreply.github.com> Co-authored-by: shodoco <shodoco@users.noreply.github.com>
This commit is contained in:
co-authored by
metamergebot
shodoco
parent
a038a0dca0
commit
26a2f18c02
@@ -9,7 +9,12 @@ from types import ModuleType
|
||||
from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
from sglang.srt.arg_groups.overrides import resolving_view
|
||||
from sglang.srt.entrypoints.v1_loads import (
|
||||
_accelerator_name,
|
||||
_num_accelerators_per_dp_rank,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.managers.load_snapshot import ShmLoadSnapshotWriter
|
||||
from sglang.srt.managers.utils import compute_num_reserved_tokens
|
||||
from sglang.srt.runtime_context import (
|
||||
get_disagg,
|
||||
@@ -57,6 +62,11 @@ def _build_server_args(
|
||||
if get_serving().smg_grpc_mode or get_serving().grpc_mode
|
||||
else get_serving().grpc_port
|
||||
)
|
||||
# `/v1/loads` reads this rank's slot of the scheduler's shared-memory
|
||||
# snapshots directly; the ZMQ transport (multi-node DP) has no file to read.
|
||||
shm_writer = scheduler.load_snapshot_writer
|
||||
if not isinstance(shm_writer, ShmLoadSnapshotWriter):
|
||||
shm_writer = None
|
||||
return ext.ServerArgs(
|
||||
model_path=get_model().model_path,
|
||||
served_model_name=get_serving().served_model_name,
|
||||
@@ -118,6 +128,11 @@ def _build_server_args(
|
||||
# can serve them statically (no scheduler round-trip).
|
||||
version=__version__,
|
||||
max_total_num_tokens=scheduler.max_total_num_tokens,
|
||||
load_snapshot_path=shm_writer.path if shm_writer else None,
|
||||
load_snapshot_dp_size=shm_writer.dp_size if shm_writer else 0,
|
||||
load_snapshot_dp_rank=shm_writer.dp_rank if shm_writer else 0,
|
||||
accelerator=_accelerator_name(),
|
||||
num_accelerators=_num_accelerators_per_dp_rank(),
|
||||
# Every process writes its prometheus_client samples under this
|
||||
# directory (set before the schedulers start); `/metrics` merges them.
|
||||
metrics=(
|
||||
|
||||
Generated
+1
@@ -3767,6 +3767,7 @@ dependencies = [
|
||||
"axum",
|
||||
"bytemuck",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"core_affinity",
|
||||
"dynamo-parsers",
|
||||
"dynamo-protocols",
|
||||
|
||||
@@ -34,6 +34,8 @@ uuid = { workspace = true }
|
||||
|
||||
arc-swap = "1"
|
||||
axum = { version = "0.8.9", features = ["json", "tokio"] }
|
||||
# Already in the tree via dynamo-renderer; `/v1/loads` timestamps.
|
||||
chrono = { version = "0.4", default-features = false, features = ["clock"] }
|
||||
# Safe POD slice casts (feature buffers viewed as bytes for the shm copy).
|
||||
bytemuck = "1"
|
||||
core_affinity = "0.8"
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
pub(crate) mod app;
|
||||
mod info;
|
||||
mod loads;
|
||||
mod metrics;
|
||||
mod native_api;
|
||||
mod openai;
|
||||
|
||||
@@ -7,7 +7,7 @@ use std::time::Duration;
|
||||
|
||||
use axum::Router;
|
||||
|
||||
use super::{info, metrics, native_api, openai};
|
||||
use super::{info, loads, metrics, native_api, openai};
|
||||
use crate::HttpLayer;
|
||||
use crate::api_server::core::CoreHandle;
|
||||
use crate::api_server::disaggregation::bootstrap as pd_bootstrap;
|
||||
@@ -80,6 +80,7 @@ pub async fn serve(
|
||||
// Each endpoint module registers its own routes and merges here.
|
||||
let mut router = Router::new()
|
||||
.merge(info::routes())
|
||||
.merge(loads::routes())
|
||||
.merge(native_api::routes())
|
||||
.merge(openai::routes());
|
||||
if server_args.metrics.is_some() && metrics_listener {
|
||||
|
||||
@@ -0,0 +1,563 @@
|
||||
//! `GET /v1/loads`: the port of Python `entrypoints/v1_loads.py`. Each
|
||||
//! scheduler publishes its per-DP-rank load snapshot into a shared-memory file
|
||||
//! (`managers/load_snapshot.py`, `ShmLoadSnapshotWriter`); this reads that file
|
||||
//! and renders Python's `LoadSnapshot.to_dict`, as JSON or Prometheus text.
|
||||
//!
|
||||
//! Unlike the Python server, which fronts every DP rank, a Rust listener serves
|
||||
//! only its own scheduler's rank, so it reports itself as a single-rank engine:
|
||||
//! its own slot, as rank 0.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::fs::File;
|
||||
use std::os::unix::fs::FileExt;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::extract::{Query, State};
|
||||
use axum::http::{StatusCode, header};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use axum::routing::get;
|
||||
use axum::{Json, Router};
|
||||
use rustix::fs::{FlockOperation, flock};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use super::app::AppState;
|
||||
use super::native_api::native_error;
|
||||
|
||||
/// The file header, Python's `HEADER_STRUCT` (`<4sHHI`): magic, version,
|
||||
/// dp_size, slot_size. Each rank's slot follows at `HEADER_LEN + rank *
|
||||
/// slot_size`: a little-endian `u32` length, then the msgpack snapshot.
|
||||
const MAGIC: &[u8; 4] = b"SLNS";
|
||||
const VERSION: u16 = 2;
|
||||
const HEADER_LEN: u64 = 12;
|
||||
const SLOT_LEN_PREFIX: u64 = 4;
|
||||
|
||||
/// The zero value a field takes when the snapshot omits it.
|
||||
#[derive(Clone, Copy)]
|
||||
enum Zero {
|
||||
Int,
|
||||
Float,
|
||||
Null,
|
||||
}
|
||||
|
||||
impl Zero {
|
||||
fn value(self) -> Value {
|
||||
match self {
|
||||
Zero::Int => Value::from(0),
|
||||
Zero::Float => Value::from(0.0),
|
||||
Zero::Null => Value::Null,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `LoadSnapshot`'s flat fields in declaration order. The snapshot is encoded
|
||||
/// with `omit_defaults`, so an absent key takes its zero.
|
||||
const CORE: &[(&str, Zero)] = &[
|
||||
("timestamp", Zero::Float),
|
||||
("dp_rank", Zero::Int),
|
||||
("num_running_reqs", Zero::Int),
|
||||
("num_waiting_reqs", Zero::Int),
|
||||
("num_waiting_uncached_tokens", Zero::Int),
|
||||
("num_used_tokens", Zero::Int),
|
||||
("num_total_tokens", Zero::Int),
|
||||
("num_active_tokens", Zero::Int),
|
||||
("num_prealloc_ready_tokens", Zero::Int),
|
||||
("max_total_num_tokens", Zero::Int),
|
||||
("max_running_requests", Zero::Int),
|
||||
("token_usage", Zero::Float),
|
||||
("gen_throughput", Zero::Float),
|
||||
("cache_hit_rate", Zero::Float),
|
||||
("utilization", Zero::Float),
|
||||
("total_prefill_uncached_tokens", Zero::Int),
|
||||
("total_prefill_busy_us", Zero::Int),
|
||||
("decode_moments", Zero::Null),
|
||||
];
|
||||
|
||||
/// One optional sub-struct of `LoadSnapshot`, encoded as a positional array.
|
||||
/// `None` marks a field without a default.
|
||||
struct Section {
|
||||
field: &'static str,
|
||||
include: &'static str,
|
||||
prometheus_prefix: &'static str,
|
||||
fields: &'static [(&'static str, Option<Zero>)],
|
||||
}
|
||||
|
||||
const SECTIONS: &[Section] = &[
|
||||
Section {
|
||||
field: "memory",
|
||||
include: "memory",
|
||||
prometheus_prefix: "memory",
|
||||
fields: &[
|
||||
("weight_gb", None),
|
||||
("kv_cache_gb", None),
|
||||
("graph_gb", None),
|
||||
("token_capacity", None),
|
||||
],
|
||||
},
|
||||
Section {
|
||||
field: "speculative",
|
||||
include: "spec",
|
||||
prometheus_prefix: "spec",
|
||||
fields: &[("accept_length", None), ("accept_rate", None)],
|
||||
},
|
||||
Section {
|
||||
field: "lora",
|
||||
include: "lora",
|
||||
prometheus_prefix: "lora",
|
||||
fields: &[
|
||||
("slots_used", None),
|
||||
("slots_total", None),
|
||||
("utilization", None),
|
||||
],
|
||||
},
|
||||
Section {
|
||||
field: "disaggregation",
|
||||
include: "disagg",
|
||||
prometheus_prefix: "disagg",
|
||||
fields: &[
|
||||
("mode", None),
|
||||
("prefill_bootstrap_queue_reqs", Some(Zero::Int)),
|
||||
("prefill_inflight_queue_reqs", Some(Zero::Int)),
|
||||
("decode_prealloc_queue_reqs", Some(Zero::Int)),
|
||||
("decode_transfer_queue_reqs", Some(Zero::Int)),
|
||||
("decode_retracted_queue_reqs", Some(Zero::Int)),
|
||||
("kv_transfer_speed_gb_s", Some(Zero::Float)),
|
||||
("kv_transfer_latency_ms", Some(Zero::Float)),
|
||||
],
|
||||
},
|
||||
Section {
|
||||
field: "queues",
|
||||
include: "queues",
|
||||
prometheus_prefix: "queues",
|
||||
fields: &[
|
||||
("waiting", None),
|
||||
("grammar", None),
|
||||
("paused", None),
|
||||
("retracted", None),
|
||||
("prealloc_ready", None),
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const VALID_INCLUDES: &[&str] = &["all", "core", "disagg", "lora", "memory", "queues", "spec"];
|
||||
|
||||
pub(super) fn routes() -> Router<Arc<AppState>> {
|
||||
Router::new().route("/v1/loads", get(loads))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct LoadsQuery {
|
||||
dp_rank: Option<i64>,
|
||||
include: Option<String>,
|
||||
format: Option<String>,
|
||||
}
|
||||
|
||||
async fn loads(State(state): State<Arc<AppState>>, Query(query): Query<LoadsQuery>) -> Response {
|
||||
let args = &state.server_args;
|
||||
let Some(path) = args.load_snapshot_path.as_deref() else {
|
||||
return native_error(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"load snapshots are not published to shared memory in this deployment",
|
||||
false,
|
||||
);
|
||||
};
|
||||
let include = match parse_include(query.include.as_deref()) {
|
||||
Ok(include) => include,
|
||||
Err(message) => return native_error(StatusCode::BAD_REQUEST, &message, false),
|
||||
};
|
||||
let ranks = match query.dp_rank {
|
||||
None | Some(0) => vec![args.load_snapshot_dp_rank],
|
||||
Some(_) => Vec::new(),
|
||||
};
|
||||
let loads: Vec<Map<String, Value>> =
|
||||
read_snapshots(Path::new(path), args.load_snapshot_dp_size, &ranks)
|
||||
.iter()
|
||||
.filter_map(|snapshot| to_dict(snapshot, include.as_ref()))
|
||||
.map(|mut load| {
|
||||
load.insert("dp_rank".into(), Value::from(0));
|
||||
load
|
||||
})
|
||||
.collect();
|
||||
|
||||
if query.format.as_deref() == Some("prometheus") {
|
||||
return (
|
||||
[(
|
||||
header::CONTENT_TYPE,
|
||||
"text/plain; version=0.0.4; charset=utf-8",
|
||||
)],
|
||||
prometheus(&loads),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
let mut body = Map::new();
|
||||
body.insert("timestamp".into(), Value::from(utc_now_isoformat()));
|
||||
body.insert("version".into(), Value::from(args.version.as_str()));
|
||||
body.insert("accelerator".into(), args.accelerator.clone().into());
|
||||
body.insert(
|
||||
"num_accelerators".into(),
|
||||
Value::from(args.num_accelerators),
|
||||
);
|
||||
body.insert(
|
||||
"loads".into(),
|
||||
Value::Array(loads.into_iter().map(Value::Object).collect()),
|
||||
);
|
||||
Json(Value::Object(body)).into_response()
|
||||
}
|
||||
|
||||
/// Python `to_dict`'s include rules: absent or `all` means every section.
|
||||
/// Returns the selected section names, `None` for all of them.
|
||||
fn parse_include(include: Option<&str>) -> Result<Option<BTreeSet<String>>, String> {
|
||||
let Some(include) = include.filter(|include| !include.is_empty()) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let names: BTreeSet<String> = include
|
||||
.split(',')
|
||||
.map(|name| name.trim().to_owned())
|
||||
.collect();
|
||||
if names.contains("all") {
|
||||
return Ok(None);
|
||||
}
|
||||
let invalid: Vec<&String> = names
|
||||
.iter()
|
||||
.filter(|name| !VALID_INCLUDES.contains(&name.as_str()))
|
||||
.collect();
|
||||
if !invalid.is_empty() {
|
||||
return Err(format!(
|
||||
"Invalid include sections: {invalid:?}. Valid options: {VALID_INCLUDES:?}"
|
||||
));
|
||||
}
|
||||
Ok(Some(names))
|
||||
}
|
||||
|
||||
/// The decoded snapshots of `ranks`, read under the shared lock Python's
|
||||
/// reader takes. A missing or mismatched file, or an empty or malformed slot,
|
||||
/// yields nothing for that rank, as Python's reader does.
|
||||
fn read_snapshots(path: &Path, dp_size: usize, ranks: &[usize]) -> Vec<rmpv::Value> {
|
||||
let Ok(file) = File::open(path) else {
|
||||
return Vec::new();
|
||||
};
|
||||
if flock(&file, FlockOperation::LockShared).is_err() {
|
||||
return Vec::new();
|
||||
}
|
||||
let mut header = [0u8; HEADER_LEN as usize];
|
||||
if file.read_exact_at(&mut header, 0).is_err() {
|
||||
return Vec::new();
|
||||
}
|
||||
let version = u16::from_le_bytes([header[4], header[5]]);
|
||||
let file_dp_size = usize::from(u16::from_le_bytes([header[6], header[7]]));
|
||||
let slot_size = u64::from(u32::from_le_bytes([
|
||||
header[8], header[9], header[10], header[11],
|
||||
]));
|
||||
let file_len = file.metadata().map(|meta| meta.len()).unwrap_or(0);
|
||||
if &header[..4] != MAGIC
|
||||
|| version != VERSION
|
||||
|| file_dp_size != dp_size
|
||||
|| slot_size < SLOT_LEN_PREFIX
|
||||
|| file_len < HEADER_LEN + dp_size as u64 * slot_size
|
||||
{
|
||||
tracing::warn!(path = %path.display(), "load snapshot header mismatch");
|
||||
return Vec::new();
|
||||
}
|
||||
ranks
|
||||
.iter()
|
||||
.filter_map(|&rank| {
|
||||
let offset = HEADER_LEN + rank as u64 * slot_size;
|
||||
let mut len = [0u8; SLOT_LEN_PREFIX as usize];
|
||||
file.read_exact_at(&mut len, offset).ok()?;
|
||||
let len = u64::from(u32::from_le_bytes(len));
|
||||
if len == 0 || len > slot_size - SLOT_LEN_PREFIX {
|
||||
return None;
|
||||
}
|
||||
let mut payload = vec![0u8; len as usize];
|
||||
file.read_exact_at(&mut payload, offset + SLOT_LEN_PREFIX)
|
||||
.ok()?;
|
||||
rmpv::decode::read_value(&mut payload.as_slice()).ok()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Python `LoadSnapshot.to_dict`: every flat field, then each present section
|
||||
/// the include set selects (`None` selects all). `None` for a payload Python's
|
||||
/// decoder would reject.
|
||||
fn to_dict(
|
||||
snapshot: &rmpv::Value,
|
||||
include: Option<&BTreeSet<String>>,
|
||||
) -> Option<Map<String, Value>> {
|
||||
let fields = snapshot.as_map()?;
|
||||
let get = |name: &str| {
|
||||
fields
|
||||
.iter()
|
||||
.find(|(key, _)| key.as_str() == Some(name))
|
||||
.map(|(_, value)| value)
|
||||
};
|
||||
let mut load = Map::new();
|
||||
for &(name, zero) in CORE {
|
||||
let value = match get(name) {
|
||||
Some(value) => json(value)?,
|
||||
None => zero.value(),
|
||||
};
|
||||
load.insert(name.to_owned(), value);
|
||||
}
|
||||
for section in SECTIONS {
|
||||
let selected = include.is_none_or(|include| include.contains(section.include));
|
||||
let Some(values) = get(section.field).filter(|value| !value.is_nil()) else {
|
||||
continue;
|
||||
};
|
||||
let values = values.as_array()?;
|
||||
let mut object = Map::new();
|
||||
for (i, &(name, zero)) in section.fields.iter().enumerate() {
|
||||
let value = match (values.get(i), zero) {
|
||||
(Some(value), _) => json(value)?,
|
||||
(None, Some(zero)) => zero.value(),
|
||||
(None, None) => return None,
|
||||
};
|
||||
object.insert(name.to_owned(), value);
|
||||
}
|
||||
if selected {
|
||||
load.insert(section.field.to_owned(), Value::Object(object));
|
||||
}
|
||||
}
|
||||
Some(load)
|
||||
}
|
||||
|
||||
fn json(value: &rmpv::Value) -> Option<Value> {
|
||||
Some(match value {
|
||||
rmpv::Value::Nil => Value::Null,
|
||||
rmpv::Value::Boolean(b) => Value::from(*b),
|
||||
rmpv::Value::Integer(n) => n
|
||||
.as_i64()
|
||||
.map(Value::from)
|
||||
.or_else(|| n.as_u64().map(Value::from))?,
|
||||
rmpv::Value::F32(x) => Value::from(f64::from(*x)),
|
||||
rmpv::Value::F64(x) => Value::from(*x),
|
||||
rmpv::Value::String(s) => Value::from(s.as_str()?),
|
||||
rmpv::Value::Array(items) => Value::Array(items.iter().map(json).collect::<Option<_>>()?),
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Python `_format_loads_prometheus`: one gauge per numeric field, samples
|
||||
/// labelled by `dp_rank`, metrics in first-seen order.
|
||||
fn prometheus(loads: &[Map<String, Value>]) -> String {
|
||||
let mut metrics: Vec<(String, Vec<String>)> = Vec::new();
|
||||
let mut add = |name: String, rank: &str, value: &Value| {
|
||||
let sample = format!("{name}{{dp_rank=\"{rank}\"}} {}", prometheus_value(value));
|
||||
match metrics.iter_mut().find(|(seen, _)| *seen == name) {
|
||||
Some((_, samples)) => samples.push(sample),
|
||||
None => metrics.push((name, vec![sample])),
|
||||
}
|
||||
};
|
||||
for load in loads {
|
||||
let rank = load
|
||||
.get("dp_rank")
|
||||
.map(Value::to_string)
|
||||
.unwrap_or_default();
|
||||
for (key, value) in load.iter().filter(|(key, _)| *key != "dp_rank") {
|
||||
match value {
|
||||
Value::Object(section) => {
|
||||
let prefix = SECTIONS
|
||||
.iter()
|
||||
.find(|s| s.field == key)
|
||||
.map_or(key.as_str(), |s| s.prometheus_prefix);
|
||||
for (sub_key, sub_value) in section.iter().filter(|(_, v)| v.is_number()) {
|
||||
add(format!("sglang_{prefix}_{sub_key}"), &rank, sub_value);
|
||||
}
|
||||
}
|
||||
Value::Number(_) => add(format!("sglang_{key}"), &rank, value),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut lines = Vec::new();
|
||||
for (name, samples) in metrics {
|
||||
lines.push(format!("# TYPE {name} gauge"));
|
||||
lines.extend(samples);
|
||||
}
|
||||
lines.join("\n") + "\n"
|
||||
}
|
||||
|
||||
/// Python's `str()` of a number: integers bare, floats with a decimal point.
|
||||
fn prometheus_value(value: &Value) -> String {
|
||||
match value.as_f64() {
|
||||
Some(x) if !value.is_i64() && !value.is_u64() => format!("{x:?}"),
|
||||
_ => value.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// `datetime.now(timezone.utc).isoformat()`.
|
||||
fn utc_now_isoformat() -> String {
|
||||
chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Micros, false)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::message::config::ServerArgs;
|
||||
use serde_json::json;
|
||||
|
||||
fn fixture() -> Value {
|
||||
serde_json::from_str(include_str!("../../../testdata/load_snapshot.json")).unwrap()
|
||||
}
|
||||
|
||||
/// The fixture's payloads in Python's file format, one slot per rank.
|
||||
struct SnapshotFile(std::path::PathBuf);
|
||||
|
||||
impl SnapshotFile {
|
||||
fn new(dp_size: usize) -> Self {
|
||||
let slot_size = 16 * 1024u32;
|
||||
let mut bytes = Vec::from(&MAGIC[..]);
|
||||
bytes.extend(VERSION.to_le_bytes());
|
||||
bytes.extend((dp_size as u16).to_le_bytes());
|
||||
bytes.extend(slot_size.to_le_bytes());
|
||||
for payload in fixture()["payloads"].as_array().unwrap() {
|
||||
let hex = payload.as_str().unwrap();
|
||||
let payload: Vec<u8> = (0..hex.len())
|
||||
.step_by(2)
|
||||
.map(|i| u8::from_str_radix(&hex[i..i + 2], 16).unwrap())
|
||||
.collect();
|
||||
let mut slot = (payload.len() as u32).to_le_bytes().to_vec();
|
||||
slot.extend(payload);
|
||||
slot.resize(slot_size as usize, 0);
|
||||
bytes.extend(slot);
|
||||
}
|
||||
let path = std::env::temp_dir().join(format!("sglang-loads-{}", uuid::Uuid::new_v4()));
|
||||
std::fs::write(&path, bytes).unwrap();
|
||||
Self(path)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for SnapshotFile {
|
||||
fn drop(&mut self) {
|
||||
let _ = std::fs::remove_file(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
/// The JSON and Prometheus renderings match what Python produces from the
|
||||
/// same bytes, key order and float formatting included.
|
||||
#[test]
|
||||
fn renders_snapshots_as_python_does() {
|
||||
let file = SnapshotFile::new(2);
|
||||
let snapshots = read_snapshots(&file.0, 2, &[0, 1]);
|
||||
assert_eq!(snapshots.len(), 2);
|
||||
let expected = fixture();
|
||||
for name in ["all", "core", "core,queues"] {
|
||||
let include = parse_include(Some(name)).unwrap();
|
||||
let loads: Vec<_> = snapshots
|
||||
.iter()
|
||||
.map(|s| to_dict(s, include.as_ref()).unwrap())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
serde_json::to_string(&loads).unwrap(),
|
||||
serde_json::to_string(&expected["to_dict"][name]).unwrap(),
|
||||
"to_dict({name})"
|
||||
);
|
||||
assert_eq!(
|
||||
prometheus(&loads),
|
||||
expected["prometheus"][name],
|
||||
"prometheus({name})"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn get(server_args: ServerArgs, query: &str) -> (StatusCode, String) {
|
||||
let (intake_tx, _intake_rx) = flume::unbounded();
|
||||
let (abort_tx, _abort_rx) = flume::unbounded();
|
||||
let state = Arc::new(AppState {
|
||||
core: crate::api_server::core::CoreHandle::new(
|
||||
intake_tx,
|
||||
abort_tx,
|
||||
crate::api_server::core::CoreConfig {
|
||||
response_capacity: 8,
|
||||
response_activity: Default::default(),
|
||||
startup_ready: true,
|
||||
is_disaggregation: false,
|
||||
mm_limits: Default::default(),
|
||||
metadata: crate::api_server::core::CoreMetadata::default(),
|
||||
},
|
||||
),
|
||||
server_args: Arc::new(server_args),
|
||||
chat_formatter: None,
|
||||
});
|
||||
let query = Query::try_from_uri(&format!("/v1/loads?{query}").parse().unwrap()).unwrap();
|
||||
let response = loads(State(state), query).await;
|
||||
let status = response.status();
|
||||
let body = axum::body::to_bytes(response.into_body(), 64 * 1024)
|
||||
.await
|
||||
.unwrap();
|
||||
(status, String::from_utf8(body.to_vec()).unwrap())
|
||||
}
|
||||
|
||||
/// The endpoint wraps the listener's own slot, as rank 0, in Python's
|
||||
/// envelope, and degrades as Python's reader does; it rejects unknown
|
||||
/// sections and answers 503 when snapshots are not in shared memory.
|
||||
#[tokio::test]
|
||||
async fn serves_own_rank_in_python_envelope() {
|
||||
let file = SnapshotFile::new(2);
|
||||
let args = |path: &Path, dp_size, dp_rank| ServerArgs {
|
||||
load_snapshot_path: Some(path.display().to_string()),
|
||||
load_snapshot_dp_size: dp_size,
|
||||
load_snapshot_dp_rank: dp_rank,
|
||||
version: "1.2.3".into(),
|
||||
accelerator: Some("GPU".into()),
|
||||
num_accelerators: 4,
|
||||
..Default::default()
|
||||
};
|
||||
let json = |body: &str| serde_json::from_str::<Value>(body).unwrap();
|
||||
let expected = fixture();
|
||||
|
||||
let (status, body) = get(args(&file.0, 2, 0), "include=core").await;
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
let body = json(&body);
|
||||
let keys: Vec<_> = body.as_object().unwrap().keys().collect();
|
||||
assert_eq!(
|
||||
keys,
|
||||
[
|
||||
"timestamp",
|
||||
"version",
|
||||
"accelerator",
|
||||
"num_accelerators",
|
||||
"loads"
|
||||
]
|
||||
);
|
||||
let timestamp = body["timestamp"].as_str().unwrap();
|
||||
assert!(
|
||||
timestamp.len() == 32 && timestamp.ends_with("+00:00"),
|
||||
"{timestamp}"
|
||||
);
|
||||
assert_eq!(
|
||||
(&body["version"], &body["accelerator"]),
|
||||
(&json!("1.2.3"), &json!("GPU"))
|
||||
);
|
||||
assert_eq!(body["num_accelerators"], 4);
|
||||
assert_eq!(body["loads"], json!([expected["to_dict"]["core"][0]]));
|
||||
|
||||
// The rank-1 listener reports the rank-1 slot, renumbered to 0.
|
||||
let (_, body) = get(args(&file.0, 2, 1), "dp_rank=0").await;
|
||||
let mut rank1 = expected["to_dict"]["all"][1].clone();
|
||||
rank1["dp_rank"] = json!(0);
|
||||
assert_eq!(json(&body)["loads"], json!([rank1]));
|
||||
|
||||
let ranks = |body: &str| json(body)["loads"].as_array().unwrap().len();
|
||||
assert_eq!(ranks(&get(args(&file.0, 2, 1), "dp_rank=1").await.1), 0);
|
||||
assert_eq!(
|
||||
ranks(&get(args(&file.0, 3, 0), "").await.1),
|
||||
0,
|
||||
"dp_size mismatch"
|
||||
);
|
||||
let missing = Path::new("/nonexistent/sglang_loads.shm");
|
||||
assert_eq!(ranks(&get(args(missing, 2, 0), "").await.1), 0);
|
||||
|
||||
let (status, body) = get(args(&file.0, 2, 0), "format=prometheus").await;
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
let rank0 = expected["to_dict"]["all"][0].as_object().unwrap().clone();
|
||||
assert_eq!(body, prometheus(&[rank0]));
|
||||
|
||||
let (status, body) = get(args(&file.0, 2, 0), "include=core,bogus").await;
|
||||
assert_eq!(status, StatusCode::BAD_REQUEST);
|
||||
assert!(body.contains("bogus"), "{body}");
|
||||
|
||||
let (status, _) = get(ServerArgs::default(), "").await;
|
||||
assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE);
|
||||
}
|
||||
}
|
||||
@@ -173,6 +173,15 @@ pub struct ServerArgs {
|
||||
/// and the scheduler-derived KV token capacity, reported by `/server_info`.
|
||||
pub version: String,
|
||||
pub max_total_num_tokens: u64,
|
||||
/// The scheduler's shared-memory load snapshot file, its DP rank count and
|
||||
/// this listener's own slot, which `/v1/loads` reads (`None` when snapshots
|
||||
/// go over ZMQ instead), plus the accelerator name and per-rank count that
|
||||
/// endpoint reports.
|
||||
pub load_snapshot_path: Option<String>,
|
||||
pub load_snapshot_dp_size: usize,
|
||||
pub load_snapshot_dp_rank: usize,
|
||||
pub accelerator: Option<String>,
|
||||
pub num_accelerators: u32,
|
||||
/// Prometheus metrics (`--enable-metrics`); `None` serves no `/metrics`.
|
||||
pub metrics: Option<MetricsConfig>,
|
||||
}
|
||||
@@ -213,6 +222,11 @@ impl ServerArgs {
|
||||
num_reserved_tokens,
|
||||
version,
|
||||
max_total_num_tokens,
|
||||
load_snapshot_path,
|
||||
load_snapshot_dp_size,
|
||||
load_snapshot_dp_rank,
|
||||
accelerator,
|
||||
num_accelerators,
|
||||
metrics,
|
||||
))]
|
||||
// The parameter list IS the schema; one keyword per field, all required.
|
||||
@@ -250,6 +264,11 @@ impl ServerArgs {
|
||||
num_reserved_tokens: u64,
|
||||
version: String,
|
||||
max_total_num_tokens: u64,
|
||||
load_snapshot_path: Option<String>,
|
||||
load_snapshot_dp_size: usize,
|
||||
load_snapshot_dp_rank: usize,
|
||||
accelerator: Option<String>,
|
||||
num_accelerators: u32,
|
||||
metrics: Option<MetricsConfig>,
|
||||
) -> Self {
|
||||
Self {
|
||||
@@ -285,6 +304,11 @@ impl ServerArgs {
|
||||
num_reserved_tokens,
|
||||
version,
|
||||
max_total_num_tokens,
|
||||
load_snapshot_path,
|
||||
load_snapshot_dp_size,
|
||||
load_snapshot_dp_rank,
|
||||
accelerator,
|
||||
num_accelerators,
|
||||
metrics,
|
||||
}
|
||||
}
|
||||
@@ -328,6 +352,11 @@ impl Default for ServerArgs {
|
||||
num_reserved_tokens: 0,
|
||||
version: String::new(),
|
||||
max_total_num_tokens: 0,
|
||||
load_snapshot_path: None,
|
||||
load_snapshot_dp_size: 0,
|
||||
load_snapshot_dp_rank: 0,
|
||||
accelerator: None,
|
||||
num_accelerators: 0,
|
||||
metrics: None,
|
||||
}
|
||||
}
|
||||
|
||||
+190
@@ -0,0 +1,190 @@
|
||||
{
|
||||
"payloads": [
|
||||
"de0016a974696d657374616d70cb3ff4000000000000b06e756d5f72756e6e696e675f7265717307b06e756d5f77616974696e675f7265717303bb6e756d5f77616974696e675f756e6361636865645f746f6b656e73cd1000af6e756d5f757365645f746f6b656e73ce00014000b06e756d5f746f74616c5f746f6b656e73ce00016000b16e756d5f6163746976655f746f6b656e73ce000157c0b96e756d5f707265616c6c6f635f72656164795f746f6b656e73cd0200b46d61785f746f74616c5f6e756d5f746f6b656e73ce00100000b46d61785f72756e6e696e675f7265717565737473cd0100ab746f6b656e5f7573616765cb3fb4000000000000ae67656e5f7468726f756768707574cb40934a0000000000ae63616368655f6869745f72617465cb3fe4000000000000ab7574696c697a6174696f6ecb3fe0000000000000bd746f74616c5f70726566696c6c5f756e6361636865645f746f6b656e73ce3ade68b1b5746f74616c5f70726566696c6c5f627573795f7573ce075bcd15ae6465636f64655f6d6f6d656e747393cb4059000000000000cb4004000000000000cb3fc0000000000000a66d656d6f727994cb404e400000000000cb405e100000000000cb4000000000000000ce00100000ab73706563756c617469766592cb400c000000000000cb3fe8000000000000a46c6f7261930108cb3fc0000000000000ae6469736167677265676174696f6e98a66465636f64650000020000cb4029000000000000cb0000000000000000a6717565756573950301000204",
|
||||
"de0001a764705f72616e6b01"
|
||||
],
|
||||
"to_dict": {
|
||||
"all": [
|
||||
{
|
||||
"timestamp": 1.25,
|
||||
"dp_rank": 0,
|
||||
"num_running_reqs": 7,
|
||||
"num_waiting_reqs": 3,
|
||||
"num_waiting_uncached_tokens": 4096,
|
||||
"num_used_tokens": 81920,
|
||||
"num_total_tokens": 90112,
|
||||
"num_active_tokens": 88000,
|
||||
"num_prealloc_ready_tokens": 512,
|
||||
"max_total_num_tokens": 1048576,
|
||||
"max_running_requests": 256,
|
||||
"token_usage": 0.078125,
|
||||
"gen_throughput": 1234.5,
|
||||
"cache_hit_rate": 0.625,
|
||||
"utilization": 0.5,
|
||||
"total_prefill_uncached_tokens": 987654321,
|
||||
"total_prefill_busy_us": 123456789,
|
||||
"decode_moments": [
|
||||
100.0,
|
||||
2.5,
|
||||
0.125
|
||||
],
|
||||
"memory": {
|
||||
"weight_gb": 60.5,
|
||||
"kv_cache_gb": 120.25,
|
||||
"graph_gb": 2.0,
|
||||
"token_capacity": 1048576
|
||||
},
|
||||
"speculative": {
|
||||
"accept_length": 3.5,
|
||||
"accept_rate": 0.75
|
||||
},
|
||||
"lora": {
|
||||
"slots_used": 1,
|
||||
"slots_total": 8,
|
||||
"utilization": 0.125
|
||||
},
|
||||
"disaggregation": {
|
||||
"mode": "decode",
|
||||
"prefill_bootstrap_queue_reqs": 0,
|
||||
"prefill_inflight_queue_reqs": 0,
|
||||
"decode_prealloc_queue_reqs": 2,
|
||||
"decode_transfer_queue_reqs": 0,
|
||||
"decode_retracted_queue_reqs": 0,
|
||||
"kv_transfer_speed_gb_s": 12.5,
|
||||
"kv_transfer_latency_ms": 0.0
|
||||
},
|
||||
"queues": {
|
||||
"waiting": 3,
|
||||
"grammar": 1,
|
||||
"paused": 0,
|
||||
"retracted": 2,
|
||||
"prealloc_ready": 4
|
||||
}
|
||||
},
|
||||
{
|
||||
"timestamp": 0.0,
|
||||
"dp_rank": 1,
|
||||
"num_running_reqs": 0,
|
||||
"num_waiting_reqs": 0,
|
||||
"num_waiting_uncached_tokens": 0,
|
||||
"num_used_tokens": 0,
|
||||
"num_total_tokens": 0,
|
||||
"num_active_tokens": 0,
|
||||
"num_prealloc_ready_tokens": 0,
|
||||
"max_total_num_tokens": 0,
|
||||
"max_running_requests": 0,
|
||||
"token_usage": 0.0,
|
||||
"gen_throughput": 0.0,
|
||||
"cache_hit_rate": 0.0,
|
||||
"utilization": 0.0,
|
||||
"total_prefill_uncached_tokens": 0,
|
||||
"total_prefill_busy_us": 0,
|
||||
"decode_moments": null
|
||||
}
|
||||
],
|
||||
"core": [
|
||||
{
|
||||
"timestamp": 1.25,
|
||||
"dp_rank": 0,
|
||||
"num_running_reqs": 7,
|
||||
"num_waiting_reqs": 3,
|
||||
"num_waiting_uncached_tokens": 4096,
|
||||
"num_used_tokens": 81920,
|
||||
"num_total_tokens": 90112,
|
||||
"num_active_tokens": 88000,
|
||||
"num_prealloc_ready_tokens": 512,
|
||||
"max_total_num_tokens": 1048576,
|
||||
"max_running_requests": 256,
|
||||
"token_usage": 0.078125,
|
||||
"gen_throughput": 1234.5,
|
||||
"cache_hit_rate": 0.625,
|
||||
"utilization": 0.5,
|
||||
"total_prefill_uncached_tokens": 987654321,
|
||||
"total_prefill_busy_us": 123456789,
|
||||
"decode_moments": [
|
||||
100.0,
|
||||
2.5,
|
||||
0.125
|
||||
]
|
||||
},
|
||||
{
|
||||
"timestamp": 0.0,
|
||||
"dp_rank": 1,
|
||||
"num_running_reqs": 0,
|
||||
"num_waiting_reqs": 0,
|
||||
"num_waiting_uncached_tokens": 0,
|
||||
"num_used_tokens": 0,
|
||||
"num_total_tokens": 0,
|
||||
"num_active_tokens": 0,
|
||||
"num_prealloc_ready_tokens": 0,
|
||||
"max_total_num_tokens": 0,
|
||||
"max_running_requests": 0,
|
||||
"token_usage": 0.0,
|
||||
"gen_throughput": 0.0,
|
||||
"cache_hit_rate": 0.0,
|
||||
"utilization": 0.0,
|
||||
"total_prefill_uncached_tokens": 0,
|
||||
"total_prefill_busy_us": 0,
|
||||
"decode_moments": null
|
||||
}
|
||||
],
|
||||
"core,queues": [
|
||||
{
|
||||
"timestamp": 1.25,
|
||||
"dp_rank": 0,
|
||||
"num_running_reqs": 7,
|
||||
"num_waiting_reqs": 3,
|
||||
"num_waiting_uncached_tokens": 4096,
|
||||
"num_used_tokens": 81920,
|
||||
"num_total_tokens": 90112,
|
||||
"num_active_tokens": 88000,
|
||||
"num_prealloc_ready_tokens": 512,
|
||||
"max_total_num_tokens": 1048576,
|
||||
"max_running_requests": 256,
|
||||
"token_usage": 0.078125,
|
||||
"gen_throughput": 1234.5,
|
||||
"cache_hit_rate": 0.625,
|
||||
"utilization": 0.5,
|
||||
"total_prefill_uncached_tokens": 987654321,
|
||||
"total_prefill_busy_us": 123456789,
|
||||
"decode_moments": [
|
||||
100.0,
|
||||
2.5,
|
||||
0.125
|
||||
],
|
||||
"queues": {
|
||||
"waiting": 3,
|
||||
"grammar": 1,
|
||||
"paused": 0,
|
||||
"retracted": 2,
|
||||
"prealloc_ready": 4
|
||||
}
|
||||
},
|
||||
{
|
||||
"timestamp": 0.0,
|
||||
"dp_rank": 1,
|
||||
"num_running_reqs": 0,
|
||||
"num_waiting_reqs": 0,
|
||||
"num_waiting_uncached_tokens": 0,
|
||||
"num_used_tokens": 0,
|
||||
"num_total_tokens": 0,
|
||||
"num_active_tokens": 0,
|
||||
"num_prealloc_ready_tokens": 0,
|
||||
"max_total_num_tokens": 0,
|
||||
"max_running_requests": 0,
|
||||
"token_usage": 0.0,
|
||||
"gen_throughput": 0.0,
|
||||
"cache_hit_rate": 0.0,
|
||||
"utilization": 0.0,
|
||||
"total_prefill_uncached_tokens": 0,
|
||||
"total_prefill_busy_us": 0,
|
||||
"decode_moments": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"prometheus": {
|
||||
"all": "# TYPE sglang_timestamp gauge\nsglang_timestamp{dp_rank=\"0\"} 1.25\nsglang_timestamp{dp_rank=\"1\"} 0.0\n# TYPE sglang_num_running_reqs gauge\nsglang_num_running_reqs{dp_rank=\"0\"} 7\nsglang_num_running_reqs{dp_rank=\"1\"} 0\n# TYPE sglang_num_waiting_reqs gauge\nsglang_num_waiting_reqs{dp_rank=\"0\"} 3\nsglang_num_waiting_reqs{dp_rank=\"1\"} 0\n# TYPE sglang_num_waiting_uncached_tokens gauge\nsglang_num_waiting_uncached_tokens{dp_rank=\"0\"} 4096\nsglang_num_waiting_uncached_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_num_used_tokens gauge\nsglang_num_used_tokens{dp_rank=\"0\"} 81920\nsglang_num_used_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_num_total_tokens gauge\nsglang_num_total_tokens{dp_rank=\"0\"} 90112\nsglang_num_total_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_num_active_tokens gauge\nsglang_num_active_tokens{dp_rank=\"0\"} 88000\nsglang_num_active_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_num_prealloc_ready_tokens gauge\nsglang_num_prealloc_ready_tokens{dp_rank=\"0\"} 512\nsglang_num_prealloc_ready_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_max_total_num_tokens gauge\nsglang_max_total_num_tokens{dp_rank=\"0\"} 1048576\nsglang_max_total_num_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_max_running_requests gauge\nsglang_max_running_requests{dp_rank=\"0\"} 256\nsglang_max_running_requests{dp_rank=\"1\"} 0\n# TYPE sglang_token_usage gauge\nsglang_token_usage{dp_rank=\"0\"} 0.078125\nsglang_token_usage{dp_rank=\"1\"} 0.0\n# TYPE sglang_gen_throughput gauge\nsglang_gen_throughput{dp_rank=\"0\"} 1234.5\nsglang_gen_throughput{dp_rank=\"1\"} 0.0\n# TYPE sglang_cache_hit_rate gauge\nsglang_cache_hit_rate{dp_rank=\"0\"} 0.625\nsglang_cache_hit_rate{dp_rank=\"1\"} 0.0\n# TYPE sglang_utilization gauge\nsglang_utilization{dp_rank=\"0\"} 0.5\nsglang_utilization{dp_rank=\"1\"} 0.0\n# TYPE sglang_total_prefill_uncached_tokens gauge\nsglang_total_prefill_uncached_tokens{dp_rank=\"0\"} 987654321\nsglang_total_prefill_uncached_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_total_prefill_busy_us gauge\nsglang_total_prefill_busy_us{dp_rank=\"0\"} 123456789\nsglang_total_prefill_busy_us{dp_rank=\"1\"} 0\n# TYPE sglang_memory_weight_gb gauge\nsglang_memory_weight_gb{dp_rank=\"0\"} 60.5\n# TYPE sglang_memory_kv_cache_gb gauge\nsglang_memory_kv_cache_gb{dp_rank=\"0\"} 120.25\n# TYPE sglang_memory_graph_gb gauge\nsglang_memory_graph_gb{dp_rank=\"0\"} 2.0\n# TYPE sglang_memory_token_capacity gauge\nsglang_memory_token_capacity{dp_rank=\"0\"} 1048576\n# TYPE sglang_spec_accept_length gauge\nsglang_spec_accept_length{dp_rank=\"0\"} 3.5\n# TYPE sglang_spec_accept_rate gauge\nsglang_spec_accept_rate{dp_rank=\"0\"} 0.75\n# TYPE sglang_lora_slots_used gauge\nsglang_lora_slots_used{dp_rank=\"0\"} 1\n# TYPE sglang_lora_slots_total gauge\nsglang_lora_slots_total{dp_rank=\"0\"} 8\n# TYPE sglang_lora_utilization gauge\nsglang_lora_utilization{dp_rank=\"0\"} 0.125\n# TYPE sglang_disagg_prefill_bootstrap_queue_reqs gauge\nsglang_disagg_prefill_bootstrap_queue_reqs{dp_rank=\"0\"} 0\n# TYPE sglang_disagg_prefill_inflight_queue_reqs gauge\nsglang_disagg_prefill_inflight_queue_reqs{dp_rank=\"0\"} 0\n# TYPE sglang_disagg_decode_prealloc_queue_reqs gauge\nsglang_disagg_decode_prealloc_queue_reqs{dp_rank=\"0\"} 2\n# TYPE sglang_disagg_decode_transfer_queue_reqs gauge\nsglang_disagg_decode_transfer_queue_reqs{dp_rank=\"0\"} 0\n# TYPE sglang_disagg_decode_retracted_queue_reqs gauge\nsglang_disagg_decode_retracted_queue_reqs{dp_rank=\"0\"} 0\n# TYPE sglang_disagg_kv_transfer_speed_gb_s gauge\nsglang_disagg_kv_transfer_speed_gb_s{dp_rank=\"0\"} 12.5\n# TYPE sglang_disagg_kv_transfer_latency_ms gauge\nsglang_disagg_kv_transfer_latency_ms{dp_rank=\"0\"} 0.0\n# TYPE sglang_queues_waiting gauge\nsglang_queues_waiting{dp_rank=\"0\"} 3\n# TYPE sglang_queues_grammar gauge\nsglang_queues_grammar{dp_rank=\"0\"} 1\n# TYPE sglang_queues_paused gauge\nsglang_queues_paused{dp_rank=\"0\"} 0\n# TYPE sglang_queues_retracted gauge\nsglang_queues_retracted{dp_rank=\"0\"} 2\n# TYPE sglang_queues_prealloc_ready gauge\nsglang_queues_prealloc_ready{dp_rank=\"0\"} 4\n",
|
||||
"core": "# TYPE sglang_timestamp gauge\nsglang_timestamp{dp_rank=\"0\"} 1.25\nsglang_timestamp{dp_rank=\"1\"} 0.0\n# TYPE sglang_num_running_reqs gauge\nsglang_num_running_reqs{dp_rank=\"0\"} 7\nsglang_num_running_reqs{dp_rank=\"1\"} 0\n# TYPE sglang_num_waiting_reqs gauge\nsglang_num_waiting_reqs{dp_rank=\"0\"} 3\nsglang_num_waiting_reqs{dp_rank=\"1\"} 0\n# TYPE sglang_num_waiting_uncached_tokens gauge\nsglang_num_waiting_uncached_tokens{dp_rank=\"0\"} 4096\nsglang_num_waiting_uncached_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_num_used_tokens gauge\nsglang_num_used_tokens{dp_rank=\"0\"} 81920\nsglang_num_used_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_num_total_tokens gauge\nsglang_num_total_tokens{dp_rank=\"0\"} 90112\nsglang_num_total_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_num_active_tokens gauge\nsglang_num_active_tokens{dp_rank=\"0\"} 88000\nsglang_num_active_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_num_prealloc_ready_tokens gauge\nsglang_num_prealloc_ready_tokens{dp_rank=\"0\"} 512\nsglang_num_prealloc_ready_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_max_total_num_tokens gauge\nsglang_max_total_num_tokens{dp_rank=\"0\"} 1048576\nsglang_max_total_num_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_max_running_requests gauge\nsglang_max_running_requests{dp_rank=\"0\"} 256\nsglang_max_running_requests{dp_rank=\"1\"} 0\n# TYPE sglang_token_usage gauge\nsglang_token_usage{dp_rank=\"0\"} 0.078125\nsglang_token_usage{dp_rank=\"1\"} 0.0\n# TYPE sglang_gen_throughput gauge\nsglang_gen_throughput{dp_rank=\"0\"} 1234.5\nsglang_gen_throughput{dp_rank=\"1\"} 0.0\n# TYPE sglang_cache_hit_rate gauge\nsglang_cache_hit_rate{dp_rank=\"0\"} 0.625\nsglang_cache_hit_rate{dp_rank=\"1\"} 0.0\n# TYPE sglang_utilization gauge\nsglang_utilization{dp_rank=\"0\"} 0.5\nsglang_utilization{dp_rank=\"1\"} 0.0\n# TYPE sglang_total_prefill_uncached_tokens gauge\nsglang_total_prefill_uncached_tokens{dp_rank=\"0\"} 987654321\nsglang_total_prefill_uncached_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_total_prefill_busy_us gauge\nsglang_total_prefill_busy_us{dp_rank=\"0\"} 123456789\nsglang_total_prefill_busy_us{dp_rank=\"1\"} 0\n",
|
||||
"core,queues": "# TYPE sglang_timestamp gauge\nsglang_timestamp{dp_rank=\"0\"} 1.25\nsglang_timestamp{dp_rank=\"1\"} 0.0\n# TYPE sglang_num_running_reqs gauge\nsglang_num_running_reqs{dp_rank=\"0\"} 7\nsglang_num_running_reqs{dp_rank=\"1\"} 0\n# TYPE sglang_num_waiting_reqs gauge\nsglang_num_waiting_reqs{dp_rank=\"0\"} 3\nsglang_num_waiting_reqs{dp_rank=\"1\"} 0\n# TYPE sglang_num_waiting_uncached_tokens gauge\nsglang_num_waiting_uncached_tokens{dp_rank=\"0\"} 4096\nsglang_num_waiting_uncached_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_num_used_tokens gauge\nsglang_num_used_tokens{dp_rank=\"0\"} 81920\nsglang_num_used_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_num_total_tokens gauge\nsglang_num_total_tokens{dp_rank=\"0\"} 90112\nsglang_num_total_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_num_active_tokens gauge\nsglang_num_active_tokens{dp_rank=\"0\"} 88000\nsglang_num_active_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_num_prealloc_ready_tokens gauge\nsglang_num_prealloc_ready_tokens{dp_rank=\"0\"} 512\nsglang_num_prealloc_ready_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_max_total_num_tokens gauge\nsglang_max_total_num_tokens{dp_rank=\"0\"} 1048576\nsglang_max_total_num_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_max_running_requests gauge\nsglang_max_running_requests{dp_rank=\"0\"} 256\nsglang_max_running_requests{dp_rank=\"1\"} 0\n# TYPE sglang_token_usage gauge\nsglang_token_usage{dp_rank=\"0\"} 0.078125\nsglang_token_usage{dp_rank=\"1\"} 0.0\n# TYPE sglang_gen_throughput gauge\nsglang_gen_throughput{dp_rank=\"0\"} 1234.5\nsglang_gen_throughput{dp_rank=\"1\"} 0.0\n# TYPE sglang_cache_hit_rate gauge\nsglang_cache_hit_rate{dp_rank=\"0\"} 0.625\nsglang_cache_hit_rate{dp_rank=\"1\"} 0.0\n# TYPE sglang_utilization gauge\nsglang_utilization{dp_rank=\"0\"} 0.5\nsglang_utilization{dp_rank=\"1\"} 0.0\n# TYPE sglang_total_prefill_uncached_tokens gauge\nsglang_total_prefill_uncached_tokens{dp_rank=\"0\"} 987654321\nsglang_total_prefill_uncached_tokens{dp_rank=\"1\"} 0\n# TYPE sglang_total_prefill_busy_us gauge\nsglang_total_prefill_busy_us{dp_rank=\"0\"} 123456789\nsglang_total_prefill_busy_us{dp_rank=\"1\"} 0\n# TYPE sglang_queues_waiting gauge\nsglang_queues_waiting{dp_rank=\"0\"} 3\n# TYPE sglang_queues_grammar gauge\nsglang_queues_grammar{dp_rank=\"0\"} 1\n# TYPE sglang_queues_paused gauge\nsglang_queues_paused{dp_rank=\"0\"} 0\n# TYPE sglang_queues_retracted gauge\nsglang_queues_retracted{dp_rank=\"0\"} 2\n# TYPE sglang_queues_prealloc_ready gauge\nsglang_queues_prealloc_ready{dp_rank=\"0\"} 4\n"
|
||||
}
|
||||
}
|
||||
@@ -25,6 +25,7 @@ def _scheduler_for_typed_config():
|
||||
),
|
||||
rust_server_tokenizer_path=lambda: "tokenizer",
|
||||
max_total_num_tokens=1024,
|
||||
load_snapshot_writer=None,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Unit tests for LoadSnapshot SHM and ZMQ backends."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
@@ -9,15 +10,21 @@ from unittest import mock
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.managers.load_snapshot import (
|
||||
DisaggregationMetrics,
|
||||
LoadSnapshot,
|
||||
LoRAMetrics,
|
||||
MemoryMetrics,
|
||||
QueueMetrics,
|
||||
ShmLoadSnapshotReader,
|
||||
ShmLoadSnapshotWriter,
|
||||
SpeculativeMetrics,
|
||||
ZmqLoadSnapshotWriter,
|
||||
ZmqShmLoadSnapshotReader,
|
||||
_zmq_addr_for,
|
||||
create_load_snapshot_reader,
|
||||
create_load_snapshot_writer,
|
||||
should_use_zmq,
|
||||
snapshot_encoder,
|
||||
zmq_reader_owner,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_context
|
||||
@@ -492,5 +499,75 @@ class TestEndToEndZmqSimulation(CustomTestCase):
|
||||
os.unlink(shm_path)
|
||||
|
||||
|
||||
# The Rust frontend's `/v1/loads` (rust/sglang-server/src/api_server/http/loads.rs)
|
||||
# renders these snapshots from the same SHM bytes; its fixture must match Python.
|
||||
RUST_LOADS_FIXTURE = os.path.join(
|
||||
os.path.dirname(__file__),
|
||||
*[os.pardir] * 4,
|
||||
"rust/sglang-server/testdata/load_snapshot.json",
|
||||
)
|
||||
|
||||
|
||||
def build_rust_loads_fixture() -> dict:
|
||||
"""Wire payloads for a full and a minimal rank, and what Python renders from
|
||||
them. The values are arbitrary and pairwise distinct, so a field rendered
|
||||
under the wrong name or in the wrong slot shows. Regenerate the fixture
|
||||
with ``json.dump(build_rust_loads_fixture(), f, indent=2)`` after a
|
||||
``LoadSnapshot`` change, then update the Rust renderer."""
|
||||
from sglang.srt.entrypoints.v1_loads import _format_loads_prometheus
|
||||
|
||||
full = LoadSnapshot(
|
||||
timestamp=1.25,
|
||||
dp_rank=0,
|
||||
num_running_reqs=7,
|
||||
num_waiting_reqs=3,
|
||||
num_waiting_uncached_tokens=4096,
|
||||
num_used_tokens=81920,
|
||||
num_total_tokens=90112,
|
||||
num_active_tokens=88000,
|
||||
num_prealloc_ready_tokens=512,
|
||||
max_total_num_tokens=1048576,
|
||||
max_running_requests=256,
|
||||
token_usage=0.078125,
|
||||
gen_throughput=1234.5,
|
||||
cache_hit_rate=0.625,
|
||||
utilization=0.5,
|
||||
total_prefill_uncached_tokens=987654321,
|
||||
total_prefill_busy_us=123456789,
|
||||
decode_moments=[100.0, 2.5, 0.125],
|
||||
memory=MemoryMetrics(
|
||||
weight_gb=60.5, kv_cache_gb=120.25, graph_gb=2.0, token_capacity=1048576
|
||||
),
|
||||
speculative=SpeculativeMetrics(accept_length=3.5, accept_rate=0.75),
|
||||
lora=LoRAMetrics(slots_used=1, slots_total=8, utilization=0.125),
|
||||
disaggregation=DisaggregationMetrics(
|
||||
mode="decode", decode_prealloc_queue_reqs=2, kv_transfer_speed_gb_s=12.5
|
||||
),
|
||||
queues=QueueMetrics(
|
||||
waiting=3, grammar=1, paused=0, retracted=2, prealloc_ready=4
|
||||
),
|
||||
)
|
||||
snapshots = [full, LoadSnapshot(dp_rank=1)]
|
||||
includes = {"all": None, "core": {"core"}, "core,queues": {"core", "queues"}}
|
||||
return {
|
||||
"payloads": [snapshot_encoder.encode(s).hex() for s in snapshots],
|
||||
"to_dict": {
|
||||
name: [s.to_dict(include) for s in snapshots]
|
||||
for name, include in includes.items()
|
||||
},
|
||||
"prometheus": {
|
||||
name: _format_loads_prometheus(snapshots, include).body.decode()
|
||||
for name, include in includes.items()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class TestRustLoadsFixture(CustomTestCase):
|
||||
def test_fixture_matches_python(self):
|
||||
with open(RUST_LOADS_FIXTURE) as f:
|
||||
fixture = json.load(f)
|
||||
self.assertEqual(fixture, build_rust_loads_fixture())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user