Files
sglang/rust/sglang-api-codegen/src/model.rs
T

551 lines
18 KiB
Rust

//! Pass 2's input model: each message classified by the `sglang.json.v1`
//! options into the JSON shape the emitter renders. The options are read from
//! the FileDescriptorSet through prost-reflect, so the schema file is the
//! single source of truth.
use prost_reflect::{
DescriptorPool, ExtensionDescriptor, FieldDescriptor, Kind, MessageDescriptor,
};
pub struct Extensions {
message_json: ExtensionDescriptor,
field_json: ExtensionDescriptor,
oneof_json: ExtensionDescriptor,
}
impl Extensions {
pub fn resolve(pool: &DescriptorPool) -> Extensions {
let get = |name: &str| {
pool.get_extension_by_name(name)
.unwrap_or_else(|| panic!("options.proto extension missing: {name}"))
};
Extensions {
message_json: get("sglang.json.v1.message_json"),
field_json: get("sglang.json.v1.field_json"),
oneof_json: get("sglang.json.v1.oneof_json"),
}
}
}
fn opt_bool(options: &prost_reflect::DynamicMessage, name: &str) -> bool {
options
.get_field_by_name(name)
.and_then(|v| v.as_bool())
.unwrap_or(false)
}
/// An enum-typed option as its number; 0 (the unset/default variant) when absent.
fn opt_enum(options: &prost_reflect::DynamicMessage, name: &str) -> i32 {
options
.get_field_by_name(name)
.and_then(|v| v.as_enum_number())
.unwrap_or(0)
}
fn opt_str(options: &prost_reflect::DynamicMessage, name: &str) -> Option<String> {
options
.get_field_by_name(name)
.and_then(|v| v.as_str().map(str::to_owned))
.filter(|s| !s.is_empty())
}
#[derive(Debug, Clone, PartialEq)]
pub enum ScalarTy {
F64,
I32,
I64,
U32,
U64,
Bool,
Str,
}
impl ScalarTy {
pub fn rust(&self) -> &'static str {
match self {
ScalarTy::F64 => "f64",
ScalarTy::I32 => "i32",
ScalarTy::I64 => "i64",
ScalarTy::U32 => "u32",
ScalarTy::U64 => "u64",
ScalarTy::Bool => "bool",
ScalarTy::Str => "::prost::alloc::string::String",
}
}
}
#[derive(Debug, Clone)]
pub enum DefaultLit {
F64(f64),
I64(i64),
Bool(bool),
Str(String),
}
impl DefaultLit {
pub fn rust(&self) -> String {
match self {
DefaultLit::F64(v) => format!("{v:?}f64"),
DefaultLit::I64(v) => format!("{v}i64"),
DefaultLit::Bool(v) => format!("{v}"),
DefaultLit::Str(v) => format!("{v:?}.to_string()"),
}
}
}
#[derive(Debug, Clone)]
pub enum FieldKind {
/// A plain proto3 scalar (no presence). `null_default` set =
/// null_resets_default: absent AND explicit null yield the default.
Scalar {
ty: ScalarTy,
null_default: Option<DefaultLit>,
},
/// proto3 `optional` scalar. `null_is_none_default` set: absent yields the
/// default, explicit null clears to None. `null_default` set
/// (null_resets_default): absent AND null yield Some(default), and an
/// `*_or_default` accessor covers protobuf arrival (unset = None).
OptScalar {
ty: ScalarTy,
emit_null: bool,
null_is_none_default: Option<DefaultLit>,
null_default: Option<DefaultLit>,
},
/// A message-typed field: `Option<T>` presence; the value's JSON shape is
/// the message's own impl.
Message {
rust_path: String,
emit_null: bool,
},
RepeatedScalar {
ty: ScalarTy,
},
RepeatedMessage {
rust_path: String,
},
/// repeated string with raw_json: Vec<String> of JSON texts passing
/// through as an array of raw values.
RepeatedRawJson,
/// map<string, scalar>; int_key means numeric-string keys (no transform
/// needed on the JSON side, documented parity).
Map {
value: ScalarTy,
},
/// A string field holding JSON text (raw_json): the value passes through
/// verbatim on the HTTP side.
RawJson {
emit_null: bool,
},
}
#[derive(Debug, Clone)]
pub struct FieldModel {
/// The JSON key (= the proto field name).
pub json_name: String,
/// The prost struct field ident (keyword-escaped).
pub rust_ident: String,
pub kind: FieldKind,
}
#[derive(Debug, Clone)]
pub struct VariantModel {
/// The JSON discriminator value / shape source (= the proto field name).
pub json_name: String,
/// The prost oneof enum variant (PascalCase).
pub rust_variant: String,
pub kind: FieldKind,
/// For tagged unions: the payload message's fields, inlined into the
/// tagged object.
pub payload_fields: Vec<FieldModel>,
}
#[derive(Debug)]
pub enum MessageKind {
/// Single-repeated-field carrier: JSON is the bare array.
BareList { item: FieldModel },
/// Positional tuple: fields in declaration order form a bare array.
Tuple { fields: Vec<FieldModel> },
/// Single-field wrapper serializing as its sole field's value.
Transparent { field: FieldModel },
/// Single oneof { one, many }: scalar-or-list flattening.
OneOrMany {
oneof_mod: String,
one: VariantModel,
many: VariantModel,
},
/// Single oneof, internally tagged by `tag`; `passthrough` catches
/// unknown tags as a raw map (and any known-tag payload that fails to
/// parse, matching the hand-written FinishReason).
TaggedUnion {
oneof_mod: String,
oneof_ident: String,
tag: String,
variants: Vec<VariantModel>,
passthrough: Option<VariantModel>,
},
/// Single oneof, untagged: the value's shape picks the variant.
Untagged {
oneof_mod: String,
oneof_ident: String,
variants: Vec<VariantModel>,
},
/// A plain object.
Struct {
deny_unknown: bool,
fields: Vec<FieldModel>,
},
/// gRPC-only (a plain oneof container like GenerateStreamItem): no HTTP
/// JSON contract, no serde emitted.
Skip,
}
#[derive(Debug)]
pub struct MessageModel {
/// Rust type name (= proto message name).
pub rust_name: String,
pub kind: MessageKind,
}
pub fn build_models(pool: &DescriptorPool, package: &str) -> Vec<MessageModel> {
let ext = Extensions::resolve(pool);
pool.all_messages()
.filter(|m| m.full_name().starts_with(package))
// Map-entry synthetic messages have no JSON identity of their own.
.filter(|m| !m.is_map_entry())
.map(|m| message_model(&m, &ext))
.collect()
}
fn message_model(msg: &MessageDescriptor, ext: &Extensions) -> MessageModel {
let rust_name = msg.name().to_string();
let options = msg.options();
let mopts = options.get_extension(&ext.message_json);
let mopts = mopts.as_message();
// `MessageShape`: one shape per message, by construction of the enum.
let shape = mopts.map(|m| opt_enum(m, "shape")).unwrap_or(0);
let (bare_list, tuple, transparent) = (shape == 1, shape == 2, shape == 3);
let deny_unknown = mopts
.and_then(|m| m.get_field_by_name("unknown_fields"))
.and_then(|v| v.as_enum_number())
.unwrap_or(0)
== 1;
// proto3 `optional` is implemented as a synthetic oneof: those fields are
// plain fields with presence, not union members.
let oneofs: Vec<_> = msg.oneofs().filter(|o| !o.is_synthetic()).collect();
let plain_fields: Vec<FieldDescriptor> = msg
.fields()
.filter(|f| f.containing_oneof().is_none_or(|o| o.is_synthetic()))
.collect();
if bare_list {
assert_eq!(
plain_fields.len(),
1,
"{}: bare_list needs exactly one repeated field",
msg.full_name()
);
return MessageModel {
rust_name,
kind: MessageKind::BareList {
item: field_model(&plain_fields[0], ext),
},
};
}
if transparent {
assert_eq!(
plain_fields.len(),
1,
"{}: transparent needs exactly one field",
msg.full_name()
);
return MessageModel {
rust_name,
kind: MessageKind::Transparent {
field: field_model(&plain_fields[0], ext),
},
};
}
if tuple {
return MessageModel {
rust_name,
kind: MessageKind::Tuple {
fields: plain_fields.iter().map(|f| field_model(f, ext)).collect(),
},
};
}
if oneofs.len() == 1 && plain_fields.is_empty() {
let oneof = &oneofs[0];
let oopts = oneof.options();
let oopts = oopts.get_extension(&ext.oneof_json);
let oopts = oopts.as_message();
// `OneofShape`: one spelling per union, by construction of the enum.
let shape = oopts.map(|m| opt_enum(m, "shape")).unwrap_or(0);
let (untagged, one_or_many, tagged) = (shape == 1, shape == 2, shape == 3);
let tag = oopts.and_then(|m| opt_str(m, "tag"));
assert_eq!(
tagged,
tag.is_some(),
"{}: ONEOF_SHAPE_TAGGED and `tag` go together",
msg.full_name()
);
let passthrough_on = oopts
.map(|m| opt_bool(m, "unknown_variant_passthrough"))
.unwrap_or(false);
let oneof_mod = snake(&rust_name);
let oneof_ident = pascal(oneof.name());
let mut variants: Vec<VariantModel> = oneof
.fields()
.map(|f| VariantModel {
json_name: f.name().to_string(),
rust_variant: pascal(f.name()),
kind: field_model(&f, ext).kind,
payload_fields: match f.kind() {
// The raw_json passthrough arm carries a well-known type;
// its innards are never inlined.
Kind::Message(payload)
if tag.is_some() && !payload.full_name().starts_with("google.protobuf") =>
{
payload
.fields()
.map(|pf| field_model(&pf, ext))
.collect::<Vec<_>>()
}
_ => vec![],
},
})
.collect();
if one_or_many {
assert_eq!(variants.len(), 2, "{}: one_or_many", msg.full_name());
let many = variants.pop().expect("many variant");
let one = variants.pop().expect("one variant");
return MessageModel {
rust_name,
kind: MessageKind::OneOrMany {
oneof_mod,
one,
many,
},
};
}
if let Some(tag) = tag {
let passthrough = passthrough_on.then(|| {
let idx = variants
.iter()
.position(|v| matches!(v.kind, FieldKind::RawJson { .. }))
.expect("passthrough needs a raw_json string variant");
variants.remove(idx)
});
return MessageModel {
rust_name,
kind: MessageKind::TaggedUnion {
oneof_mod,
oneof_ident,
tag,
variants,
passthrough,
},
};
}
if untagged {
return MessageModel {
rust_name,
kind: MessageKind::Untagged {
oneof_mod,
oneof_ident,
variants,
},
};
}
// A plain oneof container (e.g. GenerateStreamItem) has no HTTP JSON
// contract of its own — it is gRPC-only. Emit nothing.
return MessageModel {
rust_name,
kind: MessageKind::Skip,
};
}
MessageModel {
rust_name,
kind: MessageKind::Struct {
deny_unknown,
fields: plain_fields.iter().map(|f| field_model(f, ext)).collect(),
},
}
}
fn field_model(f: &FieldDescriptor, ext: &Extensions) -> FieldModel {
let options = f.options();
let fopts = options.get_extension(&ext.field_json);
let fopts = fopts.as_message();
let get_bool = |name: &str| fopts.map(|m| opt_bool(m, name)).unwrap_or(false);
let default_lit = |m: &prost_reflect::DynamicMessage| -> Option<DefaultLit> {
// has_field_by_name: the default_* members are proto3 optional, so
// presence is meaningful.
if m.has_field_by_name("default_f64") {
return m
.get_field_by_name("default_f64")
.and_then(|v| v.as_f64())
.map(DefaultLit::F64);
}
if m.has_field_by_name("default_i64") {
return m
.get_field_by_name("default_i64")
.and_then(|v| v.as_i64())
.map(DefaultLit::I64);
}
if m.has_field_by_name("default_bool") {
return m
.get_field_by_name("default_bool")
.and_then(|v| v.as_bool())
.map(DefaultLit::Bool);
}
if m.has_field_by_name("default_string") {
return m
.get_field_by_name("default_string")
.and_then(|v| v.as_str().map(str::to_owned))
.map(DefaultLit::Str);
}
None
};
let raw_json = get_bool("raw_json");
let emit_null = get_bool("emit_null_when_absent");
// `NullPolicy`: one policy per field, by construction of the enum.
let null_policy = fopts.map(|m| opt_enum(m, "null_policy")).unwrap_or(0);
let (null_resets, null_is_none) = (null_policy == 1, null_policy == 2);
let default = fopts.and_then(default_lit);
let kind = if raw_json {
if f.is_list() {
FieldKind::RepeatedRawJson
} else {
assert!(
matches!(f.kind(), Kind::String),
"{}: raw_json requires a string field",
f.full_name()
);
FieldKind::RawJson { emit_null }
}
} else if f.is_map() {
let value_ty = match f.kind() {
Kind::Message(entry) => scalar_ty(
&entry
.get_field_by_name("value")
.expect("map value field")
.kind(),
),
_ => unreachable!("map fields are message-kinded"),
};
FieldKind::Map { value: value_ty }
} else if f.is_list() {
match f.kind() {
Kind::Message(m) => FieldKind::RepeatedMessage {
rust_path: m.name().to_string(),
},
other => FieldKind::RepeatedScalar {
ty: scalar_ty(&other),
},
}
} else {
match f.kind() {
Kind::Message(m) => FieldKind::Message {
rust_path: m.name().to_string(),
emit_null,
},
other => {
let ty = scalar_ty(&other);
if f.supports_presence() {
FieldKind::OptScalar {
ty,
emit_null,
null_is_none_default: null_is_none.then(|| {
default
.clone()
.expect("NULL_POLICY_IS_NONE requires a default_*")
}),
null_default: null_resets.then(|| {
default
.clone()
.expect("NULL_POLICY_RESETS_DEFAULT requires a default_*")
}),
}
} else {
FieldKind::Scalar {
ty,
null_default: null_resets.then(|| {
default
.clone()
.expect("NULL_POLICY_RESETS_DEFAULT requires a default_*")
}),
}
}
}
}
};
FieldModel {
json_name: f.name().to_string(),
rust_ident: escape_ident(f.name()),
kind,
}
}
fn scalar_ty(kind: &Kind) -> ScalarTy {
match kind {
Kind::Double | Kind::Float => ScalarTy::F64,
Kind::Int64 | Kind::Sint64 | Kind::Sfixed64 => ScalarTy::I64,
Kind::Int32 | Kind::Sint32 | Kind::Sfixed32 => ScalarTy::I32,
Kind::Uint32 | Kind::Fixed32 => ScalarTy::U32,
Kind::Uint64 | Kind::Fixed64 => ScalarTy::U64,
Kind::Bool => ScalarTy::Bool,
Kind::String => ScalarTy::Str,
other => panic!("unsupported scalar kind {other:?}"),
}
}
/// prost's keyword escaping for field idents.
fn escape_ident(name: &str) -> String {
const KEYWORDS: &[&str] = &[
"as", "break", "const", "continue", "crate", "dyn", "else", "enum", "extern", "false",
"fn", "for", "if", "impl", "in", "let", "loop", "match", "mod", "move", "mut", "pub",
"ref", "return", "self", "static", "struct", "super", "trait", "true", "type", "unsafe",
"use", "where", "while",
];
if KEYWORDS.contains(&name) {
format!("r#{name}")
} else {
name.to_string()
}
}
pub fn pascal(name: &str) -> String {
name.split('_')
.filter(|s| !s.is_empty())
.map(|s| {
let mut chars = s.chars();
match chars.next() {
Some(first) => first.to_ascii_uppercase().to_string() + chars.as_str(),
None => String::new(),
}
})
.collect()
}
pub fn snake(name: &str) -> String {
let mut out = String::new();
for (i, ch) in name.chars().enumerate() {
if ch.is_ascii_uppercase() {
if i > 0 {
out.push('_');
}
out.push(ch.to_ascii_lowercase());
} else {
out.push(ch);
}
}
out
}