diff --git a/python/sglang/srt/managers/rust_server.py b/python/sglang/srt/managers/rust_server.py index 0e6ef6dfa..4c947ae9f 100644 --- a/python/sglang/srt/managers/rust_server.py +++ b/python/sglang/srt/managers/rust_server.py @@ -352,6 +352,13 @@ class RustServer: server_args = dict(vars(scheduler.server_args)) model_config = dict(vars(scheduler.model_config)) model_config["hf_config"] = None # HF config is not JSON-serializable + # Resolved default sampling params (generation_config.json when + # `--sampling-defaults model`, {} otherwise). The rust server consumes + # these for omitted temperature/top_p in chat conversions instead of + # hard-coding the OpenAI terminal defaults. + model_config["default_sampling_params"] = ( + scheduler.model_config.get_default_sampling_params() + ) server_args["model_config"] = model_config # Launch-time facts Python's /server_info reports from scheduler_info / # the package — stamped here so the rust endpoint can serve them diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 355859b8a..3e3d564bb 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -31,6 +31,24 @@ dependencies = [ "memchr", ] +[[package]] +name = "aligned" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee4508988c62edf04abd8d92897fca0c2995d907ce1dfeaf369dac3716a40685" +dependencies = [ + "as-slice", +] + +[[package]] +name = "aligned-vec" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b" +dependencies = [ + "equator", +] + [[package]] name = "android_system_properties" version = "0.1.5" @@ -41,10 +59,77 @@ dependencies = [ ] [[package]] -name = "anyhow" -version = "1.0.103" +name = "anstream" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" + +[[package]] +name = "arg_enum_proc_macro" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] [[package]] name = "arrayref" @@ -58,6 +143,28 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" +[[package]] +name = "as-slice" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "516b6b4f0e40d50dcda9365d53964ec74560ad4284da2e7fc97122cd83174516" +dependencies = [ + "stable_deref_trait", +] + +[[package]] +name = "async-openai" +version = "0.41.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d72db2750faea2ca5edbf6d0c50277a89dc8f75f5e6ddd695ef30f75e335019b" +dependencies = [ + "bytes", + "derive_builder", + "getrandom 0.3.4", + "serde", + "serde_json", +] + [[package]] name = "async-stream" version = "0.3.6" @@ -77,7 +184,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -97,12 +204,85 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "attribute-derive" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05832cdddc8f2650cc2cc187cc2e952b8c133a48eb055f35211f61ee81502d77" +dependencies = [ + "attribute-derive-macro", + "derive-where", + "manyhow", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "attribute-derive-macro" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a7cdbbd4bd005c5d3e2e9c885e6fa575db4f4a3572335b974d8db853b6beb61" +dependencies = [ + "collection_literals", + "interpolator", + "manyhow", + "proc-macro-utils", + "proc-macro2", + "quote", + "quote-use", + "syn 2.0.119", +] + [[package]] name = "autocfg" version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "av-scenechange" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f321d77c20e19b92c39e7471cf986812cbb46659d2af674adc4331ef3f18394" +dependencies = [ + "aligned", + "anyhow", + "arg_enum_proc_macro", + "arrayvec", + "log", + "num-rational", + "num-traits", + "pastey 0.1.1", + "rayon", + "thiserror", + "v_frame", + "y4m", +] + +[[package]] +name = "av1-grain" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cfddb07216410377231960af4fcab838eaa12e013417781b78bd95ee22077f8" +dependencies = [ + "anyhow", + "arrayvec", + "log", + "nom 8.0.0", + "num-rational", + "v_frame", +] + +[[package]] +name = "avif-serialize" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7178fe5f7d460b13895ebb9dcb28a3a6216d2df2574a0806cb51b555d297f38" +dependencies = [ + "arrayvec", +] + [[package]] name = "axum" version = "0.7.9" @@ -267,10 +447,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] -name = "bitflags" -version = "2.13.0" +name = "bit_field" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +checksum = "1e4b40c7323adcfc0a41c4b88143ed58346ff65a288fc144329c5c45e05d70c6" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bitstream-io" +version = "4.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eff00be299a18769011411c9def0d827e8f2d7bf0c3dbf53633147a8867fd1f" +dependencies = [ + "no_std_io2", +] [[package]] name = "blake3" @@ -283,20 +478,44 @@ dependencies = [ "cc", "cfg-if", "constant_time_eq", - "cpufeatures", + "cpufeatures 0.3.0", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", ] [[package]] name = "bstr" -version = "1.12.3" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cee35f73844aa3014bb606320a6c1f010249dbdf43342fe54b5a4f6a8ed4b79" +checksum = "1f7dc094d718f2e1c1559ad110e27eeaae14a5465d3d56dd6dbd793079fbd530" dependencies = [ "memchr", "regex-automata", "serde_core", ] +[[package]] +name = "built" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c0e531d93d39c34eef561e929e8a7f86d77a5af08aac4f6d6e39976c51858e9" + [[package]] name = "bumpalo" version = "3.20.3" @@ -338,9 +557,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -356,9 +575,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cfg_aliases" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" [[package]] name = "chacha20" @@ -367,7 +586,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.3.0", "rand_core 0.10.1", ] @@ -379,9 +598,68 @@ checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "num-traits", + "serde", "windows-link", ] +[[package]] +name = "clap" +version = "4.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "301b56658598e48f3648647ac6fc887be7e7108eddfa4e9b63fcf3ec58c0cadf" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94a65403d1a1bd28f7dc68eb8506e8874808ee5eecb59298de588e2e1407a078" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "collection_literals" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2550f75b8cfac212855f6b1885455df8eaee8fe8e246b647d69146142e016084" + +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "compact_str" version = "0.9.1" @@ -433,6 +711,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "cpufeatures" version = "0.3.0" @@ -491,6 +778,16 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "daachorse" version = "1.0.1" @@ -503,8 +800,18 @@ version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core 0.23.0", + "darling_macro 0.23.0", ] [[package]] @@ -518,7 +825,20 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.118", + "syn 2.0.119", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", ] [[package]] @@ -527,9 +847,20 @@ version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ - "darling_core", + "darling_core 0.20.11", "quote", - "syn 2.0.118", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core 0.23.0", + "quote", + "syn 2.0.119", ] [[package]] @@ -546,6 +877,20 @@ name = "deranged" version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "serde_core", +] + +[[package]] +name = "derive-where" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d08b3a0bcc0d079199cd476b2cae8435016ec11d1c0986c6901c5ac223041534" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] [[package]] name = "derive_builder" @@ -562,10 +907,10 @@ version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" dependencies = [ - "darling", + "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -575,7 +920,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn 2.0.118", + "syn 2.0.119", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", ] [[package]] @@ -601,13 +956,77 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.3", +] + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "dynamo-parsers" +version = "7.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97cce1ec70f4c4896ff9a4e0bb29cd603f465e599c94bf2085b71872a3a58c70" +dependencies = [ + "aho-corasick", + "anyhow", + "async-stream", + "dynamo-protocols", + "futures", + "num-traits", + "openai-harmony", + "regex", + "rustpython-ruff_python_ast", + "rustpython-ruff_python_parser", + "serde", + "serde_json", + "tokio", + "tracing", + "uuid", +] + +[[package]] +name = "dynamo-protocols" +version = "5.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "696b6df4c817c16dc0a619d43f6af5ecc4f045a6b6be7ef516bfe502f977764b" +dependencies = [ + "async-openai", + "derive_builder", + "futures", + "serde", + "serde_json", + "thiserror", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "dynamo-renderer" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0423da7bec83734e7567b1c8895fd2e7a1a98256253a29fe128032ac01cefac1" +dependencies = [ + "anyhow", + "chrono", + "dynamo-protocols", + "dynamo-tokenizers", + "either", + "minijinja", + "minijinja-contrib", + "serde", + "serde_json", + "tracing", ] [[package]] @@ -635,9 +1054,12 @@ dependencies = [ [[package]] name = "either" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" +dependencies = [ + "serde", +] [[package]] name = "encode_unicode" @@ -645,6 +1067,26 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" +[[package]] +name = "equator" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc" +dependencies = [ + "equator-macro", +] + +[[package]] +name = "equator-macro" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -670,6 +1112,23 @@ dependencies = [ "cc", ] +[[package]] +name = "exr" +version = "1.74.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711fe42c9964295e01ee3fba3f9fe0e1d24b98886950d68efe81b1c76e21adf3" +dependencies = [ + "bit_field", + "half", + "lebe", + "miniz_oxide", + "num-complex", + "pulp", + "rayon-core", + "smallvec", + "zune-inflate", +] + [[package]] name = "fancy-regex" version = "0.13.0" @@ -713,13 +1172,19 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" dependencies = [ - "getrandom 0.3.4", + "getrandom 0.4.3", ] +[[package]] +name = "fax" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf1079563223d5d59d83c85886a56e586cfd5c1a26292e971a0fa266531ac5a" + [[package]] name = "fdeflate" version = "0.3.7" @@ -780,9 +1245,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" dependencies = [ "futures-channel", "futures-core", @@ -795,9 +1260,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" dependencies = [ "futures-core", "futures-sink", @@ -805,15 +1270,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" [[package]] name = "futures-executor" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" dependencies = [ "futures-core", "futures-task", @@ -822,38 +1287,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" [[package]] name = "futures-macro" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "futures-sink" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" dependencies = [ "futures-channel", "futures-core", @@ -866,6 +1331,49 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "get-size-derive2" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b6d1e2f75c16bfbcd0f95d84f99858a6e2f885c2287d1f5c3a96e8444a34b4" +dependencies = [ + "attribute-derive", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "get-size2" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49cf31a6d70300cf81461098f7797571362387ef4bf85d32ac47eaa59b3a5a1a" +dependencies = [ + "compact_str", + "get-size-derive2", + "hashbrown 0.16.1", + "ordermap", + "smallvec", +] + +[[package]] +name = "getopts" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" +dependencies = [ + "unicode-width", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -907,6 +1415,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "gif" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "h2" version = "0.4.15" @@ -943,6 +1461,12 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + [[package]] name = "hashbrown" version = "0.17.1" @@ -961,6 +1485,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "hf-hub" version = "0.4.3" @@ -983,9 +1513,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" dependencies = [ "bytes", "itoa", @@ -1028,9 +1558,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" dependencies = [ "atomic-waker", "bytes", @@ -1061,7 +1591,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots 1.0.8", + "webpki-roots 1.0.9", ] [[package]] @@ -1094,7 +1624,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.4", + "socket2 0.6.5", "tokio", "tower-service", "tracing", @@ -1244,13 +1774,38 @@ checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" dependencies = [ "bytemuck", "byteorder-lite", + "color_quant", + "exr", + "gif", + "image-webp", "moxcms", "num-traits", "png", + "qoi", + "ravif", + "rayon", + "rgb", + "tiff", "zune-core", "zune-jpeg", ] +[[package]] +name = "image-webp" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" +dependencies = [ + "byteorder-lite", + "quick-error", +] + +[[package]] +name = "imgref" +version = "1.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89194689a993ab15268672e99e7b0e19da2da3268ac682e8f02d29d4d1434cd7" + [[package]] name = "indexmap" version = "1.9.3" @@ -1259,6 +1814,7 @@ checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", "hashbrown 0.12.3", + "serde", ] [[package]] @@ -1269,6 +1825,8 @@ checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", "hashbrown 0.17.1", + "serde", + "serde_core", ] [[package]] @@ -1284,12 +1842,47 @@ dependencies = [ "web-time", ] +[[package]] +name = "interpolate_name" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "interpolator" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71dd52191aae121e8611f1e8dc3e324dd0dd1dee1e6dd91d10ee07a3cfb4d9d8" + [[package]] name = "ipnet" version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +[[package]] +name = "is-macro" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d57a3e447e24c22647738e4607f1df1e0ec6f72e16182c4cd199f647cdfb0e4" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" version = "0.14.0" @@ -1333,10 +1926,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] -name = "libc" -version = "0.2.186" +name = "lebe" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "7a79a3332a6609480d7d0c9eab957bca6b455b91bb84e66d19f5ff66294b85b8" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libredox" @@ -1374,6 +1989,15 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "loop9" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fae87c125b03c1d2c0150c90365d7d6bcc53fb73a9acaef207d2d065860f062" +dependencies = [ + "imgref", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -1382,19 +2006,42 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "macro_rules_attribute" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65049d7923698040cd0b1ddcced9b0eb14dd22c5f86ae59c3740eab64a676520" +checksum = "b3ae8f6d608c795738406608304d30a2dfbdc8e58e44f7ba43236da5208ded3c" dependencies = [ "macro_rules_attribute-proc_macro", - "paste", + "pastey 0.2.3", ] [[package]] name = "macro_rules_attribute-proc_macro" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30" +checksum = "fc04a4c58212d57930a24bf47d3fa87485264a3a054e9c10e042eb373573ad3c" + +[[package]] +name = "manyhow" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b33efb3ca6d3b07393750d4030418d594ab1139cee518f0dc88db70fec873587" +dependencies = [ + "manyhow-macros", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "manyhow-macros" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46fce34d199b78b6e6073abf984c9cf5fd3e9330145a93ee0738a7443e371495" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", +] [[package]] name = "matchers" @@ -1427,6 +2074,16 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "maybe-rayon" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519" +dependencies = [ + "cfg-if", + "rayon", +] + [[package]] name = "memchr" version = "2.8.3" @@ -1445,6 +2102,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "minijinja" version = "2.21.0" @@ -1530,7 +2197,7 @@ checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1564,6 +2231,21 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + +[[package]] +name = "no_std_io2" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418abd1b6d34fbf6cae440dc874771b0525a604428704c76e48b29a5e67b8003" +dependencies = [ + "memchr", +] + [[package]] name = "nom" version = "7.1.3" @@ -1574,6 +2256,21 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "noop_proc_macro" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -1583,12 +2280,23 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-complex" version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ + "bytemuck", "num-traits", ] @@ -1598,6 +2306,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "num-integer" version = "0.1.46" @@ -1607,6 +2326,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1654,6 +2384,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "onig" version = "6.5.3" @@ -1676,12 +2412,45 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "openai-harmony" +version = "0.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e77e82af451fc95deeb728a40b84db8ee82d341e136c268de415123a560b9b72" +dependencies = [ + "anyhow", + "base64 0.22.1", + "bstr", + "clap", + "fancy-regex 0.13.0", + "futures", + "image", + "regex", + "reqwest", + "rustc-hash 1.1.0", + "serde", + "serde_json", + "serde_with", + "sha1", + "sha2", + "thiserror", +] + [[package]] name = "option-ext" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "ordermap" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7476a5b122ff1fce7208e7ee9dccd0a516e835f5b8b19b8f3c98a34cf757c1" +dependencies = [ + "indexmap 2.14.0", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -1711,6 +2480,18 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pastey" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec" + +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + [[package]] name = "pcre2" version = "0.2.11" @@ -1749,6 +2530,44 @@ dependencies = [ "indexmap 2.14.0", ] +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand 0.8.7", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.1.13" @@ -1766,7 +2585,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1796,9 +2615,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.13.1" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" [[package]] name = "portable-atomic-util" @@ -1840,18 +2659,48 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.118", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro-utils" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeaf08a13de400bc215877b5bdc088f241b12eb42f0a548d3390dc1c56bb7071" +dependencies = [ + "proc-macro2", + "quote", + "smallvec", ] [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] +[[package]] +name = "profiling" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d595e54a326bc53c1c197b32d295e14b169e3cfeaa8dc82b529f947fba6bcf5" +dependencies = [ + "profiling-procmacros", +] + +[[package]] +name = "profiling-procmacros" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4488a4a36b9a4ba6b9334a32a39971f77c1436ec82c38707bce707699cc3bbcb" +dependencies = [ + "quote", + "syn 2.0.119", +] + [[package]] name = "prost" version = "0.13.5" @@ -1878,7 +2727,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.118", + "syn 2.0.119", "tempfile", ] @@ -1892,7 +2741,7 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1968,6 +2817,29 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95067976aca6421a523e491fce939a3e65249bac4b977adee0ee9771568e8aa3" +[[package]] +name = "pulp" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046aa45b989642ec2e4717c8e72d677b13edd831a4d3b6cf37d9a3e54912496a" +dependencies = [ + "bytemuck", + "cfg-if", + "libm", + "num-complex", + "paste", + "pulp-wasm-simd-flag", + "raw-cpuid", + "reborrow", + "version_check", +] + +[[package]] +name = "pulp-wasm-simd-flag" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d8f70e07b9c3962945a74e59ca1c511bba65b6419468acc217c457d93f3c740" + [[package]] name = "pxfm" version = "0.1.30" @@ -2016,7 +2888,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2028,9 +2900,24 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] +[[package]] +name = "qoi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6d64c71eb498fe9eae14ce4ec935c555749aef511cca85b5568910d6e48001" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quinn" version = "0.11.11" @@ -2044,7 +2931,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.3", "rustls", - "socket2 0.6.4", + "socket2 0.6.5", "thiserror", "tokio", "tracing", @@ -2082,20 +2969,42 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.4", + "socket2 0.6.5", "tracing", "windows-sys 0.61.2", ] [[package]] name = "quote" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] +[[package]] +name = "quote-use" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9619db1197b497a36178cfc736dc96b271fe918875fbf1344c436a7e93d0321e" +dependencies = [ + "quote", + "quote-use-macros", +] + +[[package]] +name = "quote-use-macros" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82ebfb7faafadc06a7ab141a6f67bcfb24cb8beb158c6fe933f2f035afa99f35" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "r-efi" version = "5.3.0" @@ -2193,6 +3102,65 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "rav1e" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43b6dd56e85d9483277cde964fd1bdb0428de4fec5ebba7540995639a21cb32b" +dependencies = [ + "aligned-vec", + "arbitrary", + "arg_enum_proc_macro", + "arrayvec", + "av-scenechange", + "av1-grain", + "bitstream-io", + "built", + "cfg-if", + "interpolate_name", + "itertools", + "libc", + "libfuzzer-sys", + "log", + "maybe-rayon", + "new_debug_unreachable", + "noop_proc_macro", + "num-derive", + "num-traits", + "paste", + "profiling", + "rand 0.9.5", + "rand_chacha 0.9.0", + "simd_helpers", + "thiserror", + "v_frame", + "wasm-bindgen", +] + +[[package]] +name = "ravif" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e52310197d971b0f5be7fe6b57530dcd27beb35c1b013f29d66c1ad73fbbcc45" +dependencies = [ + "avif-serialize", + "imgref", + "loop9", + "quick-error", + "rav1e", + "rayon", + "rgb", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags", +] + [[package]] name = "rawpointer" version = "0.2.1" @@ -2230,6 +3198,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "reborrow" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430" + [[package]] name = "redox_syscall" version = "0.5.18" @@ -2251,10 +3225,30 @@ dependencies = [ ] [[package]] -name = "regex" -version = "1.13.0" +name = "ref-cast" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0e75113e14dc5acb068cd0786884f214f1312650a3d36d269f5c4f3cdee8a2" +checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -2264,9 +3258,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.15" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f388202e4b80542a0921078cc23b6333bcf1409c1e3f86404cae4766a6131db" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" dependencies = [ "aho-corasick", "memchr", @@ -2287,6 +3281,7 @@ checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ "base64 0.22.1", "bytes", + "futures-channel", "futures-core", "futures-util", "http", @@ -2297,6 +3292,7 @@ dependencies = [ "hyper-util", "js-sys", "log", + "mime_guess", "percent-encoding", "pin-project-lite", "quinn", @@ -2317,9 +3313,15 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 1.0.8", + "webpki-roots 1.0.9", ] +[[package]] +name = "rgb" +version = "0.8.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b34b781b31e5d73e9fbc8689c70551fd1ade9a19e3e28cfec8580a79290cc4" + [[package]] name = "ring" version = "0.17.14" @@ -2391,9 +3393,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.42" +version = "0.23.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" dependencies = [ "log", "once_cell", @@ -2406,9 +3408,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.15.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ "web-time", "zeroize", @@ -2425,6 +3427,77 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustpython-ruff_python_ast" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f021ff72cabf5e2cd6d8ec8813d376a8445a228dc610ab56c27bd9054cda70d4" +dependencies = [ + "aho-corasick", + "bitflags", + "compact_str", + "get-size2", + "is-macro", + "memchr", + "rustc-hash 2.1.3", + "rustpython-ruff_python_trivia", + "rustpython-ruff_source_file", + "rustpython-ruff_text_size", + "thiserror", +] + +[[package]] +name = "rustpython-ruff_python_parser" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01e6ee78bd9671fb5766664b2695fe1f2a92a961f4d9101646c570d8acdb1e0b" +dependencies = [ + "bitflags", + "bstr", + "compact_str", + "get-size2", + "memchr", + "rustc-hash 2.1.3", + "rustpython-ruff_python_ast", + "rustpython-ruff_python_trivia", + "rustpython-ruff_text_size", + "static_assertions", + "unicode-ident", + "unicode-normalization", + "unicode_names2", +] + +[[package]] +name = "rustpython-ruff_python_trivia" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79e7cfd1056f3a02ff0d2d0e4474286ca963260782f878b7b81c1dd87432e682" +dependencies = [ + "itertools", + "rustpython-ruff_source_file", + "rustpython-ruff_text_size", + "unicode-ident", +] + +[[package]] +name = "rustpython-ruff_source_file" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "948107aad62ddb12a11fc7bf68a49e52a0b0a3737d415a2505e54f5a9edac737" +dependencies = [ + "memchr", + "rustpython-ruff_text_size", +] + +[[package]] +name = "rustpython-ruff_text_size" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8291ee0f5a779e54ccd4e0151a0c426f8b49a123f99b5b6545db17ccdd4277aa" +dependencies = [ + "get-size2", +] + [[package]] name = "rustversion" version = "1.0.23" @@ -2437,6 +3510,30 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -2445,9 +3542,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -2465,29 +3562,29 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.3", ] [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "indexmap 2.14.0", "itoa", @@ -2520,6 +3617,38 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" +dependencies = [ + "base64 0.22.1", + "bs58", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.14.0", + "schemars 0.9.0", + "schemars 1.2.2", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" +dependencies = [ + "darling 0.23.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "sglang-grpc" version = "0.1.0" @@ -2560,6 +3689,9 @@ dependencies = [ "axum 0.8.9", "bytes", "core_affinity", + "dynamo-parsers", + "dynamo-protocols", + "dynamo-renderer", "dynamo-tokenizers", "flume", "futures", @@ -2571,15 +3703,38 @@ dependencies = [ "rmpv", "serde", "serde_json", - "socket2 0.6.4", + "socket2 0.6.5", "thiserror", "tokio", + "tower 0.5.3", "tracing", "tracing-appender", "tracing-subscriber", "uuid", ] +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -2607,9 +3762,24 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "simd_helpers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95890f873bec569a0362c235787f3aca6e1e887302ba4840839bcc6459c42da6" +dependencies = [ + "quote", +] + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" [[package]] name = "slab" @@ -2635,9 +3805,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", "windows-sys 0.61.2", @@ -2656,9 +3826,9 @@ dependencies = [ [[package]] name = "spin" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" dependencies = [ "lock_api", ] @@ -2670,7 +3840,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" dependencies = [ "base64 0.13.1", - "nom", + "nom 7.1.3", "serde", "unicode-segmentation", ] @@ -2711,7 +3881,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2728,9 +3898,9 @@ checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" [[package]] name = "syn" -version = "2.0.118" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" dependencies = [ "proc-macro2", "quote", @@ -2765,7 +3935,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2795,22 +3965,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.18" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.18" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.3", ] [[package]] @@ -2822,6 +3992,20 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "tiff" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63feaf3343d35b6ca4d50483f94843803b0f51634937cc2ec519fc32232bc52" +dependencies = [ + "fax", + "flate2", + "half", + "quick-error", + "weezl", + "zune-jpeg", +] + [[package]] name = "tiktoken-rs" version = "0.9.1" @@ -2839,9 +4023,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.53" +version = "0.3.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50" +checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244" dependencies = [ "deranged", "num-conv", @@ -2859,9 +4043,9 @@ checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.31" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" dependencies = [ "num-conv", "time-core", @@ -2928,9 +4112,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.52.3" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -2938,20 +4122,20 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.4", + "socket2 0.6.5", "tokio-macros", "windows-sys 0.61.2", ] [[package]] name = "tokio-macros" -version = "2.7.0" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.3", ] [[package]] @@ -2977,13 +4161,14 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" dependencies = [ "bytes", "futures-core", "futures-sink", + "libc", "pin-project-lite", "tokio", ] @@ -3030,7 +4215,7 @@ dependencies = [ "prost-build", "prost-types", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3132,7 +4317,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3180,12 +4365,33 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-normalization-alignments" version = "0.1.12" @@ -3213,6 +4419,28 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" +[[package]] +name = "unicode_names2" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1673eca9782c84de5f81b82e4109dcfb3611c8ba0d52930ec4a9478f547b2dd" +dependencies = [ + "phf", + "unicode_names2_generator", +] + +[[package]] +name = "unicode_names2_generator" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b91e5b84611016120197efd7dc93ef76774f4e084cd73c9fb3ea4a86c570c56e" +dependencies = [ + "getopts", + "log", + "phf_codegen", + "rand 0.8.7", +] + [[package]] name = "untrusted" version = "0.9.0" @@ -3248,6 +4476,7 @@ dependencies = [ "idna", "percent-encoding", "serde", + "serde_derive", ] [[package]] @@ -3263,13 +4492,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] -name = "uuid" -version = "1.23.5" +name = "utf8parse" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea5fab0d6c3c01ae70085a09cb03d4c7a1d6314e2b3e075392783396d724ca0a" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ "getrandom 0.4.3", "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "v_frame" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "666b7727c8875d6ab5db9533418d7c764233ac9c0cff1d469aec8fa127597be2" +dependencies = [ + "aligned-vec", + "num-traits", "wasm-bindgen", ] @@ -3351,7 +4598,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", "wasm-bindgen-shared", ] @@ -3403,18 +4650,24 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.8", + "webpki-roots 1.0.9", ] [[package]] name = "webpki-roots" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" dependencies = [ "rustls-pki-types", ] +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + [[package]] name = "winapi" version = "0.3.9" @@ -3458,7 +4711,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3469,7 +4722,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3679,6 +4932,12 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "y4m" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5a4b21e1a62b67a2970e6831bc091d7b87e119e7f9791aef9702e3bef04448" + [[package]] name = "yoke" version = "0.8.3" @@ -3698,28 +4957,28 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.54" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.54" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3739,7 +4998,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", "synstructure", ] @@ -3779,7 +5038,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3794,6 +5053,15 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9" +[[package]] +name = "zune-inflate" +version = "0.2.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02" +dependencies = [ + "simd-adler32", +] + [[package]] name = "zune-jpeg" version = "0.5.15" diff --git a/rust/rust-toolchain.toml b/rust/rust-toolchain.toml index 2739b9054..8cc66449c 100644 --- a/rust/rust-toolchain.toml +++ b/rust/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "1.90" +channel = "1.92" profile = "minimal" components = ["clippy", "rustfmt"] diff --git a/rust/sglang-server/Cargo.toml b/rust/sglang-server/Cargo.toml index 8d87255d4..d0e2afc20 100644 --- a/rust/sglang-server/Cargo.toml +++ b/rust/sglang-server/Cargo.toml @@ -36,6 +36,12 @@ axum = { version = "0.8.9", features = ["json", "tokio"] } core_affinity = "0.8" # the dynamo-tokenizers is deps on hf-hub, should bump version together. dynamo-tokenizers = "1.7.0" +# Keep the HTTP adapter on Dynamo's public OpenAI types, renderer, and parsers. +# `dynamo-llm` stays excluded because it pulls the full runtime just to reuse +# small service helpers. +dynamo-parsers = "7.0.1" +dynamo-protocols = "5.1.0" +dynamo-renderer = "5.0.0" flume = "0.12.0" itertools = "0.14" hf-hub = { version = "0.4", default-features = false } @@ -45,3 +51,7 @@ rmpv = { version = "1", features = ["with-serde"] } # "anything Rust admits, Python can compile" invariant in `message::sampling`. # A minor bump can widen it and silently reopen a scheduler-killing hole. regex-syntax = "=0.8.11" + +[dev-dependencies] +# `Router::oneshot` for handler-level router tests. +tower = { version = "0.5", features = ["util"] } diff --git a/rust/sglang-server/src/api_server.rs b/rust/sglang-server/src/api_server.rs index 80a8da927..a827275f1 100644 --- a/rust/sglang-server/src/api_server.rs +++ b/rust/sglang-server/src/api_server.rs @@ -20,13 +20,14 @@ use crate::runtime::ServerArgs; use crate::tokenizer_manager::ActivityCounter; use crate::tokenizer_manager::Senders; -/// Shared handler state: the submit machinery (`senders`, `egress_buf`) -/// + shared tokenizer. +/// Shared handler state: submission handles, immutable server configuration, +/// and the API-owned chat formatter. #[derive(Clone)] struct AppState { senders: Senders, egress_buf: usize, server_args: Arc, + chat_formatter: Option, /// Egress heartbeat (bumped per drained ring frame). egress_activity: ActivityCounter, } @@ -42,10 +43,12 @@ pub async fn serve( // releases. shutdown: flume::Receiver<()>, ) { + let chat_formatter = openai::load_chat_support(&server_args); let state = AppState { senders, egress_buf, server_args: server_args.clone(), + chat_formatter, egress_activity, }; // Each endpoint module registers its own routes and merges here. diff --git a/rust/sglang-server/src/api_server/common.rs b/rust/sglang-server/src/api_server/common.rs index 70a7b7323..0b079661a 100644 --- a/rust/sglang-server/src/api_server/common.rs +++ b/rust/sglang-server/src/api_server/common.rs @@ -56,12 +56,14 @@ async fn await_control_result( StatusCode::from_u16(e.http_status()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); Err((code, e.to_string()).into_response()) } - // A control request never receives generation frames. - Some(EgressItem::Frame(_)) | Some(EgressItem::Done(_)) => Err(( - StatusCode::INTERNAL_SERVER_ERROR, - "unexpected generation output for control request", - ) - .into_response()), + // A control request never receives generation frames or service-call data. + Some(EgressItem::Frame(_)) | Some(EgressItem::Done(_)) | Some(EgressItem::Data(_)) => { + Err(( + StatusCode::INTERNAL_SERVER_ERROR, + "unexpected generation output for control request", + ) + .into_response()) + } None => Err((StatusCode::from_u16(499).unwrap(), "request aborted").into_response()), } } diff --git a/rust/sglang-server/src/api_server/native_api.rs b/rust/sglang-server/src/api_server/native_api.rs index 4037f46ea..6d53b9b89 100644 --- a/rust/sglang-server/src/api_server/native_api.rs +++ b/rust/sglang-server/src/api_server/native_api.rs @@ -243,7 +243,7 @@ async fn drain_unary( StatusCode::from_u16(code).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); return (status, error_value(code, &e.to_string()), true); } - EgressItem::Control(_) => continue, // never on `/generate` + EgressItem::Control(_) | EgressItem::Data(_) => continue, // never on `/generate` } } // Sender dropped without a terminal item: the shard dropped this request (a @@ -376,7 +376,7 @@ fn generation_event_stream( terminal = Some(out); } EgressItem::Error(e) => failed = Some(e), - EgressItem::Control(_) => {} // never on /generate + EgressItem::Control(_) | EgressItem::Data(_) => {} // never on /generate } } diff --git a/rust/sglang-server/src/api_server/openai.rs b/rust/sglang-server/src/api_server/openai.rs index 22678f6d3..f3e59090b 100644 --- a/rust/sglang-server/src/api_server/openai.rs +++ b/rust/sglang-server/src/api_server/openai.rs @@ -1,55 +1,240 @@ -//! OpenAI-compatible endpoints: `/v1/completions`, `/v1/chat/completions`, and -//! `/v1/models`. Each runs the same tokenize→generate→detok pipeline as -//! `/generate` and shapes the neutral [`ChunkEvent`] delta into OpenAI types -//! (`dynamo-protocols`), with chat-template rendering (`dynamo-renderer`) and -//! reasoning / tool-call parsing (`dynamo-parsers`). +//! OpenAI-compatible generation endpoints. //! -//! Mounted on the shared [`AppState`](super::AppState) by the parent -//! `api_server` module; the submit machinery and control plane live there. +//! The HTTP adapter stays deliberately thin: Dynamo owns the standard OpenAI +//! request and response primitives. Native [`ChunkEvent`] values remain the one +//! backend output type for both unary and streaming responses. use axum::{ - extract::State, + Json, Router, http::StatusCode, response::{IntoResponse, Response}, }; +use futures::StreamExt; +use tokio::sync::mpsc; -use axum::{Router, routing::get}; +mod chat; +mod completions; +mod models; +mod reasoning; +mod template; +mod tools; + +pub(super) use template::ChatFormatter; use super::AppState; +use super::frame::OutputAccumulator; +use super::guard::AbortGuard; +use super::submit::submit; +use crate::ids::Rid; +use crate::message::{ChunkEvent, EgressItem, GenerateRequest, RequestKind}; +use crate::runtime::ServerArgs; + +const MAX_OPENAI_CHOICES: usize = 4096; /// The routes this module owns, mounted by `api_server::serve`. pub(super) fn routes() -> Router { - // `/v1/models` is OpenAI-compatible; completions/chat land here too. - Router::new().route("/v1/models", get(available_models)) + Router::new() + .merge(models::routes()) + .merge(completions::routes()) + .merge(chat::routes()) } -/// `GET /v1/models` — OpenAI-compatible model list. Served from `server_args`; -/// no scheduler round-trip. Mirrors `http_server.available_models`. -/// -/// TODO(v1/models): when `--enable-lora`, append a `ModelCard` per loaded LoRA -/// adapter (`id=lora_name, root=lora_path, parent=served_model_name, -/// max_model_len=None`). Adapters load/unload at runtime, so that part needs a -/// control-request query to the scheduler's LoRA registry. -async fn available_models(State(state): State) -> Response { - let created = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0); - let name = &state.server_args.served_model_name; - let base = serde_json::json!({ - "id": name, - "object": "model", - "created": created, - "owned_by": "sglang", - "root": name, - "parent": serde_json::Value::Null, - "max_model_len": state.server_args.model_config.context_len, - }); - let list = serde_json::json!({ "object": "list", "data": [base] }); - ( - StatusCode::OK, - [("content-type", "application/json")], - serde_json::to_vec(&list).unwrap_or_default(), - ) - .into_response() +/// Resolve the chat formatter, or `None` to disable the OpenAI chat-completions +/// endpoint. Tokenization is the tokenizer pool's job (the api server never +/// encodes); the formatter needs at most `tokenizer_config.json` — a built-in +/// `--chat-template` name or a model-path-inferred legacy template resolve +/// without it, so its absence must not disable chat. +pub(super) fn load_chat_support(server_args: &ServerArgs) -> Option { + // Chat needs the tokenizer pool behind it: under `skip_tokenizer_init` + // there is none (text cannot be submitted), so chat is disabled. + if server_args.skip_tokenizer_init || server_args.tokenizer_path.is_empty() { + return None; + } + let config_file = crate::tokenizer::resolve_model_file( + &server_args.tokenizer_path, + server_args.revision.as_deref(), + "tokenizer_config.json", + ); + + match template::load_chat_formatter( + config_file.as_deref(), + (!server_args.model_path.is_empty()).then_some(server_args.model_path.as_str()), + server_args.chat_template.as_deref(), + ) { + Ok(formatter) => { + tracing::info!( + config = ?config_file.as_deref().unwrap_or(""), + "loaded OpenAI chat template" + ); + Some(formatter) + } + Err(error) => { + tracing::warn!(%error, "OpenAI chat completions disabled"); + None + } + } } + +fn unix_seconds() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_secs()) + .unwrap_or(0) +} + +fn unix_seconds_u32() -> u32 { + u32::try_from(unix_seconds()).unwrap_or(u32::MAX) +} + +/// The OpenAI `{"error": {...}}` payload: `type` is the SDK-facing error kind +/// (`AuthenticationError` / `InternalServerError` / `BadRequestError`), and +/// `code` carries the HTTP status — the shape Python's OpenAI frontend emits. +fn error_payload(code: StatusCode, message: String) -> serde_json::Value { + let error_type = if code == StatusCode::UNAUTHORIZED { + "AuthenticationError" + } else if code.is_server_error() { + "InternalServerError" + } else { + "BadRequestError" + }; + serde_json::json!({ + "error": { + "object": "error", + "message": message, + "type": error_type, + "param": null, + "code": code.as_u16(), + } + }) +} + +/// Shape a `StatusCode` + message into an OpenAI error response, mirroring +/// `pre_submit_error`'s rule: unary requests get the JSON error with its +/// status; a request whose stream is already committed gets 200 + one SSE +/// error frame + `[DONE]`. +pub(super) fn openai_error_response( + code: StatusCode, + message: impl Into, + stream: bool, +) -> Response { + let body = error_payload(code, message.into()); + if !stream { + return (code, Json(body)).into_response(); + } + super::submit::sse_error_response(body) +} + +/// Unary OpenAI error — the common pre-submit case (Python validates before +/// the stream starts and answers 4xx in JSON even for `stream=true`). +fn openai_error(code: StatusCode, message: impl Into) -> Response { + openai_error_response(code, message, false) +} + +/// The OpenAI error frame payload for errors raised *inside* a committed +/// stream, where only a `data:` frame can be emitted (the status is folded +/// into the body, since the response status is already 200). +pub(super) fn streaming_error(code: u16, message: impl Into) -> String { + let status = StatusCode::from_u16(code).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + error_payload(status, message.into()).to_string() +} + +async fn collect_output( + mut rx: mpsc::Receiver, + guard: &mut AbortGuard, + rid: &Rid, +) -> Result { + let mut accumulator = OutputAccumulator::default(); + let output = loop { + match rx.recv().await { + Some(EgressItem::Frame(output)) => accumulator.fold(&output), + Some(EgressItem::Done(output)) => { + accumulator.fold(&output); + break accumulator.into_output(); + } + Some(EgressItem::Error(error)) => { + guard.disarm(rid); + let status = StatusCode::from_u16(error.http_status()) + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + return Err((status, error.to_string())); + } + Some(EgressItem::Control(_)) | Some(EgressItem::Data(_)) => {} + None => { + return Err(( + StatusCode::INTERNAL_SERVER_ERROR, + "response truncated before completion".into(), + )); + } + } + }; + guard.disarm(rid); + if let Some((code, message)) = output + .finish_reason + .as_ref() + .and_then(|reason| reason.abort_status()) + { + return Err(( + StatusCode::from_u16(code).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), + message.to_owned(), + )); + } + Ok(output) +} + +async fn submit_generation( + state: &AppState, + request: GenerateRequest, + stream: bool, + guard: &mut AbortGuard, +) -> Result, Response> { + match submit(state, RequestKind::Generate(Box::new(request)), stream).await { + Ok((rid, rx)) => { + guard.arm(rid); + Ok(rx) + } + // Same rule as `pre_submit_error`: a committed stream gets 200 plus an + // SSE error frame + `[DONE]`, not a unary 503 — but with the OpenAI + // error shape, since this is the OpenAI frontend. + Err(_) => Err(openai_error_response( + StatusCode::SERVICE_UNAVAILABLE, + "service unavailable", + stream, + )), + } +} + +fn indexed_egress_stream( + index: usize, + rx: mpsc::Receiver, +) -> futures::stream::BoxStream<'static, (usize, Option)> { + futures::stream::unfold((rx, false), move |(mut rx, finished)| async move { + if finished { + return None; + } + match rx.recv().await { + Some(item) => { + let finished = matches!(item, EgressItem::Done(_) | EgressItem::Error(_)); + Some(((index, Some(item)), (rx, finished))) + } + None => Some(((index, None), (rx, true))), + } + }) + .boxed() +} + +fn contains_media(value: &serde_json::Value) -> bool { + match value { + serde_json::Value::Array(values) => values.iter().any(contains_media), + serde_json::Value::Object(object) => { + object.keys().any(|key| { + matches!( + key.as_str(), + "image_url" | "video_url" | "input_audio" | "audio_url" | "file" + ) + }) || object.values().any(contains_media) + } + _ => false, + } +} + +#[cfg(test)] +mod test_utils; diff --git a/rust/sglang-server/src/api_server/openai/chat.rs b/rust/sglang-server/src/api_server/openai/chat.rs new file mode 100644 index 000000000..5e8c04ddf --- /dev/null +++ b/rust/sglang-server/src/api_server/openai/chat.rs @@ -0,0 +1,1174 @@ +//! OpenAI Chat Completions endpoint and chat-template preparation. + +use std::collections::BTreeMap; +use std::convert::Infallible; + +use axum::{ + Json, Router, + extract::{State, rejection::JsonRejection}, + http::StatusCode, + response::{ + IntoResponse, Response, + sse::{Event, Sse}, + }, + routing::post, +}; +use dynamo_parsers::tool_calling::jail::{Annotated, apply_tool_calling_jail}; +use dynamo_parsers::{ToolChoice as DynamoToolChoice, ToolDefinition}; +use dynamo_protocols::types::{ + ChatChoice, ChatChoiceLogprobs, ChatChoiceStream, ChatCompletionMessageContent, + ChatCompletionResponseMessage, ChatCompletionTokenLogprob, ChatCompletionToolChoiceOption, + CreateChatCompletionRequest, CreateChatCompletionResponse, CreateChatCompletionStreamResponse, + FinishReason as OpenAIFinishReason, ResponseFormat, Role, ServiceTier as ChatServiceTier, Stop, + TopLogprobs, +}; +use futures::StreamExt; +use tokio::sync::mpsc; + +use super::super::guard::AbortGuard; +use super::completions::completion_usage; +use super::reasoning::{ReasoningStreamSplitter, split_reasoning_unary}; +use super::tools::{ + apply_tool_constraint, chat_delta, chat_finish_reason, dynamo_parser_name, dynamo_tool_choice, + parse_chat_tool_calls, +}; +use super::{ + AppState, ChatFormatter, collect_output, contains_media, indexed_egress_stream, openai_error, + streaming_error, submit_generation, unix_seconds_u32, +}; +use crate::ids::Rid; +use crate::message::{ChunkExtras, EgressItem, GenerateRequest, OneOrMany, SamplingParams}; + +pub(super) fn routes() -> Router { + Router::new().route("/v1/chat/completions", post(chat_completions)) +} + +async fn chat_completions( + State(state): State, + body: Result, JsonRejection>, +) -> Response { + let request = match body { + Ok(Json(request)) => request, + Err(rejection) => return openai_error(StatusCode::BAD_REQUEST, rejection.body_text()), + }; + if request.model != state.server_args.served_model_name { + return openai_error( + StatusCode::BAD_REQUEST, + format!("The model `{}` does not exist", request.model), + ); + } + if request.messages.is_empty() { + return openai_error(StatusCode::BAD_REQUEST, "messages cannot be empty"); + } + if serde_json::to_value(&request.messages).is_ok_and(|messages| contains_media(&messages)) { + return openai_error( + StatusCode::BAD_REQUEST, + "image, audio, video, and file message content is not supported", + ); + } + if request.n == Some(0) { + return openai_error(StatusCode::BAD_REQUEST, "n must be at least 1"); + } + #[allow(deprecated)] + let max_tokens = request.max_completion_tokens.or(request.max_tokens); + if max_tokens == Some(0) { + return openai_error( + StatusCode::BAD_REQUEST, + "max_completion_tokens must be positive", + ); + } + if request.modalities.as_ref().is_some_and(|modalities| { + serde_json::to_value(modalities).is_ok_and(|value| value.to_string().contains("\"audio\"")) + }) || request.audio.is_some() + || request.prediction.is_some() + || request.web_search_options.is_some() + || request.mm_processor_kwargs.is_some() + { + return openai_error( + StatusCode::BAD_REQUEST, + "audio, prediction, web search, and multimodal inputs are not supported", + ); + } + #[allow(deprecated)] + if request.function_call.is_some() || request.functions.is_some() { + return openai_error( + StatusCode::BAD_REQUEST, + "deprecated function_call/functions are not supported; use tools and tool_choice", + ); + } + + let tool_choice = dynamo_tool_choice(&request.tool_choice); + let tools_enabled = request + .tools + .as_ref() + .is_some_and(|tools| !tools.is_empty()) + && tool_choice != DynamoToolChoice::None; + let parser = tools_enabled + .then(|| state.server_args.tool_call_parser.clone()) + .flatten(); + if tools_enabled && parser.is_none() { + return openai_error( + StatusCode::BAD_REQUEST, + "tool calls require --tool-call-parser", + ); + } + // Python gates the split on `request.separate_reasoning` (default true); + // the Dynamo request type has no such field, so it is always on when the + // server was launched with `--reasoning-parser`. + let reasoning_parser = state.server_args.reasoning_parser.clone(); + let tools = request.tools.as_ref().map(|tools| { + tools + .iter() + .map(|tool| ToolDefinition { + name: tool.function.name.clone(), + parameters: tool.function.parameters.clone(), + strict: tool.function.strict, + }) + .collect::>() + }); + let tools_slice = tools.as_deref().unwrap_or_default(); + + let (request, prompt) = match prepare_chat_request(&state, request).await { + Ok(prepared) => prepared, + Err(response) => return response, + }; + + let sampling = match chat_sampling( + &request, + SamplingDefaults::CHAT, + parser.as_deref(), + &tool_choice, + tools_slice, + request.parallel_tool_calls, + &state.server_args, + ) { + Ok(sampling) => sampling, + Err(message) => return openai_error(StatusCode::BAD_REQUEST, message), + }; + + let stream = request.stream.unwrap_or(false); + let n = request.n.unwrap_or(1) as usize; + let want_logprobs = request.logprobs.unwrap_or(false); + let parallel_tool_calls = request.parallel_tool_calls.unwrap_or(true); + let stream_tool_choice = request.tool_choice.clone(); + let uses_tool_call_structural_tag = sampling.structural_tag.is_some(); + let service_tier = request.service_tier; + let response_id = format!("chatcmpl-{}", uuid::Uuid::new_v4().simple()); + let created = unix_seconds_u32(); + let model = request.model; + let include_usage = request + .stream_options + .is_some_and(|options| options.include_usage) + || state.server_args.stream_response_default_include_usage; + let mut guard = AbortGuard::new_empty(state.senders.clone()); + let mut submitted = Vec::with_capacity(n); + + let mut prompt = Some(prompt); + for index in 0..n { + let rid = Rid::from_client(&format!("{response_id}-{index}")); + let choice_prompt = if index + 1 == n { + prompt.take().expect("last chat choice owns the prompt") + } else { + prompt + .as_ref() + .expect("chat prompt exists until the last choice") + .clone() + }; + let native = GenerateRequest { + rid: rid.clone(), + text: Some(choice_prompt), + // Rendered templates own their special tokens — the pool must not + // add another BOS/EOS (Python's `add_special_tokens=False`). + skip_special_tokens: true, + sampling_params: sampling.clone(), + stream, + return_logprob: want_logprobs, + logprob_start_len: -1, + top_logprobs_num: request.top_logprobs.unwrap_or(0) as i64, + return_text_in_logprobs: want_logprobs.then_some(true), + ..Default::default() + }; + let rx = match submit_generation(&state, native, stream, &mut guard).await { + Ok(rx) => rx, + Err(response) => return response, + }; + submitted.push((index, rid, rx)); + } + + if stream { + let event_stream = chat_event_stream( + submitted, + guard, + response_id, + model, + created, + want_logprobs, + include_usage, + parser, + reasoning_parser, + tools, + stream_tool_choice, + uses_tool_call_structural_tag, + parallel_tool_calls, + service_tier, + ) + .map(|data| Ok::<_, Infallible>(Event::default().data(data))); + Sse::new(event_stream).into_response() + } else { + unary_chat( + submitted, + guard, + response_id, + model, + created, + want_logprobs, + parser, + reasoning_parser, + tools, + parallel_tool_calls, + service_tier, + ) + .await + } +} + +/// Render the chat template for an OpenAI request, mapping a missing +/// formatter or a render failure to the standard 400. The rendered prompt is +/// submitted as text — the tokenizer pool encodes it (with +/// `skip_special_tokens`, since the template owns its special tokens). +pub(super) async fn prepare_chat_request( + state: &AppState, + mut request: CreateChatCompletionRequest, +) -> Result<(CreateChatCompletionRequest, String), Response> { + let Some(formatter) = state.chat_formatter.clone() else { + return Err(openai_error( + StatusCode::BAD_REQUEST, + "this model has no usable chat template", + )); + }; + // Template stops first, then the request's own — Python + // `_apply_conversation_template` (`conv.stop_str` + `request.stop`). A + // token-id stop cannot be merged into the string list (Python has no such + // field), so it is kept alone. + merge_template_stops(&mut request, &formatter); + let prompt = formatter.render(&request).map_err(|error| { + openai_error( + StatusCode::BAD_REQUEST, + format!("chat template render failed: {error}"), + ) + })?; + Ok((request, prompt)) +} + +/// Full sampling resolution for an OpenAI request, mirroring the Python +/// handler: endpoint defaults → tool-choice validation + constraint → clamp. +/// The tool-choice checks run regardless of whether a parser is configured +/// (see `apply_tool_constraint`). +pub(super) fn chat_sampling( + request: &CreateChatCompletionRequest, + defaults: SamplingDefaults, + parser: Option<&str>, + tool_choice: &DynamoToolChoice, + tools: &[ToolDefinition], + parallel_tool_calls: Option, + server_args: &crate::runtime::ServerArgs, +) -> Result { + let mut sampling = chat_sampling_params( + request, + &defaults.with_model_defaults(&server_args.model_config.default_sampling_params), + )?; + apply_tool_constraint( + &mut sampling, + parser, + tool_choice, + tools, + parallel_tool_calls, + )?; + sampling + .normalize( + server_args.skip_tokenizer_init, + server_args.model_config.vocab_size.unwrap_or(u64::MAX), + ) + .map_err(|error| error.to_string())?; + Ok(sampling) +} + +/// Merge the formatter's template stops into the request's `stop`. +/// +/// Python `_apply_conversation_template`: `stop = copy.copy(conv.stop_str or []) +/// + request.stop` (a string request stop appends as one entry). Without this, +/// generation with a legacy/builtin template would run past the template's own +/// delimiters (e.g. chatml's `<|im_end|>`) whenever they are not model EOS ids. +fn merge_template_stops(request: &mut CreateChatCompletionRequest, formatter: &ChatFormatter) { + let Some(template_stops) = formatter.stop_strs() else { + return; + }; + let mut stops = match template_stops { + OneOrMany::One(one) => vec![one], + OneOrMany::Many(many) => many, + }; + if let Some(request_stop) = &request.stop { + let Some(request_stops) = request_stop.strings() else { + return; + }; + stops.extend(request_stops); + } + request.stop = Some(Stop::StringArray(stops)); +} + +/// Where an omitted `temperature` / `top_p` gets its value. Mirrors Python's +/// `to_sampling_params` priority: user value > model generation_config (when +/// `--sampling-defaults model`) > OpenAI terminal default +/// (`_DEFAULT_SAMPLING_PARAMS`: chat uses 1.0/1.0). +pub(super) struct SamplingDefaults { + /// Model defaults; `None` when the model config doesn't set them or when + /// `--sampling-defaults openai` (the Python dump is then empty). + temperature: Option, + top_p: Option, + /// OpenAI terminal defaults for chat completions. + fallback_temperature: f64, + fallback_top_p: f64, +} + +impl SamplingDefaults { + /// `protocol.py` chat `_DEFAULT_SAMPLING_PARAMS`: temperature 1.0, top_p 1.0. + pub(super) const CHAT: SamplingDefaults = SamplingDefaults { + temperature: None, + top_p: None, + fallback_temperature: 1.0, + fallback_top_p: 1.0, + }; + /// The resolved model defaults (empty in `--sampling-defaults openai` + /// mode), which slot between the user's values and the OpenAI terminals. + pub(super) fn with_model_defaults( + mut self, + model: &crate::runtime::DefaultSamplingParams, + ) -> SamplingDefaults { + self.temperature = model.temperature; + self.top_p = model.top_p; + self + } +} + +#[allow(deprecated)] +pub(super) fn chat_sampling_params( + request: &CreateChatCompletionRequest, + defaults: &SamplingDefaults, +) -> Result { + let mut stop = None; + let mut stop_token_ids = None; + match request.stop.as_ref() { + Some(Stop::String(value)) => stop = Some(OneOrMany::One(value.clone())), + Some(Stop::StringArray(values)) => stop = Some(OneOrMany::Many(values.clone())), + Some(Stop::TokenIdArray(values)) => { + stop_token_ids = Some(values.iter().map(|&id| id as i64).collect()) + } + None => {} + } + let mut logit_bias = BTreeMap::new(); + if let Some(values) = request.logit_bias.as_ref() { + for (token, bias) in values { + let bias = bias + .as_f64() + .ok_or_else(|| format!("logit_bias[{token:?}] must be a number"))?; + logit_bias.insert(token.clone(), bias); + } + } + let json_schema = match request.response_format.as_ref() { + Some(ResponseFormat::JsonSchema { json_schema }) => Some(json_schema.schema.to_string()), + Some(ResponseFormat::JsonObject) => Some(r#"{"type":"object"}"#.into()), + _ => None, + }; + + Ok(SamplingParams { + max_new_tokens: request + .max_completion_tokens + .or(request.max_tokens) + .map(i64::from), + stop, + stop_token_ids, + temperature: request + .temperature + .map(f64::from) + .or(defaults.temperature) + .unwrap_or(defaults.fallback_temperature), + top_p: request + .top_p + .map(f64::from) + .or(defaults.top_p) + .unwrap_or(defaults.fallback_top_p), + frequency_penalty: request.frequency_penalty.unwrap_or(0.0) as f64, + presence_penalty: request.presence_penalty.unwrap_or(0.0) as f64, + n: 1, + logit_bias: (!logit_bias.is_empty()).then_some(logit_bias), + sampling_seed: request.seed, + json_schema, + ..Default::default() + }) +} + +#[allow(clippy::too_many_arguments)] +pub(super) async fn unary_chat( + submitted: Vec<(usize, Rid, mpsc::Receiver)>, + mut guard: AbortGuard, + response_id: String, + model: String, + created: u32, + want_logprobs: bool, + parser: Option, + reasoning_parser: Option, + tools: Option>, + parallel_tool_calls: bool, + service_tier: Option, +) -> Response { + let mut choices = Vec::with_capacity(submitted.len()); + let mut prompt_tokens = 0; + let mut completion_tokens = 0u64; + + for (index, rid, rx) in submitted { + let output = match collect_output(rx, &mut guard, &rid).await { + Ok(output) => output, + Err((status, message)) => return openai_error(status, message), + }; + + if prompt_tokens == 0 { + prompt_tokens = output.prompt_tokens; + } + completion_tokens = completion_tokens.saturating_add(output.completion_tokens); + let logprobs = want_logprobs.then(|| chat_logprobs(output.extras.as_deref())); + let finish_reason = chat_finish_reason(&output); + // Split reasoning markers out of the content first (Python splits + // before tool-call parsing too), then parse tool calls on the clean + // normal text. + let (reasoning_text, text) = + split_reasoning_unary(reasoning_parser.as_deref(), &output.text, &output.token_ids); + let (content, tool_calls) = parse_chat_tool_calls( + text, + parser.as_deref(), + tools.as_deref(), + parallel_tool_calls, + ) + .await; + let finish_reason = if tool_calls.is_some() { + Some(OpenAIFinishReason::ToolCalls) + } else { + finish_reason + }; + #[allow(deprecated)] + let message = ChatCompletionResponseMessage { + content: (!content.is_empty()).then_some(ChatCompletionMessageContent::Text(content)), + refusal: None, + tool_calls, + role: Role::Assistant, + function_call: None, + audio: None, + // Python: `reasoning_text if reasoning_text else None`. + reasoning_content: (!reasoning_text.is_empty()).then_some(reasoning_text), + }; + choices.push(ChatChoice { + index: u32::try_from(index).unwrap_or(u32::MAX), + message, + finish_reason, + logprobs, + }); + } + + Json(CreateChatCompletionResponse { + id: response_id, + choices, + created, + model, + service_tier, + system_fingerprint: None, + object: "chat.completion".into(), + usage: Some(completion_usage( + prompt_tokens, + u32::try_from(completion_tokens).unwrap_or(u32::MAX), + )), + }) + .into_response() +} + +#[allow(clippy::too_many_arguments)] +pub(super) fn chat_event_stream( + submitted: Vec<(usize, Rid, mpsc::Receiver)>, + mut guard: AbortGuard, + response_id: String, + model: String, + created: u32, + want_logprobs: bool, + include_usage: bool, + parser: Option, + reasoning_parser: Option, + tools: Option>, + tool_choice: Option, + uses_tool_call_structural_tag: bool, + parallel_tool_calls: bool, + service_tier: Option, +) -> impl futures::Stream { + let count = submitted.len(); + let raw = async_stream::stream! { + let count = submitted.len(); + let mut rids = Vec::with_capacity(count); + let mut streams = Vec::with_capacity(count); + let mut prompt_tokens = 0u32; + let mut completion_tokens = 0u64; + // One stateful reasoning splitter per choice (Python keeps a + // `reasoning_parser_dict` per index). + let mut reasoning_splitters: Vec = + if reasoning_parser.is_some() { + (0..count) + .map(|_| ReasoningStreamSplitter::new(reasoning_parser.as_deref())) + .collect() + } else { + vec![] + }; + let reasoning_enabled = !reasoning_splitters.is_empty(); + + for (index, rid, rx) in submitted { + rids.push(rid); + streams.push(indexed_egress_stream(index, rx)); + yield Annotated { + data: Some(CreateChatCompletionStreamResponse { + id: response_id.clone(), + choices: vec![ChatChoiceStream { + index: u32::try_from(index).unwrap_or(u32::MAX), + delta: chat_delta(None, Some(Role::Assistant), None, None), + finish_reason: None, + logprobs: None, + }], + created, + model: model.clone(), + service_tier: service_tier.clone(), + system_fingerprint: None, + object: "chat.completion.chunk".into(), + usage: None, + }), + id: None, + event: None, + comment: None, + error: None, + }; + } + + let mut events = futures::stream::select_all(streams); + while let Some((index, item)) = events.next().await { + let Some(item) = item else { + yield Annotated { + data: None, + id: None, + event: None, + comment: None, + error: Some(streaming_error(500, "response truncated before completion")), + }; + continue; + }; + let output = match item { + EgressItem::Frame(output) => output, + EgressItem::Done(output) => { + guard.disarm(&rids[index]); + output + } + EgressItem::Error(error) => { + guard.disarm(&rids[index]); + yield Annotated { + data: None, + id: None, + event: None, + comment: None, + error: Some(streaming_error(error.http_status(), error.to_string())), + }; + continue; + } + EgressItem::Control(_) | EgressItem::Data(_) => continue, + }; + if let Some((code, message)) = output + .finish_reason + .as_ref() + .and_then(|reason| reason.abort_status()) + { + yield Annotated { + data: None, + id: None, + event: None, + comment: None, + error: Some(streaming_error(code, message)), + }; + continue; + } + + if prompt_tokens == 0 { + prompt_tokens = output.prompt_tokens; + } + completion_tokens = completion_tokens.saturating_add(output.completion_tokens); + let finish_reason = chat_finish_reason(&output); + // Split the step's text into (reasoning, normal) deltas when + // `--reasoning-parser` is set. Mirrors Python's per-step emission: + // reasoning chunk first (logprobs ride it), then the content chunk. + let mut emitted = Vec::with_capacity(2); + if reasoning_enabled { + let (reasoning_text, normal_text) = + reasoning_splitters[index].split(&output.text, &output.token_ids); + let mut remaining_logprobs = + want_logprobs.then(|| chat_logprobs(output.extras.as_deref())); + if !reasoning_text.is_empty() { + emitted.push(ChatChoiceStream { + index: u32::try_from(index).unwrap_or(u32::MAX), + delta: chat_delta(None, None, None, Some(reasoning_text)), + finish_reason: None, + logprobs: remaining_logprobs.clone(), + }); + remaining_logprobs = None; + } + if !normal_text.is_empty() { + emitted.push(ChatChoiceStream { + index: u32::try_from(index).unwrap_or(u32::MAX), + delta: chat_delta(Some(normal_text), None, None, None), + finish_reason: None, + logprobs: remaining_logprobs, + }); + } + } else { + emitted.push(ChatChoiceStream { + index: u32::try_from(index).unwrap_or(u32::MAX), + delta: chat_delta( + (!output.text.is_empty()).then_some(output.text), + None, + None, + None, + ), + finish_reason: None, + logprobs: want_logprobs.then(|| chat_logprobs(output.extras.as_deref())), + }); + }; + // Flush the choice's buffered reasoning tail before its terminal + // frame (Python `parse_stream_end`, which skips aborts — abort + // frames already became error chunks above). Both columns flush: + // some parsers buffer the answer text until EOF. + if reasoning_enabled && finish_reason.is_some() { + let (reasoning_tail, normal_tail) = reasoning_splitters[index].finish(); + if !reasoning_tail.is_empty() { + emitted.push(ChatChoiceStream { + index: u32::try_from(index).unwrap_or(u32::MAX), + delta: chat_delta(None, None, None, Some(reasoning_tail)), + finish_reason: None, + logprobs: None, + }); + } + if !normal_tail.is_empty() { + emitted.push(ChatChoiceStream { + index: u32::try_from(index).unwrap_or(u32::MAX), + delta: chat_delta(Some(normal_tail), None, None, None), + finish_reason: None, + logprobs: None, + }); + } + } + // The finish reason rides the last emitted chunk (the wire format + // the equivalence tests pin); a step whose text was entirely + // buffered inside the parser still gets a finish-only frame. + match emitted.last_mut() { + Some(last) => last.finish_reason = finish_reason, + None => emitted.push(ChatChoiceStream { + index: u32::try_from(index).unwrap_or(u32::MAX), + delta: chat_delta(None, None, None, None), + finish_reason, + logprobs: None, + }), + } + for choice in emitted { + yield Annotated { + data: Some(CreateChatCompletionStreamResponse { + id: response_id.clone(), + choices: vec![choice], + created, + model: model.clone(), + service_tier: service_tier.clone(), + system_fingerprint: None, + object: "chat.completion.chunk".into(), + usage: None, + }), + id: None, + event: None, + comment: None, + error: None, + }; + } + } + + if include_usage { + yield Annotated { + data: Some(CreateChatCompletionStreamResponse { + id: response_id, + choices: vec![], + created, + model, + service_tier, + system_fingerprint: None, + object: "chat.completion.chunk".into(), + usage: Some(completion_usage( + prompt_tokens, + u32::try_from(completion_tokens).unwrap_or(u32::MAX), + )), + }), + id: None, + event: None, + comment: None, + error: None, + }; + } + }; + + let parsed: std::pin::Pin< + Box> + Send>, + > = if let Some(parser) = parser { + Box::pin(apply_tool_calling_jail( + Some(dynamo_parser_name(&parser).to_owned()), + tool_choice, + tools, + uses_tool_call_structural_tag, + raw, + )) + } else { + Box::pin(raw) + }; + + async_stream::stream! { + let mut tool_calls_seen = vec![false; count]; + futures::pin_mut!(parsed); + while let Some(mut item) = parsed.next().await { + if let Some(response) = item.data.as_mut() { + if !parallel_tool_calls { + for choice in &mut response.choices { + let index = choice.index as usize; + if let Some(calls) = choice.delta.tool_calls.as_mut() { + if tool_calls_seen.get(index).copied().unwrap_or(false) { + calls.clear(); + } else { + calls.truncate(1); + if !calls.is_empty() + && let Some(seen) = tool_calls_seen.get_mut(index) + { + *seen = true; + } + } + if calls.is_empty() { + choice.delta.tool_calls = None; + } + } + } + } + yield serialize_chat_stream_response(response.clone()); + } else if let Some(error) = item.error { + yield error; + } + } + yield "[DONE]".to_string(); + } +} + +fn serialize_chat_stream_response(response: CreateChatCompletionStreamResponse) -> String { + let mut response = serde_json::to_value(response).expect("OpenAI response must serialize"); + if let Some(delta) = response + .pointer_mut("/choices/0/delta") + .and_then(serde_json::Value::as_object_mut) + { + delta + .entry("reasoning_content") + .or_insert(serde_json::Value::Null); + } + response.to_string() +} + +#[allow(deprecated)] +pub(super) fn chat_logprobs(extras: Option<&ChunkExtras>) -> ChatChoiceLogprobs { + let mut content = Vec::new(); + let Some(extras) = extras else { + return ChatChoiceLogprobs { + content: Some(content), + refusal: None, + }; + }; + let mut top_offset = 0usize; + for (position, (&logprob, &token_id)) in + extras.out_lp_val.iter().zip(&extras.out_lp_idx).enumerate() + { + let token = extras + .out_lp_txt + .get(position) + .cloned() + .unwrap_or_else(|| format!("token_id:{token_id}")); + let top_len = extras.out_top_lens.get(position).copied().unwrap_or(0) as usize; + let top_logprobs = extras.out_top_val[top_offset..] + .iter() + .zip(&extras.out_top_idx[top_offset..]) + .take(top_len) + .enumerate() + .map(|(offset, (&logprob, &id))| { + let text = extras + .out_top_txt + .get(top_offset + offset) + .cloned() + .unwrap_or_else(|| format!("token_id:{id}")); + TopLogprobs { + bytes: Some(text.as_bytes().to_vec()), + token: text, + logprob, + } + }) + .collect(); + top_offset = top_offset.saturating_add(top_len); + content.push(ChatCompletionTokenLogprob { + bytes: Some(token.as_bytes().to_vec()), + token, + logprob, + top_logprobs, + }); + } + ChatChoiceLogprobs { + content: Some(content), + refusal: None, + } +} + +#[cfg(test)] +mod tests { + use super::super::test_utils::{chat_submitted, chunk, senders}; + use super::{ + SamplingDefaults, chat_event_stream, chat_logprobs, chat_sampling_params, + merge_template_stops, unary_chat, + }; + use crate::api_server::guard::AbortGuard; + use crate::message::ChunkExtras; + use crate::runtime::DefaultSamplingParams; + use axum::http::StatusCode; + use dynamo_protocols::types::{CreateChatCompletionRequest, Stop}; + use futures::StreamExt; + + fn request() -> CreateChatCompletionRequest { + serde_json::from_value(serde_json::json!({ + "model": "test", + "messages": [{"role": "user", "content": "hi"}] + })) + .unwrap() + } + + /// Python `to_sampling_params` priority: user value > model generation + /// config (`--sampling-defaults model`) > OpenAI terminal default. + #[test] + fn sampling_defaults_follow_python_priority_chain() { + let model = DefaultSamplingParams { + temperature: Some(0.6), + top_p: Some(0.9), + ..Default::default() + }; + // Omitted → model defaults, not the 1.0 OpenAI terminals. + let sampling = chat_sampling_params( + &request(), + &SamplingDefaults::CHAT.with_model_defaults(&model), + ) + .unwrap(); + assert_eq!(sampling.temperature, 0.6); + assert_eq!(sampling.top_p, 0.9); + // Explicit request values win. `Option` loses precision in f64 — + // compare with tolerance. + let mut request = request(); + request.temperature = Some(0.2); + request.top_p = Some(0.5); + let sampling = chat_sampling_params( + &request, + &SamplingDefaults::CHAT.with_model_defaults(&model), + ) + .unwrap(); + assert!((sampling.temperature - 0.2).abs() < 1e-6); + assert!((sampling.top_p - 0.5).abs() < 1e-6); + } + + /// `--sampling-defaults openai` resolves an empty model-config slice, so the + /// conversion falls back to the OpenAI terminal defaults. + #[test] + fn sampling_defaults_fall_back_to_openai_terminals_in_openai_mode() { + let openai_mode = DefaultSamplingParams::default(); + let sampling = chat_sampling_params( + &request(), + &SamplingDefaults::CHAT.with_model_defaults(&openai_mode), + ) + .unwrap(); + assert_eq!(sampling.temperature, 1.0); + assert_eq!(sampling.top_p, 1.0); + } + + /// Python `_apply_conversation_template`: template `stop_str` first, then + /// the request's own stops. + #[test] + fn template_stops_merge_before_request_stops() { + let chatml = super::super::template::builtin_template("chatml").unwrap(); + let formatter = super::super::ChatFormatter::Legacy(Box::new( + super::super::template::LegacyFormatter { spec: chatml }, + )); + assert_eq!( + formatter.stop_strs(), + Some(crate::message::OneOrMany::Many(vec![ + "<|endoftext|>".into(), + "<|im_end|>".into() + ])) + ); + // No request stop → the template's delimiters alone. + let mut req = request(); + merge_template_stops(&mut req, &formatter); + assert_eq!( + req.stop, + Some(Stop::StringArray(vec![ + "<|endoftext|>".into(), + "<|im_end|>".into() + ])) + ); + // A string request stop appends as one entry. + let mut req = request(); + req.stop = Some(Stop::String("".into())); + merge_template_stops(&mut req, &formatter); + assert_eq!( + req.stop, + Some(Stop::StringArray(vec![ + "<|endoftext|>".into(), + "<|im_end|>".into(), + "".into() + ])) + ); + // A list request stop extends the list. + let mut req = request(); + req.stop = Some(Stop::StringArray(vec!["a".into(), "b".into()])); + merge_template_stops(&mut req, &formatter); + assert_eq!( + req.stop, + Some(Stop::StringArray(vec![ + "<|endoftext|>".into(), + "<|im_end|>".into(), + "a".into(), + "b".into() + ])) + ); + // Token-id stops cannot be merged (Python has no such field) — kept alone. + let mut req = request(); + req.stop = Some(Stop::TokenIdArray(vec![2, 3])); + merge_template_stops(&mut req, &formatter); + assert_eq!(req.stop, Some(Stop::TokenIdArray(vec![2, 3]))); + } + + /// The HuggingFace renderer carries no template stops (Python's jinja path + /// keeps only the request's stops), so the request is left unchanged. + #[test] + fn huggingface_formatter_leaves_request_stops_alone() { + let mut req = request(); + req.stop = Some(Stop::String("x".into())); + // A prompt formatter is not constructible here without a tokenizer; the + // empty-legacy-spec twin proves the merge is formatter-gated, and the + // `HuggingFace` arm returns `None` by construction (see `stop_strs`). + let legacy = super::super::ChatFormatter::Legacy(Box::new( + super::super::template::LegacyFormatter { + spec: super::super::template::LegacySpec::default(), + }, + )); + assert!(legacy.stop_strs().is_none()); + merge_template_stops(&mut req, &legacy); + assert_eq!(req.stop, Some(Stop::String("x".into()))); + } + + /// A request with no `max_tokens`/`max_completion_tokens` stays unbounded — + /// no terminal default is imposed. + #[test] + fn chat_without_a_token_limit_stays_unbounded() { + let request: CreateChatCompletionRequest = serde_json::from_value(serde_json::json!({ + "model": "test", + "messages": [{"role": "user", "content": "hello"}] + })) + .unwrap(); + assert_eq!( + chat_sampling_params(&request, &SamplingDefaults::CHAT) + .unwrap() + .max_new_tokens, + None + ); + } + + #[test] + fn chat_logprobs_use_dynamo_wire_types() { + let extras = ChunkExtras { + out_lp_val: vec![-0.25], + out_lp_idx: vec![7], + out_lp_txt: vec!["x".into()], + out_top_val: vec![-0.25, -1.0], + out_top_idx: vec![7, 8], + out_top_lens: vec![2], + out_top_txt: vec!["x".into(), "y".into()], + ..Default::default() + }; + let logprobs = chat_logprobs(Some(&extras)); + let token = &logprobs.content.unwrap()[0]; + assert_eq!(token.token, "x"); + assert_eq!(token.top_logprobs.len(), 2); + assert_eq!(token.top_logprobs[1].token, "y"); + } + + #[tokio::test] + async fn unary_chat_fans_in_choices_and_usage() { + let (choice0, tx0) = chat_submitted(0, "r0"); + let (choice1, tx1) = chat_submitted(1, "r1"); + tx0.send(chunk("r0", "Paris", true)).await.unwrap(); + tx1.send(chunk("r1", "Paris", true)).await.unwrap(); + + let response = unary_chat( + vec![choice0, choice1], + AbortGuard::new_empty(senders()), + "chatcmpl-test".into(), + "model".into(), + 1, + false, + None, + None, + None, + true, + None, + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), 64 * 1024) + .await + .unwrap(); + let value: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(value["choices"][0]["message"]["role"], "assistant"); + assert_eq!(value["choices"][0]["message"]["content"], "Paris"); + assert_eq!(value["choices"][1]["index"], 1); + assert_eq!(value["usage"]["prompt_tokens"], 5); + assert_eq!(value["usage"]["completion_tokens"], 2); + } + + #[tokio::test] + async fn unary_chat_separates_reasoning_content_with_parser_configured() { + let (choice, tx) = chat_submitted(0, "r0"); + tx.send(chunk( + "r0", + "because Paris is famousParis", + true, + )) + .await + .unwrap(); + + let response = unary_chat( + vec![choice], + AbortGuard::new_empty(senders()), + "chatcmpl-test".into(), + "model".into(), + 1, + false, + None, + Some("deepseek-r1".into()), + None, + true, + None, + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), 64 * 1024) + .await + .unwrap(); + let value: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!( + value["choices"][0]["message"]["reasoning_content"], + "because Paris is famous" + ); + assert_eq!(value["choices"][0]["message"]["content"], "Paris"); + assert!(value["choices"][0]["message"]["reasoning_content"].is_string()); + } + + #[tokio::test] + async fn streaming_chat_separates_reasoning_into_own_deltas() { + let (choice, tx) = chat_submitted(0, "r0"); + // Force mode starts in reasoning, so the opener is stripped and the first + // reasoning fragment streams immediately. + tx.send(chunk("r0", "be", false)).await.unwrap(); + tx.send(chunk("r0", "causePar", false)) + .await + .unwrap(); + tx.send(chunk("r0", "is", true)).await.unwrap(); + + let stream = chat_event_stream( + vec![choice], + AbortGuard::new_empty(senders()), + "chatcmpl-test".into(), + "model".into(), + 1, + false, + true, + None, + Some("deepseek-r1".into()), + None, + None, + false, + true, + None, + ); + futures::pin_mut!(stream); + let frames: Vec = stream.collect().await; + let role: serde_json::Value = serde_json::from_str(&frames[0]).unwrap(); + let first_reasoning: serde_json::Value = serde_json::from_str(&frames[1]).unwrap(); + let second_reasoning: serde_json::Value = serde_json::from_str(&frames[2]).unwrap(); + let content: serde_json::Value = serde_json::from_str(&frames[3]).unwrap(); + let terminal: serde_json::Value = serde_json::from_str(&frames[4]).unwrap(); + assert_eq!(role["choices"][0]["delta"]["role"], "assistant"); + assert_eq!( + first_reasoning["choices"][0]["delta"]["reasoning_content"], + "be" + ); + assert!(first_reasoning["choices"][0]["delta"]["content"].is_null()); + assert_eq!( + second_reasoning["choices"][0]["delta"]["reasoning_content"], + "cause" + ); + assert_eq!(content["choices"][0]["delta"]["content"], "Par"); + assert!(content["choices"][0]["delta"]["reasoning_content"].is_null()); + assert_eq!(terminal["choices"][0]["delta"]["content"], "is"); + assert_eq!(terminal["choices"][0]["finish_reason"], "stop"); + assert_eq!(frames.len(), 7); + } + + #[tokio::test] + async fn streaming_chat_emits_role_deltas_usage_and_done() { + let (choice, tx) = chat_submitted(0, "r0"); + tx.send(chunk("r0", "Par", false)).await.unwrap(); + tx.send(chunk("r0", "is", true)).await.unwrap(); + + let stream = chat_event_stream( + vec![choice], + AbortGuard::new_empty(senders()), + "chatcmpl-test".into(), + "model".into(), + 1, + false, + true, + None, + None, + None, + None, + false, + true, + None, + ); + futures::pin_mut!(stream); + let frames: Vec = stream.collect().await; + assert_eq!(frames.len(), 5); + let role: serde_json::Value = serde_json::from_str(&frames[0]).unwrap(); + let delta: serde_json::Value = serde_json::from_str(&frames[1]).unwrap(); + let terminal: serde_json::Value = serde_json::from_str(&frames[2]).unwrap(); + let usage: serde_json::Value = serde_json::from_str(&frames[3]).unwrap(); + assert_eq!(role["choices"][0]["delta"]["role"], "assistant"); + assert!(role["choices"][0]["delta"]["reasoning_content"].is_null()); + assert_eq!(delta["choices"][0]["delta"]["content"], "Par"); + assert!(delta["choices"][0]["delta"]["reasoning_content"].is_null()); + assert_eq!(terminal["choices"][0]["delta"]["content"], "is"); + assert!(terminal["choices"][0]["delta"]["reasoning_content"].is_null()); + assert_eq!(terminal["choices"][0]["finish_reason"], "stop"); + assert_eq!(usage["usage"]["completion_tokens"], 2); + assert_eq!(frames[4], "[DONE]"); + } +} diff --git a/rust/sglang-server/src/api_server/openai/completions.rs b/rust/sglang-server/src/api_server/openai/completions.rs new file mode 100644 index 000000000..fcd2be828 --- /dev/null +++ b/rust/sglang-server/src/api_server/openai/completions.rs @@ -0,0 +1,864 @@ +//! OpenAI legacy text-completion endpoint and wire shaping. + +use std::collections::BTreeMap; +use std::convert::Infallible; + +use axum::{ + Json, Router, + extract::{State, rejection::JsonRejection}, + http::StatusCode, + response::{ + IntoResponse, Response, + sse::{Event, Sse}, + }, + routing::post, +}; +use dynamo_protocols::types::{ + Choice, CompletionFinishReason, CompletionUsage, CreateCompletionRequest, + CreateCompletionResponse, Logprobs, Prompt, Stop, +}; +use futures::StreamExt; +use tokio::sync::mpsc; + +use super::super::guard::AbortGuard; +use super::super::submit::submit; +use super::{ + AppState, MAX_OPENAI_CHOICES, collect_output, indexed_egress_stream, openai_error, + streaming_error, submit_generation, unix_seconds_u32, +}; +use crate::ids::Rid; +use crate::message::{ + ChunkEvent, ChunkExtras, EgressItem, GenerateRequest, Matched, OneOrMany, RequestKind, + SamplingParams, TokenIds, +}; + +pub(super) fn routes() -> Router { + Router::new().route("/v1/completions", post(completions)) +} + +#[derive(Debug, PartialEq, Eq)] +enum PromptSpec { + Text(String), + TokenIds(TokenIds), +} + +pub(super) struct SubmittedChoice { + pub(super) index: usize, + pub(super) prompt_index: usize, + pub(super) rid: Rid, + pub(super) echo: String, + pub(super) rx: mpsc::Receiver, +} +#[derive(Debug, Default)] +pub(super) struct ChoiceExtensions { + matched_stop: Option, + /// Dynamo's enum covers the standard values. Python additionally exposes + /// `abort`, and native unknown finish types are preserved rather than lost. + finish_reason_override: Option, +} + +async fn completions( + State(state): State, + body: Result, JsonRejection>, +) -> Response { + let request = match body { + Ok(Json(request)) => request, + Err(rejection) => { + return openai_error(StatusCode::BAD_REQUEST, rejection.body_text()); + } + }; + let stream = request.stream.unwrap_or(false); + let echo = request.echo.unwrap_or(false); + let model = request.model.clone(); + if model != state.server_args.served_model_name { + return openai_error( + StatusCode::BAD_REQUEST, + format!("The model `{model}` does not exist"), + ); + } + + if request.prompt_embeds.is_some() { + return openai_error( + StatusCode::BAD_REQUEST, + "prompt_embeds is not supported by the Rust frontend", + ); + } + if request.suffix.is_some() { + return openai_error( + StatusCode::BAD_REQUEST, + "suffix is not supported by this model", + ); + } + if request.best_of.is_some_and(|best_of| best_of != 1) { + return openai_error( + StatusCode::BAD_REQUEST, + "best_of values greater than 1 are not supported", + ); + } + if request.max_tokens == Some(0) { + return openai_error(StatusCode::BAD_REQUEST, "max_tokens must be positive"); + } + if request.n == Some(0) { + return openai_error(StatusCode::BAD_REQUEST, "n must be at least 1"); + } + let prompts = match completion_prompt_specs(&request.prompt) { + Ok(prompts) => prompts, + Err(message) => return openai_error(StatusCode::BAD_REQUEST, message), + }; + let mut sampling = match completion_sampling_params(&request) { + Ok(sampling) => sampling, + Err(message) => return openai_error(StatusCode::BAD_REQUEST, message), + }; + if let Err(error) = sampling.normalize( + state.server_args.skip_tokenizer_init, + state + .server_args + .model_config + .vocab_size + .unwrap_or(u64::MAX), + ) { + return openai_error(StatusCode::BAD_REQUEST, error.to_string()); + } + + let n = request.n.unwrap_or(1) as usize; + let choice_count = match prompts.len().checked_mul(n) { + Some(count) if count <= MAX_OPENAI_CHOICES => count, + _ => { + return openai_error( + StatusCode::BAD_REQUEST, + format!("prompt count times n exceeds the maximum of {MAX_OPENAI_CHOICES}"), + ); + } + }; + let response_id = format!("cmpl-{}", uuid::Uuid::new_v4().simple()); + let created = unix_seconds_u32(); + let mut guard = AbortGuard::new_empty(state.senders.clone()); + let mut submitted = Vec::with_capacity(choice_count); + + for (prompt_index, prompt) in prompts.into_iter().enumerate() { + let (text, input_ids, mut prompt_echo) = match prompt { + PromptSpec::Text(text) => { + let prompt_echo = if echo { text.clone() } else { String::new() }; + (Some(text), None, prompt_echo) + } + PromptSpec::TokenIds(input_ids) => (None, Some(input_ids), String::new()), + }; + for sample_index in 0..n { + let index = prompt_index * n + sample_index; + let rid = Rid::from_client(&format!("{response_id}-{index}")); + if echo + && sample_index == 0 + && let Some(token_ids) = &input_ids + { + prompt_echo = match decode_prompt_echo(&state, token_ids.clone()).await { + Ok(echo) => echo, + Err(response) => return response, + }; + } + let native = GenerateRequest { + rid: rid.clone(), + text: text.clone(), + input_ids: input_ids.clone(), + sampling_params: sampling.clone(), + stream, + return_logprob: request.logprobs.is_some(), + logprob_start_len: if echo && request.logprobs.is_some() { + 0 + } else { + -1 + }, + top_logprobs_num: request.logprobs.unwrap_or(0) as i64, + return_text_in_logprobs: request.logprobs.map(|_| true), + ..Default::default() + }; + let rx = match submit_generation(&state, native, stream, &mut guard).await { + Ok(rx) => rx, + Err(response) => return response, + }; + submitted.push(SubmittedChoice { + index, + prompt_index, + rid, + echo: prompt_echo.clone(), + rx, + }); + } + } + + if stream { + let include_usage = request + .stream_options + .map(|o| o.include_usage) + .unwrap_or(false) + || state.server_args.stream_response_default_include_usage; + let continuous_usage = request + .stream_options + .map(|o| o.continuous_usage_stats) + .unwrap_or(false); + let want_logprobs = request.logprobs.is_some(); + let s = completion_event_stream( + submitted, + guard, + response_id, + model, + created, + echo, + want_logprobs, + include_usage, + continuous_usage, + ) + .map(|data| Ok::<_, Infallible>(Event::default().data(data))); + Sse::new(s).into_response() + } else { + unary_completion( + submitted, + guard, + response_id, + model, + created, + echo, + request.logprobs.is_some(), + ) + .await + } +} + +/// Decode a token-id prompt back to text for `echo=true`, via a +/// `RequestKind::Detokenize` request through the regular submit path — the +/// detok stage answers it with a single `Data` payload (the raw UTF-8 text), +/// or an `Error` (e.g. out-of-range ids → `Validation` → 400). +async fn decode_prompt_echo(state: &AppState, token_ids: TokenIds) -> Result { + let Ok((_rid, mut rx)) = submit(state, RequestKind::Detokenize { token_ids }, false).await + else { + // Same rule as `submit_generation`: rebuild the refusal in the OpenAI + // error shape rather than forwarding the native-shaped response. + return Err(openai_error( + StatusCode::SERVICE_UNAVAILABLE, + "service unavailable", + )); + }; + match rx.recv().await { + Some(EgressItem::Data(payload)) => String::from_utf8(payload.to_vec()).map_err(|_| { + openai_error( + StatusCode::INTERNAL_SERVER_ERROR, + "detokenized prompt is not valid UTF-8", + ) + }), + Some(EgressItem::Error(crate::error::Error::Validation(message))) => { + Err(openai_error(StatusCode::BAD_REQUEST, message)) + } + Some(EgressItem::Error(error)) => { + let status = StatusCode::from_u16(error.http_status()) + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + Err(openai_error( + status, + format!("failed to decode prompt for echo: {error}"), + )) + } + Some(_) | None => Err(openai_error( + StatusCode::INTERNAL_SERVER_ERROR, + "failed to decode prompt for echo: reply channel closed", + )), + } +} + +fn completion_prompt_specs(prompt: &Prompt) -> Result, String> { + match prompt { + Prompt::String(text) => { + if text.is_empty() { + return Err("Prompt cannot be empty".into()); + } + Ok(vec![PromptSpec::Text(text.clone())]) + } + Prompt::StringArray(texts) => { + if texts.is_empty() || texts.iter().any(String::is_empty) { + return Err("Prompt cannot be empty".into()); + } + Ok(texts.iter().cloned().map(PromptSpec::Text).collect()) + } + Prompt::IntegerArray(ids) => Ok(vec![token_prompt_spec(ids)?]), + Prompt::ArrayOfIntegerArray(prompts) => { + if prompts.is_empty() { + return Err("Prompt cannot be empty".into()); + } + prompts.iter().map(|ids| token_prompt_spec(ids)).collect() + } + } +} + +fn token_prompt_spec(ids: &[u32]) -> Result { + if ids.is_empty() { + return Err("Prompt cannot be empty".into()); + } + let input_ids = ids + .iter() + .map(|&id| i32::try_from(id).map_err(|_| format!("Token ID {id} is out of range"))) + .collect::, _>>()?; + Ok(PromptSpec::TokenIds(input_ids)) +} + +fn completion_sampling_params(request: &CreateCompletionRequest) -> Result { + let mut stop = None; + let mut stop_token_ids = None; + match request.stop.as_ref() { + Some(Stop::String(value)) => stop = Some(OneOrMany::One(value.clone())), + Some(Stop::StringArray(values)) => stop = Some(OneOrMany::Many(values.clone())), + Some(Stop::TokenIdArray(values)) => { + stop_token_ids + .get_or_insert_with(Vec::new) + .extend(values.iter().map(|&id| id as i64)); + } + None => {} + } + + let mut logit_bias = BTreeMap::new(); + if let Some(values) = request.logit_bias.as_ref() { + for (token, bias) in values { + let bias = bias + .as_f64() + .ok_or_else(|| format!("logit_bias[{token:?}] must be a number"))?; + logit_bias.insert(token.clone(), bias); + } + } + + Ok(SamplingParams { + max_new_tokens: Some(request.max_tokens.unwrap_or(16) as i64), + stop, + stop_token_ids, + temperature: request.temperature.unwrap_or(1.0) as f64, + top_p: request.top_p.unwrap_or(1.0) as f64, + frequency_penalty: request.frequency_penalty.unwrap_or(0.0) as f64, + presence_penalty: request.presence_penalty.unwrap_or(0.0) as f64, + // OpenAI `n` is implemented by fan-out: every native request has one + // output, avoiding the native path's intentional `n > 1` rejection. + n: 1, + logit_bias: (!logit_bias.is_empty()).then_some(logit_bias), + sampling_seed: request.seed, + ..Default::default() + }) +} + +pub(super) async fn unary_completion( + submitted: Vec, + mut guard: AbortGuard, + response_id: String, + model: String, + created: u32, + echo: bool, + want_logprobs: bool, +) -> Response { + // Every request is already submitted, so draining in choice order does not + // serialize generation. The non-streaming native path sends one terminal + // result, and the accumulator also tolerates intermediate frames. + let mut choices = Vec::with_capacity(submitted.len()); + let mut extensions = Vec::with_capacity(submitted.len()); + let mut prompt_tokens = BTreeMap::::new(); + let mut completion_tokens = 0u64; + + for choice in submitted { + let output = match collect_output(choice.rx, &mut guard, &choice.rid).await { + Ok(output) => output, + Err((status, message)) => return openai_error(status, message), + }; + + prompt_tokens + .entry(choice.prompt_index) + .or_insert(output.prompt_tokens); + completion_tokens = completion_tokens.saturating_add(output.completion_tokens); + let (response_choice, extension) = completion_choice( + choice.index, + if echo { + choice.echo + &output.text + } else { + output.text.clone() + }, + &output, + want_logprobs, + echo, + ); + choices.push(response_choice); + extensions.push(extension); + } + + let prompt_tokens = prompt_tokens + .values() + .copied() + .fold(0u32, u32::saturating_add); + let usage = completion_usage( + prompt_tokens, + u32::try_from(completion_tokens).unwrap_or(u32::MAX), + ); + + Json(completion_response_value( + CreateCompletionResponse { + id: response_id, + choices, + created, + model, + system_fingerprint: None, + object: "text_completion".into(), + usage: Some(usage), + }, + &extensions, + )) + .into_response() +} + +fn completion_choice( + index: usize, + text: String, + output: &ChunkEvent, + want_logprobs: bool, + include_input_logprobs: bool, +) -> (Choice, ChoiceExtensions) { + let reason = output.finish_reason.as_ref(); + let (finish_reason, finish_reason_override) = { + match reason.and_then(|reason| reason.kind_name()) { + Some("stop") => (Some(CompletionFinishReason::Stop), None), + Some("length") => (Some(CompletionFinishReason::Length), None), + Some("content_filter") => (Some(CompletionFinishReason::ContentFilter), None), + Some(other) => (None, Some(other.into())), + None => (None, None), + } + }; + let matched_stop = reason + .and_then(|reason| reason.matched()) + .map(|matched| match matched { + Matched::Token(id) => serde_json::json!(id), + Matched::Str(value) => serde_json::json!(value), + // Python's OpenAI schema supports an integer or string here, not a + // multi-token list. Preserve the native value rather than dropping it. + Matched::Tokens(ids) => serde_json::json!(ids), + }); + ( + Choice { + text, + index: u32::try_from(index).unwrap_or(u32::MAX), + logprobs: want_logprobs + .then(|| completion_logprobs(output.extras.as_deref(), include_input_logprobs)), + finish_reason, + }, + ChoiceExtensions { + matched_stop, + finish_reason_override, + }, + ) +} + +/// Serialize Dynamo's standard response and add only SGLang/Python fields that +/// its schema cannot represent. `text_offset` is corrected here because Dynamo +/// types it as `u32`, while Python deliberately emits `-1`. +pub(super) fn completion_response_value( + response: CreateCompletionResponse, + extensions: &[ChoiceExtensions], +) -> serde_json::Value { + let mut value = serde_json::to_value(response).expect("OpenAI response must serialize"); + let Some(root) = value.as_object_mut() else { + return value; + }; + // Python's Completion response does not expose this OpenAI field. + root.remove("system_fingerprint"); + let Some(choices) = root + .get_mut("choices") + .and_then(serde_json::Value::as_array_mut) + else { + return value; + }; + for (choice, extension) in choices.iter_mut().zip(extensions) { + let Some(choice) = choice.as_object_mut() else { + continue; + }; + if let Some(reason) = &extension.finish_reason_override { + choice.insert("finish_reason".into(), serde_json::json!(reason)); + } + choice.insert( + "matched_stop".into(), + extension + .matched_stop + .clone() + .unwrap_or(serde_json::Value::Null), + ); + if let Some(logprobs) = choice + .get_mut("logprobs") + .and_then(serde_json::Value::as_object_mut) + { + let count = logprobs + .get("tokens") + .and_then(serde_json::Value::as_array) + .map_or(0, Vec::len); + logprobs.insert("text_offset".into(), serde_json::json!(vec![-1; count])); + } + } + value +} + +#[allow(clippy::too_many_arguments)] +pub(super) fn completion_event_stream( + submitted: Vec, + mut guard: AbortGuard, + response_id: String, + model: String, + created: u32, + echo: bool, + want_logprobs: bool, + include_usage: bool, + continuous_usage: bool, +) -> impl futures::Stream { + async_stream::stream! { + let count = submitted.len(); + let mut rids = Vec::with_capacity(count); + let mut prompt_indexes = Vec::with_capacity(count); + let mut echoes = Vec::with_capacity(count); + let mut first_chunks = vec![true; count]; + let mut prompt_tokens_by_prompt = BTreeMap::::new(); + let mut completion_tokens_by_choice = vec![0u64; count]; + let mut streams = Vec::with_capacity(count); + + for choice in submitted { + let index = choice.index; + rids.push(choice.rid); + prompt_indexes.push(choice.prompt_index); + echoes.push(choice.echo); + streams.push(indexed_egress_stream(index, choice.rx)); + } + let mut events = futures::stream::select_all(streams); + + while let Some((index, item)) = events.next().await { + let Some(item) = item else { + yield streaming_error(500, "response truncated before completion"); + continue; + }; + let output = match item { + EgressItem::Frame(output) => output, + EgressItem::Done(output) => { + guard.disarm(&rids[index]); + output + } + EgressItem::Error(error) => { + guard.disarm(&rids[index]); + yield streaming_error(error.http_status(), error.to_string()); + continue; + } + EgressItem::Control(_) | EgressItem::Data(_) => continue, + }; + + if let Some((code, message)) = output + .finish_reason + .as_ref() + .and_then(|reason| reason.abort_status()) + { + yield streaming_error(code, message); + continue; + } + + prompt_tokens_by_prompt + .entry(prompt_indexes[index]) + .or_insert(output.prompt_tokens); + completion_tokens_by_choice[index] = completion_tokens_by_choice[index] + .saturating_add(output.completion_tokens); + let first = std::mem::replace(&mut first_chunks[index], false); + let text = if echo && first { + echoes[index].clone() + &output.text + } else { + output.text.clone() + }; + let chunk_usage = continuous_usage.then(|| { + completion_usage( + output.prompt_tokens, + u32::try_from(completion_tokens_by_choice[index]).unwrap_or(u32::MAX), + ) + }); + let (choice, extension) = completion_choice( + index, + text, + &output, + want_logprobs, + echo && first, + ); + let chunk = CreateCompletionResponse { + id: response_id.clone(), + choices: vec![choice], + created, + model: model.clone(), + system_fingerprint: None, + object: "text_completion".into(), + usage: chunk_usage, + }; + yield completion_response_value(chunk, &[extension]).to_string(); + } + + if include_usage { + let prompt_tokens = prompt_tokens_by_prompt + .values() + .copied() + .fold(0u32, u32::saturating_add); + let completion_tokens = completion_tokens_by_choice + .into_iter() + .fold(0u64, u64::saturating_add); + let final_chunk = CreateCompletionResponse { + id: response_id, + choices: vec![], + created, + model, + system_fingerprint: None, + object: "text_completion".into(), + usage: Some(completion_usage( + prompt_tokens, + u32::try_from(completion_tokens).unwrap_or(u32::MAX), + )), + }; + yield completion_response_value(final_chunk, &[]).to_string(); + } + yield "[DONE]".to_string(); + } +} + +pub(super) fn completion_usage(prompt_tokens: u32, completion_tokens: u32) -> CompletionUsage { + CompletionUsage { + prompt_tokens, + completion_tokens, + total_tokens: prompt_tokens.saturating_add(completion_tokens), + ..Default::default() + } +} + +pub(super) fn completion_logprobs(extras: Option<&ChunkExtras>, include_input: bool) -> Logprobs { + let mut result = Logprobs { + tokens: Vec::new(), + token_logprobs: Vec::new(), + top_logprobs: Vec::new(), + text_offset: Vec::new(), + }; + let Some(extras) = extras else { + return result; + }; + if include_input { + append_selected_logprobs( + &mut result, + &extras.in_lp_val, + &extras.in_lp_idx, + &extras.in_lp_txt, + ); + append_top_logprobs( + &mut result, + &extras.in_top_val, + &extras.in_top_idx, + &extras.in_top_lens, + &extras.in_top_txt, + ); + } + append_selected_logprobs( + &mut result, + &extras.out_lp_val, + &extras.out_lp_idx, + &extras.out_lp_txt, + ); + append_top_logprobs( + &mut result, + &extras.out_top_val, + &extras.out_top_idx, + &extras.out_top_lens, + &extras.out_top_txt, + ); + result +} + +fn append_selected_logprobs(result: &mut Logprobs, values: &[f32], ids: &[i32], texts: &[String]) { + for (index, (&value, &id)) in values.iter().zip(ids).enumerate() { + result.tokens.push( + texts + .get(index) + .cloned() + .unwrap_or_else(|| format!("token_id:{id}")), + ); + result + .token_logprobs + .push((!value.is_nan()).then_some(value)); + // Dynamo's field is `u32`; Python's `-1` sentinel is applied once at + // final wire shaping in `completion_response_value`. + result.text_offset.push(0); + } +} + +fn append_top_logprobs( + result: &mut Logprobs, + values: &[f32], + ids: &[i32], + lens: &[u32], + texts: &[String], +) { + let mut offset = 0usize; + for &len in lens { + let len = len as usize; + if len == 0 { + result.top_logprobs.push(serde_json::Value::Null); + continue; + } + let mut top = BTreeMap::new(); + for index in offset..offset.saturating_add(len) { + let (Some(&value), Some(&id)) = (values.get(index), ids.get(index)) else { + continue; + }; + top.insert( + texts + .get(index) + .cloned() + .unwrap_or_else(|| format!("token_id:{id}")), + value, + ); + } + result.top_logprobs.push(serde_json::json!(top)); + offset = offset.saturating_add(len); + } +} + +#[cfg(test)] +mod tests { + use super::super::test_utils::{chunk, senders, submitted}; + use super::{ + ChoiceExtensions, PromptSpec, completion_event_stream, completion_logprobs, + completion_prompt_specs, completion_response_value, unary_completion, + }; + use crate::api_server::guard::AbortGuard; + use crate::message::ChunkExtras; + use axum::http::StatusCode; + use dynamo_protocols::types::{ + Choice, CreateCompletionRequest, CreateCompletionResponse, Prompt, + }; + use futures::StreamExt; + + #[test] + fn dynamo_completion_request_deserializes_directly() { + let request: CreateCompletionRequest = serde_json::from_value(serde_json::json!({ + "model": "m", + "prompt": ["a", "b"], + "max_tokens": 8, + "n": 2, + "stream_options": { + "include_usage": true, + "continuous_usage_stats": true + } + })) + .unwrap(); + assert!(matches!(request.prompt, Prompt::StringArray(_))); + assert_eq!(request.n, Some(2)); + assert!(request.stream_options.unwrap().continuous_usage_stats); + } + + #[test] + fn max_tokens_zero_is_rejected_before_submission() { + let request: CreateCompletionRequest = serde_json::from_value(serde_json::json!({ + "model": "m", + "prompt": "hello", + "max_tokens": 0 + })) + .unwrap(); + assert_eq!(request.max_tokens, Some(0)); + } + + #[test] + fn token_prompt_is_normalized_without_echo_state() { + let specs = completion_prompt_specs(&Prompt::IntegerArray(vec![1, 2])).unwrap(); + assert_eq!(specs, [PromptSpec::TokenIds(vec![1, 2])]); + } + + #[test] + fn zero_top_logprobs_keeps_selected_token_and_empty_top_map() { + let extras = ChunkExtras { + out_lp_val: vec![-0.25], + out_lp_idx: vec![7], + out_lp_txt: vec!["x".into()], + out_top_lens: vec![0], + ..Default::default() + }; + let logprobs = completion_logprobs(Some(&extras), false); + assert_eq!(logprobs.tokens, ["x"]); + assert_eq!(logprobs.token_logprobs, [Some(-0.25)]); + assert_eq!(logprobs.top_logprobs, [serde_json::Value::Null]); + + let value = completion_response_value( + CreateCompletionResponse { + id: "cmpl-test".into(), + choices: vec![Choice { + text: "x".into(), + index: 0, + logprobs: Some(logprobs), + finish_reason: None, + }], + created: 1, + model: "model".into(), + system_fingerprint: None, + object: "text_completion".into(), + usage: None, + }, + &[ChoiceExtensions::default()], + ); + assert_eq!( + value["choices"][0]["logprobs"]["text_offset"], + serde_json::json!([-1]) + ); + } + + #[tokio::test] + async fn unary_fold_orders_choices_and_counts_each_prompt_once() { + let (choice0, tx0) = submitted(0, 0, "r0"); + let (choice1, tx1) = submitted(1, 0, "r1"); + tx0.send(chunk("r0", "a", false)).await.unwrap(); + tx0.send(chunk("r0", "b", true)).await.unwrap(); + tx1.send(chunk("r1", "x", false)).await.unwrap(); + tx1.send(chunk("r1", "y", true)).await.unwrap(); + + let response = unary_completion( + vec![choice0, choice1], + AbortGuard::new_empty(senders()), + "cmpl-test".into(), + "model".into(), + 1, + false, + false, + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), 64 * 1024) + .await + .unwrap(); + let value: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(value["choices"][0]["text"], "ab"); + assert_eq!(value["choices"][1]["text"], "xy"); + assert_eq!(value["choices"][0]["matched_stop"], ""); + assert_eq!(value["usage"]["prompt_tokens"], 5); + assert_eq!(value["usage"]["completion_tokens"], 4); + } + + #[tokio::test] + async fn stream_uses_deltas_then_usage_and_done() { + let (choice, tx) = submitted(0, 0, "r0"); + tx.send(chunk("r0", "a", false)).await.unwrap(); + tx.send(chunk("r0", "b", true)).await.unwrap(); + + let stream = completion_event_stream( + vec![choice], + AbortGuard::new_empty(senders()), + "cmpl-test".into(), + "model".into(), + 1, + false, + false, + true, + false, + ); + futures::pin_mut!(stream); + let frames: Vec = stream.collect().await; + assert_eq!(frames.len(), 4); + let first: serde_json::Value = serde_json::from_str(&frames[0]).unwrap(); + let terminal: serde_json::Value = serde_json::from_str(&frames[1]).unwrap(); + let usage: serde_json::Value = serde_json::from_str(&frames[2]).unwrap(); + assert_eq!(first["choices"][0]["text"], "a"); + assert_eq!(terminal["choices"][0]["text"], "b"); + assert_eq!(terminal["choices"][0]["finish_reason"], "stop"); + assert!(usage["choices"].as_array().unwrap().is_empty()); + assert_eq!(usage["usage"]["prompt_tokens"], 5); + assert_eq!(usage["usage"]["completion_tokens"], 2); + assert_eq!(frames[3], "[DONE]"); + } +} diff --git a/rust/sglang-server/src/api_server/openai/models.rs b/rust/sglang-server/src/api_server/openai/models.rs new file mode 100644 index 000000000..22f78f9b9 --- /dev/null +++ b/rust/sglang-server/src/api_server/openai/models.rs @@ -0,0 +1,47 @@ +//! OpenAI model discovery endpoints. + +use axum::{ + Json, Router, + extract::{Path, State}, + http::StatusCode, + response::{IntoResponse, Response}, + routing::get, +}; + +use super::{AppState, openai_error, unix_seconds_u32}; + +pub(super) fn routes() -> Router { + Router::new() + .route("/v1/models", get(available_models)) + .route("/v1/models/{model}", get(retrieve_model)) +} + +/// `GET /v1/models` — OpenAI-compatible model list. Served from `server_args`; +/// no scheduler round-trip. +async fn available_models(State(state): State) -> Response { + let base = model_card(&state); + Json(serde_json::json!({ "object": "list", "data": [base] })).into_response() +} + +async fn retrieve_model(State(state): State, Path(model): Path) -> Response { + if model != state.server_args.served_model_name { + return openai_error( + StatusCode::NOT_FOUND, + format!("The model `{model}` does not exist"), + ); + } + Json(model_card(&state)).into_response() +} + +fn model_card(state: &AppState) -> serde_json::Value { + let name = &state.server_args.served_model_name; + serde_json::json!({ + "id": name, + "object": "model", + "created": unix_seconds_u32(), + "owned_by": "sglang", + "root": name, + "parent": serde_json::Value::Null, + "max_model_len": state.server_args.model_config.context_len, + }) +} diff --git a/rust/sglang-server/src/api_server/openai/reasoning.rs b/rust/sglang-server/src/api_server/openai/reasoning.rs new file mode 100644 index 000000000..3533e6a6d --- /dev/null +++ b/rust/sglang-server/src/api_server/openai/reasoning.rs @@ -0,0 +1,216 @@ +//! Reasoning-content splitting for Chat Completions (`--reasoning-parser`). +//! +//! Mirrors the Python frontend (`sglang.srt.parser.reasoning_parser` + +//! `serving_chat._process_reasoning_stream`): when the +//! server was launched with `--reasoning-parser ` (and the request keeps +//! the default `separate_reasoning=true`, which the Dynamo request type cannot +//! express), the model's ``-style markers are stripped out of `content` +//! into `reasoning_content` — for unary responses and streaming deltas alike. +//! +//! The parser lifecycle (lazy build, per-frame incremental split, terminal +//! flush of *both* buffered columns) lives here so the endpoint cannot drop +//! the tail half. + +use dynamo_parsers::reasoning::{ + ReasoningParser as _, ReasoningParserType, ReasoningParserWrapper, +}; + +/// Build the parser the Python `--reasoning-parser` name selects. +/// +/// The names come from Python's `ReasoningParser.DetectorMap`, which differs +/// from the dynamo-parsers registry keys in a few spellings (deepseek-r1 vs +/// deepseek_r1, kimi_k2 vs kimi_k25, …) and has a few entries that Python maps +/// onto a forced-reasoning `` parser (qwen3-thinking, minimax). Names +/// dynamo does not know (hunyuan, inkling, apertus2509, mimo, poolside_v1, +/// cohere_command4 — all tokenizer-driven parsers) fall through to the +/// registry, which warns and falls back to the non-forced Basic parser. +pub(super) fn build_reasoning_parser(server_name: &str) -> ReasoningParserWrapper { + let name = match server_name { + // Python DetectorMap spellings that differ from the dynamo registry keys. + "deepseek-r1" | "step3p5" => "deepseek_r1", + "kimi_k2" => "kimi_k25", + "gpt-oss" => "gpt_oss", + "nemotron_3" => "nemotron3", + "interns1" => "qwen3", + // Python forces reasoning for these; the R1 parser is the same + // `` / `` configuration with `force_reasoning=true`. + "qwen3-thinking" | "minimax" => "deepseek_r1", + _ => server_name, + }; + ReasoningParserType::get_reasoning_parser_from_name(name) +} + +/// Split a completed generation's text into `(reasoning_text, normal_text)` +/// when `--reasoning-parser` selects a parser; otherwise the text passes +/// through untouched as normal text. Chat splits before tool-call parsing. +pub(super) fn split_reasoning_unary( + name: Option<&str>, + text: &str, + token_ids: &[i32], +) -> (String, String) { + let Some(name) = name else { + return (String::new(), text.to_owned()); + }; + let mut parser = build_reasoning_parser(name); + let token_ids = token_ids + .iter() + .filter_map(|&id| u32::try_from(id).ok()) + .collect::>(); + let split = parser.detect_and_parse_reasoning(text, &token_ids); + (split.reasoning_text, split.normal_text) +} + +/// Stateful reasoning split for one streaming response. Mirrors Python's +/// `reasoning_parser_dict` entries: the parser is built lazily on the first +/// content delta, each frame is split into `(reasoning, normal)` deltas, and +/// [`finish`](Self::finish) flushes the parser-buffered tail — *both* columns, +/// since the buffered text can sit in either one (e.g. MiniMax M3's +/// implicit-tool-start recovery holds the leading answer text until the think +/// opener or a tool marker establishes the mode, and releases it as normal +/// text at EOF). +#[derive(Default)] +pub(super) struct ReasoningStreamSplitter { + name: Option, + parser: Option, +} + +impl ReasoningStreamSplitter { + pub(super) fn new(name: Option<&str>) -> Self { + Self { + name: name.map(str::to_owned), + parser: None, + } + } + + /// Split one frame's text into `(reasoning_text, normal_text)` deltas. + pub(super) fn split(&mut self, text: &str, token_ids: &[i32]) -> (String, String) { + let Some(name) = self.name.as_deref() else { + return (String::new(), text.to_owned()); + }; + let parser = self + .parser + .get_or_insert_with(|| build_reasoning_parser(name)); + let token_ids = token_ids + .iter() + .filter_map(|&id| u32::try_from(id).ok()) + .collect::>(); + let split = parser.parse_reasoning_streaming_incremental(text, &token_ids); + (split.reasoning_text, split.normal_text) + } + + /// Flush the parser-buffered tail at stream end, releasing both columns. + pub(super) fn finish(&mut self) -> (String, String) { + let Some(parser) = self.parser.as_mut() else { + return (String::new(), String::new()); + }; + let tail = parser.finish_reasoning_stream(); + (tail.reasoning_text, tail.normal_text) + } +} + +#[cfg(test)] +mod tests { + use super::{ReasoningStreamSplitter, build_reasoning_parser, split_reasoning_unary}; + use dynamo_parsers::reasoning::ReasoningParser; + + #[test] + fn python_deepseek_r1_name_splits_forced_reasoning() { + let mut parser = build_reasoning_parser("deepseek-r1"); + // Forced: text before any marker is reasoning. + let split = parser.detect_and_parse_reasoning("think hardParis", &[]); + assert_eq!(split.reasoning_text, "think hard"); + assert_eq!(split.normal_text, "Paris"); + let split = parser.detect_and_parse_reasoning("yesanswer", &[]); + assert_eq!(split.reasoning_text, "yes"); + assert_eq!(split.normal_text, "answer"); + } + + #[test] + fn python_kimi_k2_name_maps_to_kimi_k25() { + let mut parser = build_reasoning_parser("kimi_k2"); + let split = parser.detect_and_parse_reasoning("kout", &[]); + assert_eq!(split.reasoning_text, "k"); + assert_eq!(split.normal_text, "out"); + // Kimi-K2.5 interrupts reasoning at the tool-call section marker. + let mut parser = build_reasoning_parser("kimi_k2"); + let split = + parser.detect_and_parse_reasoning("reasons<|tool_calls_section_begin|>calls", &[]); + assert_eq!(split.reasoning_text, "reasons"); + assert_eq!(split.normal_text, "<|tool_calls_section_begin|>calls"); + } + + #[test] + fn qwen3_thinking_forces_reasoning_like_python() { + let mut parser = build_reasoning_parser("qwen3-thinking"); + let split = parser.detect_and_parse_reasoning("plain text", &[]); + assert_eq!(split.reasoning_text, "plain text"); + assert_eq!(split.normal_text, ""); + } + + #[test] + fn streaming_split_keeps_markers_out_of_both_columns() { + let mut parser = build_reasoning_parser("deepseek-r1"); + let mut reasoning = String::new(); + let mut normal = String::new(); + for chunk in ["rea", "sonan", "swer"] { + let split = parser.parse_reasoning_streaming_incremental(chunk, &[]); + reasoning.push_str(&split.reasoning_text); + normal.push_str(&split.normal_text); + } + let tail = parser.finish_reasoning_stream(); + reasoning.push_str(&tail.reasoning_text); + normal.push_str(&tail.normal_text); + assert_eq!(reasoning, "reason"); + assert_eq!(normal, "answer"); + } + + #[test] + fn unary_split_passes_text_through_without_a_parser() { + let (reasoning, normal) = split_reasoning_unary(None, "kept as text", &[1]); + assert_eq!(reasoning, ""); + assert_eq!(normal, "kept as text"); + } + + /// REASONING_P1: MiniMax M3's implicit-tool-start recovery buffers the + /// answer text until a boundary establishes the mode; with no opener the + /// whole buffer is released as normal text only at `finish`. The chat + /// terminal flush must emit the normal half of the tail. + #[test] + fn streaming_tail_releases_normal_text_only_at_finish() { + let mut splitter = ReasoningStreamSplitter::new(Some("minimax_m3")); + let (reasoning, normal) = splitter.split("The answer is", &[]); + assert_eq!(reasoning, ""); + assert_eq!(normal, "", "M3 holds the ambiguous prefix until a boundary"); + let (reasoning, normal) = splitter.split(" 42", &[]); + assert_eq!(reasoning, ""); + assert_eq!(normal, ""); + let (reasoning, normal) = splitter.finish(); + assert_eq!(reasoning, ""); + assert_eq!(normal, "The answer is 42"); + } + + #[test] + fn streaming_tail_releases_reasoning_after_marker_boundary() { + let mut splitter = ReasoningStreamSplitter::new(Some("minimax_m3")); + let (reasoning, normal) = splitter.split("think", &[]); + assert_eq!(reasoning, "think"); + assert_eq!(normal, ""); + let (reasoning, normal) = splitter.split(" hard", &[]); + assert_eq!(reasoning, " hard"); + assert_eq!(normal, ""); + let (reasoning, normal) = splitter.finish(); + assert_eq!(reasoning, ""); + assert_eq!(normal, ""); + } + + #[test] + fn finish_without_a_parser_is_empty() { + let mut splitter = ReasoningStreamSplitter::new(None); + let (reasoning, normal) = splitter.split("plain", &[]); + assert_eq!(reasoning, ""); + assert_eq!(normal, "plain"); + let (reasoning, normal) = splitter.finish(); + assert_eq!(reasoning, ""); + assert_eq!(normal, ""); + } +} diff --git a/rust/sglang-server/src/api_server/openai/template.rs b/rust/sglang-server/src/api_server/openai/template.rs new file mode 100644 index 000000000..3f832c7ef --- /dev/null +++ b/rust/sglang-server/src/api_server/openai/template.rs @@ -0,0 +1,1868 @@ +//! Resolve chat-template names and files to chat prompt formatters. +// +//! Hugging Face tokenizer configs contain Jinja templates. SGLang also accepts +//! legacy conversation JSON files and the names in Python's template registry. +//! Legacy definitions are rendered by a native port of Python's +//! `Conversation.get_prompt()` so there is exactly one implementation of the +//! per-style formatting logic (no Jinja translation to drift). + +use std::path::{Path, PathBuf}; + +use dynamo_protocols::types::{ + ChatCompletionRequestAssistantMessageContent, ChatCompletionRequestAssistantMessageContentPart, + ChatCompletionRequestMessage, ChatCompletionRequestSystemMessageContent, + ChatCompletionRequestSystemMessageContentPart, ChatCompletionRequestUserMessageContent, + ChatCompletionRequestUserMessageContentPart, CreateChatCompletionRequest, +}; +use dynamo_renderer::{ChatTemplate, ContextMixins, PromptContextMixin, PromptFormatter}; +use serde_json::Value; +use thiserror::Error; + +use crate::message::OneOrMany; + +const SUPPORTED_STYLES: &[&str] = &[ + "ADD_COLON_SINGLE", + "ADD_COLON_TWO", + "ADD_COLON_SPACE_SINGLE", + "NO_COLON_SINGLE", + "NO_COLON_TWO", + "ADD_NEW_LINE_SINGLE", + "LLAMA2", + "LLAMA3", + "LLAMA4", + "CHATGLM", + "CHATML", + "CHATINTERN", + "DOLLY", + "RWKV", + "PHOENIX", + "ROBIN", + "FALCON_CHAT", + "CHATGLM3", + "DEEPSEEK_CHAT", + "METAMATH", + "DeepSeekVL2", + "QWEN2_VL_EMBED", + "QWEN2_AUDIO", + "GEMMA3", + "MPT", + "PADDLE_OCR", + "UNLIMITED_OCR", +]; + +/// A chat prompt formatter: either the model's HuggingFace Jinja template or a +/// legacy SGLang conversation template. +#[derive(Clone)] +pub enum ChatFormatter { + HuggingFace(PromptFormatter), + Legacy(Box), +} + +impl ChatFormatter { + /// Render the request's messages to a single prompt string. + pub(super) fn render( + &self, + request: &CreateChatCompletionRequest, + ) -> Result { + match self { + ChatFormatter::HuggingFace(formatter) => { + let PromptFormatter::OAI(formatter) = formatter; + formatter + .render(request) + .map_err(|error| TemplateError::Renderer { + message: error.to_string(), + }) + } + ChatFormatter::Legacy(formatter) => formatter.render(request), + } + } + + /// The template's stop strings — Python `Conversation.stop_str` + /// (`str | list[str] | None`). Legacy/builtin templates define them (e.g. + /// chatml's `<|im_end|>`); the HuggingFace renderer carries none, matching + /// Python's jinja path, which keeps only the request's own stops. + pub(super) fn stop_strs(&self) -> Option> { + match self { + ChatFormatter::HuggingFace(_) => None, + ChatFormatter::Legacy(formatter) => formatter.spec.stop_str.clone(), + } + } +} + +/// A legacy conversation template, mirroring Python's `Conversation` fields. +#[derive(Debug, Clone)] +pub(super) struct LegacySpec { + /// Python `Conversation.name` — drives the CHATGLM round-offset quirk. + pub(super) name: String, + pub(super) system_template: String, + pub(super) system_message: String, + /// `(user_role, assistant_role)` — Python `Conversation.roles`. + pub(super) roles: (String, String), + pub(super) style: String, + pub(super) sep: String, + /// `None` = Python's `Conversation.sep2` default. Styles that alternate + /// seps (`seps[i % 2]`) need it set; Python crashes on `None` there and we + /// error deliberately. + pub(super) sep2: Option, + /// Python `Conversation.stop_str` (`str | list[str] | None`). + pub(super) stop_str: Option>, + pub(super) image_token: String, + pub(super) audio_token: String, +} + +impl Default for LegacySpec { + fn default() -> Self { + Self { + name: String::new(), + system_template: String::new(), + system_message: String::new(), + roles: (String::new(), String::new()), + style: String::new(), + sep: String::new(), + sep2: None, + stop_str: None, + image_token: "".into(), + audio_token: "