Compare commits
85
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
216880034a | ||
|
|
6da0a40b02 | ||
|
|
caf3403086 | ||
|
|
617f19ba61 | ||
|
|
ba3318ffbd | ||
|
|
8279d3d6df | ||
|
|
ef4cccb6a0 | ||
|
|
98b950366d | ||
|
|
87ab6c0155 | ||
|
|
0308920bdb | ||
|
|
57db2a4ae9 | ||
|
|
7fd7fd58a5 | ||
|
|
71a72b8fea | ||
|
|
6c9518fcd8 | ||
|
|
3a4ab451ac | ||
|
|
5d38fc96a9 | ||
|
|
261ebbc55b | ||
|
|
53fb9be7ee | ||
|
|
e5be84f795 | ||
|
|
39ceba8a3d | ||
|
|
b9d259441c | ||
|
|
e7dc175e94 | ||
|
|
6612af9e0e | ||
|
|
71c6d8bc55 | ||
|
|
dbc661602e | ||
|
|
054dcee896 | ||
|
|
73a25267ac | ||
|
|
bcad628d30 | ||
|
|
baba18bc7d | ||
|
|
0afcb334a4 | ||
|
|
33e6381b7d | ||
|
|
9c4ff32562 | ||
|
|
7d5759b2d9 | ||
|
|
ac11e9162d | ||
|
|
84fc632088 | ||
|
|
8d9bce30a7 | ||
|
|
5f2ffa70b8 | ||
|
|
f4ef648b16 | ||
|
|
6469cf4197 | ||
|
|
3f8751c6ed | ||
|
|
e4f1310d6f | ||
|
|
a42ab6fce0 | ||
|
|
fa8ad18b5c | ||
|
|
7f1edbba80 | ||
|
|
99c317b131 | ||
|
|
2b2ede8211 | ||
|
|
b4dbcebbec | ||
|
|
30ba5818e7 | ||
|
|
c9bfb24bcc | ||
|
|
3d109d50be | ||
|
|
4eabf5e228 | ||
|
|
7d67d57804 | ||
|
|
491989284f | ||
|
|
53ea8fb2a7 | ||
|
|
efdbfd2cc3 | ||
|
|
8342b89e85 | ||
|
|
c92938d718 | ||
|
|
7578cdd218 | ||
|
|
21f7ce37dd | ||
|
|
236686454c | ||
|
|
1f6ca2cf68 | ||
|
|
bde1fcbf51 | ||
|
|
c4f3e3be33 | ||
|
|
9ef65b51c2 | ||
|
|
6ba34507f1 | ||
|
|
93d610c1fd | ||
|
|
4d0efcc921 | ||
|
|
f71fafa684 | ||
|
|
1068926cd6 | ||
|
|
48fb49baef | ||
|
|
2372cdc06e | ||
|
|
7b815cc98b | ||
|
|
d06c5dde42 | ||
|
|
8d0ee5e51e | ||
|
|
75758cb0a0 | ||
|
|
60e80ac7d1 | ||
|
|
4ef799aaa6 | ||
|
|
1be762368b | ||
|
|
f577b61b52 | ||
|
|
4792b61332 | ||
|
|
62f1ed099e | ||
|
|
bfd9bb2e03 | ||
|
|
6deac260bf | ||
|
|
842ad7c4d4 | ||
|
|
b605d2d370 |
@@ -8,4 +8,17 @@ FROM lmsysorg/sglang:dev-dsv41
|
||||
# checkout 含 .git,setuptools-scm 可打戳(main 系分支 version 显示 dev 属正常)
|
||||
COPY . /sgl-workspace/sglang
|
||||
|
||||
# Rust gateway(sgl-model-gateway)编译,与 b300 离线版 deploy/b300-build/Dockerfile 同构。
|
||||
# 基底自带 rustup stable,RUSTUP_TOOLCHAIN=stable 绕过 rust-toolchain.toml 的 1.90 钉版;
|
||||
# python 扩展必须显式 -p,产物重命名为 sglang_router_rs.abi3.so 覆盖 python 包内同文件。
|
||||
# 海外 runner 直连 crates.io 无网络问题;cache mount 海外环境同样无害。
|
||||
RUN --mount=type=cache,target=/cargo-home \
|
||||
--mount=type=cache,target=/cargo-target \
|
||||
cd /sgl-workspace/sglang/sgl-model-gateway \
|
||||
&& CARGO_HOME=/cargo-home CARGO_TARGET_DIR=/cargo-target RUSTUP_TOOLCHAIN=stable \
|
||||
cargo build --release -p sgl-model-gateway -p sgl-model-gateway-python \
|
||||
&& cp /cargo-target/release/sgl-model-gateway /opt/sglang/bin/sgl-model-gateway \
|
||||
&& cp /cargo-target/release/libsglang_router_rs.so \
|
||||
/opt/sglang/lib/python3.12/site-packages/sglang_router/sglang_router_rs.abi3.so
|
||||
|
||||
RUN pip install --no-cache-dir -e /sgl-workspace/sglang/python
|
||||
|
||||
@@ -1,9 +1,19 @@
|
||||
# 优化版(2026-09-24):解决清华源限流下每次构建重下 torch 的问题。
|
||||
# 优化版(2026-10-08 v3):torch/nvidia 全家桶永远复用基底,全量路径零重下。
|
||||
# 要点:
|
||||
# 1. pyproject build-system.requires 含 torch==2.13.0,pip 隔离构建环境每次都重下 ~6GB。
|
||||
# 基底镜像已装 torch 2.13 → 用 --no-build-isolation 复用,构建环境零下载。
|
||||
# 2. 基底缺 setuptools-scm(版本打戳要用),单层预装 + pip cache mount,只下载一次。
|
||||
# 3. RUN 的 pip cache mount 持久化在宿主机,依赖解析命中的包不再重复下载。
|
||||
# 2. 老全量路径让 pip 解析全部依赖:bare `torchvision` 等会让解析器弃用基底已装的
|
||||
# 本地版本(torch 2.13.0+cu130),改用 PyPI 的 torch==2.13.0,连带重下整套 nvidia
|
||||
# 依赖(9/23 实测 nvidia-nccl-cu13 等 6GB+,限流下 2h40m);且基底 NCCL 是
|
||||
# 刻意装高的 2.30.7(torch 元数据钉 ==2.29.7),任何全量解析都会把它降级回去。
|
||||
# → v3 不再让 pip 碰重型依赖:check_deps.py 在构建期对照 pyproject 与基底
|
||||
# 已装版本,把增量分三类——重型(torch*/nvidia-*/cuda-*)永不进解析、
|
||||
# 不满足即让构建失败并提示决策(显式失败好过静默重装);
|
||||
# kernel wheel(sglang-kernel/flashinfer*/sgl-deep-gemm)要跟随源码版本,
|
||||
# 钉版变动时以 pip --no-deps 单独装(防止顺藤摸瓜解析 torch);
|
||||
# 其余轻量依赖只装未满足的增量。
|
||||
# 3. 基底缺 setuptools-scm(版本打戳要用),单层预装 + pip cache mount,只下载一次。
|
||||
# 4. RUN 的 pip cache mount 持久化在宿主机,轻量依赖增量不再重复下载。
|
||||
FROM uhub.service.ucloud.cn/umirror/sglang:dev-dsv41
|
||||
|
||||
ARG PIP_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
@@ -13,9 +23,55 @@ ENV PIP_INDEX_URL=${PIP_INDEX}
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install "setuptools>=61" "setuptools-scm>=8" "setuptools-rust>=1.11" wheel
|
||||
|
||||
# 依赖增量计算脚本(对照 pyproject 与基底已装版本,产出轻量/kernel 增量清单)
|
||||
# 注意 build context 是 /data/ymk 且 .dockerignore 排除了 build/,只能从 sglang/ 树内 COPY
|
||||
COPY sglang/deploy/b300-build/check_deps.py /tmp/check_deps.py
|
||||
|
||||
# 整个源码树(含 .git,用于 setuptools-scm 版本打戳)
|
||||
COPY sglang/ /sgl-workspace/sglang/
|
||||
|
||||
# editable 安装,全量依赖解析(pyproject.toml 变化时走这个)
|
||||
# Rust gateway(sgl-model-gateway)编译。基底自带 rustup stable(实测 cargo 1.98.1),
|
||||
# RUSTUP_TOOLCHAIN=stable 绕过仓库 rust-toolchain.toml 钉的 1.90(避免重复下载工具链);
|
||||
# b300 直连 crates.io(稀疏索引 + static.crates.io 均通),无需任何镜像源。
|
||||
# cache mount 持久化 registry/target:冷编译 ~15min(二进制 ~11min + python 扩展 ~4min),
|
||||
# 增量重建秒~分钟级。python 扩展必须显式 -p(workspace 根包默认不编译扩展),
|
||||
# 产物 libsglang_router_rs.so 需重命名为 sglang_router_rs.abi3.so 覆盖 python 包内同文件
|
||||
# (与 H20 线上部署方式一致)。2026-10-09 b300 实测验证。
|
||||
RUN --mount=type=cache,target=/cargo-home \
|
||||
--mount=type=cache,target=/cargo-target \
|
||||
cd /sgl-workspace/sglang/sgl-model-gateway \
|
||||
&& CARGO_HOME=/cargo-home CARGO_TARGET_DIR=/cargo-target RUSTUP_TOOLCHAIN=stable \
|
||||
cargo build --release -p sgl-model-gateway -p sgl-model-gateway-python \
|
||||
&& cp /cargo-target/release/sgl-model-gateway /opt/sglang/bin/sgl-model-gateway \
|
||||
&& cp /cargo-target/release/libsglang_router_rs.so \
|
||||
/opt/sglang/lib/python3.12/site-packages/sglang_router/sglang_router_rs.abi3.so
|
||||
|
||||
# 计算依赖增量(重型 torch/nvidia/cuda 永不进入清单;不满足则 check_deps 非零、构建失败)
|
||||
RUN python /tmp/check_deps.py /sgl-workspace/sglang/python/pyproject.toml \
|
||||
/tmp/missing_light.txt /tmp/missing_kernel.txt \
|
||||
&& echo "== kernel wheel delta (pip --no-deps) ==" && cat /tmp/missing_kernel.txt \
|
||||
&& echo "== lightweight dep delta ==" && cat /tmp/missing_light.txt
|
||||
|
||||
# kernel wheel 增量(如 sglang-kernel 升版):--no-deps 防止 pip 顺藤摸瓜解析 torch
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install --no-build-isolation -e /sgl-workspace/sglang/python
|
||||
if grep -q '[^[:space:]]' /tmp/missing_kernel.txt; then \
|
||||
pip install --no-deps -r /tmp/missing_kernel.txt; \
|
||||
else \
|
||||
echo "== no kernel wheel delta"; \
|
||||
fi
|
||||
|
||||
# 轻量依赖增量(通常为空或几个小包),重型全家桶原样复用基底。
|
||||
# ⚠️ 必须 --no-deps:即使目标只是 xgrammar 这种小包,只要依赖图里出现 torch,
|
||||
# pip 就会校验 torch+cu130 元数据钉死的 nvidia-nccl-cu13==2.29.7,发现基底装的是
|
||||
# 2.30.7 便"好心"下载 206MB 降级(10-08 v3 首跑实测踩中)。基底的传递依赖本来就
|
||||
# 齐全,增量小包的二级依赖也一样;若有漏网,b300 冒烟会在 import 阶段显式暴露。
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
if grep -q '[^[:space:]]' /tmp/missing_light.txt; then \
|
||||
pip install --no-deps -r /tmp/missing_light.txt; \
|
||||
else \
|
||||
echo "== no lightweight delta, skipping dependency install"; \
|
||||
fi
|
||||
|
||||
# editable 安装 sglang 本体,--no-deps 保证 pip 完全不碰依赖图
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install --no-deps --no-build-isolation -e /sgl-workspace/sglang/python
|
||||
|
||||
@@ -1,15 +1,65 @@
|
||||
# fast 路径:python/pyproject.toml 未变化时使用(build.sh 自动判断)。
|
||||
# --no-deps 跳过全部依赖解析(连 index 元数据请求都省掉),纯源码迭代秒级完成。
|
||||
# 依赖有变化时必须走 Dockerfile 全量路径(build.sh 按 pyproject sha256 自动切换)。
|
||||
# fast 路径(2026-10-08 v3.1):python/pyproject.toml 未变化时使用(build.sh 自动判断)。
|
||||
# 与 Dockerfile 全量路径同构:同样跑 check_deps.py 增量安装。
|
||||
# 原因:全量构建若装过 kernel wheel 增量(如 sglang-kernel 0.4.7),随后 pyproject 未变的
|
||||
# fast 构建若只 --no-deps,会静默退回基底的旧 kernel(0.4.6.post1)——镜像内容漂移。
|
||||
# check_deps 本地比对 ~0.5s,增量命中 pip cache 秒级,成本可忽略。
|
||||
FROM uhub.service.ucloud.cn/umirror/sglang:dev-dsv41
|
||||
|
||||
ARG PIP_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
ENV PIP_INDEX_URL=${PIP_INDEX}
|
||||
|
||||
# 构建后端(pyproject build-system.requires,torch 除外——基底已有)
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install "setuptools>=61" "setuptools-scm>=8" "setuptools-rust>=1.11" wheel
|
||||
|
||||
# 依赖增量计算脚本(对照 pyproject 与基底已装版本,产出轻量/kernel 增量清单)
|
||||
# 注意 build context 是 /data/ymk 且 .dockerignore 排除了 build/,只能从 sglang/ 树内 COPY
|
||||
COPY sglang/deploy/b300-build/check_deps.py /tmp/check_deps.py
|
||||
|
||||
# 整个源码树(含 .git,用于 setuptools-scm 版本打戳)
|
||||
COPY sglang/ /sgl-workspace/sglang/
|
||||
|
||||
# Rust gateway(sgl-model-gateway)编译。基底自带 rustup stable(实测 cargo 1.98.1),
|
||||
# RUSTUP_TOOLCHAIN=stable 绕过仓库 rust-toolchain.toml 钉的 1.90(避免重复下载工具链);
|
||||
# b300 直连 crates.io(稀疏索引 + static.crates.io 均通),无需任何镜像源。
|
||||
# cache mount 持久化 registry/target:冷编译 ~15min,增量重建秒~分钟级。
|
||||
# python 扩展必须显式 -p(workspace 根包默认不编译扩展),产物 libsglang_router_rs.so
|
||||
# 需重命名为 sglang_router_rs.abi3.so 覆盖 python 包内同文件(与 H20 线上部署方式一致)。
|
||||
RUN --mount=type=cache,target=/cargo-home \
|
||||
--mount=type=cache,target=/cargo-target \
|
||||
cd /sgl-workspace/sglang/sgl-model-gateway \
|
||||
&& CARGO_HOME=/cargo-home CARGO_TARGET_DIR=/cargo-target RUSTUP_TOOLCHAIN=stable \
|
||||
cargo build --release -p sgl-model-gateway -p sgl-model-gateway-python \
|
||||
&& cp /cargo-target/release/sgl-model-gateway /opt/sglang/bin/sgl-model-gateway \
|
||||
&& cp /cargo-target/release/libsglang_router_rs.so \
|
||||
/opt/sglang/lib/python3.12/site-packages/sglang_router/sglang_router_rs.abi3.so
|
||||
|
||||
# 计算依赖增量(重型 torch/nvidia/cuda 永不进入清单;不满足则 check_deps 非零、构建失败)
|
||||
RUN python /tmp/check_deps.py /sgl-workspace/sglang/python/pyproject.toml \
|
||||
/tmp/missing_light.txt /tmp/missing_kernel.txt \
|
||||
&& echo "== kernel wheel delta (pip --no-deps) ==" && cat /tmp/missing_kernel.txt \
|
||||
&& echo "== lightweight dep delta ==" && cat /tmp/missing_light.txt
|
||||
|
||||
# kernel wheel 增量(如 sglang-kernel 升版):--no-deps 防止 pip 顺藤摸瓜解析 torch
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
if grep -q '[^[:space:]]' /tmp/missing_kernel.txt; then \
|
||||
pip install --no-deps -r /tmp/missing_kernel.txt; \
|
||||
else \
|
||||
echo "== no kernel wheel delta"; \
|
||||
fi
|
||||
|
||||
# 轻量依赖增量(通常为空或几个小包),重型全家桶原样复用基底。
|
||||
# ⚠️ 必须 --no-deps:即使目标只是 xgrammar 这种小包,只要依赖图里出现 torch,
|
||||
# pip 就会校验 torch+cu130 元数据钉死的 nvidia-nccl-cu13==2.29.7,发现基底装的是
|
||||
# 2.30.7 便"好心"下载 206MB 降级(10-08 v3 首跑实测踩中)。基底的传递依赖本来就
|
||||
# 齐全,增量小包的二级依赖也一样;若有漏网,b300 冒烟会在 import 阶段显式暴露。
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
if grep -q '[^[:space:]]' /tmp/missing_light.txt; then \
|
||||
pip install --no-deps -r /tmp/missing_light.txt; \
|
||||
else \
|
||||
echo "== no lightweight delta, skipping dependency install"; \
|
||||
fi
|
||||
|
||||
# editable 安装 sglang 本体,--no-deps 保证 pip 完全不碰依赖图
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install --no-deps --no-build-isolation -e /sgl-workspace/sglang/python
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compute dependency deltas for a b300 sglang image build, in three classes.
|
||||
|
||||
Reads pyproject.toml, evaluates every requirement against the *installed*
|
||||
distributions in the base image, and writes two requirements files:
|
||||
|
||||
argv[2] (light): unsatisfied lightweight requirements → `pip install -r`
|
||||
argv[3] (kernel): unsatisfied kernel wheels (sglang-kernel, flashinfer*,
|
||||
sgl-deep-gemm) → `pip install --no-deps -r`
|
||||
These track the sglang source being built and are meant
|
||||
to be upgraded from the index when pyproject bumps them;
|
||||
--no-deps keeps pip from walking torch's metadata.
|
||||
|
||||
The PyTorch stack (torch/torchvision/torchaudio/torchcodec/
|
||||
torch_memory_saver), the nvidia-* CUDA family and cuda-* packages are NEVER
|
||||
emitted. If one of them is missing or its installed version no longer
|
||||
satisfies the pinned specifier (e.g. upstream bumps nvidia-cutlass-dsl), the
|
||||
script fails the build loudly instead of letting pip download ~6GB or
|
||||
"fix" the base image's deliberately newer nvidia-nccl-cu13 2.30.7 back to
|
||||
torch's metadata pin ==2.29.7 (any pip full-resolution would do both —
|
||||
2026-09-23/24 builds did, throttled to 2h40m).
|
||||
"""
|
||||
|
||||
import sys
|
||||
import tomllib
|
||||
from importlib import metadata as md
|
||||
|
||||
from packaging.requirements import Requirement
|
||||
|
||||
# Never resolved by pip: reused from the base image or the build fails.
|
||||
HEAVY_PREFIXES = ("torch", "nvidia-", "cuda-")
|
||||
# Kernel wheels: upgraded via pip --no-deps when pyproject pins move.
|
||||
KERNEL_PREFIXES = ("sglang-kernel", "flashinfer", "sgl-deep-gemm")
|
||||
|
||||
|
||||
def norm(name: str) -> str:
|
||||
return name.lower().replace("_", "-")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
pyproject, light_out, kernel_out = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
with open(pyproject, "rb") as f:
|
||||
deps = tomllib.load(f)["project"]["dependencies"]
|
||||
|
||||
light: list[str] = []
|
||||
kernel: list[str] = []
|
||||
heavy_unsat: list[str] = []
|
||||
for dep in deps:
|
||||
req = Requirement(dep)
|
||||
if req.marker is not None and not req.marker.evaluate():
|
||||
continue
|
||||
try:
|
||||
installed = md.version(req.name)
|
||||
except md.PackageNotFoundError:
|
||||
installed = None
|
||||
satisfied = installed is not None and req.specifier.contains(
|
||||
installed, prereleases=True
|
||||
)
|
||||
if satisfied:
|
||||
continue
|
||||
name = norm(req.name)
|
||||
if name.startswith(HEAVY_PREFIXES):
|
||||
heavy_unsat.append(f"{dep} (installed: {installed})")
|
||||
elif name.startswith(KERNEL_PREFIXES):
|
||||
kernel.append(dep)
|
||||
else:
|
||||
light.append(dep)
|
||||
|
||||
if heavy_unsat:
|
||||
print("=" * 70, file=sys.stderr)
|
||||
print("FATAL: heavy base-image deps no longer satisfy pyproject:", file=sys.stderr)
|
||||
for item in heavy_unsat:
|
||||
print(f" {item}", file=sys.stderr)
|
||||
print(
|
||||
"Refusing to let pip resolve these (would re-download ~6GB and/or\n"
|
||||
"downgrade NCCL). Decide explicitly: bump the base image, or adjust\n"
|
||||
"the pin in pyproject.toml.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
with open(light_out, "w") as f:
|
||||
f.write("\n".join(light) + ("\n" if light else ""))
|
||||
with open(kernel_out, "w") as f:
|
||||
f.write("\n".join(kernel) + ("\n" if kernel else ""))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,27 @@
|
||||
cd /data/ymk/ds41
|
||||
HDR="Authorization=Bearer deepseek-v4-flash-0731-key"
|
||||
PORT=30040
|
||||
MODEL=/data/models/DeepSeek-V4.1-Flash
|
||||
|
||||
# 一律 --warmup-requests 0:默认 warmup 会把 prompt#0 种进 L1/L2 污染 L3 测量(已踩过)
|
||||
run() {
|
||||
echo "===== [$(date +%H:%M:%S)] RUN: $1 (np=$2) ====="
|
||||
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--warmup-requests 0 --num-prompts "$2" --max-concurrency "$2" --seed "$3" --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR" 2>&1 | grep -E 'Input token throughput|Mean TTFT|P99 TTFT|Successful'
|
||||
sleep 2
|
||||
}
|
||||
flush() { curl -sf -m 10 -X POST http://127.0.0.1:$PORT/flush_cache > /dev/null && echo "[$(date +%H:%M:%S)] flushed"; sleep 3; }
|
||||
|
||||
echo "===== PRIME COLD mc8(冷基线 + 写入 L3) ====="
|
||||
run PRIME_COLD 8 1541391
|
||||
flush
|
||||
run MC1_L3 1 1541391
|
||||
flush
|
||||
run MC2_L3 2 1541391
|
||||
flush
|
||||
run MC4_L3 4 1541391
|
||||
flush
|
||||
run MC8_L3 8 1541391
|
||||
echo "===== [$(date +%H:%M:%S)] ALL DONE ====="
|
||||
@@ -0,0 +1,204 @@
|
||||
#!/bin/bash
|
||||
# bench_suite_v2.sh — DeepSeek-V4.1-Flash on b300-01,2026-10-09 新版基线方法
|
||||
#
|
||||
# 相对 v1 的方法变化:
|
||||
# 1. 服务刚起数据不准 → 每个测试模式自带 warmup 轮(结果丢弃,标记 WARM)
|
||||
# 2. prefill 冷/热双轮:冷轮写入 hicache(L1/L3),同 seed 热轮测纯 cache-hit TPS
|
||||
# 3. decode 阶梯并发:步长 32,上限 = cuda-graph-max-bs-decode 的 80%;
|
||||
# 每档先 warm(灌 L1)再测,prefill 全 cache hit;
|
||||
# 报告 TPOT + 整实例输出 TPS + 推算实际并发(= 输出TPS × TPOT / 1000)
|
||||
# 4. l3 模式(3p1d 经 router):大语料溢出 L1 后重放,验证 L3 命中
|
||||
#
|
||||
# 在宿主机执行,bench 客户端通过 docker exec 进容器跑(需要 /data 挂载)。
|
||||
#
|
||||
# 用法:
|
||||
# bench_suite_v2.sh warmup <container> <port>
|
||||
# bench_suite_v2.sh prefill <tag> <container> <port>
|
||||
# bench_suite_v2.sh decode <tag> <container> <port> [cg_max_bs=256]
|
||||
# bench_suite_v2.sh l3 <tag> <container> <port> [num_prompts=24]
|
||||
# bench_suite_v2.sh report <log>
|
||||
# 产物: /data/ymk/ds41/bench_v2_<tag>.log
|
||||
set -u
|
||||
|
||||
BASE=/data/ymk/ds41
|
||||
MODEL=${MODEL:-/data/models/DeepSeek-V4.1-Flash}
|
||||
HDR="Authorization=Bearer deepseek-v4-flash-0731-key"
|
||||
|
||||
usage() { sed -n 2,20p "$0" >&2; exit 1; }
|
||||
|
||||
# $1=container $2=port $3=tag -> 生成可执行脚本路径并 echo
|
||||
emit_common() {
|
||||
cat <<EOF
|
||||
cd $BASE
|
||||
HDR="$HDR"
|
||||
PORT=$2
|
||||
run() {
|
||||
local name="\$1"; shift
|
||||
echo "===== [\$(date +%H:%M:%S)] START: \$name ====="
|
||||
python3 -m sglang.bench_serving "\$@"
|
||||
echo "===== [\$(date +%H:%M:%S)] END(exit=\$?): \$name ====="
|
||||
sleep 5
|
||||
}
|
||||
EOF
|
||||
}
|
||||
|
||||
# ---------------- warmup ----------------
|
||||
cmd_warmup() {
|
||||
local c=${1:?container} port=${2:?port}
|
||||
local gen=$BASE/.bench_v2_warmup.sh
|
||||
{ emit_common "$c" "$port"; cat <<'EOF'
|
||||
run WARMUP_a --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
|
||||
--dataset-name random-ids --random-input-len 4096 --random-output-len 16 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 777001 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
run WARMUP_b --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
|
||||
--dataset-name random-ids --random-input-len 16000 --random-output-len 1 \
|
||||
--num-prompts 2 --max-concurrency 1 --seed 777002 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
echo "===== [$(date +%H:%M:%S)] ALL DONE ====="
|
||||
EOF
|
||||
} | sed "s|MODELPH|$MODEL|g" > "$gen"
|
||||
chmod +x "$gen"
|
||||
docker exec "$c" bash "$gen"
|
||||
}
|
||||
|
||||
# ---------------- prefill(冷/热双轮) ----------------
|
||||
cmd_prefill() {
|
||||
local tag=${1:?tag} c=${2:?container} port=${3:?port}
|
||||
local p=$(date +%H%M%S) gen=$BASE/.bench_v2_gen_${tag}.sh
|
||||
{ emit_common "$c" "$port"
|
||||
# warmup(丢弃)
|
||||
cat <<'EOF'
|
||||
run WARMUP --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
|
||||
--dataset-name random-ids --random-input-len 4096 --random-output-len 16 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 777000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
EOF
|
||||
# 冷轮
|
||||
for stage in "bs1_16000 16000 1 4 1" "bs1_160000 160000 1 4 2" "bs16_4096 4096 16 32 3"; do
|
||||
set -- $stage
|
||||
cat <<EOF
|
||||
run COLD_$1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len $2 --random-output-len 1 \\
|
||||
--num-prompts $4 --max-concurrency $3 --seed ${p}$5 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
done
|
||||
# 热轮(同 seed → cache hit)
|
||||
for stage in "bs1_16000 16000 1 4 1" "bs1_160000 160000 1 4 2" "bs16_4096 4096 16 32 3"; do
|
||||
set -- $stage
|
||||
cat <<EOF
|
||||
run HIT_$1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len $2 --random-output-len 1 \\
|
||||
--num-prompts $4 --max-concurrency $3 --seed ${p}$5 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
done
|
||||
echo 'echo "===== [$(date +%H:%M:%S)] ALL DONE ====="'
|
||||
} | sed "s|MODELPH|$MODEL|g" > "$gen"
|
||||
chmod +x "$gen"
|
||||
nohup docker exec "$c" bash "$gen" > "$BASE/bench_v2_${tag}.log" 2>&1 &
|
||||
echo "launched pid=$! seeds=${p}1..3 log=$BASE/bench_v2_${tag}.log"
|
||||
}
|
||||
|
||||
# ---------------- decode 阶梯 ----------------
|
||||
cmd_decode() {
|
||||
local tag=${1:?tag} c=${2:?container} port=${3:?port} cg=${4:-256}
|
||||
local maxc=$(( cg * 80 / 100 / 32 * 32 )) # 80% 向下取整到 32 的倍数
|
||||
local p=$(date +%H%M%S) gen=$BASE/.bench_v2_gen_${tag}.sh
|
||||
echo "ladder: 32..$maxc step 32 (cg_max_bs=$cg)"
|
||||
{ emit_common "$c" "$port"
|
||||
cat <<'EOF'
|
||||
run WARMUP --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 128 \
|
||||
--num-prompts 16 --max-concurrency 16 --seed 777000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
EOF
|
||||
# bs1 延迟地板(warm + measure)
|
||||
cat <<EOF
|
||||
run WARM_bs1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
|
||||
--num-prompts 4 --max-concurrency 1 --seed ${p}01 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
run DEC_bs1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
|
||||
--num-prompts 4 --max-concurrency 1 --seed ${p}01 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
local cc
|
||||
for (( cc=32; cc<=maxc; cc+=32 )); do
|
||||
local np=$(( cc * 2 ))
|
||||
cat <<EOF
|
||||
run WARM_c${cc} --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
|
||||
--num-prompts $np --max-concurrency $cc --seed ${p}1${cc} --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
run DEC_c${cc} --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
|
||||
--num-prompts $np --max-concurrency $cc --seed ${p}1${cc} --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
done
|
||||
echo 'echo "===== [$(date +%H:%M:%S)] ALL DONE ====="'
|
||||
} | sed "s|MODELPH|$MODEL|g" > "$gen"
|
||||
chmod +x "$gen"
|
||||
nohup docker exec "$c" bash "$gen" > "$BASE/bench_v2_${tag}.log" 2>&1 &
|
||||
echo "launched pid=$! log=$BASE/bench_v2_${tag}.log"
|
||||
}
|
||||
|
||||
# ---------------- L3 语料溢出(3p1d 经 router) ----------------
|
||||
cmd_l3() {
|
||||
local tag=${1:?tag} c=${2:?container} port=${3:?port} np=${4:-48}
|
||||
local p=$(date +%H%M%S) gen=$BASE/.bench_v2_gen_${tag}.sh
|
||||
{ emit_common "$c" "$port"
|
||||
cat <<EOF
|
||||
run WARMUP --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 4096 --random-output-len 16 \\
|
||||
--num-prompts 8 --max-concurrency 8 --seed 777000 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
run L3_COLD_160k --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \\
|
||||
--num-prompts $np --max-concurrency 8 --seed ${p}1 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
echo "===== [\$(date +%H:%M:%S)] DRAIN: sleep 120 等 L2→L3 备份排空 ====="
|
||||
sleep 120
|
||||
run L3_HIT_160k --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \\
|
||||
--num-prompts $np --max-concurrency 8 --seed ${p}1 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
echo 'echo "===== [$(date +%H:%M:%S)] ALL DONE ====="'
|
||||
} > "$gen"
|
||||
chmod +x "$gen"
|
||||
nohup docker exec "$c" bash "$gen" > "$BASE/bench_v2_${tag}.log" 2>&1 &
|
||||
echo "launched pid=$! corpus=${np}x160k log=$BASE/bench_v2_${tag}.log"
|
||||
}
|
||||
|
||||
# ---------------- report ----------------
|
||||
cmd_report() {
|
||||
local log=${1:?log required}
|
||||
awk '
|
||||
/START:/ { name=$4; sub(/:$/, "", name); tps=""; itps=""; tpot=""; ttft=""; acc="" }
|
||||
/Input token throughput/ { itps=$NF }
|
||||
/Output token throughput/ { tps=$NF }
|
||||
/Mean TPOT/ { tpot=$NF }
|
||||
/Mean TTFT/ { ttft=$NF }
|
||||
/[Aa]ccept/ { acc=$0; gsub(/^ +/, "", acc) }
|
||||
/END\(exit/ {
|
||||
line=sprintf("%-22s in_tps=%-10s out_tps=%-10s tpot_ms=%-8s ttft_ms=%-8s", name, itps, tps, tpot, ttft)
|
||||
if (tps != "" && tpot != "" && tpot+0 > 0)
|
||||
line=line sprintf(" eff_concurrency=%.1f", tps * tpot / 1000)
|
||||
print line
|
||||
if (acc != "") print " " acc
|
||||
}
|
||||
' "$log"
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
warmup) shift; cmd_warmup "$@" ;;
|
||||
prefill) shift; cmd_prefill "$@" ;;
|
||||
decode) shift; cmd_decode "$@" ;;
|
||||
l3) shift; cmd_l3 "$@" ;;
|
||||
report) shift; cmd_report "$@" ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
@@ -0,0 +1,61 @@
|
||||
cd /data/ymk/ds41
|
||||
HDR="Authorization=Bearer deepseek-v4-flash-0731-key"
|
||||
PORT=30040
|
||||
MODEL=/data/models/DeepSeek-V4.1-Flash
|
||||
|
||||
run() {
|
||||
local name="$1"; shift
|
||||
echo "===== [$(date +%H:%M:%S)] START: $name ====="
|
||||
python3 -m sglang.bench_serving "$@" 2>&1 | grep -vE '^Namespace|warn'
|
||||
echo "===== [$(date +%H:%M:%S)] END(exit=$?): $name ====="
|
||||
sleep 3
|
||||
}
|
||||
|
||||
r160() { # $1=name $2=seed $3=num_prompts $4=mc
|
||||
run "$1" --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--num-prompts "$3" --max-concurrency "$4" --seed "$2" --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
}
|
||||
|
||||
# WARMUP
|
||||
run WARMUP --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 4096 --random-output-len 16 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 777000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
|
||||
# R0 纯计算极限(集合 A 首次出现,L1/L2/L3 全空)
|
||||
r160 R0_COMPUTE_COLD 1541391 8 8
|
||||
# R1 L1 极限(A 全量在 L1 设备池:1.28M < 1.6M)
|
||||
r160 R1_L1_HIT 1541391 8 8
|
||||
# R2 填充集 B(12×160k=1.92M > L1 1.6M),把 A 挤出 L1 到 L2
|
||||
r160 R2_FILLER_EVICT 999000 12 8
|
||||
# R3 L2 极限(A 已被挤出 L1,从 L2 host 池 load-back)
|
||||
r160 R3_L2_HIT 1541391 8 8
|
||||
# R4 flush L1+L2(L3 保留),走 L3 prefetch
|
||||
echo "===== [$(date +%H:%M:%S)] FLUSH L1+L2 ====="
|
||||
curl -sf -m 10 -X POST http://127.0.0.1:$PORT/flush_cache && echo "flush ok"
|
||||
sleep 5
|
||||
r160 R4_L3_HIT 1541391 8 8
|
||||
# R5 嫁接后 L2 复核
|
||||
r160 R5_L2_AFTER_GRAFT 1541391 8 8
|
||||
|
||||
# R6 混合负载:新集合 C(纯计算,4×160k mc4)+ 集合 A(命中,8×160k mc8)并发打
|
||||
echo "===== [$(date +%H:%M:%S)] START: R6_MIX_PARALLEL ====="
|
||||
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--num-prompts 4 --max-concurrency 4 --seed 555000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR" > /data/ymk/ds41/.tier_r6_coldclient.log 2>&1 &
|
||||
COLD_PID=$!
|
||||
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 1541391 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR" > /data/ymk/ds41/.tier_r6_hitclient.log 2>&1 &
|
||||
HIT_PID=$!
|
||||
wait $COLD_PID $HIT_PID
|
||||
echo "===== [$(date +%H:%M:%S)] END: R6_MIX_PARALLEL ====="
|
||||
echo "--- R6 cold client (compute) ---"
|
||||
grep -E 'Input token throughput|Mean TTFT|Request throughput' /data/ymk/ds41/.tier_r6_coldclient.log
|
||||
echo "--- R6 hit client (cache) ---"
|
||||
grep -E 'Input token throughput|Mean TTFT|Request throughput' /data/ymk/ds41/.tier_r6_hitclient.log
|
||||
echo "===== [$(date +%H:%M:%S)] ALL DONE ====="
|
||||
@@ -0,0 +1,27 @@
|
||||
cd /data/ymk/ds41
|
||||
HDR="Authorization=Bearer deepseek-v4-flash-0731-key"
|
||||
PORT=30040
|
||||
MODEL=/data/models/DeepSeek-V4.1-Flash
|
||||
|
||||
echo "===== [$(date +%H:%M:%S)] FLUSH L1+L2(A 只剩 L3) ====="
|
||||
curl -sf -m 10 -X POST http://127.0.0.1:$PORT/flush_cache && echo "flush ok"
|
||||
sleep 5
|
||||
|
||||
echo "===== [$(date +%H:%M:%S)] START: R7_MIX_L3_PLUS_COMPUTE ====="
|
||||
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--num-prompts 4 --max-concurrency 4 --seed 666000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR" > /data/ymk/ds41/.tier_r7_coldclient.log 2>&1 &
|
||||
COLD_PID=$!
|
||||
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 1541391 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR" > /data/ymk/ds41/.tier_r7_hitclient.log 2>&1 &
|
||||
HIT_PID=$!
|
||||
wait $COLD_PID $HIT_PID
|
||||
echo "===== [$(date +%H:%M:%S)] END: R7 ====="
|
||||
echo "--- R7 cold client (compute, 4x160k mc4) ---"
|
||||
grep -E 'Input token throughput|Mean TTFT|Request throughput' /data/ymk/ds41/.tier_r7_coldclient.log
|
||||
echo "--- R7 hit client (L3, 8x160k mc8) ---"
|
||||
grep -E 'Input token throughput|Mean TTFT|Request throughput' /data/ymk/ds41/.tier_r7_hitclient.log
|
||||
echo "===== [$(date +%H:%M:%S)] ALL DONE ====="
|
||||
@@ -28,6 +28,7 @@ x-p-environment: &p-environment
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
# 【L3 dummy client 变体 2026-10-09 深夜,定型=shm+best_effort】sglang→standalone_storage,数据面经本机 mooncake_client(50052 IPC+进程映射),不碰 TCP 传输引擎
|
||||
# 实测 24x160k 热/冷:shm+best_effort +26.4%(唯一正收益);wait_complete 因 all-or-nothing discard 等待白等反而 -17.7%。TCP 传输引擎两组皆负。
|
||||
# 【L3 cache-hit 测试变体 2026-10-09】基于 dockerserve-3p1d-otlp.yml,仅改 P:
|
||||
# --max-total-tokens 800000(L1 缩到 ~0.8M token)+ --hicache-ratio 1.0(L2=L1=0.8M token。⚠️ 教训:ratio 0.05 时 L2 太小,build_storage_backup_spec 拿不到 host 页会静默跳过 L3 写入,热轮无数据可中;V4 hicache 不支持 --hicache-size)
|
||||
# 使 ~2.6M token/P 的语料(48×160k)溢出 L1+L2(1.6M),热轮强制走 L3 mooncake 命中。
|
||||
# 【2026-10-10 L2 放大】hicache-ratio 1.0→4.0(L2=3.2M token/P,宿主可用内存 ~1.9TB 充裕):
|
||||
# #38348 fail-soft 移植后 bench 显示收益被容量掩盖——24x160k 下 host_capacity 拒绝
|
||||
# 0.75M~2.97M token/P(L2 装不下,prefetch 未发起即被拒退回重算)。放大后验证:
|
||||
# ①热轮请求 L3/L2 命中不被容量拒绝;②L3 hit 嫁接进 L2 树后在 L2 长期留存。
|
||||
# 与主模板同容器名/端口,用前必须先 down 主栈。D 与 router 不变。
|
||||
# 原注释: 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram + OTLP trace(2026-09-27)
|
||||
# 基于 dockerserve-3p1d-soak.yml 增加: --enable-trace --trace-modules request,mooncake
|
||||
# --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name ds41-3p1d-<实例名>
|
||||
# 依赖: b300-01 k8s cw jaeger-trace(jaegertracing/all-in-one:1.76.0,badger 6h TTL,hostNetwork 4317)
|
||||
# 安全性: BatchSpanProcessor 导出失败只丢 span;Jaeger 宕机不影响推理服务
|
||||
# 调优旋钮(默认未开): SGLANG_TRACE_ASYNC=1(异步 span 创建), SGLANG_TRACE_LEVEL(默认 3)
|
||||
# router(ds41-router-3p1d,rust)无 OTLP 支持,不产生 span,追踪从各 sglang HTTP 层开始
|
||||
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-baba18bc7d-local-20261010-2310
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python:/sgl-workspace/sglang/python
|
||||
- /data/ymk/ds41/fake-cargo:/root/.cargo/bin/cargo:ro
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
RUSTUP_TOOLCHAIN: "1.92-x86_64-unknown-linux-gnu"
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store-dummy.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 4.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p1
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 4.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p2
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 4.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p3
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
# G9 P0 验证(2026-10-11):D 单独升镜像拿 /query_cache_hit 修复(71c6d8bc5),
|
||||
# 并开 decode radix + hicache L3 客户端(复用 dummy store,只读查询)。
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-6612af9e0-local-20261011-0740
|
||||
container_name: ds41-3p1d-d1
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
RUSTUP_TOOLCHAIN: "1.92-x86_64-unknown-linux-gnu"
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store-dummy.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
--disaggregation-decode-enable-radix-cache
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-d1
|
||||
@@ -0,0 +1,239 @@
|
||||
# 【G9 P0 验证栈:真实 store 3p1d 2026-10-11】基于 dockerserve-3p1d-otlp.yml。
|
||||
# 用途:验证 P0 改动(/v1/loads 排队 token + drain rate 字段、P 侧 /query_cache_hit)。
|
||||
# 与主模板同容器名/端口,用前必须先 down 主栈;router(ds41-router-3p1d)独立不在本文件。
|
||||
# 2026-10-11 改造要点(vs 原 l3test):
|
||||
# - 镜像升 6612af9e0(含 G9 P0 + /query_cache_hit KeyError 修复)+ scheduler.py
|
||||
# 热补丁(L3 查询门限从 enable_decode_hicache 放宽到 enable_hicache_storage,
|
||||
# 否则 P 节点永远查不到 L3)。
|
||||
# - 每实例独立 MOONCAKE_LOCAL_HOSTNAME(a/b/c/d),extra_hosts 互配全部 rank 别名
|
||||
# (真实 store 跨实例副本写要求对端 rank 主机名可解析,见 realstore-x2 注释;
|
||||
# rank 命名规则 {hostname}-d{dp}t{tp}p{pp}c{attn_cp},commit 84fc632088;
|
||||
# D 为 dp2 dp-attention(attn_tp=1),后缀推断为 -d0t0p0c0/-d1t1p0c0,未实证)。
|
||||
# - ⚠️ D 侧 cache 方案实证否决(2026-10-11):SWA 模型下
|
||||
# --disaggregation-decode-enable-radix-cache 先要 SGLANG_ENABLE_UNIFIED_RADIX_TREE=1,
|
||||
# 开了又报 "supports only device-resident cache and is incompatible with
|
||||
# --enable-hierarchical-cache"(kv_cache_builder.py 硬 ValueError)——D 挂不了 L3。
|
||||
# 结论:/query_cache_hit 放 P(L3 全局统一,P 有完整 hicache),D 保持生产配置。
|
||||
# - D mem-fraction 0.80:曾因崩溃 d1 泄漏 ~61GiB 临时降 0.75,全栈 down 掉旧容器后
|
||||
# 驱动即回收(GPU 全零),已恢复 0.80。D max-running 保持 256(512 时 SWA 池挤爆 KV 池)。
|
||||
# P 侧保持 l3test 原状:--max-total-tokens 800000 + ratio 1.0(L1/L2 各 ~0.8M token),
|
||||
# write_through 保证 prefill 后 L3 有数据,供 /query_cache_hit 对拍。
|
||||
# 原注释: 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram + OTLP trace(2026-09-27)
|
||||
# 依赖: mooncake-master 容器(50051,起栈前 restart 清旧 segment 注册);
|
||||
# b300-01 k8s cw jaeger-trace(4317,导出失败只丢 span 不影响服务)
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-e7dc175e9-local-20261011-0940
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-rank-hosts: &rank-hosts
|
||||
# mooncake rank 主机名别名:P(tp2/cp2 interleave)-d0t0p0c0/-d0t1p0c1;
|
||||
# D(dp2 dp-attention)-d0t0p0c0/-d1t1p0c0(推断,起栈后核对 master 注册名)
|
||||
- "localhost-a-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-a-d0t1p0c1:127.0.0.1"
|
||||
- "localhost-b-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-b-d0t1p0c1:127.0.0.1"
|
||||
- "localhost-c-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-c-d0t1p0c1:127.0.0.1"
|
||||
- "localhost-d-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-d-d1t1p0c0:127.0.0.1"
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p1
|
||||
extra_hosts: *rank-hosts
|
||||
environment:
|
||||
<<: *p-environment
|
||||
MOONCAKE_LOCAL_HOSTNAME: localhost-a
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p1
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p2
|
||||
extra_hosts: *rank-hosts
|
||||
environment:
|
||||
<<: *p-environment
|
||||
MOONCAKE_LOCAL_HOSTNAME: localhost-b
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p2
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p3
|
||||
extra_hosts: *rank-hosts
|
||||
environment:
|
||||
<<: *p-environment
|
||||
MOONCAKE_LOCAL_HOSTNAME: localhost-c
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p3
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-d1
|
||||
extra_hosts: *rank-hosts
|
||||
# ⚠️ D 保持生产配置(2026-10-11 实证):SWA 模型下
|
||||
# --disaggregation-decode-enable-radix-cache 与 --enable-hierarchical-cache 互斥
|
||||
# (kv_cache_builder.py 硬 ValueError,"supports only device-resident cache"),
|
||||
# D 无法挂 L3 —— /query_cache_hit 改由 P 节点回答(L3 全局统一,P 有完整 hicache)。
|
||||
# MOONCAKE_LOCAL_HOSTNAME 保留备用(无害)。
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
MOONCAKE_LOCAL_HOSTNAME: localhost-d
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-d1
|
||||
@@ -35,6 +35,7 @@ x-p-environment: &p-environment
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
@@ -35,6 +35,7 @@ x-p-environment: &p-environment
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ x-p-environment: &p-environment
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ x-p-environment: &p-environment
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
container_name: ds41-sglang-b4
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-92632a60b-20260922-2100
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
container_name: ds41-cpdspark-nr
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
# 单独 D 节点基线(非 PD role):3p1d 的 d1 配置去掉 disaggregation,直接对外服务
|
||||
# 用途:测单 D 节点 decode 基线(阶梯并发 32..80%*cuda-graph-max-bs-decode,
|
||||
# 报 TPOT / 实际并发 / 整实例输出 TPS;prefill 走 L1 radix cache-hit)
|
||||
# GPU 6,7;端口 30001;无 hicache 依赖(L1 即可支撑重复 prompt 命中)
|
||||
# 2026-10-09 建:配合 bench_suite_v2.sh decode
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
d1s:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-d1-standalone
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
@@ -0,0 +1,61 @@
|
||||
# 单独 P 节点基线(非 PD role):3p1d 的 p1 配置去掉 disaggregation,直接对外服务
|
||||
# 用途:测单 P 节点 prefill 基线吞吐(冷)+ hicache cache-hit TPS(热)
|
||||
# GPU 0,1;端口 30000;依赖 mooncake-master(hicache L3,127.0.0.1:50051)
|
||||
# 2026-10-09 建:配合 bench_suite_v2.sh prefill
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
p1s:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-p1-standalone
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
@@ -22,6 +22,7 @@ services:
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
- SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1
|
||||
- SGLANG_HICACHE_MOONCAKE_CONFIG_PATH=/data/ymk/ds41/mooncake-store.json
|
||||
- SGLANG_HICACHE_PREFETCH_IO_THREADS=4
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
|
||||
@@ -18,6 +18,7 @@ services:
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
- SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1
|
||||
- SGLANG_HICACHE_MOONCAKE_CONFIG_PATH=/data/ymk/ds41/mooncake-store.json
|
||||
- SGLANG_HICACHE_PREFETCH_IO_THREADS=4
|
||||
- SGLANG_DISAGGREGATION_QUEUE_SIZE=16
|
||||
- SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=32
|
||||
volumes:
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
# 【跨 P 实例 L3 读写验证 2026-10-10】dockerserve-tier-cp2-realstore.yml 的第二实例:
|
||||
# 与 ds41-tier-t2(A:GPU 0,1,端口 30040)同时跑,共用 mooncake master 50051。
|
||||
# 关键差异:
|
||||
# - GPU 2,3,端口 30041,容器名 ds41-tier-t3
|
||||
# - MOONCAKE_LOCAL_HOSTNAME=localhost-b:mooncake 客户端 hostname 前缀按实例区分
|
||||
# (rank 后缀 -d0t0p0c0/-d0t1p0c1 由 84fc632088 拼在后面的),否则两实例 rank 同名
|
||||
# → master preferred-segment 分不出段归属 → 副本落错进程段 → 跨进程回退 TCP(实例内)
|
||||
# 且跨实例段地址冲突。extra_hosts 别名同步加 -b 前缀(P2P handshake 要求可解析,
|
||||
# ac11e9162d)。
|
||||
# 验证路径:A prime 写 L3 → flush B 的 L1/L2 → B 读同 seed prompts = 跨实例 TCP P2P 命中。
|
||||
x-common: &common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
shm_size: 32gb
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python:/sgl-workspace/sglang/python
|
||||
- /data/ymk/ds41/fake-cargo:/root/.cargo/bin/cargo:ro
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
t3:
|
||||
<<: *common
|
||||
container_name: ds41-tier-t3
|
||||
extra_hosts:
|
||||
- "localhost-b-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-b-d0t1p0c1:127.0.0.1"
|
||||
# 跨实例 P2P:对端实例(realstore compose 的 t2)rank 主机名也要能解析
|
||||
- "localhost-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-d0t1p0c1:127.0.0.1"
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: 1
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_TIMING: 1
|
||||
MOONCAKE_LOCAL_HOSTNAME: localhost-b
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: 4
|
||||
CUDA_VISIBLE_DEVICES: 2,3
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--chunked-prefill-size 32768
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30041
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 4.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--max-total-tokens 1600000
|
||||
@@ -0,0 +1,73 @@
|
||||
# 【L3 真实 store A/B 2026-10-10】同 dockerserve-tier-cp2.yml,唯一差异:
|
||||
# L3 从 dummy standalone(mooncake-store-dummy.json,50052 守护进程)
|
||||
# 切到真实 mooncake store(mooncake-store.json:master 50051 + P2PHANDSHAKE + tcp,
|
||||
# 客户端注册 300GB 自身内存段)——与 k8s 线上 ds41-3p1d 的 L3 模式一致。
|
||||
# 目的:区分 "sglang L3 路径本身慢" 还是 "dummy 后端 per-key 路径慢"(mc 扫描 A/B)。
|
||||
x-common: &common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
shm_size: 32gb
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python:/sgl-workspace/sglang/python
|
||||
- /data/ymk/ds41/fake-cargo:/root/.cargo/bin/cargo:ro
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
t2:
|
||||
<<: *common
|
||||
container_name: ds41-tier-t2
|
||||
extra_hosts:
|
||||
- "localhost-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-d0t1p0c1:127.0.0.1"
|
||||
# 跨实例 P2P:对端实例(x2 compose 的 t3)rank 主机名也要能解析,
|
||||
# 否则 master 把副本分配到对端段后本端 open segment 失败(TRANSFER_FAIL)
|
||||
- "localhost-b-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-b-d0t1p0c1:127.0.0.1"
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: 1
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_TIMING: 1
|
||||
# L3 预取传输并行 lane 数(crc32(rid) 分 lane,每 lane 独立 gloo 组;
|
||||
# 实例内所有 rank 必须一致)。2026-10-10 A/B 见 l3-hicache-deep-dive 文档。
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: 4
|
||||
CUDA_VISIBLE_DEVICES: 0,1
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--chunked-prefill-size 32768
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30040
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 4.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--max-total-tokens 1600000
|
||||
@@ -0,0 +1,65 @@
|
||||
# 【prefill 分层极限测试 2026-10-10】单实例 cp2(tp2+ep2+prefill-cp interleave)+ dspark b5
|
||||
# + hicache L2(ratio 4.0=6.4M token) + L3(shm dummy) + best_effort
|
||||
# 目的:测 纯计算 / L1 / L2 / L3 四档 prefill 极限吞吐 + 混合负载异步聚合验证
|
||||
# L1=1.6M token(装得下 8×160k=1.28M 工作集);L2 6.4M;L3 shm 64GB
|
||||
# GPU 0,1(用前停 p3d1 栈);端口 30040;共用 mooncake-store(50052)/master(50051)
|
||||
x-common: &common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
shm_size: 32gb
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python:/sgl-workspace/sglang/python
|
||||
- /data/ymk/ds41/fake-cargo:/root/.cargo/bin/cargo:ro
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
t2:
|
||||
<<: *common
|
||||
container_name: ds41-tier-t2
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: 1
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store-dummy.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: 4
|
||||
SGLANG_HICACHE_TIMING: 1
|
||||
SGLANG_HICACHE_STORAGE_BATCH_SIZE: 512
|
||||
CUDA_VISIBLE_DEVICES: 0,1
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--chunked-prefill-size 32768
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30040
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 4.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--max-total-tokens 1600000
|
||||
@@ -0,0 +1,454 @@
|
||||
# ds41-3p1d: DeepSeek-V4.1-Flash 3P1D PD 分离(cs32k + OTLP trace),RBG 版
|
||||
# 拓扑: 3×prefill(tp2+ep2+cp2, cs32k, dspark, hicache L3) + 1×decode(tp2+ep2+dp2, cg256 mr320, dspark,
|
||||
# load-balance-method=total_requests 按请求数分流) + mooncake-master(hicache L3 store) + router(mini-lb PD)
|
||||
# 全部钉 b300-01(单机 8 卡,hostNetwork + hostPID——mooncake NVLink-intra 跨 Pod CUDA IPC 必需,
|
||||
# 等同 docker compose 的 pid: host;参考 AGENTS.md「部署四要素」)
|
||||
# 镜像已侧载 containerd: docker.io/ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831(IfNotPresent)
|
||||
# 参数来源: deploy/b300-ds41/dockerserve-3p1d-otlp-cs32k.yml(docker compose 版)
|
||||
# 对外: 仅 ClusterIP Service ds41-3p1d:50002(router 用普通 Pod 网络,不占宿主机端口,
|
||||
# 天然不对宿主机/外网暴露);new-api-gw 用集群 DNS http://ds41-3p1d:50002/v1 访问
|
||||
# trace: OTLP → 127.0.0.1:4317(同节点 jaeger-trace cw,badger 6h 轮窗)
|
||||
apiVersion: workloads.x-k8s.io/v1alpha2
|
||||
kind: RoleBasedGroup
|
||||
metadata:
|
||||
name: ds41-3p1d
|
||||
labels:
|
||||
app: ds41-3p1d
|
||||
sglang-model: ds-v41-flash
|
||||
spec:
|
||||
roleTemplates:
|
||||
- name: prefill-base
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
sglang-model: ds-v41-flash
|
||||
sglang-role: prefill
|
||||
spec:
|
||||
hostNetwork: true
|
||||
hostPID: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: b300-01
|
||||
containers:
|
||||
- name: sglang
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec sglang serve \
|
||||
--trust-remote-code \
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash \
|
||||
--tp 2 --ep-size 2 \
|
||||
--enable-prefill-cp --cp-strategy interleave \
|
||||
--chunked-prefill-size 32768 --max-prefill-tokens 32768 \
|
||||
--mem-fraction-static 0.75 \
|
||||
--attention-backend dsv4 \
|
||||
--moe-runner-backend flashinfer_mxfp4 \
|
||||
--cuda-graph-max-bs-decode 32 \
|
||||
--reasoning-parser auto --tool-call-parser auto \
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' \
|
||||
--max-running-requests 256 \
|
||||
--prefill-max-requests 8 \
|
||||
--tokenizer-worker-num 8 \
|
||||
--host 0.0.0.0 --port "$SGLANG_PORT" \
|
||||
--enable-cache-report --enable-metrics \
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5 \
|
||||
--enable-hierarchical-cache \
|
||||
--hicache-ratio 2 \
|
||||
--hicache-mem-layout page_first_direct \
|
||||
--hicache-io-backend direct \
|
||||
--hicache-write-policy write_through \
|
||||
--hicache-storage-backend mooncake \
|
||||
--hicache-storage-prefetch-policy wait_complete \
|
||||
--hicache-size 0 \
|
||||
--disaggregation-mode prefill \
|
||||
--disaggregation-transfer-backend mooncake \
|
||||
--disaggregation-bootstrap-port "$SGLANG_BOOTSTRAP_PORT" \
|
||||
--enable-trace \
|
||||
--trace-modules request,mooncake \
|
||||
--otlp-traces-endpoint 127.0.0.1:4317 \
|
||||
--otlp-service-name "$OTLP_NAME"
|
||||
env:
|
||||
- name: SGLANG_TORCH_PROFILER_DIR
|
||||
value: /data/ymk/prof/torch
|
||||
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||
value: "1"
|
||||
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||
value: static
|
||||
- name: MC_INTRANODE_NVLINK
|
||||
value: "true"
|
||||
- name: MC_INTRA_NVLINK
|
||||
value: "true"
|
||||
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||
value: "1"
|
||||
- name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH
|
||||
value: /data/ymk/ds41/mooncake-store.json
|
||||
- name: SGLANG_HICACHE_PREFETCH_IO_THREADS
|
||||
value: "4"
|
||||
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||
value: "16"
|
||||
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||
value: "32"
|
||||
- name: TZ
|
||||
value: Asia/Shanghai
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: metrics
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 90
|
||||
resources:
|
||||
requests:
|
||||
cpu: "32"
|
||||
memory: 220Gi
|
||||
nvidia.com/gpu: "2"
|
||||
limits:
|
||||
cpu: "96"
|
||||
memory: 950Gi
|
||||
nvidia.com/gpu: "2"
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
- name: sglang-cache
|
||||
mountPath: /root/.cache/sglang
|
||||
- name: dshm
|
||||
mountPath: /dev/shm
|
||||
volumes:
|
||||
- name: data
|
||||
hostPath:
|
||||
path: /data
|
||||
type: Directory
|
||||
- name: sglang-cache
|
||||
hostPath:
|
||||
path: /data/ymk/cache/sglang
|
||||
type: Directory
|
||||
- name: dshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 32Gi
|
||||
- name: decode-base
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
sglang-model: ds-v41-flash
|
||||
sglang-role: decode
|
||||
spec:
|
||||
hostNetwork: true
|
||||
hostPID: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: b300-01
|
||||
containers:
|
||||
- name: sglang
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec sglang serve \
|
||||
--trust-remote-code \
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash \
|
||||
--tp 2 --ep-size 2 --dp-size 2 \
|
||||
--enable-dp-attention --enable-dp-lm-head \
|
||||
--mem-fraction-static 0.83 \
|
||||
--attention-backend dsv4 \
|
||||
--moe-runner-backend flashinfer_mxfp4 \
|
||||
--cuda-graph-max-bs-decode 256 \
|
||||
--load-balance-method total_requests \
|
||||
--reasoning-parser auto --tool-call-parser auto \
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' \
|
||||
--max-running-requests 320 \
|
||||
--tokenizer-worker-num 8 \
|
||||
--host 0.0.0.0 --port 30030 \
|
||||
--enable-cache-report --enable-metrics \
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5 \
|
||||
--disaggregation-mode decode \
|
||||
--disaggregation-transfer-backend mooncake \
|
||||
--disaggregation-bootstrap-port 9001 \
|
||||
--enable-trace \
|
||||
--trace-modules request,mooncake \
|
||||
--otlp-traces-endpoint 127.0.0.1:4317 \
|
||||
--otlp-service-name ds41-3p1d-d1
|
||||
env:
|
||||
- name: SGLANG_TORCH_PROFILER_DIR
|
||||
value: /data/ymk/prof/torch
|
||||
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||
value: "1"
|
||||
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||
value: static
|
||||
- name: MC_INTRANODE_NVLINK
|
||||
value: "true"
|
||||
- name: MC_INTRA_NVLINK
|
||||
value: "true"
|
||||
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||
value: "1"
|
||||
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||
value: "16"
|
||||
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||
value: "32"
|
||||
- name: TZ
|
||||
value: Asia/Shanghai
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 30030
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: metrics
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 90
|
||||
resources:
|
||||
requests:
|
||||
cpu: "32"
|
||||
memory: 100Gi
|
||||
nvidia.com/gpu: "2"
|
||||
limits:
|
||||
cpu: "96"
|
||||
memory: 600Gi
|
||||
nvidia.com/gpu: "2"
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
- name: sglang-cache
|
||||
mountPath: /root/.cache/sglang
|
||||
- name: dshm
|
||||
mountPath: /dev/shm
|
||||
volumes:
|
||||
- name: data
|
||||
hostPath:
|
||||
path: /data
|
||||
type: Directory
|
||||
- name: sglang-cache
|
||||
hostPath:
|
||||
path: /data/ymk/cache/sglang
|
||||
type: Directory
|
||||
- name: dshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 32Gi
|
||||
roles:
|
||||
- name: mooncake-master
|
||||
replicas: 1
|
||||
standalonePattern:
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-mooncake-master
|
||||
spec:
|
||||
hostNetwork: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: b300-01
|
||||
containers:
|
||||
- name: mooncake-master
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec /opt/sglang/bin/mooncake_master \
|
||||
--port 50051 \
|
||||
--enable_http_metadata_server=true \
|
||||
--http_metadata_server_port=18080 \
|
||||
--eviction_high_watermark_ratio=0.90
|
||||
readinessProbe:
|
||||
tcpSocket:
|
||||
port: 50051
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
failureThreshold: 60
|
||||
resources:
|
||||
requests:
|
||||
cpu: "1"
|
||||
memory: 8Gi
|
||||
limits:
|
||||
cpu: "4"
|
||||
memory: 64Gi
|
||||
- name: p1
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- mooncake-master
|
||||
standalonePattern:
|
||||
templateRef:
|
||||
name: prefill-base
|
||||
patch:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-p1
|
||||
spec:
|
||||
containers:
|
||||
- name: sglang
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: "0,1"
|
||||
- name: SGLANG_PORT
|
||||
value: "30000"
|
||||
- name: SGLANG_BOOTSTRAP_PORT
|
||||
value: "8998"
|
||||
- name: OTLP_NAME
|
||||
value: ds41-3p1d-p1
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 30000
|
||||
- name: p2
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- mooncake-master
|
||||
standalonePattern:
|
||||
templateRef:
|
||||
name: prefill-base
|
||||
patch:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-p2
|
||||
spec:
|
||||
containers:
|
||||
- name: sglang
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: "2,3"
|
||||
- name: SGLANG_PORT
|
||||
value: "30010"
|
||||
- name: SGLANG_BOOTSTRAP_PORT
|
||||
value: "8999"
|
||||
- name: OTLP_NAME
|
||||
value: ds41-3p1d-p2
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 30010
|
||||
- name: p3
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- mooncake-master
|
||||
standalonePattern:
|
||||
templateRef:
|
||||
name: prefill-base
|
||||
patch:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-p3
|
||||
spec:
|
||||
containers:
|
||||
- name: sglang
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: "4,5"
|
||||
- name: SGLANG_PORT
|
||||
value: "30020"
|
||||
- name: SGLANG_BOOTSTRAP_PORT
|
||||
value: "9000"
|
||||
- name: OTLP_NAME
|
||||
value: ds41-3p1d-p3
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 30020
|
||||
- name: d1
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- mooncake-master
|
||||
standalonePattern:
|
||||
templateRef:
|
||||
name: decode-base
|
||||
patch:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-d1
|
||||
spec:
|
||||
containers:
|
||||
- name: sglang
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: "6,7"
|
||||
- name: router
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- p1
|
||||
- p2
|
||||
- p3
|
||||
- d1
|
||||
standalonePattern:
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-router
|
||||
spec:
|
||||
# 入口不走 hostNetwork:普通 Pod 网络 + ClusterIP,天然不对宿主机/外网暴露。
|
||||
# P/D 地址用节点内网 IP(hostNetwork 监听在 192.168.1.243 上,Calico 可达)。
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: b300-01
|
||||
containers:
|
||||
- name: router
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec sglang-router launch \
|
||||
--mini-lb --pd-disaggregation \
|
||||
--host 0.0.0.0 --port 50002 \
|
||||
--prefill http://192.168.1.243:30000 8998 \
|
||||
--prefill http://192.168.1.243:30010 8999 \
|
||||
--prefill http://192.168.1.243:30020 9000 \
|
||||
--decode http://192.168.1.243:30030
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 50002
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 30
|
||||
resources:
|
||||
requests:
|
||||
cpu: "1"
|
||||
memory: 4Gi
|
||||
limits:
|
||||
cpu: "8"
|
||||
memory: 16Gi
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: ds41-3p1d
|
||||
labels:
|
||||
app: ds41-3p1d
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
rbg.workloads.x-k8s.io/group-name: ds41-3p1d
|
||||
rbg.workloads.x-k8s.io/role-name: router
|
||||
ports:
|
||||
- name: http
|
||||
port: 50002
|
||||
targetPort: 50002
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PodMonitor
|
||||
metadata:
|
||||
name: ds41-3p1d
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
sglang-model: ds-v41-flash
|
||||
podMetricsEndpoints:
|
||||
- interval: 15s
|
||||
path: /metrics
|
||||
port: metrics
|
||||
@@ -0,0 +1,2 @@
|
||||
#!/bin/sh
|
||||
exit 1
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"standalone_storage": true,
|
||||
"client_server_address": "127.0.0.1:50052"
|
||||
}
|
||||
@@ -18,6 +18,8 @@
|
||||
|
||||
namespace sglang {
|
||||
|
||||
using device::math::fast_mod_div_u32_t;
|
||||
|
||||
/// \brief Ratio-2 decode compressor: pair-pool, RMSNorm and the main-KV write.
|
||||
///
|
||||
/// `kv_input` and `kv_state` rows are `2 * kHeadDim` floats, kv then score.
|
||||
@@ -34,8 +36,7 @@ struct Compress2DecodeParams {
|
||||
const int64_t* __restrict__ req; // [num_tokens], req_pool_idx per token
|
||||
const void* __restrict__ raw_out_loc; // [num_tokens] LocT, the FULL slot; 0 marks a padded row
|
||||
uint8_t* __restrict__ kvcache; // [npages, kPageBytes] uint8
|
||||
/// Positions per request slot in the pair-state ring.
|
||||
uint32_t ring_size;
|
||||
fast_mod_div_u32_t ring_size; // Positions per request slot in the pair-state ring.
|
||||
float eps;
|
||||
};
|
||||
|
||||
@@ -43,21 +44,32 @@ struct Compress2DecodeParams {
|
||||
/// At (512, 64) it also keeps the nope/rope split warp-aligned, as the fp8 amax reduction requires.
|
||||
constexpr uint32_t kC2VecSize = 2;
|
||||
|
||||
/// \brief grid = num_tokens, block = kHeadDim / kC2VecSize.
|
||||
/// \brief Where a completing row finds its partner, and who parks a pending row.
|
||||
enum class C2Mode {
|
||||
/// One row per request: the partner comes from the ring, a pending row parks there.
|
||||
Decode,
|
||||
/// `draft_len` consecutive positions per request: `blockIdx.x` is the position inside the
|
||||
/// block, `blockIdx.y` the request, and every row but the first takes its partner from the
|
||||
/// previous `kv_input` row instead of the ring.
|
||||
Verify,
|
||||
/// Arbitrary rows, paired as `_low_ratio_pair_partners` pairs them: the partner is the previous
|
||||
/// row when it is the same request's previous position, else the ring. A pending row does
|
||||
/// nothing here; `c2_prefill_store_kernel` parks the ring rows after this kernel read them.
|
||||
Prefill,
|
||||
};
|
||||
|
||||
/// \brief grid = num_tokens (Decode, Prefill) or `draft_len x batch` (Verify),
|
||||
/// block = kHeadDim / kC2VecSize.
|
||||
///
|
||||
/// An odd position completes a group with its even predecessor; an even one parks in the state.
|
||||
///
|
||||
/// Under target-verify, `draft_len` consecutive positions per request: `blockIdx.x` is the
|
||||
/// position inside the block, `blockIdx.y` the request, and every row but the first takes its
|
||||
/// partner from the previous `kv_input` row instead of the ring.
|
||||
///
|
||||
/// The three reductions have different widths and are not interchangeable: the RMSNorm
|
||||
/// statistic spans the row, an fp8 store scale 64 elements, an fp4 block 16.
|
||||
///
|
||||
/// kLayout is the cache's page format: V4 and V41 store the fake-quantized value; V41_FP4
|
||||
/// stores the e2m1 codes and their e4m3 scales directly, so the fp4 rounding happens once.
|
||||
template <
|
||||
bool kVerify,
|
||||
C2Mode kMode,
|
||||
int64_t kHeadDim,
|
||||
int64_t kRopeDim,
|
||||
int32_t kPageBits,
|
||||
@@ -91,16 +103,21 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
|
||||
|
||||
const auto tx = threadIdx.x;
|
||||
// Verify gives each request a CTA column; decode a flat grid of one row each.
|
||||
const auto row = kVerify ? blockIdx.y * gridDim.x + blockIdx.x : blockIdx.x;
|
||||
const auto row = kMode == C2Mode::Verify ? blockIdx.y * gridDim.x + blockIdx.x : blockIdx.x;
|
||||
PDLWaitPrimary<kUsePDL>();
|
||||
// Slots fit in int32 whatever width the scheduler hands them in.
|
||||
const auto raw_out_loc = static_cast<int32_t>(static_cast<const LocT*>(params.raw_out_loc)[row]);
|
||||
// CUDA graph padding is a completely inert row: do not read schedule, input,
|
||||
// state, or RoPE data, and do not publish output or update the cache.
|
||||
if (raw_out_loc == 0) return PDLTriggerSecondary<kUsePDL>();
|
||||
const auto pos = static_cast<const PosT*>(params.positions)[row];
|
||||
// A completing row reads the slot left by `pos - 1`;
|
||||
// a pending row writes its own slot, so reads and writes stay disjoint.
|
||||
const auto* loc_ptr = static_cast<const LocT*>(params.raw_out_loc);
|
||||
const auto* pos_ptr = static_cast<const PosT*>(params.positions);
|
||||
const auto raw_out_loc = loc_ptr[row];
|
||||
// Prefill rows are many, so a CUDA graph padding row exits before any load.
|
||||
// Decode and verify rows are few: every row issues its loads to keep more in flight.
|
||||
if constexpr (kMode == C2Mode::Prefill) {
|
||||
if (raw_out_loc == 0) return PDLTriggerSecondary<kUsePDL>();
|
||||
}
|
||||
const auto pos = pos_ptr[row];
|
||||
// Prefill leaves a pending row's latent unwritten; its index-K slot is -1, so nothing reads it.
|
||||
if constexpr (kMode == C2Mode::Prefill) {
|
||||
if ((pos & 1) == 0) return PDLTriggerSecondary<kUsePDL>();
|
||||
}
|
||||
const auto rid = params.req[row];
|
||||
|
||||
fp32_vec_t kv_new, score_new;
|
||||
@@ -108,21 +125,32 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
|
||||
score_new.load(params.kv_input + row * kStride, tx + kCTASize);
|
||||
|
||||
const auto ring = static_cast<int64_t>(rid) * params.ring_size;
|
||||
const auto read_row = ring + (pos - 1 + params.ring_size) % params.ring_size;
|
||||
const auto write_row = ring + pos % params.ring_size;
|
||||
const auto write_offset = static_cast<uint32_t>(pos) % params.ring_size;
|
||||
const auto read_offset = write_offset == 0 ? params.ring_size - 1 : write_offset - 1;
|
||||
const auto write_row = ring + write_offset;
|
||||
const auto read_row = ring + read_offset;
|
||||
|
||||
fp32_vec_t kv_old, score_old;
|
||||
const float* partner = params.kv_state + read_row * kStride;
|
||||
if constexpr (kVerify) {
|
||||
if constexpr (kMode == C2Mode::Verify) {
|
||||
if (blockIdx.x != 0) partner = params.kv_input + static_cast<int64_t>(row - 1) * kStride;
|
||||
}
|
||||
if constexpr (kMode == C2Mode::Prefill) {
|
||||
if (row > 0 && params.req[row - 1] == rid && pos_ptr[row - 1] == pos - 1 && loc_ptr[row - 1] != 0) {
|
||||
partner = params.kv_input + static_cast<int64_t>(row - 1) * kStride;
|
||||
}
|
||||
}
|
||||
kv_old.load(partner, tx);
|
||||
score_old.load(partner, tx + kCTASize);
|
||||
|
||||
if ((pos & 1) == 0) {
|
||||
kv_new.store(params.kv_state + write_row * kStride, tx);
|
||||
score_new.store(params.kv_state + write_row * kStride, tx + kCTASize);
|
||||
return PDLTriggerSecondary<kUsePDL>();
|
||||
if constexpr (kMode != C2Mode::Prefill) {
|
||||
// CUDA graph padding case
|
||||
if (raw_out_loc == 0) return PDLTriggerSecondary<kUsePDL>();
|
||||
if ((pos & 1) == 0) {
|
||||
kv_new.store(params.kv_state + write_row * kStride, tx);
|
||||
score_new.store(params.kv_state + write_row * kStride, tx + kCTASize);
|
||||
return PDLTriggerSecondary<kUsePDL>();
|
||||
}
|
||||
}
|
||||
|
||||
constexpr uint32_t kNumWarps = kCTASize / kWarpThreads;
|
||||
@@ -203,9 +231,9 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
|
||||
}
|
||||
}
|
||||
|
||||
const int32_t out_loc = static_cast<uint32_t>(raw_out_loc) >> 1;
|
||||
if constexpr (kLayout == KVLayout::V41_FP4) {
|
||||
// The fp4 cache takes the rotated bf16 value as is: its row quantizer is the fake quant, minus the dequant.
|
||||
const int32_t out_loc = raw_out_loc >> 1;
|
||||
const auto kv_row = Paged::row(params.kvcache, out_loc);
|
||||
return deepseek_v4::v41::store_row<kLayout>(kv_row.data, kv_row.scale, tx, staged);
|
||||
}
|
||||
@@ -228,7 +256,6 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
|
||||
}
|
||||
|
||||
// `raw_out_loc / ratio`; ratio 2 makes it a shift.
|
||||
const int32_t out_loc = raw_out_loc >> 1;
|
||||
const auto kv_row = Paged::row(params.kvcache, out_loc);
|
||||
|
||||
if constexpr (kLayout == KVLayout::V41) {
|
||||
@@ -264,21 +291,63 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
|
||||
}
|
||||
}
|
||||
|
||||
/// \brief Prefill ring write-back, launched after `flash_c2_decode_kernel<C2Mode::Prefill>`.
|
||||
///
|
||||
/// Each request's last `ring_size` rows park their `| kv | score |` at `req * ring_size +
|
||||
/// pos % ring_size` for a later step's first partner. It cannot run inside the pooling kernel:
|
||||
/// a chunk whose first row is odd and whose last row is even reads and writes the same ring
|
||||
/// slot. grid = `ring_size x num_reqs`, CTA (k, i) parks the k-th row from the end of request i,
|
||||
/// one float4 per thread; those rows are consecutive positions, so their slots are distinct.
|
||||
template <int64_t kHeadDim, typename PosT, typename LocT, bool kUsePDL>
|
||||
__global__ __launch_bounds__(kHeadDim / 2) void c2_prefill_store_kernel(
|
||||
const Compress2DecodeParams params,
|
||||
const int32_t* __restrict__ extend_start_loc,
|
||||
const int32_t* __restrict__ extend_seq_lens) {
|
||||
using namespace device;
|
||||
constexpr int64_t kStride = kHeadDim * 2;
|
||||
using vec_t = AlignedVector<float, 4>;
|
||||
static_assert(kStride == 4 * (kHeadDim / 2));
|
||||
|
||||
// The pooling kernel reads the ring rows this one overwrites.
|
||||
PDLWaitPrimary<kUsePDL>();
|
||||
const auto bx = static_cast<int32_t>(blockIdx.x);
|
||||
const auto len = extend_seq_lens[blockIdx.y];
|
||||
const auto start_loc = extend_start_loc[blockIdx.y];
|
||||
if (bx >= len) return;
|
||||
const auto row = start_loc + (len - 1 - bx);
|
||||
const auto rid = params.req[row];
|
||||
const auto out = static_cast<const LocT*>(params.raw_out_loc)[row];
|
||||
const auto pos = static_cast<uint32_t>(static_cast<const PosT*>(params.positions)[row]);
|
||||
const auto dst_row = rid * params.ring_size + pos % params.ring_size;
|
||||
if (out == 0) return;
|
||||
vec_t buf;
|
||||
buf.load(params.kv_input + row * kStride, threadIdx.x);
|
||||
PDLTriggerSecondary<kUsePDL>();
|
||||
buf.store(params.kv_state + dst_row * kStride, threadIdx.x);
|
||||
}
|
||||
|
||||
template <int64_t kHeadDim, int64_t kRopeDim, uint32_t kPageSize, deepseek_v4::KVLayout kLayout, bool kUsePDL>
|
||||
struct FlashCompress2Kernel {
|
||||
static constexpr uint32_t kBlockSize = kHeadDim / kC2VecSize;
|
||||
static constexpr int32_t kPageBits = std::bit_width(kPageSize) - 1;
|
||||
static constexpr int64_t kPageBytes = deepseek_v4::kv_page_bytes<kLayout>(kPageSize);
|
||||
static_assert(kLayout != deepseek_v4::KVLayout::V4 || kPageBytes == host::div_ceil(584ll * kPageSize, 576) * 576);
|
||||
template <bool kVerify, typename PosT, typename LocT>
|
||||
template <C2Mode kMode, typename PosT, typename LocT>
|
||||
static constexpr auto kernel =
|
||||
flash_c2_decode_kernel<kVerify, kHeadDim, kRopeDim, kPageBits, PosT, LocT, kLayout, kUsePDL>;
|
||||
flash_c2_decode_kernel<kMode, kHeadDim, kRopeDim, kPageBits, PosT, LocT, kLayout, kUsePDL>;
|
||||
template <typename PosT, typename LocT>
|
||||
static constexpr auto store_kernel = c2_prefill_store_kernel<kHeadDim, PosT, LocT, kUsePDL>;
|
||||
|
||||
/// \brief The (`positions`, `raw_out_loc`) dtype pair, resolved at run time.
|
||||
template <bool kVerify>
|
||||
template <C2Mode kMode>
|
||||
static auto select(const bool pos_i32, const bool loc_i32) {
|
||||
if (pos_i32) return loc_i32 ? kernel<kVerify, int32_t, int32_t> : kernel<kVerify, int32_t, int64_t>;
|
||||
return loc_i32 ? kernel<kVerify, int64_t, int32_t> : kernel<kVerify, int64_t, int64_t>;
|
||||
if (pos_i32) return loc_i32 ? kernel<kMode, int32_t, int32_t> : kernel<kMode, int32_t, int64_t>;
|
||||
return loc_i32 ? kernel<kMode, int64_t, int32_t> : kernel<kMode, int64_t, int64_t>;
|
||||
}
|
||||
|
||||
static auto select_store(const bool pos_i32, const bool loc_i32) {
|
||||
if (pos_i32) return loc_i32 ? store_kernel<int32_t, int32_t> : store_kernel<int32_t, int64_t>;
|
||||
return loc_i32 ? store_kernel<int64_t, int32_t> : store_kernel<int64_t, int64_t>;
|
||||
}
|
||||
|
||||
// The sum of squares reduces through a fixed-size shared array, so the CTA must be whole warps.
|
||||
@@ -302,7 +371,7 @@ struct FlashCompress2Kernel {
|
||||
const tvm::ffi::TensorView kvcache,
|
||||
const int64_t ring_size,
|
||||
const int64_t draft_len) {
|
||||
launch(
|
||||
launch<false>(
|
||||
kv_input,
|
||||
kv_state,
|
||||
kv_output,
|
||||
@@ -317,7 +386,62 @@ struct FlashCompress2Kernel {
|
||||
draft_len);
|
||||
}
|
||||
|
||||
/// \brief The extend-batch pooling kernel, then the ring write-back.
|
||||
///
|
||||
/// Rows may mix requests, prefix offsets and padding; a pending (even) row's `kv_output`
|
||||
/// row is left unwritten. `extend_start_loc` / `extend_seq_lens` are int32 `[num_reqs]`:
|
||||
/// request i owns rows `[start[i], start[i] + len[i])`; zero-length requests are skipped.
|
||||
static void run_prefill_fusion(
|
||||
const tvm::ffi::TensorView kv_input,
|
||||
const tvm::ffi::TensorView kv_state,
|
||||
const tvm::ffi::TensorView kv_output,
|
||||
const tvm::ffi::TensorView norm_weight,
|
||||
const tvm::ffi::TensorView positions,
|
||||
const tvm::ffi::TensorView req,
|
||||
const tvm::ffi::TensorView raw_out_loc,
|
||||
const float eps,
|
||||
const tvm::ffi::TensorView freqs_cis,
|
||||
const tvm::ffi::TensorView kvcache,
|
||||
const int64_t ring_size,
|
||||
const tvm::ffi::TensorView extend_start_loc,
|
||||
const tvm::ffi::TensorView extend_seq_lens) {
|
||||
using namespace host;
|
||||
auto B = SymbolicSize{"num_reqs"};
|
||||
auto device_ = SymbolicDevice{};
|
||||
device_.set_options<kDLGPU>();
|
||||
TensorMatcher({B}).with_dtype<int32_t>().with_device(device_).verify(extend_start_loc);
|
||||
TensorMatcher({B}).with_dtype<int32_t>().with_device(device_).verify(extend_seq_lens);
|
||||
const auto requests = StoreRequests{
|
||||
.extend_start_loc = static_cast<const int32_t*>(extend_start_loc.data_ptr()),
|
||||
.extend_seq_lens = static_cast<const int32_t*>(extend_seq_lens.data_ptr()),
|
||||
.num_reqs = static_cast<uint32_t>(B.unwrap()),
|
||||
};
|
||||
launch<true>(
|
||||
kv_input,
|
||||
kv_state,
|
||||
kv_output,
|
||||
norm_weight,
|
||||
positions,
|
||||
req,
|
||||
raw_out_loc,
|
||||
eps,
|
||||
ring_size,
|
||||
freqs_cis,
|
||||
kvcache,
|
||||
1,
|
||||
&requests);
|
||||
}
|
||||
|
||||
private:
|
||||
struct StoreRequests {
|
||||
const int32_t* extend_start_loc;
|
||||
const int32_t* extend_seq_lens;
|
||||
uint32_t num_reqs;
|
||||
};
|
||||
|
||||
/// \tparam kPrefill `C2Mode::Prefill` plus the ring write-back of `requests`; otherwise
|
||||
/// Decode, or Verify when `draft_len > 1`.
|
||||
template <bool kPrefill>
|
||||
static void launch(
|
||||
const tvm::ffi::TensorView kv_input,
|
||||
const tvm::ffi::TensorView kv_state,
|
||||
@@ -330,7 +454,8 @@ struct FlashCompress2Kernel {
|
||||
const int64_t ring_size,
|
||||
const tvm::ffi::TensorView freqs_cis,
|
||||
const tvm::ffi::TensorView kvcache,
|
||||
const int64_t draft_len) {
|
||||
const int64_t draft_len,
|
||||
const StoreRequests* requests = nullptr) {
|
||||
using namespace host;
|
||||
|
||||
auto N = SymbolicSize{"num_tokens"};
|
||||
@@ -358,7 +483,7 @@ struct FlashCompress2Kernel {
|
||||
|
||||
const auto num_tokens = static_cast<uint32_t>(N.unwrap());
|
||||
if (num_tokens == 0) return;
|
||||
const auto is_verify = draft_len > 1;
|
||||
const auto is_verify = !kPrefill && draft_len > 1;
|
||||
CHECK_HOST(ring_size > 0 && draft_len >= 1);
|
||||
CHECK_HOST(!is_verify || num_tokens % draft_len == 0);
|
||||
CHECK_HOST(!is_verify || ring_size > draft_len)
|
||||
@@ -373,19 +498,28 @@ struct FlashCompress2Kernel {
|
||||
.req = static_cast<const int64_t*>(req.data_ptr()),
|
||||
.raw_out_loc = raw_out_loc.data_ptr(),
|
||||
.kvcache = static_cast<uint8_t*>(kvcache.data_ptr()),
|
||||
.ring_size = static_cast<uint32_t>(ring_size),
|
||||
.ring_size = static_cast<fast_mod_div_u32_t>(ring_size),
|
||||
.eps = eps,
|
||||
};
|
||||
// `LaunchKernel` is move-only, so each arm builds its own.
|
||||
const auto pos_i32 = pos_dtype.is_type<int32_t>();
|
||||
const auto loc_i32 = loc_dtype.is_type<int32_t>();
|
||||
if (is_verify) {
|
||||
if constexpr (kPrefill) {
|
||||
const auto k = select<C2Mode::Prefill>(pos_i32, loc_i32);
|
||||
LaunchKernel(num_tokens, kBlockSize, device_.unwrap()) //
|
||||
.enable_pdl(kUsePDL)(k, params);
|
||||
if (requests->num_reqs == 0) return;
|
||||
const auto store = select_store(pos_i32, loc_i32);
|
||||
const auto grid = dim3{static_cast<uint32_t>(ring_size), requests->num_reqs};
|
||||
LaunchKernel(grid, kHeadDim / 2, device_.unwrap()) //
|
||||
.enable_pdl(kUsePDL)(store, params, requests->extend_start_loc, requests->extend_seq_lens);
|
||||
} else if (is_verify) {
|
||||
const auto block = static_cast<uint32_t>(draft_len);
|
||||
const auto k = select<true>(pos_i32, loc_i32);
|
||||
const auto k = select<C2Mode::Verify>(pos_i32, loc_i32);
|
||||
LaunchKernel(dim3{block, num_tokens / block}, kBlockSize, device_.unwrap()) //
|
||||
.enable_pdl(kUsePDL)(k, params);
|
||||
} else {
|
||||
const auto k = select<false>(pos_i32, loc_i32);
|
||||
const auto k = select<C2Mode::Decode>(pos_i32, loc_i32);
|
||||
LaunchKernel(num_tokens, kBlockSize, device_.unwrap()) //
|
||||
.enable_pdl(kUsePDL)(k, params);
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
#include <sgl_kernel/tensor.h>
|
||||
#include <sgl_kernel/utils.h>
|
||||
|
||||
#include <sgl_kernel/math.cuh>
|
||||
#include <sgl_kernel/runtime.cuh>
|
||||
#include <sgl_kernel/type.cuh>
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
@@ -21,31 +22,7 @@ namespace sglang {
|
||||
|
||||
using device::distributed::PushWorkSpace;
|
||||
using device::distributed::Semaphore;
|
||||
|
||||
// Runtime uint32 division as a multiply-high and a shift (round-up magic,
|
||||
// exact below 2^31); cuda::fast_mod_div needs a newer CCCL than CUDA 13 bundles.
|
||||
struct fast_mod_div_u32_t {
|
||||
uint32_t divisor;
|
||||
uint32_t magic;
|
||||
uint32_t shift;
|
||||
|
||||
__host__ explicit fast_mod_div_u32_t(uint32_t d) : divisor(d), magic(0), shift(0) {
|
||||
if (d > 1) {
|
||||
const uint32_t log2_ceil = 32 - std::countl_zero(d - 1);
|
||||
const uint32_t p = 31 + log2_ceil;
|
||||
magic = static_cast<uint32_t>(((uint64_t{1} << p) + d - 1) / d);
|
||||
shift = p - 32;
|
||||
}
|
||||
}
|
||||
|
||||
__device__ friend uint32_t operator/(uint32_t n, const fast_mod_div_u32_t& fd) {
|
||||
return fd.divisor == 1 ? n : __umulhi(n, fd.magic) >> fd.shift;
|
||||
}
|
||||
|
||||
__device__ friend uint32_t operator%(uint32_t n, const fast_mod_div_u32_t& fd) {
|
||||
return n - (n / fd) * fd.divisor;
|
||||
}
|
||||
};
|
||||
using device::math::fast_mod_div_u32_t;
|
||||
|
||||
template <uint32_t kWorldSize>
|
||||
struct NVLinkCommPushParams {
|
||||
@@ -168,8 +145,7 @@ PUSH_KERNEL void nvlink_push_kernel(const __grid_constant__ NVLinkCommPushParams
|
||||
ptx::st_multimem_16B(vec, dst_ptr_mc, vid);
|
||||
}
|
||||
} else /* reduce-scatter only */ {
|
||||
const auto token_id = vid / params.vecs_per_token_div;
|
||||
const auto offset = vid % params.vecs_per_token_div;
|
||||
const auto [token_id, offset] = math::div(vid, params.vecs_per_token_div);
|
||||
// Both by a compile-time constant, so this is a mask and a shift.
|
||||
const auto dst_rank = token_id % kWorldSize;
|
||||
const auto dst_token_id = token_id / kWorldSize;
|
||||
|
||||
@@ -112,6 +112,53 @@ SGL_DEVICE float fma_f32_bf16(bf16_t a, float b, float acc) {
|
||||
return fmaf(cast<fp32_t>(a), b, acc);
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief Fast uint32 division and modulo by a compile-time constant divisor.
|
||||
*
|
||||
* This struct is only exact when the divisor is less than `2^31`. Larger dividends
|
||||
* will lead to incorrect results. Be careful when using this struct.
|
||||
*/
|
||||
struct fast_mod_div_u32_t {
|
||||
public:
|
||||
explicit constexpr fast_mod_div_u32_t(uint32_t d) : m_divisor(d) {
|
||||
if (d > 1) {
|
||||
const uint32_t log2_ceil = 32 - std::countl_zero(d - 1);
|
||||
const uint32_t p = 31 + log2_ceil;
|
||||
m_magic = static_cast<uint32_t>(((uint64_t{1} << p) + d - 1) / d);
|
||||
m_shift = p - 32;
|
||||
} else {
|
||||
m_magic = 0;
|
||||
m_shift = 0;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
SGL_DEVICE friend uint32_t operator/(T n, const fast_mod_div_u32_t& d) {
|
||||
static_assert(std::is_unsigned_v<T> && sizeof(T) <= sizeof(uint32_t));
|
||||
return d.m_divisor == 1 ? n : __umulhi(n, d.m_magic) >> d.m_shift;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
SGL_DEVICE friend uint32_t operator%(T n, const fast_mod_div_u32_t& d) {
|
||||
static_assert(std::is_unsigned_v<T> && sizeof(T) <= sizeof(uint32_t));
|
||||
return n - (n / d) * d.m_divisor;
|
||||
}
|
||||
|
||||
constexpr operator uint32_t() const {
|
||||
return m_divisor;
|
||||
}
|
||||
|
||||
private:
|
||||
uint32_t m_divisor;
|
||||
uint32_t m_magic;
|
||||
uint32_t m_shift;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
SGL_DEVICE uint2 div(T n, const fast_mod_div_u32_t& d) {
|
||||
return {n / d, n % d};
|
||||
}
|
||||
|
||||
} // namespace device::math
|
||||
|
||||
} // namespace sglang
|
||||
|
||||
@@ -273,6 +273,67 @@ def store_fp4_index_k_cache(
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _gather_fp4_index_k_kernel(
|
||||
cache,
|
||||
cache_stride,
|
||||
slots,
|
||||
payload,
|
||||
scales,
|
||||
n,
|
||||
page_size: tl.constexpr,
|
||||
BLOCK_ROWS: tl.constexpr,
|
||||
):
|
||||
# int32 view of a page: page_size * 16 payload words, then page_size scale words.
|
||||
rows = tl.program_id(0) * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS)
|
||||
valid = rows < n
|
||||
slot = tl.load(slots + rows, valid, 0).to(tl.int64)
|
||||
page = slot // page_size
|
||||
offset = slot - page * page_size
|
||||
page_base = cache + page * cache_stride
|
||||
words = tl.arange(0, 16)
|
||||
payload_words = tl.load(
|
||||
page_base[:, None] + offset[:, None] * 16 + words[None, :], valid[:, None]
|
||||
)
|
||||
tl.store(
|
||||
payload + rows[:, None].to(tl.int64) * 16 + words[None, :],
|
||||
payload_words,
|
||||
valid[:, None],
|
||||
)
|
||||
tl.store(scales + rows, tl.load(page_base + page_size * 16 + offset, valid), valid)
|
||||
|
||||
|
||||
def gather_fp4_index_k(
|
||||
cache: torch.Tensor, slots: torch.Tensor, *, page_size: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Packed fp4 rows at ``slots`` of the index-K ``cache`` (``[pages, page_size *
|
||||
68]`` uint8, ``store_fp4_index_k_cache``'s layout): (payload int8 ``[n, 64]``,
|
||||
ue8m0 scales packed int32 ``[n]``), the layout ``quantize_fp4_indexer_tensor``
|
||||
returns."""
|
||||
assert cache.shape[1] == page_size * INDEX_K_SLOT_BYTES and cache.stride(1) == 1
|
||||
# The kernel addresses slots as `slots + rows`, so a strided view would be read
|
||||
# as if its elements were packed.
|
||||
assert slots.is_contiguous()
|
||||
words = cache.view(torch.int32)
|
||||
n = slots.shape[0]
|
||||
payload = torch.empty((n, 16), device=cache.device, dtype=torch.int32)
|
||||
scales = torch.empty((n,), device=cache.device, dtype=torch.int32)
|
||||
if n > 0:
|
||||
block_rows = 64
|
||||
_gather_fp4_index_k_kernel[(triton.cdiv(n, block_rows),)](
|
||||
words,
|
||||
words.stride(0),
|
||||
slots,
|
||||
payload,
|
||||
scales,
|
||||
n,
|
||||
page_size,
|
||||
BLOCK_ROWS=block_rows,
|
||||
num_warps=4,
|
||||
)
|
||||
return payload.view(torch.int8), scales
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _index_k_rope_pack_kernel(
|
||||
X,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Fused ratio-1 and ratio-2 decode compressors: RMSNorm, RoPE and the FlashMLA
|
||||
cache write in one launch.
|
||||
"""Fused ratio-1 and ratio-2 compressors: RMSNorm, RoPE and the FlashMLA cache
|
||||
write in one launch. The ratio-1 kernel is per token, so decode and extend share
|
||||
it; ratio 2 has a decode/verify entry point and an extend one.
|
||||
|
||||
Ratio 1 takes the bf16 ``wkv`` projection as is: RoPE uses the token's own
|
||||
position and the compressed slot equals the FULL slot. Ratio 2 pair-pools the
|
||||
@@ -17,6 +18,7 @@ import torch
|
||||
from sglang.kernels.jit.utils import (
|
||||
cache_once,
|
||||
is_arch_support_pdl,
|
||||
is_hip_runtime,
|
||||
load_jit,
|
||||
make_cpp_args,
|
||||
)
|
||||
@@ -119,11 +121,12 @@ def _jit_c2_module(
|
||||
is_arch_support_pdl(),
|
||||
)
|
||||
return load_jit(
|
||||
make_name("c2_decode"),
|
||||
make_name("c2"),
|
||||
*args,
|
||||
cuda_files=["deepseek_v4/c2.cuh"],
|
||||
cuda_wrappers=[
|
||||
("decode_fusion", f"FlashCompress2Kernel<{args}>::run_decode_fusion"),
|
||||
("prefill_fusion", f"FlashCompress2Kernel<{args}>::run_prefill_fusion"),
|
||||
],
|
||||
)
|
||||
|
||||
@@ -149,6 +152,12 @@ def c2_decode_norm_rope_store(
|
||||
|
||||
The cache store uses ``raw_out_loc // 2`` as its slot.
|
||||
|
||||
A padded graph row is marked by ``raw_out_loc == 0`` and writes nothing, but
|
||||
decode and verify rows issue their ring load before that check, so a padded
|
||||
row's ``req`` and ``positions`` must still address an in-bounds ring slot.
|
||||
The decode graph buffers zero both tails (``PaddingPolicy.ZERO``); a runner
|
||||
that left stale request indices there would read out of the ring pool.
|
||||
|
||||
:param freqs_cis: ``[max_pos, rope_dim]`` fp32, real/imag interleaved --
|
||||
``torch.view_as_real(freqs).flatten(-2)``. Indexed
|
||||
in-kernel at ``positions - 1``, the position the latent
|
||||
@@ -181,3 +190,60 @@ def c2_decode_norm_rope_store(
|
||||
draft_len,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def c2_prefill_norm_rope_store(
|
||||
kv_input: torch.Tensor,
|
||||
kv_state: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
req: torch.Tensor,
|
||||
raw_out_loc: torch.Tensor,
|
||||
eps: float,
|
||||
freqs_cis: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
extend_start_loc: torch.Tensor,
|
||||
extend_seq_lens: torch.Tensor,
|
||||
*,
|
||||
page_size: int,
|
||||
ring_size: int,
|
||||
layout: Union[KVLayout, str] = KVLayout.V4,
|
||||
out: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
""":func:`c2_decode_norm_rope_store` for an extend batch, then the ring write-back.
|
||||
|
||||
Rows may mix requests, prefix offsets and CUDA graph padding (``raw_out_loc
|
||||
== 0``). An odd position pairs with the previous row when that row is the
|
||||
same request's previous position, else with its ring slot. Afterwards each
|
||||
request's last ``ring_size`` rows are saved to the ring, which a later step
|
||||
reads for its first partner.
|
||||
|
||||
:param extend_start_loc: int32 ``[num_reqs]``, each request's first row.
|
||||
:param extend_seq_lens: int32 ``[num_reqs]``, each request's row count.
|
||||
:return: ``out``, ``[num_tokens, head_dim]`` bf16. Only the rows of odd
|
||||
positions are written; the rest are left as they were.
|
||||
"""
|
||||
num_tokens, fused_dim = kv_input.shape
|
||||
head_dim = fused_dim // 2
|
||||
if out is None:
|
||||
out = kv_input.new_empty((num_tokens, head_dim), dtype=torch.bfloat16)
|
||||
|
||||
layout = KVLayout.parse(layout)
|
||||
module = _jit_c2_module(head_dim, freqs_cis.shape[-1], page_size, layout)
|
||||
module.prefill_fusion(
|
||||
kv_input,
|
||||
kv_state,
|
||||
out,
|
||||
norm_weight,
|
||||
positions,
|
||||
req,
|
||||
raw_out_loc,
|
||||
eps,
|
||||
freqs_cis,
|
||||
# HIP's fp8_e4m3_t is uint8_t, so the kernel matches the fp8 pool as bytes
|
||||
k_cache.view(torch.uint8) if is_hip_runtime() else k_cache,
|
||||
ring_size,
|
||||
extend_start_loc,
|
||||
extend_seq_lens,
|
||||
)
|
||||
return out
|
||||
|
||||
@@ -36,7 +36,7 @@ def _build_swa_token_ids_kernel(
|
||||
tl.store(out_ptr + out_off + i, swa_id)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["top_k"])
|
||||
@triton.jit(do_not_specialize=["num_tokens", "num_reqs", "top_k"])
|
||||
def _combine_topk_swa_indices_kernel(
|
||||
combined_indices_ptr,
|
||||
combined_indices_stride,
|
||||
@@ -49,59 +49,76 @@ def _combine_topk_swa_indices_kernel(
|
||||
gather_lens_ptr,
|
||||
compressed_base_ptr,
|
||||
swa_base_ptr,
|
||||
num_tokens,
|
||||
num_reqs,
|
||||
top_k,
|
||||
COMPRESS_RATIO: tl.constexpr,
|
||||
WINDOW_SIZE: tl.constexpr,
|
||||
PADDED_TOP_K: tl.constexpr,
|
||||
WIDTH: tl.constexpr,
|
||||
PADDED_WIDTH: tl.constexpr,
|
||||
BLOCK_T: tl.constexpr,
|
||||
SEARCH_STEPS: tl.constexpr,
|
||||
):
|
||||
batch_idx = tl.program_id(0)
|
||||
worker_id = tl.program_id(1)
|
||||
num_workers = tl.num_programs(1)
|
||||
# BLOCK_T consecutive tokens per program, so the grid scales with tokens, not requests.
|
||||
token = tl.program_id(0) * BLOCK_T + tl.arange(0, BLOCK_T)
|
||||
valid = token < num_tokens
|
||||
|
||||
# query_start_loc may be a global tensor; rebase to chunk-local offsets
|
||||
# by subtracting the chunk's starting value.
|
||||
base = tl.load(query_start_loc_ptr)
|
||||
query_start = tl.load(query_start_loc_ptr + batch_idx) - base
|
||||
query_end = tl.load(query_start_loc_ptr + batch_idx + 1) - base
|
||||
seq_len = tl.load(seq_lens_ptr + batch_idx)
|
||||
gather_len = tl.load(gather_lens_ptr + batch_idx)
|
||||
compressed_base = tl.load(compressed_base_ptr + batch_idx)
|
||||
swa_base = tl.load(swa_base_ptr + batch_idx)
|
||||
# Each token's request: the last r with query_start_loc[r] - base <= token.
|
||||
lo = tl.zeros([BLOCK_T], dtype=tl.int32)
|
||||
hi = tl.full([BLOCK_T], 0, dtype=tl.int32) + num_reqs
|
||||
for _ in tl.static_range(SEARCH_STEPS):
|
||||
mid = (lo + hi + 1) // 2
|
||||
start = tl.load(query_start_loc_ptr + mid, mask=lo < hi, other=0) - base
|
||||
go_right = (lo < hi) & (start <= token)
|
||||
lo = tl.where(go_right, mid, lo)
|
||||
hi = tl.where((lo < hi) & ~go_right, mid - 1, hi)
|
||||
batch_idx = tl.minimum(lo, num_reqs - 1)
|
||||
# Rows past the last request's queries belong to no request: -1 rows, length 0.
|
||||
owned = valid & (token < tl.load(query_start_loc_ptr + num_reqs) - base)
|
||||
|
||||
pos = tl.load(query_pos_ptr + token, mask=owned, other=0)
|
||||
seq_len = tl.load(seq_lens_ptr + batch_idx, mask=owned, other=0)
|
||||
gather_len = tl.load(gather_lens_ptr + batch_idx, mask=owned, other=0)
|
||||
compressed_base = tl.load(compressed_base_ptr + batch_idx, mask=owned, other=0)
|
||||
swa_base = tl.load(swa_base_ptr + batch_idx, mask=owned, other=0)
|
||||
# SWA portion of the gathered buffer starts from position
|
||||
# (seq_len - gather_len), not 0. The +pos-gather_start formula maps a
|
||||
# query's window back into the workspace's SWA region.
|
||||
gather_start = seq_len - gather_len
|
||||
|
||||
for token_idx in range(query_start + worker_id, query_end, num_workers):
|
||||
pos = tl.load(query_pos_ptr + token_idx)
|
||||
# -1 entries inside the top-k span stay -1 (attention skips them).
|
||||
# top_k=0 disables the compressed portion for SWA-only layers.
|
||||
topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, top_k)
|
||||
swa_len = tl.minimum(pos + 1, WINDOW_SIZE)
|
||||
# -1 entries inside the top-k span stay -1 (attention skips them).
|
||||
# top_k=0 disables the compressed portion for SWA-only layers.
|
||||
topk_len = tl.where(owned, tl.minimum((pos + 1) // COMPRESS_RATIO, top_k), 0)
|
||||
swa_len = tl.where(owned, tl.minimum(pos + 1, WINDOW_SIZE), 0)
|
||||
|
||||
combined_row = token_idx.to(tl.int64) * combined_indices_stride
|
||||
topk_row = token_idx.to(tl.int64) * topk_indices_stride
|
||||
combined_row = token.to(tl.int64)[:, None] * combined_indices_stride
|
||||
topk_row = token.to(tl.int64)[:, None] * topk_indices_stride
|
||||
|
||||
offset = tl.arange(0, PADDED_TOP_K)
|
||||
mask = offset < topk_len
|
||||
topk_vals = tl.load(
|
||||
topk_indices_ptr + topk_row + offset,
|
||||
mask=mask,
|
||||
)
|
||||
tl.store(
|
||||
combined_indices_ptr + combined_row + offset,
|
||||
tl.where(topk_vals >= 0, topk_vals + compressed_base, -1),
|
||||
mask=mask,
|
||||
)
|
||||
# Whole rows, -1 padded, so the output needs no fill pass:
|
||||
# [top-k (-1 holes kept) | window | -1 up to WIDTH].
|
||||
offset = tl.arange(0, PADDED_WIDTH)[None, :]
|
||||
in_topk = offset < topk_len[:, None]
|
||||
topk_vals = tl.load(
|
||||
topk_indices_ptr + topk_row + offset, mask=valid[:, None] & in_topk, other=-1
|
||||
)
|
||||
# Workspace SWA index: swa_base[r] + (gather_offset_in_buffer).
|
||||
# For positions [pos - swa_len + 1, pos], the buffer offsets are
|
||||
# [pos - swa_len + 1 - gather_start, pos - gather_start].
|
||||
swa_start = swa_base + pos - swa_len + 1 - gather_start
|
||||
swa_vals = swa_start[:, None] + (offset - topk_len[:, None])
|
||||
in_swa = offset < (topk_len + swa_len)[:, None]
|
||||
vals = tl.where(
|
||||
in_topk,
|
||||
tl.where(topk_vals >= 0, topk_vals + compressed_base[:, None], -1),
|
||||
tl.where(in_swa, swa_vals, -1),
|
||||
)
|
||||
tl.store(
|
||||
combined_indices_ptr + combined_row + offset,
|
||||
vals,
|
||||
mask=valid[:, None] & (offset < WIDTH),
|
||||
)
|
||||
|
||||
offset = tl.arange(0, WINDOW_SIZE)
|
||||
# Workspace SWA index: swa_base[r] + (gather_offset_in_buffer).
|
||||
# For positions [pos - swa_len + 1, pos], the buffer offsets are
|
||||
# [pos - swa_len + 1 - gather_start, pos - gather_start].
|
||||
tl.store(
|
||||
combined_indices_ptr + combined_row + topk_len + offset,
|
||||
swa_base + offset + pos - swa_len + 1 - gather_start,
|
||||
mask=offset < swa_len,
|
||||
)
|
||||
|
||||
tl.store(combined_lens_ptr + token_idx, topk_len + swa_len)
|
||||
tl.store(combined_lens_ptr + token, topk_len + swa_len, mask=valid)
|
||||
|
||||
@@ -1882,6 +1882,34 @@ def hc_expand(x: torch.Tensor, n: int) -> torch.Tensor:
|
||||
return x.repeat(1, n)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _hc_broadcast_kernel(
|
||||
x_ptr, y_ptr, H, x_stride_m, HC: tl.constexpr, BLOCK_H: tl.constexpr
|
||||
):
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
offs = tl.program_id(1) * BLOCK_H + tl.arange(0, BLOCK_H)
|
||||
mask = offs < H
|
||||
x = tl.load(x_ptr + row * x_stride_m + offs, mask=mask)
|
||||
y_row = y_ptr + row * (HC * H)
|
||||
for k in tl.static_range(HC):
|
||||
tl.store(y_row + k * H + offs, x, mask=mask)
|
||||
|
||||
|
||||
def hc_broadcast(x: torch.Tensor, hc_mult: int) -> torch.Tensor:
|
||||
"""[T, H] -> [T, hc_mult, H], every copy equal to x: the initial mHC residual."""
|
||||
if not x.is_cuda or x.stride(-1) != 1:
|
||||
return x.unsqueeze(1).repeat(1, hc_mult, 1)
|
||||
num_tokens, hidden = x.shape
|
||||
y = x.new_empty(num_tokens, hc_mult, hidden)
|
||||
if num_tokens == 0:
|
||||
return y
|
||||
block_h = 1024
|
||||
_hc_broadcast_kernel[(num_tokens, triton.cdiv(hidden, block_h))](
|
||||
x, y, hidden, x.stride(0), HC=hc_mult, BLOCK_H=block_h, num_warps=4
|
||||
)
|
||||
return y
|
||||
|
||||
|
||||
def hc_contract(x: torch.Tensor, n: int) -> torch.Tensor:
|
||||
return x.unflatten(-1, (n, -1)).mean(dim=-2)
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ class TransferKVChunk:
|
||||
# Set when the staging worker first counts this chunk toward the per-room
|
||||
# outstanding count; stays set across re-enqueue on a watermark defer.
|
||||
staging_counted: bool = False
|
||||
# Mori early-send: CUDA event to synchronize before RDMA (optional).
|
||||
# Early-send: GPU event to synchronize before reading KV (optional).
|
||||
wait_event: Optional[object] = None
|
||||
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ from sglang.srt.managers.overlap_utils import RelayPayload
|
||||
from sglang.srt.mem_cache.common import maybe_cache_unfinished_req
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo
|
||||
from sglang.srt.utils.common import is_pin_memory_available
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -140,9 +141,14 @@ class ScheduleBatchDisaggregationDecodeMixin:
|
||||
error_message, HTTPStatus.INTERNAL_SERVER_ERROR
|
||||
)
|
||||
req.grammar.finished = req.finished()
|
||||
# Non-blocking H2D: with overlap this runs on the schedule stream after
|
||||
# its wait on the in-flight forward, so a blocking copy would stall the
|
||||
# host until that forward ends and leave the GPU idle.
|
||||
last_tokens_tensor = torch.tensor(
|
||||
last_tokens, dtype=torch.int64, device=self.device
|
||||
)
|
||||
last_tokens,
|
||||
dtype=torch.int64,
|
||||
pin_memory=is_pin_memory_available(self.device),
|
||||
).to(self.device, non_blocking=True)
|
||||
|
||||
spec_info = self.spec_algorithm.build_disagg_draft_input(
|
||||
self,
|
||||
|
||||
@@ -960,6 +960,8 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
dst_layer_ids: Optional[List[int]] = None,
|
||||
dst_device_data_indices: Optional[npt.NDArray[np.int32]] = None,
|
||||
dst_device_data_ptrs: Optional[set[int]] = None,
|
||||
dst_item_lens: list[int] | None = None,
|
||||
bootstrap_room: Optional[int] = None,
|
||||
) -> int:
|
||||
"""
|
||||
Generic KV cache transfer supporting both MHA and MLA architectures.
|
||||
@@ -1011,6 +1013,21 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
len(dst_data_ptrs),
|
||||
allow_positional_fallback=self.pp_size == 1,
|
||||
)
|
||||
if dst_item_lens is not None:
|
||||
for i, j in pairs:
|
||||
if item_lens[i] != dst_item_lens[j]:
|
||||
assert bootstrap_room is not None
|
||||
failure_reason = (
|
||||
f"{state_type} item length mismatch for paired "
|
||||
f"entries src[{i}]={item_lens[i]} "
|
||||
f"dst[{j}]={dst_item_lens[j]}"
|
||||
)
|
||||
logger.error(failure_reason)
|
||||
self.conclude_failure(
|
||||
bootstrap_room=bootstrap_room,
|
||||
failure_reason=failure_reason,
|
||||
)
|
||||
return -1
|
||||
layers_params = [
|
||||
(src_data_ptrs[i], dst_data_ptrs[j], item_lens[i]) for i, j in pairs
|
||||
]
|
||||
@@ -1020,6 +1037,24 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
src_data_ptrs, dst_data_ptrs, state_type
|
||||
)
|
||||
)
|
||||
if dst_item_lens is not None:
|
||||
_, mapped_dst_lens, _ = self.get_mla_kv_ptrs_with_pp(
|
||||
item_lens, dst_item_lens, state_type
|
||||
)
|
||||
for layer_id in range(layers_current_pp_stage):
|
||||
if item_lens[layer_id] != mapped_dst_lens[layer_id]:
|
||||
assert bootstrap_room is not None
|
||||
failure_reason = (
|
||||
f"{state_type} item length mismatch for positional "
|
||||
f"entry {layer_id}: prefill={item_lens[layer_id]} "
|
||||
f"decode={mapped_dst_lens[layer_id]}"
|
||||
)
|
||||
logger.error(failure_reason)
|
||||
self.conclude_failure(
|
||||
bootstrap_room=bootstrap_room,
|
||||
failure_reason=failure_reason,
|
||||
)
|
||||
return -1
|
||||
layers_params = [
|
||||
(
|
||||
src_kv_ptrs[layer_id],
|
||||
@@ -2110,9 +2145,13 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
),
|
||||
src_layer_ids=src_state_layer_ids,
|
||||
dst_layer_ids=dst_state_layer_ids,
|
||||
dst_item_lens=dst_item_lens,
|
||||
bootstrap_room=req.room,
|
||||
)
|
||||
or rc
|
||||
)
|
||||
if self.request_status.get(req.room, KVPoll.Failed) == KVPoll.Failed:
|
||||
return rc
|
||||
elif st in (StateType.MINIMAX_INDEX_K, StateType.MINIMAX_DENSE_KV):
|
||||
# Compacted layer lists require equal TP and PP=1 on both peers.
|
||||
if self.pp_size is not None and self.pp_size > 1:
|
||||
@@ -2376,6 +2415,19 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
self._staging_outstanding[kv_chunk.room] += 1
|
||||
kv_chunk.staging_counted = True
|
||||
|
||||
# Under overlap scheduling an early cached-prefix send can
|
||||
# precede completion of the previous prefill forward. Wait on
|
||||
# the worker, not the scheduler, before reading source KV.
|
||||
# A room that already failed skips the wait; the status check
|
||||
# below then drops the chunk, including on an abort during it.
|
||||
if kv_chunk.wait_event is not None:
|
||||
if (
|
||||
kv_chunk.room in self.request_status
|
||||
and self.check_status(kv_chunk.room) != KVPoll.Failed
|
||||
):
|
||||
kv_chunk.wait_event.synchronize()
|
||||
kv_chunk.wait_event = None
|
||||
|
||||
if (
|
||||
kv_chunk.room not in self.request_status
|
||||
or self.check_status(kv_chunk.room) == KVPoll.Failed
|
||||
@@ -2588,6 +2640,13 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
executor,
|
||||
target_rank_registration_info,
|
||||
)
|
||||
if (
|
||||
self.request_status.get(
|
||||
kv_chunk.room, KVPoll.Failed
|
||||
)
|
||||
== KVPoll.Failed
|
||||
):
|
||||
break
|
||||
if state_rc != 0:
|
||||
with self.session_lock:
|
||||
self.session_failures[
|
||||
@@ -2919,6 +2978,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
state_indices: Optional[List] = None,
|
||||
num_kv_tokens: Optional[int] = None,
|
||||
trace_ctx: Optional[Union[TraceReqContext, TraceNullContext]] = None,
|
||||
wait_event: Optional[object] = None,
|
||||
):
|
||||
assert self.disaggregation_mode == DisaggregationMode.PREFILL
|
||||
assert not is_last_chunk or (is_last_chunk and aux_index is not None)
|
||||
@@ -2938,12 +2998,22 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
# add further chunks into the transfer queue.
|
||||
return
|
||||
|
||||
# NOTE(shangming): sharding according to the dst_infos to make sure
|
||||
# requests with the same dst_sessions will be added into the same
|
||||
# queue, which enables early abort with failed sessions.
|
||||
# Shard by dst session ports AND bootstrap_room. The port term alone
|
||||
# degenerates to a constant when all requests target a single decode
|
||||
# instance (its session ports are fixed), collapsing every transfer
|
||||
# onto 1/16 of the lanes and throttling P->D intake (measured at
|
||||
# client concurrency 300: #transfer-req piled at 100+ while decode
|
||||
# itself stayed healthy). Mixing in bootstrap_room keeps all chunks
|
||||
# of one request on the same lane (per-request FIFO order preserved)
|
||||
# while spreading distinct rooms across all lanes. Abort/cleanup is
|
||||
# unaffected: chunks are dropped per-room at dequeue via
|
||||
# request_status (see the worker loop) and per-request via
|
||||
# failed_sessions before send — no lane-level purge exists, and the
|
||||
# deferred abort ack is room-scoped (_staging_outstanding). Same
|
||||
# trade-off as mori/conn.py which shards by bootstrap_room alone.
|
||||
dst_infos = self.transfer_infos[bootstrap_room].keys()
|
||||
session_port_sum = sum(int(session.rsplit(":", 1)[1]) for session in dst_infos)
|
||||
shard_idx = session_port_sum % len(self.transfer_queues)
|
||||
shard_idx = (session_port_sum + bootstrap_room) % len(self.transfer_queues)
|
||||
|
||||
if trace_ctx is None:
|
||||
trace_ctx = TraceNullContext()
|
||||
@@ -2958,6 +3028,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
state_indices=state_indices,
|
||||
num_kv_tokens=num_kv_tokens,
|
||||
trace_ctx=trace_ctx,
|
||||
wait_event=wait_event,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -3064,6 +3135,8 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
|
||||
if should_skip:
|
||||
return
|
||||
|
||||
wait_event = getattr(self, "_early_send_wait_event", None)
|
||||
self._early_send_wait_event = None
|
||||
if not is_last_chunk:
|
||||
self.kv_mgr.add_transfer_request(
|
||||
self.bootstrap_room,
|
||||
@@ -3072,6 +3145,7 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
|
||||
False,
|
||||
num_kv_tokens=num_kv_tokens,
|
||||
trace_ctx=self.trace_ctx.copy_for_thread(),
|
||||
wait_event=wait_event,
|
||||
)
|
||||
else:
|
||||
self.kv_mgr.add_transfer_request(
|
||||
@@ -3083,6 +3157,7 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
|
||||
state_indices=state_indices,
|
||||
num_kv_tokens=num_kv_tokens,
|
||||
trace_ctx=self.trace_ctx.copy_for_thread(),
|
||||
wait_event=wait_event,
|
||||
)
|
||||
self._record_transfer_indices(kv_indices, state_indices)
|
||||
|
||||
|
||||
@@ -851,7 +851,11 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
advance_logprob_pt(i, req)
|
||||
continue
|
||||
|
||||
maybe_cache_unfinished_req(req, self.tree_cache)
|
||||
self.checkpoint_disagg_prefill(req)
|
||||
# G9: the disagg-prefill path bypasses batch_result_processor's
|
||||
# prefill loop, so record the drain event here — final chunk
|
||||
# done, request heading to the inflight (KV transfer) queue.
|
||||
self.record_prefill_drain(req.seqlen)
|
||||
self.disagg_prefill_inflight_queue.append(req)
|
||||
if self.spec_algorithm.is_eagle() and draft_input is not None:
|
||||
req.output_topk_p = draft_input.topk_p[i]
|
||||
@@ -1208,7 +1212,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
chunked_req_to_exclude = set()
|
||||
if (req := self.chunked_req) is not None:
|
||||
chunked_req_to_exclude.add(req)
|
||||
maybe_cache_unfinished_req(req, self.tree_cache, chunked=True)
|
||||
self.checkpoint_disagg_prefill(req, chunked=True)
|
||||
|
||||
if not self.check_bootstrap(req):
|
||||
if is_aborted(req):
|
||||
@@ -1503,10 +1507,21 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
else:
|
||||
self.disagg_prefill_pending_chunk_rids.add(req.rid)
|
||||
|
||||
def checkpoint_disagg_prefill(
|
||||
self: Scheduler, req: Req, chunked: bool = False
|
||||
) -> None:
|
||||
"""Publish the request's computed prefix to the tree cache.
|
||||
|
||||
Name shim for the #42035-era call structure: this branch predates
|
||||
#41520's ``checkpoint_kv_cache`` / ``insert_req`` cache API, so the
|
||||
equivalent here is ``maybe_cache_unfinished_req``.
|
||||
"""
|
||||
maybe_cache_unfinished_req(req, self.tree_cache, chunked=chunked)
|
||||
|
||||
def optimistic_release_and_requeue(self: Scheduler, req: Req) -> None:
|
||||
"""Release KV cache and requeue an optimistic prefill request."""
|
||||
max_attempts = get_disagg().optimistic_prefill_attempts
|
||||
maybe_cache_unfinished_req(req, self.tree_cache)
|
||||
self.checkpoint_disagg_prefill(req)
|
||||
# The cached prefix is evictable once the KV is released. Its length
|
||||
# (capped at what a retry can match) seeds the retry's storage baseline,
|
||||
# so an evicted prefix is looked up in L3 once before it is recomputed.
|
||||
@@ -1519,10 +1534,11 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
)
|
||||
)
|
||||
self._release_aborted_request(req)
|
||||
# Mamba insertion donates the checkpoint and clears its sequence marker.
|
||||
release_kv_cache(
|
||||
req, self.tree_cache, is_insert=not self.tree_cache.supports_mamba()
|
||||
)
|
||||
# The checkpoint above (maybe_cache_unfinished_req) already handed the
|
||||
# prefill KV to the tree; the request is not finished, so the release
|
||||
# only frees the rest. (#42194 semantics; our tree predates #41520's
|
||||
# mamba-donation rework, mamba models are not run on this branch.)
|
||||
release_kv_cache(req, self.tree_cache, is_insert=False)
|
||||
req.reset_for_retract()
|
||||
req.output_ids = array("q")
|
||||
req.start_send_idx = 0
|
||||
|
||||
@@ -1776,6 +1776,18 @@ async def openai_v1_chat_completions(
|
||||
)
|
||||
|
||||
|
||||
@app.post("/query_cache_hit", dependencies=[Depends(validate_json_request)])
|
||||
async def query_cache_hit(request: ChatCompletionRequest, raw_request: Request):
|
||||
"""G9: probe prefix-cache hit (L1/L2 + L3 storage) for a chat request.
|
||||
|
||||
Deployed on decode engines; the PD router calls it before dispatch to
|
||||
decide conditional bypass. Accepts a normal chat body or `input_ids`.
|
||||
"""
|
||||
return await raw_request.app.state.openai_serving_chat.query_cache_hit(
|
||||
request, raw_request
|
||||
)
|
||||
|
||||
|
||||
@app.post(
|
||||
"/v1/embeddings",
|
||||
response_class=ORJSONResponse,
|
||||
|
||||
@@ -1140,6 +1140,73 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
return "text", processed_messages.prompt_ids
|
||||
return "input_ids", processed_messages.prompt_ids
|
||||
|
||||
async def query_cache_hit(
|
||||
self, request: ChatCompletionRequest, raw_request: Request = None
|
||||
) -> ORJSONResponse:
|
||||
"""G9 probe: report how much of this chat request's prompt is already
|
||||
cached (per-rank-best L1/L2 radix + shared L3 storage) without running
|
||||
any prefill. The router uses it for conditional PD-bypass decisions.
|
||||
|
||||
Multimodal requests are rejected (v1): their sentinel-token expansion
|
||||
costs ~10-100ms per image for only ~2% of traffic; the router skips
|
||||
them for bypass anyway.
|
||||
"""
|
||||
try:
|
||||
internal_req, _ = self._convert_to_internal_request(request, raw_request)
|
||||
except Exception as e:
|
||||
return ORJSONResponse(
|
||||
status_code=400,
|
||||
content={"error": f"failed to convert chat request: {e}"},
|
||||
)
|
||||
if internal_req.contains_mm_input():
|
||||
return ORJSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "multimodal requests are not supported by /query_cache_hit"},
|
||||
)
|
||||
try:
|
||||
# 只取 input_ids,不走 _tokenize_one_request——它要求 rid_to_state
|
||||
# 里已有 ReqState(正常生成路径在 tokenize 前注册),纯查询没有。
|
||||
# dsv41 系 chat encoding 直接渲染 prompt_ids;否则对渲染后的
|
||||
# text 走 tokenizer(_tokenize_texts 与真实路径同一入口)。
|
||||
if internal_req.input_ids is not None:
|
||||
input_ids = list(internal_req.input_ids)
|
||||
elif internal_req.text:
|
||||
input_ids, _ = await self.tokenizer_manager._tokenize_texts(
|
||||
internal_req.text, False
|
||||
)
|
||||
else:
|
||||
return ORJSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "request produced neither input_ids nor text"},
|
||||
)
|
||||
from sglang.srt.managers.io_struct import QueryCacheHitReqInput
|
||||
|
||||
result = await self.tokenizer_manager.query_cache_hit(
|
||||
QueryCacheHitReqInput(
|
||||
input_ids=list(input_ids),
|
||||
extra_key=getattr(internal_req, "extra_key", None),
|
||||
cache_salt=getattr(internal_req, "cache_salt", None),
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
return ORJSONResponse(
|
||||
status_code=500, content={"error": f"cache-hit query failed: {e}"}
|
||||
)
|
||||
if result.error is not None:
|
||||
return ORJSONResponse(status_code=500, content={"error": result.error})
|
||||
hit = result.l1_hit_tokens + result.l2_hit_tokens + result.l3_hit_tokens
|
||||
return ORJSONResponse(
|
||||
content={
|
||||
"total_tokens": result.total_tokens,
|
||||
"l1_hit_tokens": result.l1_hit_tokens,
|
||||
"l2_hit_tokens": result.l2_hit_tokens,
|
||||
"l3_hit_tokens": result.l3_hit_tokens,
|
||||
"hit_tokens": hit,
|
||||
"new_tokens": max(0, result.total_tokens - hit),
|
||||
"dp_rank": result.dp_rank,
|
||||
}
|
||||
)
|
||||
|
||||
def _convert_to_internal_request(
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
|
||||
@@ -758,6 +758,16 @@ class Envs:
|
||||
SGLANG_HICACHE_HF3FS_CONFIG_PATH = EnvStr(None)
|
||||
SGLANG_HICACHE_DECODE_OFFLOAD_STRIDE = EnvInt(None)
|
||||
SGLANG_HICACHE_SKIP_HOST_DUPLICATE_RECLAIM = EnvBool(False)
|
||||
# Per-stage timing logs for the L3 storage prefetch path (query / transfer /
|
||||
# ack reduce / drain / graft / load-back). Emits "HC_TIMING ..." lines.
|
||||
SGLANG_HICACHE_TIMING = EnvBool(False)
|
||||
# Pages per batched storage IO call (query / get / put chunking).
|
||||
SGLANG_HICACHE_STORAGE_BATCH_SIZE = EnvInt(128)
|
||||
# Number of parallel prefetch IO lanes (worker threads) per controller.
|
||||
# Operations are sharded by crc32(request_id) so every rank maps a request
|
||||
# to the same lane; each lane owns dedicated completion sync groups.
|
||||
# MUST be identical across all TP/CP/PP ranks of an instance.
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS = EnvInt(4)
|
||||
SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR = EnvStr(None)
|
||||
# File-backend LRU eviction (opt-in; sizes accept SI/IEC suffixes, "0" disables).
|
||||
SGLANG_HICACHE_FILE_BACKEND_MAX_SIZE = EnvStr(None)
|
||||
|
||||
@@ -206,6 +206,7 @@ def _create_flashmla_metadata():
|
||||
# only for that shape and go stale silently if FlashMLA retunes it.
|
||||
_FLASHMLA_SCHED_BLOCK_SIZE_N = 64
|
||||
_FLASHMLA_SCHED_FIXED_OVERHEAD = 5
|
||||
_FLASHMLA_SCHED_MAX_SHARED_BYTES = 48 << 10
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=None)
|
||||
@@ -242,6 +243,10 @@ def _maybe_precompute_flashmla_sched_meta(
|
||||
|
||||
b, s_q = q.shape[0], q.shape[1]
|
||||
num_sm_parts = max(_num_sms(q.device.index) // s_q, 1)
|
||||
# Skip when shared memory exceeds the limit.
|
||||
smem_bytes = 4 * (b * 5 + 1 + num_sm_parts * 8)
|
||||
if smem_bytes > _FLASHMLA_SCHED_MAX_SHARED_BYTES:
|
||||
return
|
||||
meta = torch.empty((num_sm_parts, META_INTS), dtype=torch.int32, device=q.device)
|
||||
num_splits = torch.empty((b + 1,), dtype=torch.int32, device=q.device)
|
||||
flashmla_sched_meta(
|
||||
@@ -2543,7 +2548,7 @@ class DeepseekV4AttnBackend(
|
||||
req_to_token=self.req_to_token,
|
||||
full_to_swa=self.token_to_kv_pool.full_to_swa_index_mapping,
|
||||
swa_window_size=SWA_WINDOW,
|
||||
swa_page_size=self.token_to_kv_pool.swa_kv_pool.page_size,
|
||||
swa_page_size=self.token_to_kv_pool.get_swa_key_page_size(),
|
||||
num_qo_tokens=num_qo_tokens,
|
||||
max_seq_len=max(seq_lens_cpu_list),
|
||||
total_swa=total_swa,
|
||||
@@ -2930,6 +2935,22 @@ class DeepseekV4AttnBackend(
|
||||
self._low_ratio_compress_fused(
|
||||
layer, x, req, pos, draft_len=self.speculative_num_draft_tokens
|
||||
)
|
||||
elif (
|
||||
layer.compressor.use_fused_compress
|
||||
and forward_batch.forward_mode.is_extend_without_speculative()
|
||||
and forward_batch.extend_start_loc is not None
|
||||
):
|
||||
# DP padding extends extend_seq_lens but not extend_start_loc, whose
|
||||
# length is the request count; padded rows belong to no request.
|
||||
start_loc = forward_batch.extend_start_loc
|
||||
seq_lens = forward_batch.extend_seq_lens[: start_loc.shape[0]]
|
||||
self._low_ratio_compress_fused(
|
||||
layer,
|
||||
x,
|
||||
req,
|
||||
pos,
|
||||
extend_offsets=(start_loc.to(torch.int32), seq_lens.to(torch.int32)),
|
||||
)
|
||||
else:
|
||||
self._low_ratio_compress_torch(
|
||||
layer,
|
||||
@@ -3001,13 +3022,18 @@ class DeepseekV4AttnBackend(
|
||||
fuse_index_store=_is_sm100_or_newer(),
|
||||
)
|
||||
|
||||
def _low_ratio_compress_fused(self, layer, x, req, pos, *, draft_len=1) -> None:
|
||||
def _low_ratio_compress_fused(
|
||||
self, layer, x, req, pos, *, draft_len=1, extend_offsets=None
|
||||
) -> None:
|
||||
"""Decode or target-verify; an extend batch passes ``extend_offsets``, the int32
|
||||
(first row, row count) of each request."""
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer_rope import (
|
||||
index_k_norm_rope_pack_store,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.low_ratio_compress import (
|
||||
c1_decode_norm_rope_store,
|
||||
c2_decode_norm_rope_store,
|
||||
c2_prefill_norm_rope_store,
|
||||
)
|
||||
|
||||
pool = self.token_to_kv_pool
|
||||
@@ -3040,7 +3066,7 @@ class DeepseekV4AttnBackend(
|
||||
# CompressStatePool stores each request's pending pairs in a position ring.
|
||||
# KVAndScore rows use | kv | score |, addressed as req * ring_size + pos % ring_size.
|
||||
state = pool.get_attention_compress_states(layer_id)
|
||||
latent = c2_decode_norm_rope_store(
|
||||
args = (
|
||||
compressor.project_fused(x),
|
||||
state.kv_score_buffer.kv_score,
|
||||
compressor.norm.weight.data,
|
||||
@@ -3050,11 +3076,23 @@ class DeepseekV4AttnBackend(
|
||||
compressor.norm.eps,
|
||||
freqs_cis,
|
||||
kv_cache,
|
||||
page_size=page_size,
|
||||
ring_size=state.ring_size,
|
||||
draft_len=draft_len,
|
||||
layout=kv_layout,
|
||||
)
|
||||
if extend_offsets is None:
|
||||
latent = c2_decode_norm_rope_store(
|
||||
*args,
|
||||
page_size=page_size,
|
||||
ring_size=state.ring_size,
|
||||
draft_len=draft_len,
|
||||
layout=kv_layout,
|
||||
)
|
||||
else:
|
||||
latent = c2_prefill_norm_rope_store(
|
||||
*args,
|
||||
*extend_offsets,
|
||||
page_size=page_size,
|
||||
ring_size=state.ring_size,
|
||||
layout=kv_layout,
|
||||
)
|
||||
out_loc = core.c2_out_loc
|
||||
|
||||
indexer = layer.indexer
|
||||
@@ -3760,7 +3798,7 @@ class DeepseekV4AttnBackend(
|
||||
compress_ratio
|
||||
)
|
||||
|
||||
swa_kv_page_size = token_to_kv_pool.swa_kv_pool.page_size
|
||||
swa_kv_page_size = token_to_kv_pool.get_swa_key_page_size()
|
||||
assert swa_k_cache.ndim == 2
|
||||
# The kernel detects each cache's format from the last dim of this
|
||||
# view: 584 (V4), 528 (V4.1 fp8) or 288 (V4.1 fp4, extra cache only).
|
||||
|
||||
@@ -1472,9 +1472,10 @@ class IndexerKPool(MultiPlatformOp):
|
||||
if metadata is None:
|
||||
return None
|
||||
|
||||
assert forward_batch.seq_lens_cpu is not None
|
||||
# Spec-v2 decode/verify batches may carry no CPU seq_lens mirror (the
|
||||
# backend opts out of the D2H); test emptiness on the device shape.
|
||||
mode = forward_batch.forward_mode
|
||||
if mode.is_idle() or len(forward_batch.seq_lens_cpu) == 0:
|
||||
if mode.is_idle() or forward_batch.seq_lens.shape[0] == 0:
|
||||
return torch.full(
|
||||
(x.shape[0], self.index_topk + self.index_kpool - 1),
|
||||
-1,
|
||||
|
||||
@@ -306,7 +306,8 @@ class DeepseekSparseAttnBackend(
|
||||
extend_dummy_seqs_capped_by_req_pool: bool = True
|
||||
# Decode/verify/draft graph replay rebuilds metadata from static buffers
|
||||
# (page-table width) and never reads seq_lens_cpu / seq_lens_sum; opt out of
|
||||
# the D2H sync. The eager fallback derives lengths from GPU seq_lens.
|
||||
# the D2H sync. The eager fallback derives lengths from GPU seq_lens, and the
|
||||
# KPool indexer reads the host mirror only for extend, which always has it.
|
||||
needs_cpu_seq_lens: bool = False
|
||||
# init_cuda_graph_state sizes this for every backend, but only the TRT-LLM
|
||||
# branch of __init__ allocates one.
|
||||
@@ -337,7 +338,6 @@ class DeepseekSparseAttnBackend(
|
||||
)
|
||||
self.dsa_index_topk = get_dsa_index_topk(hf_config)
|
||||
self.dsa_index_kpool = get_dsa_index_kpool(hf_config)
|
||||
self.needs_cpu_seq_lens = self.dsa_index_kpool > 1
|
||||
self._init_kpool_metadata_fusion()
|
||||
self.max_context_len = model_runner.model_config.context_len
|
||||
self.num_q_heads = (
|
||||
|
||||
@@ -136,8 +136,8 @@ def combine_topk_swa_indices(
|
||||
|
||||
Returns int32 indices [num_tokens, padded_topk_swa] and per-token scanned-prefix
|
||||
lengths, including -1 entries skipped by attention. Width is padded to 128.
|
||||
Preallocated out_indices must contain -1 outside the written prefix;
|
||||
reuse requires chunk-invariant scanned-prefix lengths. out_lens is overwritten.
|
||||
Rows are written whole and -1 padded, so preallocated out_indices / out_lens
|
||||
are overwritten entirely; rows past the last request are -1 with length 0.
|
||||
"""
|
||||
assert topk_indices.dtype == torch.int32
|
||||
assert query_start_loc.dtype == torch.int32
|
||||
@@ -155,9 +155,9 @@ def combine_topk_swa_indices(
|
||||
num_reqs = seq_lens.shape[0]
|
||||
combined_topk = combined_topk_width(topk, window_size)
|
||||
if out_indices is None:
|
||||
combined_indices = torch.full(
|
||||
# The kernel writes whole -1 padded rows.
|
||||
combined_indices = torch.empty(
|
||||
(num_tokens, combined_topk),
|
||||
-1,
|
||||
dtype=torch.int32,
|
||||
device=topk_indices.device,
|
||||
)
|
||||
@@ -166,7 +166,7 @@ def combine_topk_swa_indices(
|
||||
assert out_indices.dtype == torch.int32
|
||||
combined_indices = out_indices
|
||||
if out_lens is None:
|
||||
combined_lens = torch.zeros(
|
||||
combined_lens = torch.empty(
|
||||
num_tokens, dtype=torch.int32, device=topk_indices.device
|
||||
)
|
||||
else:
|
||||
@@ -174,8 +174,10 @@ def combine_topk_swa_indices(
|
||||
assert out_lens.dtype == torch.int32
|
||||
combined_lens = out_lens
|
||||
|
||||
NUM_WORKERS = 128
|
||||
_combine_topk_swa_indices_kernel[(num_reqs, NUM_WORKERS)](
|
||||
if num_tokens == 0:
|
||||
return combined_indices, combined_lens
|
||||
block_t = 2 # Measured on B200 with num_warps=2, 16384 tokens, bs 1 to 64.
|
||||
_combine_topk_swa_indices_kernel[(triton.cdiv(num_tokens, block_t),)](
|
||||
combined_indices,
|
||||
combined_indices.stride(0),
|
||||
combined_lens,
|
||||
@@ -187,10 +189,16 @@ def combine_topk_swa_indices(
|
||||
gather_lens,
|
||||
compressed_base,
|
||||
swa_base,
|
||||
num_tokens,
|
||||
num_reqs,
|
||||
top_k=topk,
|
||||
COMPRESS_RATIO=compress_ratio,
|
||||
WINDOW_SIZE=window_size,
|
||||
PADDED_TOP_K=triton.next_power_of_2(topk_indices.shape[-1]),
|
||||
WIDTH=combined_topk,
|
||||
PADDED_WIDTH=triton.next_power_of_2(combined_topk),
|
||||
BLOCK_T=block_t,
|
||||
SEARCH_STEPS=max(num_reqs, 1).bit_length(),
|
||||
num_warps=2,
|
||||
)
|
||||
return combined_indices, combined_lens
|
||||
|
||||
|
||||
@@ -239,6 +239,11 @@ class EngramHasher(nn.Module):
|
||||
self.image_token_id: Optional[int] = None
|
||||
self.history: Optional[torch.Tensor] = None
|
||||
self.pad_row = 0
|
||||
# typing only
|
||||
self.token_map: torch.Tensor
|
||||
self.multipliers: torch.Tensor
|
||||
self.primes: torch.Tensor
|
||||
self.offsets: torch.Tensor
|
||||
|
||||
def init_history(self, num_req_slots: int, device) -> None:
|
||||
"""Allocate oldest-first history with a spare row for graph padding."""
|
||||
|
||||
@@ -17,6 +17,7 @@ limitations under the License.
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import zlib
|
||||
from dataclasses import dataclass
|
||||
from queue import Empty, Queue
|
||||
from typing import TYPE_CHECKING, Callable, List, NamedTuple, Optional
|
||||
@@ -32,6 +33,17 @@ from sglang.srt.mem_cache.hicache_storage import (
|
||||
count_pool_hits,
|
||||
)
|
||||
|
||||
from sglang.srt.environ import envs as _envs
|
||||
|
||||
_HC_TIMING = None
|
||||
|
||||
|
||||
def _hc_timing_enabled() -> bool:
|
||||
global _HC_TIMING
|
||||
if _HC_TIMING is None:
|
||||
_HC_TIMING = _envs.SGLANG_HICACHE_TIMING.get()
|
||||
return _HC_TIMING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
||||
from sglang.srt.mem_cache.pool_host import HostKVCache
|
||||
@@ -306,7 +318,18 @@ class HiCacheController:
|
||||
self.attn_tp_group = attn_tp_group
|
||||
self.pp_group = pp_group
|
||||
self.prefetch_hits_sync_groups: List[torch.distributed.ProcessGroup] = []
|
||||
self.prefetch_completion_sync_groups: List[torch.distributed.ProcessGroup] = []
|
||||
# One set of completion sync groups per prefetch IO lane. Each lane's
|
||||
# worker/sync threads form a serial pipeline, so the collective
|
||||
# sequence on a lane's groups is identical across ranks as long as
|
||||
# lane assignment is deterministic (see dispatch_prefetch).
|
||||
self.prefetch_completion_sync_groups_by_lane: List[
|
||||
List[torch.distributed.ProcessGroup]
|
||||
] = []
|
||||
# Read once at construction; attach/reset reuse the stored value so
|
||||
# lane count never changes underneath live sync groups.
|
||||
self.num_prefetch_io_lanes = max(
|
||||
1, _envs.SGLANG_HICACHE_PREFETCH_IO_THREADS.get()
|
||||
)
|
||||
self.mem_pool_device_allocator = token_to_kv_pool_allocator
|
||||
mem_pool_device = token_to_kv_pool_allocator.get_kvcache()
|
||||
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool
|
||||
@@ -335,8 +358,8 @@ class HiCacheController:
|
||||
self.storage_stop_event = threading.Event()
|
||||
|
||||
# Storage control queues, (re)created whenever the storage threads start.
|
||||
self.prefetch_buffer: Optional[Queue[PrefetchOperation]] = None
|
||||
self.prefetch_sync_queue: Optional[Queue[PrefetchAck]] = None
|
||||
self.prefetch_io_queues: List[Queue[PrefetchOperation]] = []
|
||||
self.prefetch_sync_queues: List[Queue[PrefetchAck]] = []
|
||||
self.prefetch_hit_queue: Optional[Queue[StorageOperation]] = None
|
||||
self.ack_prefetch_queue = Queue[PrefetchAck]()
|
||||
self.ack_backup_queue: Optional[Queue[StorageOperation]] = None
|
||||
@@ -444,27 +467,39 @@ class HiCacheController:
|
||||
self.prefetch_thread = threading.Thread(
|
||||
target=self.prefetch_thread_func, daemon=True
|
||||
)
|
||||
self.prefetch_io_aux_thread = threading.Thread(
|
||||
target=self.prefetch_io_aux_func, daemon=True
|
||||
)
|
||||
self.prefetch_sync_thread = threading.Thread(
|
||||
target=self.prefetch_sync_thread_func, daemon=True
|
||||
)
|
||||
self.prefetch_io_aux_threads = [
|
||||
threading.Thread(
|
||||
target=self.prefetch_io_aux_func, args=(lane,), daemon=True
|
||||
)
|
||||
for lane in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
self.prefetch_sync_threads = [
|
||||
threading.Thread(
|
||||
target=self.prefetch_sync_thread_func, args=(lane,), daemon=True
|
||||
)
|
||||
for lane in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
self.backup_thread = threading.Thread(
|
||||
target=self.backup_thread_func, daemon=True
|
||||
)
|
||||
self.prefetch_queue = Queue()
|
||||
self.backup_queue = Queue()
|
||||
self.prefetch_buffer = Queue()
|
||||
self.prefetch_sync_queue = Queue()
|
||||
self.prefetch_io_queues = [
|
||||
Queue() for _ in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
self.prefetch_sync_queues = [
|
||||
Queue() for _ in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
self.prefetch_hit_queue = Queue()
|
||||
self.ack_prefetch_queue = Queue()
|
||||
self.ack_backup_queue = Queue()
|
||||
self.host_mem_release_queue = Queue()
|
||||
|
||||
self.prefetch_thread.start()
|
||||
self.prefetch_io_aux_thread.start()
|
||||
self.prefetch_sync_thread.start()
|
||||
for thread in self.prefetch_io_aux_threads:
|
||||
thread.start()
|
||||
for thread in self.prefetch_sync_threads:
|
||||
thread.start()
|
||||
self.backup_thread.start()
|
||||
|
||||
def _stop_storage_threads(self):
|
||||
@@ -485,10 +520,10 @@ class HiCacheController:
|
||||
self.prefetch_queue.put_nowait(None)
|
||||
if hasattr(self, "backup_queue"):
|
||||
self.backup_queue.put_nowait(None)
|
||||
if hasattr(self, "prefetch_buffer"):
|
||||
self.prefetch_buffer.put_nowait(None)
|
||||
if hasattr(self, "prefetch_sync_queue"):
|
||||
self.prefetch_sync_queue.put_nowait(None)
|
||||
for queue in getattr(self, "prefetch_io_queues", []) or []:
|
||||
queue.put_nowait(None)
|
||||
for queue in getattr(self, "prefetch_sync_queues", []) or []:
|
||||
queue.put_nowait(None)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -498,10 +533,8 @@ class HiCacheController:
|
||||
threads.append(self.prefetch_thread)
|
||||
if hasattr(self, "backup_thread"):
|
||||
threads.append(self.backup_thread)
|
||||
if hasattr(self, "prefetch_io_aux_thread"):
|
||||
threads.append(self.prefetch_io_aux_thread)
|
||||
if hasattr(self, "prefetch_sync_thread"):
|
||||
threads.append(self.prefetch_sync_thread)
|
||||
threads.extend(getattr(self, "prefetch_io_aux_threads", []) or [])
|
||||
threads.extend(getattr(self, "prefetch_sync_threads", []) or [])
|
||||
|
||||
for t in threads:
|
||||
try:
|
||||
@@ -585,7 +618,13 @@ class HiCacheController:
|
||||
# Use dedicated gloo groups so storage prefetch sync is isolated
|
||||
# from other collectives and consistent across CPxTP participants.
|
||||
self.prefetch_hits_sync_groups = self._create_sync_groups()
|
||||
self.prefetch_completion_sync_groups = self._create_sync_groups()
|
||||
# Each IO lane gets its own completion groups: per-lane collectives
|
||||
# stay ordered because a lane is a serial pipeline and dispatch is
|
||||
# deterministic (crc32(request_id) % num_prefetch_io_lanes).
|
||||
self.prefetch_completion_sync_groups_by_lane = [
|
||||
self._create_sync_groups()
|
||||
for _ in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
|
||||
# Select the get and set functions
|
||||
self.page_get_func = self._generic_page_get
|
||||
@@ -620,9 +659,10 @@ class HiCacheController:
|
||||
except Exception:
|
||||
pass
|
||||
self._destroy_sync_groups(self.prefetch_hits_sync_groups)
|
||||
self._destroy_sync_groups(self.prefetch_completion_sync_groups)
|
||||
for lane_groups in self.prefetch_completion_sync_groups_by_lane:
|
||||
self._destroy_sync_groups(lane_groups)
|
||||
self.prefetch_hits_sync_groups = []
|
||||
self.prefetch_completion_sync_groups = []
|
||||
self.prefetch_completion_sync_groups_by_lane = []
|
||||
try:
|
||||
if (
|
||||
hasattr(self, "storage_backend")
|
||||
@@ -660,11 +700,11 @@ class HiCacheController:
|
||||
raise RuntimeError("Stop storage threads failed; detach aborted.") from e
|
||||
|
||||
# Best-effort destroy process groups created for storage ops.
|
||||
self._destroy_sync_groups(
|
||||
self.prefetch_hits_sync_groups + self.prefetch_completion_sync_groups
|
||||
)
|
||||
self._destroy_sync_groups(self.prefetch_hits_sync_groups)
|
||||
for lane_groups in self.prefetch_completion_sync_groups_by_lane:
|
||||
self._destroy_sync_groups(lane_groups)
|
||||
self.prefetch_hits_sync_groups = []
|
||||
self.prefetch_completion_sync_groups = []
|
||||
self.prefetch_completion_sync_groups_by_lane = []
|
||||
|
||||
# Best-effort close (some backends rely on GC/destructor).
|
||||
try:
|
||||
@@ -758,13 +798,17 @@ class HiCacheController:
|
||||
self.ack_load_queue.clear()
|
||||
if self.enable_storage:
|
||||
self.prefetch_thread.join()
|
||||
self.prefetch_io_aux_thread.join()
|
||||
self.prefetch_sync_thread.join()
|
||||
for thread in self.prefetch_io_aux_threads:
|
||||
thread.join()
|
||||
for thread in self.prefetch_sync_threads:
|
||||
thread.join()
|
||||
self.backup_thread.join()
|
||||
self.prefetch_queue.queue.clear()
|
||||
self.backup_queue.queue.clear()
|
||||
self.prefetch_buffer.queue.clear()
|
||||
self.prefetch_sync_queue.queue.clear()
|
||||
for queue in self.prefetch_io_queues:
|
||||
queue.queue.clear()
|
||||
for queue in self.prefetch_sync_queues:
|
||||
queue.queue.clear()
|
||||
self.prefetch_hit_queue.queue.clear()
|
||||
self.ack_prefetch_queue.queue.clear()
|
||||
self.ack_backup_queue.queue.clear()
|
||||
@@ -777,18 +821,26 @@ class HiCacheController:
|
||||
self.prefetch_thread = threading.Thread(
|
||||
target=self.prefetch_thread_func, daemon=True
|
||||
)
|
||||
self.prefetch_io_aux_thread = threading.Thread(
|
||||
target=self.prefetch_io_aux_func, daemon=True
|
||||
)
|
||||
self.prefetch_sync_thread = threading.Thread(
|
||||
target=self.prefetch_sync_thread_func, daemon=True
|
||||
)
|
||||
self.prefetch_io_aux_threads = [
|
||||
threading.Thread(
|
||||
target=self.prefetch_io_aux_func, args=(lane,), daemon=True
|
||||
)
|
||||
for lane in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
self.prefetch_sync_threads = [
|
||||
threading.Thread(
|
||||
target=self.prefetch_sync_thread_func, args=(lane,), daemon=True
|
||||
)
|
||||
for lane in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
self.backup_thread = threading.Thread(
|
||||
target=self.backup_thread_func, daemon=True
|
||||
)
|
||||
self.prefetch_thread.start()
|
||||
self.prefetch_io_aux_thread.start()
|
||||
self.prefetch_sync_thread.start()
|
||||
for thread in self.prefetch_io_aux_threads:
|
||||
thread.start()
|
||||
for thread in self.prefetch_sync_threads:
|
||||
thread.start()
|
||||
self.backup_thread.start()
|
||||
|
||||
def has_inflight_device_transfers(self) -> bool:
|
||||
@@ -1075,7 +1127,23 @@ class HiCacheController:
|
||||
count += 1
|
||||
return count
|
||||
|
||||
def _page_transfer(self, operation: PrefetchOperation) -> int:
|
||||
def dispatch_prefetch(self, operation: PrefetchOperation) -> None:
|
||||
"""Route a prefetch operation to its IO lane.
|
||||
|
||||
Lane assignment must be deterministic and identical on every rank:
|
||||
each lane owns dedicated completion sync groups, and the collective
|
||||
sequence on a lane is the filtered dispatch order. crc32 (not the
|
||||
per-process salted builtin hash) keeps the mapping stable across
|
||||
processes. All chunks of one operation stay in one lane, preserving
|
||||
the per-operation ack order the sync thread reduces over.
|
||||
"""
|
||||
lane = (
|
||||
zlib.crc32(str(operation.request_id).encode())
|
||||
% self.num_prefetch_io_lanes
|
||||
)
|
||||
self.prefetch_io_queues[lane].put(operation)
|
||||
|
||||
def _page_transfer(self, operation: PrefetchOperation, sync_queue: Queue) -> int:
|
||||
# Transfer batch by batch
|
||||
prefix_keys = operation.prefix_keys
|
||||
kv_derived_transfers = [
|
||||
@@ -1087,8 +1155,8 @@ class HiCacheController:
|
||||
completed_pages = 0
|
||||
for i in range(0, len(operation.hash_value), STORAGE_BATCH_SIZE):
|
||||
# When an error is occurred, we should keep looping and produce the same number of
|
||||
# PrefetchAck as other ranks do, because prefetch_sync_thread (i.e. consumer of
|
||||
# prefetch_sync_queue) perform reduce on the results. This is so tricky.
|
||||
# PrefetchAck as other ranks do, because the prefetch sync thread (i.e. consumer of
|
||||
# the lane sync queue) perform reduce on the results. This is so tricky.
|
||||
if all_success and operation.is_terminated():
|
||||
all_success = False
|
||||
if all_success:
|
||||
@@ -1100,6 +1168,7 @@ class HiCacheController:
|
||||
# Get one batch token, and update the completed_tokens if succeed
|
||||
extra_info = HiCacheStorageExtraInfo(prefix_keys=prefix_keys)
|
||||
|
||||
_hc_tb = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
hit_pages = self._page_transfer_kv_batch(
|
||||
operation,
|
||||
batch_hashes,
|
||||
@@ -1107,6 +1176,14 @@ class HiCacheController:
|
||||
extra_info,
|
||||
kv_derived_transfers,
|
||||
)
|
||||
if _hc_timing_enabled():
|
||||
logger.info(
|
||||
"HC_TIMING stage=xfer_batch req=%s keys=%d ms=%.1f hit=%d",
|
||||
operation.request_id,
|
||||
len(batch_hashes),
|
||||
(time.monotonic() - _hc_tb) * 1e3,
|
||||
hit_pages,
|
||||
)
|
||||
# Check termination
|
||||
if hit_pages != len(batch_hashes):
|
||||
all_success = False
|
||||
@@ -1118,7 +1195,7 @@ class HiCacheController:
|
||||
completed_tokens=completed_pages * self.page_size,
|
||||
operation=operation,
|
||||
)
|
||||
self.prefetch_sync_queue.put(ack)
|
||||
sync_queue.put(ack)
|
||||
return completed_pages
|
||||
|
||||
def _page_transfer_kv_batch(
|
||||
@@ -1160,18 +1237,33 @@ class HiCacheController:
|
||||
# Clamp to minimal number of hits.
|
||||
return min([kv_hits, *sidecar_hits.values()])
|
||||
|
||||
def prefetch_io_aux_func(self):
|
||||
def prefetch_io_aux_func(self, lane: int = 0):
|
||||
"""
|
||||
Auxiliary function conducting IO operations for prefetching.
|
||||
|
||||
Each lane has its own worker thread and queues; an operation's whole
|
||||
chunk sequence stays on one lane so the sync thread sees it in order.
|
||||
"""
|
||||
io_queue = self.prefetch_io_queues[lane]
|
||||
sync_queue = self.prefetch_sync_queues[lane]
|
||||
while not self.storage_stop_event.is_set():
|
||||
try:
|
||||
operation = self.prefetch_buffer.get(block=True, timeout=1)
|
||||
operation = io_queue.get(block=True, timeout=1)
|
||||
if operation is None:
|
||||
continue
|
||||
self._page_transfer(operation)
|
||||
_hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
self._page_transfer(operation, sync_queue)
|
||||
if _hc_timing_enabled():
|
||||
logger.info(
|
||||
"HC_TIMING stage=xfer req=%s lane=%d wait_ms=%.1f xfer_ms=%.1f",
|
||||
operation.request_id,
|
||||
lane,
|
||||
(_hc_t0 - getattr(operation, "query_done_ts", operation.start_time))
|
||||
* 1e3,
|
||||
(time.monotonic() - _hc_t0) * 1e3,
|
||||
)
|
||||
|
||||
self.prefetch_sync_queue.put(
|
||||
sync_queue.put(
|
||||
PrefetchAck(
|
||||
rid=operation.request_id,
|
||||
completed_req=True,
|
||||
@@ -1216,7 +1308,16 @@ class HiCacheController:
|
||||
for start in range(0, len(page_hashes), STORAGE_BATCH_SIZE):
|
||||
batch_hashes = page_hashes[start : start + STORAGE_BATCH_SIZE]
|
||||
extra_info = HiCacheStorageExtraInfo(prefix_keys=prefix_keys)
|
||||
_hc_tb = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
hit_page_num = self.storage_backend.batch_exists(batch_hashes, extra_info)
|
||||
if _hc_timing_enabled():
|
||||
logger.info(
|
||||
"HC_TIMING stage=query_batch req=%s keys=%d ms=%.1f hit=%d",
|
||||
operation.request_id,
|
||||
len(batch_hashes),
|
||||
(time.monotonic() - _hc_tb) * 1e3,
|
||||
hit_page_num,
|
||||
)
|
||||
hash_value.extend(batch_hashes[:hit_page_num])
|
||||
storage_query_count += hit_page_num * self.page_size
|
||||
if hit_page_num < len(batch_hashes):
|
||||
@@ -1235,10 +1336,12 @@ class HiCacheController:
|
||||
operation = self.prefetch_queue.get(block=True, timeout=1)
|
||||
if operation is None:
|
||||
continue
|
||||
_hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
if operation.is_terminated():
|
||||
hash_value, storage_hit_count = [], 0
|
||||
else:
|
||||
hash_value, storage_hit_count = self._storage_hit_query(operation)
|
||||
_hc_t1 = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
storage_hit_count_tensor = torch.tensor(
|
||||
storage_hit_count, dtype=torch.int
|
||||
)
|
||||
@@ -1248,6 +1351,18 @@ class HiCacheController:
|
||||
self.prefetch_hits_sync_groups,
|
||||
)
|
||||
storage_hit_count = storage_hit_count_tensor.item()
|
||||
if _hc_timing_enabled():
|
||||
_hc_t2 = time.monotonic()
|
||||
operation.query_done_ts = _hc_t2
|
||||
logger.info(
|
||||
"HC_TIMING stage=query req=%s wait_ms=%.1f query_ms=%.1f "
|
||||
"hit_reduce_ms=%.1f hit_tokens=%d",
|
||||
operation.request_id,
|
||||
(_hc_t0 - operation.start_time) * 1e3,
|
||||
(_hc_t1 - _hc_t0) * 1e3,
|
||||
(_hc_t2 - _hc_t1) * 1e3,
|
||||
storage_hit_count,
|
||||
)
|
||||
|
||||
# Record the TP-synced hit count; the scheduler thread decides
|
||||
# at drain time whether to revoke (below threshold) or allocate.
|
||||
@@ -1329,28 +1444,41 @@ class HiCacheController:
|
||||
except Empty:
|
||||
continue
|
||||
|
||||
def prefetch_sync_thread_func(self):
|
||||
def prefetch_sync_thread_func(self, lane: int = 0):
|
||||
"""Synchronize prefetch results across all PP and TP ranks."""
|
||||
sync_queue = self.prefetch_sync_queues[lane]
|
||||
groups = self.prefetch_completion_sync_groups_by_lane[lane]
|
||||
while not self.storage_stop_event.is_set():
|
||||
try:
|
||||
ack = self.prefetch_sync_queue.get(block=True, timeout=1)
|
||||
ack = sync_queue.get(block=True, timeout=1)
|
||||
if ack is None:
|
||||
continue
|
||||
self._reduce_prefetch_ack(ack)
|
||||
self._reduce_prefetch_ack(ack, groups)
|
||||
if _hc_timing_enabled() and ack.completed_req:
|
||||
ack.operation.final_ack_enqueue_ts = time.monotonic()
|
||||
self.ack_prefetch_queue.put(ack)
|
||||
except Empty:
|
||||
continue
|
||||
|
||||
def _reduce_prefetch_ack(self, ack: PrefetchAck) -> None:
|
||||
def _reduce_prefetch_ack(
|
||||
self, ack: PrefetchAck, groups: List[torch.distributed.ProcessGroup]
|
||||
) -> None:
|
||||
"""Synchronize all ranks to agree on a PrefetchAck."""
|
||||
if ack.completed_tokens is not None:
|
||||
# Determine the minimal successful prefix of tokens.
|
||||
_hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
completed_tokens_tensor = torch.tensor(
|
||||
ack.completed_tokens, dtype=torch.int
|
||||
)
|
||||
self._all_reduce(
|
||||
completed_tokens_tensor,
|
||||
torch.distributed.ReduceOp.MIN,
|
||||
self.prefetch_completion_sync_groups,
|
||||
groups,
|
||||
)
|
||||
ack.completed_tokens = completed_tokens_tensor.item()
|
||||
if _hc_timing_enabled():
|
||||
logger.info(
|
||||
"HC_TIMING stage=ack_reduce req=%s ms=%.1f",
|
||||
ack.operation.request_id,
|
||||
(time.monotonic() - _hc_t0) * 1e3,
|
||||
)
|
||||
|
||||
@@ -2141,6 +2141,27 @@ class SetInternalStateReqOutput(BaseReq, kw_only=True):
|
||||
updated: bool
|
||||
|
||||
|
||||
class QueryCacheHitReqInput(BaseReq, kw_only=True):
|
||||
"""G9: read-only probe of prefix-cache hit for a token sequence.
|
||||
|
||||
Sent to every DP rank of a (decode) engine; each rank answers with its own
|
||||
L1/L2 radix match plus the shared L3 storage hit continuing from it.
|
||||
"""
|
||||
|
||||
input_ids: List[int]
|
||||
extra_key: Optional[str] = None
|
||||
cache_salt: Optional[str] = None
|
||||
|
||||
|
||||
class QueryCacheHitReqOutput(BaseReq, kw_only=True):
|
||||
total_tokens: int = 0
|
||||
l1_hit_tokens: int = 0
|
||||
l2_hit_tokens: int = 0
|
||||
l3_hit_tokens: int = 0
|
||||
dp_rank: int = 0
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
class ProfileReqType(Enum):
|
||||
START_PROFILE = 1
|
||||
STOP_PROFILE = 2
|
||||
|
||||
@@ -197,6 +197,12 @@ class LoadSnapshot(msgspec.Struct, omit_defaults=True):
|
||||
num_running_reqs: int = 0
|
||||
num_waiting_reqs: int = 0
|
||||
num_waiting_uncached_tokens: int = 0
|
||||
# G9 queue-time estimation (总输入长度口径, self-consistent numerator/denominator):
|
||||
# sum of total input lengths (req.seqlen) over all pending queues, and the
|
||||
# self-learned drain rate in the same unit — completed prefills' total
|
||||
# input tokens / wall second over a sliding window.
|
||||
queued_total_input_tokens: int = 0
|
||||
drain_rate_tokens_per_s: float = 0.0
|
||||
num_used_tokens: int = 0
|
||||
num_total_tokens: int = 0
|
||||
# num_total_tokens minus tokens still awaiting a KV transfer (equal to it
|
||||
|
||||
@@ -1034,12 +1034,12 @@ class Req(ReqDllmMixin):
|
||||
else self.origin_input_ids
|
||||
) # Before image padding
|
||||
# Each decode stage's output ids. Append-only by contract:
|
||||
# _refresh_fill_ids infers how many output tokens are already in
|
||||
# refresh_fill_ids infers how many output tokens are already in
|
||||
# full_untruncated_fill_ids from lengths alone, so in-place rewrites
|
||||
# that preserve length would silently corrupt fill_ids.
|
||||
self.output_ids = array("q")
|
||||
# Full untruncated sequence: origin + output (+ DLLM mask block).
|
||||
# Kept in sync by _refresh_fill_ids; admission only updates
|
||||
# Kept in sync by refresh_fill_ids; admission only updates
|
||||
# extend_range, never mutates this array's length.
|
||||
self.full_untruncated_fill_ids = array("q")
|
||||
self.extend_range: Optional[Range] = None
|
||||
@@ -1533,7 +1533,7 @@ class Req(ReqDllmMixin):
|
||||
def get_fill_ids(self) -> array:
|
||||
return self.full_untruncated_fill_ids[: self.extend_range.end]
|
||||
|
||||
def _refresh_fill_ids(self) -> None:
|
||||
def refresh_fill_ids(self) -> None:
|
||||
"""Keep full_untruncated_fill_ids == origin_input_ids + output_ids by
|
||||
appending only the new output tokens.
|
||||
|
||||
@@ -1562,7 +1562,7 @@ class Req(ReqDllmMixin):
|
||||
self._init_fill_ids_for_dllm()
|
||||
self.determine_dllm_phase()
|
||||
else:
|
||||
self._refresh_fill_ids()
|
||||
self.refresh_fill_ids()
|
||||
|
||||
input_len = len(self.full_untruncated_fill_ids)
|
||||
|
||||
@@ -3107,7 +3107,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
for req, prefix_len in zip(
|
||||
running_batch.reqs, running_prefix_lens, strict=True
|
||||
):
|
||||
req._refresh_fill_ids()
|
||||
req.refresh_fill_ids()
|
||||
req.set_extend_range(prefix_len, prefix_len + 1)
|
||||
|
||||
# Decode tokens of the running portion live in future_map.output_tokens_buf.
|
||||
@@ -3185,7 +3185,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
# cannot stand in for it. Rows past bs are a beam tail, not requests.
|
||||
seq_lens = self.seq_lens_cpu[:bs].tolist()
|
||||
for req, seq_len in zip(self.reqs, seq_lens, strict=True):
|
||||
req._refresh_fill_ids()
|
||||
req.refresh_fill_ids()
|
||||
# end runs one past full_untruncated_fill_ids while output_ids
|
||||
# trails; safe only while decoding_reqs suppresses cache_unfinished_req.
|
||||
req.set_extend_range(seq_len - 1, seq_len)
|
||||
|
||||
@@ -161,6 +161,8 @@ from sglang.srt.managers.io_struct import (
|
||||
PauseGenerationReqInput,
|
||||
PdRoleSwitchReqInput,
|
||||
ProfileReq,
|
||||
QueryCacheHitReqInput,
|
||||
QueryCacheHitReqOutput,
|
||||
ReleaseMemoryOccupationReqInput,
|
||||
RemoveExternalCorpusReqInput,
|
||||
RemoveExternalCorpusReqOutput,
|
||||
@@ -1806,6 +1808,7 @@ class Scheduler(
|
||||
(ShutdownReq, self.handle_shutdown),
|
||||
(GetInternalStateReq, self.get_internal_state),
|
||||
(SetInternalStateReq, self.set_internal_state),
|
||||
(QueryCacheHitReqInput, self.handle_query_cache_hit),
|
||||
(RpcReqInput, self.handle_rpc_request),
|
||||
(ExpertDistributionReq, self.expert_distribution_handle),
|
||||
(LoadLoRAAdapterReqInput, self.load_lora_adapter),
|
||||
@@ -2431,6 +2434,11 @@ class Scheduler(
|
||||
self._prev_step: Optional[Tuple[int, float, bool]] = None
|
||||
self._prev_prefill_end_ts: Optional[float] = None
|
||||
self._sched_idled = False
|
||||
# G9: sliding-window drain events (monotonic_ts, total_input_tokens) for
|
||||
# the queue-time estimator. Recorded once per request when its final
|
||||
# prefill chunk completes (batch_result_processor), in the same
|
||||
# total-input-length unit as LoadSnapshot.queued_total_input_tokens.
|
||||
self.prefill_drain_window: deque = deque()
|
||||
self.load_inquirer = SchedulerLoadInquirer(
|
||||
disaggregation_mode=self.disaggregation_mode,
|
||||
server_args=self.server_args,
|
||||
@@ -2469,8 +2477,35 @@ class Scheduler(
|
||||
),
|
||||
get_total_prefill_busy_us=lambda: self.total_prefill_busy_us,
|
||||
get_decode_moment_totals=lambda: self.decode_moment_totals,
|
||||
get_prefill_drain_rate=self.get_prefill_drain_rate,
|
||||
)
|
||||
|
||||
PREFILL_DRAIN_WINDOW_S = 60.0
|
||||
|
||||
def record_prefill_drain(self, total_input_tokens: int) -> None:
|
||||
"""G9: one drain event per request whose final prefill chunk completed.
|
||||
|
||||
Tokens are the request's total input length (seqlen), matching the
|
||||
queued_total_input_tokens numerator so the router's
|
||||
wait = queued_tokens / drain_rate is unit-consistent (决策②).
|
||||
"""
|
||||
now = time.monotonic()
|
||||
self.prefill_drain_window.append((now, total_input_tokens))
|
||||
cutoff = now - self.PREFILL_DRAIN_WINDOW_S
|
||||
while self.prefill_drain_window and self.prefill_drain_window[0][0] < cutoff:
|
||||
self.prefill_drain_window.popleft()
|
||||
|
||||
def get_prefill_drain_rate(self) -> float:
|
||||
"""Completed prefill input tokens per wall second over the window."""
|
||||
now = time.monotonic()
|
||||
cutoff = now - self.PREFILL_DRAIN_WINDOW_S
|
||||
while self.prefill_drain_window and self.prefill_drain_window[0][0] < cutoff:
|
||||
self.prefill_drain_window.popleft()
|
||||
if not self.prefill_drain_window:
|
||||
return 0.0
|
||||
span = max(now - self.prefill_drain_window[0][0], 1.0)
|
||||
return sum(tokens for _, tokens in self.prefill_drain_window) / span
|
||||
|
||||
def init_output_streamer(self) -> None:
|
||||
self.output_streamer = self.get_output_streamer_class()(
|
||||
send_to_detokenizer=self.ipc_channels.send_to_detokenizer,
|
||||
@@ -2521,6 +2556,7 @@ class Scheduler(
|
||||
output_streamer=self.output_streamer,
|
||||
beam_coordinator=self.beam_coordinator,
|
||||
abort_request=self.abort_request,
|
||||
record_prefill_drain=self.record_prefill_drain,
|
||||
)
|
||||
|
||||
def init_req_max_new_tokens(self, req):
|
||||
@@ -5215,6 +5251,86 @@ class Scheduler(
|
||||
success = False
|
||||
return success
|
||||
|
||||
def handle_query_cache_hit(self, recv_req: QueryCacheHitReqInput):
|
||||
"""G9: read-only prefix-hit probe (L1/L2 radix + L3 storage).
|
||||
|
||||
The router calls this (via /query_cache_hit on the decode engine) to
|
||||
decide conditional PD-bypass. Runs inside the scheduler event loop so
|
||||
radix/storage state is consistent, and reuses the exact admission-time
|
||||
match + L3 query logic (mirrors DecodeHiCachePreallocMixin
|
||||
._build_decode_prefix_match) without creating a Req or mutating
|
||||
request state. Node access metadata (LRU) may be touched; a hit found
|
||||
here can still be evicted before the real request arrives (inherent
|
||||
TOCTOU, acceptable for an estimate).
|
||||
"""
|
||||
from sglang.srt.mem_cache.base_prefix_cache import MatchPrefixParams
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
|
||||
total = len(recv_req.input_ids)
|
||||
# 树节点键是 array("q"),RadixKey.match_at 断言两侧类型一致——
|
||||
# 查询侧必须同样用 array,否则树非空时匹配直接炸(2026-10-11 实测)。
|
||||
from array import array
|
||||
|
||||
query_ids = array("q", recv_req.input_ids)
|
||||
try:
|
||||
match_result = self.tree_cache.match_prefix(
|
||||
MatchPrefixParams(
|
||||
key=RadixKey(
|
||||
token_ids=query_ids,
|
||||
extra_key=recv_req.extra_key,
|
||||
cache_salt=recv_req.cache_salt,
|
||||
)
|
||||
)
|
||||
)
|
||||
l1_hit = len(match_result.device_indices)
|
||||
l2_hit = getattr(match_result, "host_hit_length", 0) or 0
|
||||
l3_hit = 0
|
||||
last_host_node = getattr(match_result, "last_host_node", None)
|
||||
# L3 查询条件:本调度器挂了 hicache 存储后端即可(P 节点 HiCache
|
||||
# write_through/write_back 场景),不必是 decode-hicache 模式。
|
||||
# SWA 模型的 decode radix 与 hicache 互斥(kv_cache_builder 硬限制),
|
||||
# 若只认 enable_decode_hicache,P 节点永远查不到 L3。
|
||||
can_query_l3 = getattr(self, "enable_decode_hicache", False) or getattr(
|
||||
self, "enable_hicache_storage", False
|
||||
)
|
||||
if (
|
||||
can_query_l3
|
||||
and last_host_node is not None
|
||||
and hasattr(self.tree_cache, "query_storage_hit_length")
|
||||
and (
|
||||
self.tree_cache.is_backuped(last_host_node)
|
||||
or self.tree_cache.is_root(last_host_node)
|
||||
)
|
||||
):
|
||||
matched_len = l1_hit + l2_hit
|
||||
suffix_tokens = query_ids[matched_len:]
|
||||
last_hash = self.tree_cache.get_last_hash_value(last_host_node)
|
||||
prefix_keys = (
|
||||
self.tree_cache.get_prefix_hash_values(last_host_node)
|
||||
if getattr(
|
||||
self.tree_cache, "hicache_storage_pass_prefix_keys", False
|
||||
)
|
||||
else None
|
||||
)
|
||||
l3_hit = self.tree_cache.query_storage_hit_length(
|
||||
last_host_node,
|
||||
suffix_tokens,
|
||||
last_hash,
|
||||
prefix_keys,
|
||||
extra_key=recv_req.extra_key,
|
||||
cache_salt=recv_req.cache_salt,
|
||||
)
|
||||
return QueryCacheHitReqOutput(
|
||||
total_tokens=total,
|
||||
l1_hit_tokens=l1_hit,
|
||||
l2_hit_tokens=l2_hit,
|
||||
l3_hit_tokens=l3_hit,
|
||||
dp_rank=int(get_parallel().dp_rank or 0),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"handle_query_cache_hit failed: {e}")
|
||||
return QueryCacheHitReqOutput(total_tokens=total, error=str(e))
|
||||
|
||||
def get_internal_state(self, recv_req: GetInternalStateReq):
|
||||
# Resolved config (pristine server_args + post-publish overrides) so a
|
||||
# readback reflects values changed via /set_internal_state, not startup.
|
||||
|
||||
@@ -112,6 +112,8 @@ class SchedulerBatchResultProcessor:
|
||||
output_streamer: SchedulerOutputStreamer
|
||||
beam_coordinator: BeamCoordinator
|
||||
abort_request: Callable
|
||||
# G9: scheduler-side drain-event recorder for the queue-time estimator.
|
||||
record_prefill_drain: Callable
|
||||
|
||||
def process_batch_result_prebuilt(self, batch: ScheduleBatch):
|
||||
assert self.disaggregation_mode == DisaggregationMode.DECODE
|
||||
@@ -343,6 +345,9 @@ class SchedulerBatchResultProcessor:
|
||||
|
||||
if req.inflight_middle_chunks <= 0:
|
||||
req.time_stats.set_prefill_finished_time()
|
||||
# G9: exactly once per request (final chunk only) — feed the
|
||||
# queue-time estimator with the drained total input length.
|
||||
self.record_prefill_drain(req.seqlen)
|
||||
|
||||
if sampling_mask_finish_reason is not None:
|
||||
req.to_finish = sampling_mask_finish_reason
|
||||
|
||||
@@ -54,6 +54,7 @@ class SchedulerLoadInquirer:
|
||||
get_total_prefill_uncached_tokens: Callable
|
||||
get_total_prefill_busy_us: Callable
|
||||
get_decode_moment_totals: Callable
|
||||
get_prefill_drain_rate: Callable
|
||||
|
||||
def _get_num_pending_tokens(self, chunk_deduct: int = 0) -> int:
|
||||
"""Get the total number of tokens pending prefill.
|
||||
@@ -128,9 +129,10 @@ class SchedulerLoadInquirer:
|
||||
num_used_tokens, kv_token_usage = (
|
||||
self.pool_stats_observer.get_pool_stats().get_kv_token_stats()
|
||||
)
|
||||
num_total_tokens = num_used_tokens + sum(
|
||||
queued_total_input_tokens = sum(
|
||||
req.seqlen for queue in pending_token_queues for req in queue
|
||||
)
|
||||
num_total_tokens = num_used_tokens + queued_total_input_tokens
|
||||
num_active_tokens = max(0, num_total_tokens - awaiting_kv_tokens)
|
||||
|
||||
memory = None
|
||||
@@ -220,6 +222,8 @@ class SchedulerLoadInquirer:
|
||||
num_running_reqs=num_running_reqs,
|
||||
num_waiting_reqs=num_waiting_reqs,
|
||||
num_waiting_uncached_tokens=self.get_num_waiting_uncached_tokens(),
|
||||
queued_total_input_tokens=queued_total_input_tokens,
|
||||
drain_rate_tokens_per_s=round(self.get_prefill_drain_rate(), 2),
|
||||
num_used_tokens=num_used_tokens,
|
||||
num_total_tokens=num_total_tokens,
|
||||
num_active_tokens=num_active_tokens,
|
||||
|
||||
@@ -54,6 +54,8 @@ from sglang.srt.managers.io_struct import (
|
||||
ProfileReq,
|
||||
ProfileReqOutput,
|
||||
ProfileReqType,
|
||||
QueryCacheHitReqInput,
|
||||
QueryCacheHitReqOutput,
|
||||
ReleaseMemoryOccupationReqInput,
|
||||
ReleaseMemoryOccupationReqOutput,
|
||||
RemoveExternalCorpusReqInput,
|
||||
@@ -129,6 +131,7 @@ _COMMUNICATOR_SPECS = [
|
||||
("detach_hicache_storage", DetachHiCacheStorageReqOutput),
|
||||
("profile", ProfileReqOutput),
|
||||
("get_internal_state", GetInternalStateReqOutput),
|
||||
("query_cache_hit", QueryCacheHitReqOutput),
|
||||
("set_internal_state", SetInternalStateReqOutput),
|
||||
("expert_distribution", ExpertDistributionReqOutput),
|
||||
("update_lora_adapter", LoRAUpdateOutput),
|
||||
@@ -905,6 +908,37 @@ class TokenizerControlMixin:
|
||||
self.auto_create_handle_loop()
|
||||
return await self.dumper_control_communicator(obj)
|
||||
|
||||
async def query_cache_hit(
|
||||
self: TokenizerManager, obj: QueryCacheHitReqInput
|
||||
) -> QueryCacheHitReqOutput:
|
||||
"""G9: fan the probe out to all DP ranks and keep the best hit.
|
||||
|
||||
The decode node's DP controller routes new requests to any rank, so
|
||||
the bypass decision uses the best per-rank L1/L2 (L3 is shared); the
|
||||
real request still goes through normal dispatch afterwards.
|
||||
"""
|
||||
self.auto_create_handle_loop()
|
||||
results: List[QueryCacheHitReqOutput] = await self.query_cache_hit_communicator(
|
||||
obj
|
||||
)
|
||||
ok = [r for r in results if r.error is None]
|
||||
if not ok:
|
||||
msg = (
|
||||
"; ".join(r.error or "empty response" for r in results)
|
||||
or "no scheduler responded"
|
||||
)
|
||||
return QueryCacheHitReqOutput(total_tokens=len(obj.input_ids), error=msg)
|
||||
best = max(
|
||||
ok, key=lambda r: r.l1_hit_tokens + r.l2_hit_tokens + r.l3_hit_tokens
|
||||
)
|
||||
return QueryCacheHitReqOutput(
|
||||
total_tokens=best.total_tokens,
|
||||
l1_hit_tokens=best.l1_hit_tokens,
|
||||
l2_hit_tokens=best.l2_hit_tokens,
|
||||
l3_hit_tokens=best.l3_hit_tokens,
|
||||
dp_rank=best.dp_rank,
|
||||
)
|
||||
|
||||
async def get_loads(
|
||||
self: TokenizerManager,
|
||||
include: Optional[List[str]] = None,
|
||||
|
||||
@@ -866,3 +866,15 @@ class PureSWATokenToKVPoolAllocator(SWATokenToKVPoolAllocator):
|
||||
|
||||
def is_swa_req_ring(allocator) -> bool:
|
||||
return isinstance(allocator, SWATokenToKVPoolAllocator) and allocator.swa_req_ring
|
||||
|
||||
|
||||
def swa_kept_out_of_tree(allocator) -> bool:
|
||||
"""Whether radix-tree nodes do not own the SWA state: a per-request SWA ring,
|
||||
or the DeepSeek-V4.1 encoder-replay request window, which replay rebuilds on a
|
||||
prefix hit. SWA bookkeeping must not gate prefix matching for either."""
|
||||
if not isinstance(allocator, SWATokenToKVPoolAllocator):
|
||||
return False
|
||||
if allocator.swa_req_ring:
|
||||
return True
|
||||
kvcache = getattr(allocator, "_kvcache", None)
|
||||
return getattr(kvcache, "request_window", None) is not None
|
||||
|
||||
@@ -659,6 +659,12 @@ class DeepSeekV4IndexerPool(KVCache):
|
||||
from the page layout [page_size * 64 payload | page_size * 4 scale]."""
|
||||
assert self.use_fp4_indexer, "packed readback only applies to the fp4 layout"
|
||||
buf = self.index_k_with_scale_buffer[layer_id - self.start_layer]
|
||||
if buf.is_cuda:
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer import (
|
||||
gather_fp4_index_k,
|
||||
)
|
||||
|
||||
return gather_fp4_index_k(buf, slots, page_size=self.page_size)
|
||||
slots = slots.to(torch.int64)
|
||||
p = self.page_size
|
||||
page, off = (slots // p).unsqueeze(-1), slots % p
|
||||
@@ -1855,6 +1861,11 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
# swa_kv_pool is None under the request window and unified_kv.
|
||||
return self.kv_layout
|
||||
|
||||
def get_swa_key_page_size(self) -> int:
|
||||
if self.request_window is not None:
|
||||
return self.request_window.page_size
|
||||
return self.swa_kv_pool.page_size
|
||||
|
||||
def get_swa_key_bytes_per_token(self) -> int:
|
||||
"""Last dim of the ``(pages, page_size, 1, bytes)`` view the attention
|
||||
kernel detects the SWA cache's format from."""
|
||||
|
||||
@@ -91,7 +91,7 @@ def auto_size_hicache(
|
||||
return
|
||||
requested = get_memory().hicache_ratio
|
||||
device_bytes = _estimate_hicache_bytes(params, draft_plan)
|
||||
budget = int(host_memory_budget_bytes() * fraction)
|
||||
budget = int(host_memory_budget_bytes(auto_size=True) * fraction)
|
||||
ratio = min(requested, budget * (1 - _ALLOCATION_SLACK_FRACTION) / device_bytes)
|
||||
# One collective before any pool is built: PP stages own different pool
|
||||
# counts, so a per-pool collective could deadlock.
|
||||
|
||||
@@ -20,7 +20,7 @@ if TYPE_CHECKING:
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Max pages per batched storage IO call.
|
||||
STORAGE_BATCH_SIZE = 128
|
||||
STORAGE_BATCH_SIZE = envs.SGLANG_HICACHE_STORAGE_BATCH_SIZE.get()
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -653,7 +653,7 @@ class HiRadixCache(RadixCache):
|
||||
: alloc_len // self.page_size
|
||||
]
|
||||
operation.host_indices = host_indices
|
||||
cc.prefetch_buffer.put(operation)
|
||||
cc.dispatch_prefetch(operation)
|
||||
|
||||
def _drain_ack_prefetch():
|
||||
for ack in _drain_queue(cc.ack_prefetch_queue, n_ack_prefetch):
|
||||
|
||||
@@ -15,6 +15,11 @@ def _unescape_mount_path(value: str) -> str:
|
||||
return re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), value)
|
||||
|
||||
|
||||
def _memory_stat(directory: Path) -> dict[str, int]:
|
||||
lines = (directory / "memory.stat").read_text().splitlines()
|
||||
return {key: int(value) for key, value in map(str.split, lines)}
|
||||
|
||||
|
||||
def _cgroup_memory_headroom(proc_root: Path = Path("/proc")) -> int | None:
|
||||
memberships = {}
|
||||
try:
|
||||
@@ -65,6 +70,13 @@ def _cgroup_memory_headroom(proc_root: Path = Path("/proc")) -> int | None:
|
||||
usage_name = (
|
||||
"memory.current" if filesystem == "cgroup2" else "memory.usage_in_bytes"
|
||||
)
|
||||
# File LRU pages, as MemAvailable counts; shmem sits on the anon LRU.
|
||||
# v1's unprefixed counters omit descendants, unlike its usage.
|
||||
page_cache_keys = (
|
||||
("active_file", "inactive_file")
|
||||
if filesystem == "cgroup2"
|
||||
else ("total_active_file", "total_inactive_file")
|
||||
)
|
||||
while True:
|
||||
for name in limits:
|
||||
try:
|
||||
@@ -78,7 +90,10 @@ def _cgroup_memory_headroom(proc_root: Path = Path("/proc")) -> int | None:
|
||||
# Do not silently ignore an unreadable usage file for a known
|
||||
# limit: falling back to host RAM could overrun the container.
|
||||
usage = int((directory / usage_name).read_text())
|
||||
remaining = max(0, limit - usage)
|
||||
stat = _memory_stat(directory)
|
||||
page_cache = sum(stat[key] for key in page_cache_keys)
|
||||
# Charged page cache is reclaimed under the limit, not used.
|
||||
remaining = max(0, limit - max(0, usage - page_cache))
|
||||
headroom = remaining if headroom is None else min(headroom, remaining)
|
||||
if directory == mount:
|
||||
break
|
||||
@@ -90,10 +105,25 @@ def _cgroup_memory_headroom(proc_root: Path = Path("/proc")) -> int | None:
|
||||
return headroom
|
||||
|
||||
|
||||
def available_host_memory_bytes() -> int:
|
||||
"""Conservative allocatable RAM; charged file cache is not assumed reclaimable."""
|
||||
def available_host_memory_bytes(*, allow_cgroup_fallback: bool = False) -> int:
|
||||
"""Allocatable RAM, counting reclaimable page cache as free."""
|
||||
available = psutil.virtual_memory().available
|
||||
cgroup_headroom = _cgroup_memory_headroom()
|
||||
try:
|
||||
cgroup_headroom = _cgroup_memory_headroom()
|
||||
except (OSError, RuntimeError) as exc:
|
||||
if not allow_cgroup_fallback:
|
||||
raise RuntimeError(
|
||||
"Cannot automatically size HiCache host memory; "
|
||||
"set --hicache-size explicitly (GB per rank), leaving room "
|
||||
"for other memory use within the container limit."
|
||||
) from exc
|
||||
logger.warning(
|
||||
"Cannot determine cgroup memory headroom: %s. Checking explicitly "
|
||||
"sized HiCache pools against host available memory; ensure all "
|
||||
"local ranks' pools fit within the container memory limit.",
|
||||
exc,
|
||||
)
|
||||
return available
|
||||
if cgroup_headroom is not None:
|
||||
logger.info(
|
||||
"HiCache memory headroom: host %.1f GiB, cgroup %.1f GiB",
|
||||
|
||||
@@ -15,6 +15,7 @@ import torch
|
||||
|
||||
from sglang.srt.managers.cache_controller import (
|
||||
CacheOperation,
|
||||
_hc_timing_enabled,
|
||||
)
|
||||
from sglang.srt.managers.cache_controller import (
|
||||
HiCacheController as BaseHiCacheController,
|
||||
@@ -1005,7 +1006,7 @@ class HybridCacheController(BaseHiCacheController):
|
||||
if not self._allocate_pp_prefetch_buffers(ticket, operation):
|
||||
operation.host_indices = torch.empty(0, dtype=torch.int64)
|
||||
operation.mark_terminate()
|
||||
self.prefetch_buffer.put(operation)
|
||||
self.dispatch_prefetch(operation)
|
||||
except Exception:
|
||||
logger.exception("PP HiCache ticket processing failed.")
|
||||
if operation is None or not operation.hash_value:
|
||||
@@ -1015,7 +1016,7 @@ class HybridCacheController(BaseHiCacheController):
|
||||
operation.host_indices = torch.empty(0, dtype=torch.int64)
|
||||
operation.mark_terminate()
|
||||
# Preserve every KV/sidecar ACK even when local allocation fails.
|
||||
self.prefetch_buffer.put(operation)
|
||||
self.dispatch_prefetch(operation)
|
||||
finally:
|
||||
if is_source and objects:
|
||||
self.pp_prefetch_command_queue.task_done()
|
||||
@@ -1103,15 +1104,15 @@ class HybridCacheController(BaseHiCacheController):
|
||||
)
|
||||
return host_indices, device_indices, resolved_pool_transfers
|
||||
|
||||
def _page_transfer(self, operation: PrefetchOperation) -> bool:
|
||||
def _page_transfer(self, operation: PrefetchOperation, sync_queue: Queue) -> bool:
|
||||
# KV pools and KV-derived pools first — determines actual completed page count
|
||||
kv_completed_pages = super()._page_transfer(operation)
|
||||
kv_completed_pages = super()._page_transfer(operation, sync_queue)
|
||||
|
||||
# Read non-KV derived sidecar pool, e.g. SWA, Mamba.
|
||||
self._page_transfer_sidecar(operation, kv_completed_pages)
|
||||
self._page_transfer_sidecar(operation, kv_completed_pages, sync_queue)
|
||||
|
||||
def _page_transfer_sidecar(
|
||||
self, operation: PrefetchOperation, kv_completed_pages: int
|
||||
self, operation: PrefetchOperation, kv_completed_pages: int, sync_queue: Queue
|
||||
) -> None:
|
||||
if operation.pool_transfers is None:
|
||||
return
|
||||
@@ -1143,10 +1144,10 @@ class HybridCacheController(BaseHiCacheController):
|
||||
transfers_nonkv, extra_info=extra_info
|
||||
)
|
||||
pool_hits = count_pool_hits(results)
|
||||
# Emit PrefetchAck to prefetch_sync_queue, even the operation has been canceled by the
|
||||
# Emit PrefetchAck to the lane sync queue, even the operation has been canceled by the
|
||||
# scheduler thread. The prefetch sync thread expects the same number of PrefetchAck objects
|
||||
# to perform all_reduce.
|
||||
self.prefetch_sync_queue.put(
|
||||
sync_queue.put(
|
||||
PrefetchAck(
|
||||
rid=operation.request_id,
|
||||
operation=operation,
|
||||
@@ -1420,9 +1421,11 @@ class HybridCacheController(BaseHiCacheController):
|
||||
pool.device_indices = source.device_indices
|
||||
return extra_pools
|
||||
|
||||
def _reduce_prefetch_ack(self, ack: PrefetchAck) -> None:
|
||||
def _reduce_prefetch_ack(
|
||||
self, ack: PrefetchAck, groups: List[torch.distributed.ProcessGroup]
|
||||
) -> None:
|
||||
# Handle KV-derived pool.
|
||||
super()._reduce_prefetch_ack(ack)
|
||||
super()._reduce_prefetch_ack(ack, groups)
|
||||
|
||||
# Handle other sidecar pools, e.g. SWA, Mamba.
|
||||
if ack.pool_hits is not None:
|
||||
@@ -1435,20 +1438,24 @@ class HybridCacheController(BaseHiCacheController):
|
||||
self._all_reduce(
|
||||
packed,
|
||||
torch.distributed.ReduceOp.MIN,
|
||||
self.prefetch_completion_sync_groups,
|
||||
groups,
|
||||
)
|
||||
for i, pool in enumerate(PoolName):
|
||||
ack.pool_hits[pool.value] = packed[i].item()
|
||||
|
||||
def prefetch_sync_thread_func(self) -> None:
|
||||
def prefetch_sync_thread_func(self, lane: int = 0) -> None:
|
||||
"""Synchronize progressive ACKs and publish PP ticket completion."""
|
||||
sync_queue = self.prefetch_sync_queues[lane]
|
||||
groups = self.prefetch_completion_sync_groups_by_lane[lane]
|
||||
while not self.storage_stop_event.is_set():
|
||||
try:
|
||||
ack = self.prefetch_sync_queue.get(block=True, timeout=1)
|
||||
ack = sync_queue.get(block=True, timeout=1)
|
||||
if ack is None:
|
||||
continue
|
||||
self._reduce_prefetch_ack(ack)
|
||||
self._reduce_prefetch_ack(ack, groups)
|
||||
if not getattr(ack.operation, "is_pp_broadcast", False):
|
||||
if _hc_timing_enabled() and ack.completed_req:
|
||||
ack.operation.final_ack_enqueue_ts = time.monotonic()
|
||||
self.ack_prefetch_queue.put(ack)
|
||||
continue
|
||||
|
||||
|
||||
@@ -61,12 +61,16 @@ def ranks_per_host() -> int:
|
||||
return max(launch_world_size // get_parallel().nnodes, 1)
|
||||
|
||||
|
||||
def host_memory_budget_bytes(requested_bytes: int = 0) -> int:
|
||||
def host_memory_budget_bytes(
|
||||
requested_bytes: int = 0, *, auto_size: bool = False
|
||||
) -> int:
|
||||
"""Host RAM this rank may claim for a HiCache pool.
|
||||
|
||||
Bound machine availability by the visible cgroup limits before splitting
|
||||
among local ranks. Independent engines with separate container budgets
|
||||
therefore size against their own remaining allowance.
|
||||
Auto-sizing requires successful cgroup discovery; explicit pool sizing
|
||||
can fall back to checking host availability when discovery fails.
|
||||
|
||||
Inside host_memory_budget_scope, requested_bytes is booked against the
|
||||
snapshot when it fits; the allowance before booking is returned.
|
||||
@@ -77,7 +81,10 @@ def host_memory_budget_bytes(requested_bytes: int = 0) -> int:
|
||||
_host_memory_budget.set(available - requested_bytes)
|
||||
return available
|
||||
|
||||
free = available_host_memory_bytes() - HICACHE_HOST_MEMORY_RESERVE_BYTES
|
||||
free = (
|
||||
available_host_memory_bytes(allow_cgroup_fallback=not auto_size)
|
||||
- HICACHE_HOST_MEMORY_RESERVE_BYTES
|
||||
)
|
||||
return free // ranks_per_host()
|
||||
|
||||
|
||||
|
||||
@@ -460,7 +460,18 @@ class MooncakeStore(HiCacheStorage, MooncakeBaseStore):
|
||||
)
|
||||
device_name = ""
|
||||
if self.config.standalone_storage:
|
||||
if not isinstance(mem_pool.allocator, MooncakeHostTensorAllocator):
|
||||
# V4 unified cache hands us a LogicalHostPool anchor that owns no
|
||||
# physical buffers (kv_buffer=None, allocator=None); the real
|
||||
# buffers live in side pools registered later via
|
||||
# register_mem_host_pool_v2() -> register_buffer(ptr, size).
|
||||
# Only enforce the MooncakeHostTensorAllocator requirement when
|
||||
# the anchor actually owns memory.
|
||||
_anchor_owns_buffers = (
|
||||
getattr(mem_pool, "kv_buffer", None) is not None
|
||||
)
|
||||
if _anchor_owns_buffers and not isinstance(
|
||||
getattr(mem_pool, "allocator", None), MooncakeHostTensorAllocator
|
||||
):
|
||||
raise RuntimeError(
|
||||
"MooncakeStore with standalone_storage=True requires MooncakeHostTensorAllocator. "
|
||||
"Please set standalone_storage=False "
|
||||
@@ -504,6 +515,22 @@ class MooncakeStore(HiCacheStorage, MooncakeBaseStore):
|
||||
else:
|
||||
client_hostname = self.config.local_hostname
|
||||
transfer_engine = None
|
||||
# Each rank embeds its own Mooncake client process. Giving
|
||||
# every rank the same local_hostname makes the master's
|
||||
# preferred-segment allocation unable to tell their
|
||||
# segments apart, so replicas can land on a peer rank's
|
||||
# segment and the local-memcpy fast path (which requires
|
||||
# same-process endpoints) silently falls back to TCP.
|
||||
# Make the hostname unique per rank so own-writes land on
|
||||
# the own segment and reads can use local memcpy.
|
||||
if storage_config is not None:
|
||||
client_hostname = (
|
||||
f"{client_hostname}"
|
||||
f"-d{storage_config.dp_rank}"
|
||||
f"t{storage_config.tp_rank}"
|
||||
f"p{storage_config.pp_rank}"
|
||||
f"c{storage_config.attn_cp_rank}"
|
||||
)
|
||||
|
||||
setup_kwargs = {}
|
||||
if self.config.enable_ssd_offload:
|
||||
|
||||
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Callable, Optional, Sequence
|
||||
import torch
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.mem_cache.allocator.swa import is_swa_req_ring
|
||||
from sglang.srt.mem_cache.allocator.swa import swa_kept_out_of_tree
|
||||
from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
DecLockRefParams,
|
||||
EvictParams,
|
||||
@@ -323,9 +323,9 @@ class SWAComponent(TreeComponent):
|
||||
ct = self.component_type
|
||||
state = {"len": float("inf")}
|
||||
|
||||
# A per-request SWA ring is not stored in tree nodes, so its bookkeeping
|
||||
# must not gate prefix matching.
|
||||
swa_req_ring = is_swa_req_ring(self.cache.token_to_kv_pool_allocator)
|
||||
# SWA state that tree nodes do not own (a per-request ring, or a request
|
||||
# window rebuilt by replay) must not gate prefix matching.
|
||||
swa_out_of_tree = swa_kept_out_of_tree(self.cache.token_to_kv_pool_allocator)
|
||||
|
||||
def validator(node: UnifiedTreeNode) -> bool:
|
||||
cd = node.component_data[ct]
|
||||
@@ -333,7 +333,15 @@ class SWAComponent(TreeComponent):
|
||||
# — load_back will restore SWA from host before use.
|
||||
if cd.value is None and (match_device_only or cd.host_value is None):
|
||||
state["len"] = 0
|
||||
if swa_req_ring and (node.backuped or not node.evicted):
|
||||
if swa_out_of_tree and (node.backuped or not node.evicted):
|
||||
return True
|
||||
# Fail-soft: a backuped node whose SWA is absent on *both*
|
||||
# layers (device tombstone plus no L3 backup — the hole a
|
||||
# degraded prefetch can graft) still has Full KV on host.
|
||||
# Treat only that case as a valid match boundary — SWA will
|
||||
# be rebuilt from Full KV at load-back via
|
||||
# SWARebuild/RecoverSWAWithLockedFull.
|
||||
if cd.host_value is None and node.backuped:
|
||||
return True
|
||||
return False
|
||||
state["len"] += len(node.key)
|
||||
@@ -1080,7 +1088,12 @@ class SWAComponent(TreeComponent):
|
||||
cur is not self.tree_core.root_node and n_swa < self.sliding_window_size
|
||||
):
|
||||
cd = cur.component_data[ct]
|
||||
assert cd.host_value is not None or cd.value is not None
|
||||
# Fail-soft: a degraded prefetch may insert a backuped node
|
||||
# whose SWA was never backed up (tombstoned before backup).
|
||||
# Stop collecting — SWA will be rebuilt from Full KV at
|
||||
# load-back via SWARebuild/RecoverSWAWithLockedFull.
|
||||
if cd.host_value is None and cd.value is None:
|
||||
break
|
||||
if cd.value is not None:
|
||||
# device exists, skip it
|
||||
n_swa += len(cd.value)
|
||||
|
||||
@@ -1360,7 +1360,8 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
)
|
||||
|
||||
def _insert_tail_step(self, state: _InsertWalkState) -> None:
|
||||
"""Refresh the LRUs and append terminal backup actions."""
|
||||
"""Refresh the LRUs and append the insert backup: a new-leaf write-through,
|
||||
or an SWA window publish on an existing backed node."""
|
||||
if state.target_node is not self.root_node:
|
||||
for component in self.components:
|
||||
if component.component_type == BASE_COMPONENT_TYPE:
|
||||
@@ -1371,7 +1372,9 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
|
||||
if self._should_backup_after_insert(state):
|
||||
state.pending_actions.append(
|
||||
self._build_backup_kv_action(state.target_node)
|
||||
self._build_backup_kv_action(
|
||||
state.target_node, write_back=self.is_write_back
|
||||
)
|
||||
)
|
||||
|
||||
def _split_node(
|
||||
@@ -2350,7 +2353,8 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
def _build_backup_kv_action(
|
||||
self, node: UnifiedTreeNode, write_back: bool = False
|
||||
) -> BackupKV:
|
||||
"""Build the backup action for a node and its not-yet-persisted ancestors."""
|
||||
"""Build the backup action for a node; write-through also chains its
|
||||
unbacked ancestors."""
|
||||
chain = [node]
|
||||
if not write_back:
|
||||
ancestor = node.parent
|
||||
|
||||
@@ -12,7 +12,7 @@ import torch
|
||||
|
||||
from sglang.srt.distributed.communication_tags import P2PTag
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.managers.cache_controller import CacheOperation
|
||||
from sglang.srt.managers.cache_controller import CacheOperation, _hc_timing_enabled
|
||||
from sglang.srt.mem_cache.allocator.page_interleave import (
|
||||
page_interleave_shard_size,
|
||||
)
|
||||
@@ -48,6 +48,7 @@ from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
from sglang.srt.mem_cache.storage_prefetch import StoragePrefetchRetries
|
||||
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
|
||||
from sglang.srt.mem_cache.utils import get_hash_str
|
||||
from sglang.srt.mem_cache.unified_cache.cache_action import (
|
||||
BackupKV,
|
||||
CacheAction,
|
||||
@@ -1688,6 +1689,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
for comp in self._components_tuple
|
||||
}
|
||||
success = False
|
||||
_hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
try:
|
||||
success = self._load_back_transfers(
|
||||
node_id=node_id,
|
||||
@@ -1699,6 +1701,14 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
)
|
||||
return success
|
||||
finally:
|
||||
if _hc_timing_enabled():
|
||||
logger.info(
|
||||
"HC_TIMING stage=loadback req=%s node=%d ms=%.1f success=%s",
|
||||
getattr(req, "rid", None) if req is not None else "-",
|
||||
node_id,
|
||||
(time.monotonic() - _hc_t0) * 1e3,
|
||||
success,
|
||||
)
|
||||
for comp in self._components_tuple:
|
||||
comp.finalize_load_back(req, preps[comp.component_type], success)
|
||||
|
||||
@@ -1910,11 +1920,46 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
if len(prefetch_key) < self.prefetch_threshold:
|
||||
return 0
|
||||
|
||||
# DeepSeek V4 keeps no physical KV objects in storage (logical anchor),
|
||||
# so a plain batch_exists on KV keys always misses; the usable prefix is
|
||||
# decided by the side pools. Mirror prefetch_from_storage's transfer
|
||||
# construction so the query goes through batch_exists_v2. Both hooks are
|
||||
# pure sizing/descriptor builders: prepare_prefetch only computes
|
||||
# staging_tokens, and PREFETCH-phase transfers carry placeholder keys
|
||||
# whose length sizes the TRAILING_PAGES window for batch_exists_v2.
|
||||
comp_xfers: dict[ComponentType, list[PoolTransfer]] = {}
|
||||
for ct in self.tree_components:
|
||||
if ct == BASE_COMPONENT_TYPE:
|
||||
continue
|
||||
prep = self.components[ct].prepare_prefetch(
|
||||
last_host_node_id, prefetch_tokens=len(prefetch_key)
|
||||
)
|
||||
if prep.staging_tokens == 0:
|
||||
continue
|
||||
transfers = self.tree_core.build_hicache_transfers(
|
||||
ct,
|
||||
last_host_node_id,
|
||||
CacheTransferPhase.PREFETCH,
|
||||
token_ids=prefetch_key.token_ids,
|
||||
prefetch_tokens=len(prefetch_key),
|
||||
staging_tokens=prep.staging_tokens,
|
||||
last_hash=last_hash,
|
||||
)
|
||||
if transfers:
|
||||
comp_xfers[ct] = transfers
|
||||
kv_xfer = PoolTransfer(name=PoolName.KV, host_indices=None)
|
||||
sidecar_xfers = self._build_sidecar_transfers(
|
||||
CacheTransferPhase.PREFETCH, kv_xfer, comp_xfers
|
||||
)
|
||||
aux_xfers = [x for xfers in comp_xfers.values() for x in xfers]
|
||||
aux_xfers.extend(sidecar_xfers)
|
||||
|
||||
operation = PrefetchOperation(
|
||||
CacheRequestHandle("__storage_hit_query__", 0),
|
||||
prefetch_key,
|
||||
last_hash,
|
||||
prefix_keys,
|
||||
pool_transfers=aux_xfers or None,
|
||||
)
|
||||
_, storage_hit_count = self.cache_controller._storage_hit_query(operation)
|
||||
storage_hit_count_tensor = torch.tensor(storage_hit_count, dtype=torch.int)
|
||||
@@ -2005,6 +2050,15 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
self.storage_prefetch_retries.poll_miss(req_id, storage_hit_end)
|
||||
return
|
||||
|
||||
# Recompute last_hash from the full matched prefix so the hash chain
|
||||
# is correct regardless of which node was selected as anchor. The
|
||||
# anchor's hash_value may be at a different position than matched_len
|
||||
# when the anchor has no host_value (async backup not yet complete).
|
||||
if matched_prefix_tokens and len(matched_prefix_tokens) >= self.page_size:
|
||||
prefix_hashes = get_hash_str(matched_prefix_tokens, None, self.page_size)
|
||||
if prefix_hashes:
|
||||
last_hash = prefix_hashes[-1]
|
||||
|
||||
# Buffer mode holds no tree state during the fetch: buffers are
|
||||
# operation-owned, so the anchor needs no pin.
|
||||
anchor_lock_params = (
|
||||
@@ -2236,9 +2290,14 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
last_host_node_id,
|
||||
anchor_lock_params,
|
||||
prefetch_key,
|
||||
comp_xfers,
|
||||
):
|
||||
# Hybrid all-or-nothing check failed; result already discarded.
|
||||
return
|
||||
# The check may have degraded the result (fail-soft): re-read the
|
||||
# retained KV prefix length and hash chain off the operation.
|
||||
completed_tokens = operation.completed_tokens
|
||||
hash_value = operation.hash_value
|
||||
|
||||
allocated_tokens = len(host_indices)
|
||||
if completed_tokens < allocated_tokens:
|
||||
@@ -2262,12 +2321,15 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
)
|
||||
|
||||
fetched_key = prefetch_key[:completed_tokens]
|
||||
_hc_tg = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
insert_result = self.tree_core.insert_host(
|
||||
last_host_node_id,
|
||||
fetched_key,
|
||||
host_indices[:completed_tokens],
|
||||
hash_value[: completed_tokens // self.page_size],
|
||||
)
|
||||
if _hc_timing_enabled():
|
||||
operation.graft_ms = (time.monotonic() - _hc_tg) * 1e3
|
||||
|
||||
# Apply the host-insert walk's actions before the transfer commit.
|
||||
self._apply_cache_actions(insert_result.cache_actions)
|
||||
@@ -2323,6 +2385,16 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
loaded_from_storage,
|
||||
self.cache_controller.prefetch_tokens_occupied,
|
||||
)
|
||||
if _hc_timing_enabled():
|
||||
logger.info(
|
||||
"HC_TIMING stage=prefetch_e2e req=%s e2e_ms=%.1f graft_ms=%.1f "
|
||||
"completed=%d loaded=%d",
|
||||
request.rid,
|
||||
(time.monotonic() - operation.start_time) * 1e3,
|
||||
getattr(operation, "graft_ms", -1.0),
|
||||
completed_tokens,
|
||||
loaded_from_storage,
|
||||
)
|
||||
return
|
||||
|
||||
def _check_hybrid_prefetch_result(
|
||||
@@ -2335,6 +2407,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
last_host_node_id: NodeId,
|
||||
anchor_lock_params: DecLockRefParams,
|
||||
prefetch_key: RadixKey,
|
||||
comp_xfers: dict[ComponentType, list[PoolTransfer]],
|
||||
) -> bool:
|
||||
"""Decide the length of usable prefix.
|
||||
|
||||
@@ -2345,11 +2418,16 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
the Full KV pool and every sidecar. A partial prefix is still usable
|
||||
because the sidecar is page-aligned with KV and required for every page.
|
||||
* Everything else (SWA / Mamba components, mixed DeepSeekV4 stacks):
|
||||
*all-or-nothing*. Their pools only cover a window / tail and cannot be
|
||||
truncated page by page, so any shortfall discards the whole prefetch.
|
||||
*fail-soft degrade* when the shortfall is confined to pools with a
|
||||
cheap rebuild path (SWA — the missing window is rebuilt from Full KV
|
||||
at load-back): the fetched KV prefix is retained and only the failed
|
||||
pool is discarded. Falls back to *all-or-nothing* when nothing is
|
||||
usable or a non-rebuildable pool (Mamba) falls short, since those
|
||||
pools cannot be truncated page by page.
|
||||
|
||||
Returns true if prefetch success, or false when an all-or-nothing prefetch
|
||||
was discarded (the caller should then treat the prefetch as finished).
|
||||
Returns true if prefetch success (possibly degraded), or false when an
|
||||
all-or-nothing prefetch was discarded (the caller should then treat the
|
||||
prefetch as finished).
|
||||
"""
|
||||
# Sync completed tokens and per-pool hit pages across ATTN groups, taking
|
||||
# the minimum so every rank agrees on the same usable prefix length.
|
||||
@@ -2375,57 +2453,118 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
for transfer, count in zip(pool_transfers, pool_hit_pages)
|
||||
)
|
||||
if pool_transfers and not all_succeeded:
|
||||
# Drop the KV beliefs from the first page any pool failed to serve;
|
||||
# the next insert then re-writes that span through one FULL check,
|
||||
# restoring the missing aux pages.
|
||||
# Identify which non-KV components fell short. KV-derived
|
||||
# sidecars (indices_from_pool == KV) are excluded — they share
|
||||
# the KV hit length and are always covered when KV succeeds.
|
||||
failed_components = [
|
||||
ct
|
||||
for ct, xfers in comp_xfers.items()
|
||||
if any(x.indices_from_pool != PoolName.KV for x in xfers)
|
||||
]
|
||||
# Only SWA has a cheap rebuild path (_translate_full_to_swa);
|
||||
# the missing window is rebuilt at load-back via
|
||||
# SWARebuild/RecoverSWAWithLockedFull. Mamba SSM states require
|
||||
# O(seq_len) recompute — not degradable.
|
||||
rebuildable = all(ct != ComponentType.MAMBA for ct in failed_components)
|
||||
|
||||
if completed_tokens == 0 or not rebuildable:
|
||||
# Full discard — nothing usable from any pool, or a
|
||||
# non-rebuildable pool (Mamba) fell short so the whole
|
||||
# result is unusable.
|
||||
# Drop the KV beliefs from the first page any pool failed to serve;
|
||||
# the next insert then re-writes that span through one FULL check,
|
||||
# restoring the missing aux pages.
|
||||
invalidation_hashes = (
|
||||
operation.sidecar_hash_values
|
||||
if operation.sidecar_hash_values is not None
|
||||
else hash_value
|
||||
)
|
||||
trimmed_pages = len(invalidation_hashes) - len(hash_value)
|
||||
keep_pages = trimmed_pages + completed_tokens // self.page_size
|
||||
for transfer, count in zip(pool_transfers, pool_hit_pages):
|
||||
if transfer.keys is None:
|
||||
keep_pages = 0
|
||||
elif count < len(transfer.keys):
|
||||
# Aux transfers key the chain's trailing pages.
|
||||
keep_pages = min(
|
||||
keep_pages,
|
||||
max(0, len(invalidation_hashes) - len(transfer.keys)),
|
||||
)
|
||||
self.storage_existence_cache.invalidate_beyond(
|
||||
PoolName.KV, invalidation_hashes, keep_pages=keep_pages
|
||||
)
|
||||
# The controller's prefetch IO thread already releases the untransferred
|
||||
# tail (host_indices[completed_tokens:])
|
||||
self.cache_controller.append_host_mem_release(
|
||||
host_indices=host_indices[:completed_tokens],
|
||||
extra_pools=pool_transfers if operation.pool_transfers_done else None,
|
||||
)
|
||||
self._finish_storage_prefetch(
|
||||
request, fulfilled_tokens=0, reason="storage_transfer"
|
||||
)
|
||||
if anchor_lock_params is not None:
|
||||
self.dec_host_lock_ref(last_host_node_id, anchor_lock_params)
|
||||
if self.buffer_pipeline is not None:
|
||||
self.buffer_pipeline.pop_prefix_ctx(request)
|
||||
self.buffer_pipeline.release_anchor_lock(request)
|
||||
del self.ongoing_prefetch[request]
|
||||
self.cache_controller.prefetch_tokens_occupied -= (
|
||||
self._prefetch_occupied_span(prefetch_key, host_indices)
|
||||
)
|
||||
self.prefetch_loaded_tokens_by_reqid[request] = 0
|
||||
self.prefetch_loaded_storage_start_by_reqid.pop(request, None)
|
||||
logger.warning(
|
||||
"HiCache hybrid prefetch discarded req=%s completed=%d requested=%d "
|
||||
"kv_beliefs_kept_pages=%d",
|
||||
request.rid,
|
||||
completed_tokens,
|
||||
expected_tokens,
|
||||
keep_pages,
|
||||
)
|
||||
return False
|
||||
|
||||
# Fail-soft degrade: retain the Full KV (+ KV-derived sidecar)
|
||||
# result and discard **only** the failed non-KV independent
|
||||
# pools (SWA) whose storage fetch fell short. Their host
|
||||
# buffers are released here and their component entries are
|
||||
# removed from comp_xfers so the caller's normal commit /
|
||||
# staging path never touches the freed memory. At load-back
|
||||
# time the missing SWA window is rebuilt via the existing
|
||||
# RecoverSWAWithLockedFull / SWARebuild path in swa.py.
|
||||
# Belief self-heal is scoped to the retained KV prefix.
|
||||
invalidation_hashes = (
|
||||
operation.sidecar_hash_values
|
||||
if operation.sidecar_hash_values is not None
|
||||
else hash_value
|
||||
)
|
||||
trimmed_pages = len(invalidation_hashes) - len(hash_value)
|
||||
keep_pages = trimmed_pages + completed_tokens // self.page_size
|
||||
for transfer, count in zip(pool_transfers, pool_hit_pages):
|
||||
if transfer.keys is None:
|
||||
keep_pages = 0
|
||||
elif count < len(transfer.keys):
|
||||
# Aux transfers key the chain's trailing pages.
|
||||
keep_pages = min(
|
||||
keep_pages,
|
||||
max(0, len(invalidation_hashes) - len(transfer.keys)),
|
||||
)
|
||||
self.storage_existence_cache.invalidate_beyond(
|
||||
PoolName.KV, invalidation_hashes, keep_pages=keep_pages
|
||||
PoolName.KV,
|
||||
invalidation_hashes,
|
||||
keep_pages=trimmed_pages + completed_tokens // self.page_size,
|
||||
)
|
||||
# The controller's prefetch IO thread already releases the untransferred
|
||||
# tail (host_indices[completed_tokens:])
|
||||
self.cache_controller.append_host_mem_release(
|
||||
host_indices=host_indices[:completed_tokens],
|
||||
extra_pools=pool_transfers if operation.pool_transfers_done else None,
|
||||
)
|
||||
self._finish_storage_prefetch(
|
||||
request, fulfilled_tokens=0, reason="storage_transfer"
|
||||
)
|
||||
if anchor_lock_params is not None:
|
||||
self.dec_host_lock_ref(last_host_node_id, anchor_lock_params)
|
||||
if self.buffer_pipeline is not None:
|
||||
self.buffer_pipeline.pop_prefix_ctx(request)
|
||||
self.buffer_pipeline.release_anchor_lock(request)
|
||||
del self.ongoing_prefetch[request]
|
||||
self.cache_controller.prefetch_tokens_occupied -= (
|
||||
self._prefetch_occupied_span(prefetch_key, host_indices)
|
||||
)
|
||||
self.prefetch_loaded_tokens_by_reqid[request] = 0
|
||||
self.prefetch_loaded_storage_start_by_reqid.pop(request, None)
|
||||
# Remove failed non-KV components from comp_xfers (same dict as
|
||||
# the caller's) so the commit / staging path won't touch freed
|
||||
# host buffers.
|
||||
for ct in failed_components:
|
||||
comp_xfers.pop(ct, None)
|
||||
# Retain only the KV-fetched prefix; the caller re-reads these
|
||||
# off the operation after the check passes.
|
||||
operation.completed_tokens = completed_tokens
|
||||
operation.hash_value = hash_value[: completed_tokens // self.page_size]
|
||||
logger.warning(
|
||||
"HiCache hybrid prefetch discarded req=%s completed=%d requested=%d "
|
||||
"kv_beliefs_kept_pages=%d",
|
||||
"HiCache hybrid prefetch degraded req=%s completed=%d "
|
||||
"requested=%d retained_kv=%d discarded_components=%s",
|
||||
request.rid,
|
||||
completed_tokens,
|
||||
expected_tokens,
|
||||
keep_pages,
|
||||
completed_tokens,
|
||||
[ct.value for ct in failed_components],
|
||||
)
|
||||
return False
|
||||
return True
|
||||
return True
|
||||
|
||||
def _record_storage_prefetch_hit(
|
||||
@@ -2897,7 +3036,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
self.ongoing_prefetch[request] = info._replace(host_indices=host_indices)
|
||||
if buffer_mode:
|
||||
cc.prefetch_tokens_occupied += alloc_len
|
||||
cc.prefetch_buffer.put(operation)
|
||||
cc.dispatch_prefetch(operation)
|
||||
return True
|
||||
|
||||
def _drain_and_alloc_storage_hit():
|
||||
@@ -2978,6 +3117,14 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
operation.pool_transfers_done = True
|
||||
if ack.completed_req:
|
||||
if is_current:
|
||||
if _hc_timing_enabled():
|
||||
_hc_enq = getattr(operation, "final_ack_enqueue_ts", None)
|
||||
if _hc_enq is not None:
|
||||
logger.info(
|
||||
"HC_TIMING stage=ack_drain req=%s wait_ms=%.1f",
|
||||
operation.request_id,
|
||||
(time.monotonic() - _hc_enq) * 1e3,
|
||||
)
|
||||
# check_prefetch_progress() is not called for this rid yet.
|
||||
# Let us insert the prefetch result into the radix tree.
|
||||
self._handle_prefetch_result(operation)
|
||||
|
||||
@@ -4488,9 +4488,12 @@ class DeepseekV4Model(nn.Module):
|
||||
forward_batch,
|
||||
cp_all_tokens=cp_extend,
|
||||
)
|
||||
# Only multimodal placeholders become image_token_id, so a text-only
|
||||
# batch skips this full-residual where.
|
||||
if (
|
||||
self.config.model_type == "deepseek_v41"
|
||||
and self.config.vision_n_layers > 0
|
||||
and forward_batch.contains_mm_inputs()
|
||||
):
|
||||
hidden_states = torch.where(
|
||||
(input_ids == self.config.image_token_id)[:, None, None],
|
||||
@@ -4715,7 +4718,9 @@ class DeepseekV4Model(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
hidden_states = hidden_states.unsqueeze(1).repeat(1, self.hc_mult, 1)
|
||||
from sglang.kernels.ops.layernorm.mhc import hc_broadcast
|
||||
|
||||
hidden_states = hc_broadcast(hidden_states, self.hc_mult)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
|
||||
@@ -157,7 +157,7 @@ class Session:
|
||||
or carry_fill is input_ids_unpadded
|
||||
):
|
||||
# Unexpected type or aliased with an origin array (extending it
|
||||
# below would double-append): let _refresh_fill_ids rebuild.
|
||||
# below would double-append): let refresh_fill_ids rebuild.
|
||||
carry_fill = None
|
||||
else:
|
||||
del carry_fill[self.committed_fill_len :]
|
||||
|
||||
@@ -4052,7 +4052,8 @@ def get_cuda_graph_batch_size_alignment() -> int:
|
||||
alignment *= 2
|
||||
if require_gathered_buffer():
|
||||
alignment *= get_parallel().attn_tp_size
|
||||
if alignment % get_parallel().attn_cp_size != 0:
|
||||
# TODO: unverified on NVIDIA; drop the gate once validated on CUDA.
|
||||
if not is_hip() and alignment % get_parallel().attn_cp_size != 0:
|
||||
alignment *= get_parallel().attn_cp_size
|
||||
return alignment
|
||||
|
||||
|
||||
@@ -2592,6 +2592,107 @@ fn backup_kv_action_chains_unbacked_ancestors_first() {
|
||||
assert_eq!(action.node_ids, vec![c]);
|
||||
}
|
||||
|
||||
// Mirrors UnifiedRadixCache._execute_and_commit_kv_backup: back up each chain
|
||||
// node and stamp every node its transfers cover with that node's ack.
|
||||
// It skips the inc_lock_ref the Python side takes; the asserts do not depend on it.
|
||||
fn execute_backup_kv_for_test(tc: &mut UnifiedTreeCore<Vec<i64>>, backup: &BackupKV) {
|
||||
for &node_id in &backup.node_ids {
|
||||
let (device_value, mut comp_xfers) = tc.build_backup_spec(node_id).expect("backup spec");
|
||||
if device_value.numel() == 0 && comp_xfers.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let mut publish_node_ids = Vec::new();
|
||||
for transfer in comp_xfers.values_mut().flatten() {
|
||||
transfer.host_indices = transfer.device_indices.as_ref().map(Tensor::copy);
|
||||
publish_node_ids.extend(transfer.nodes_to_load.iter().flatten().copied());
|
||||
}
|
||||
if !publish_node_ids.contains(&node_id) {
|
||||
publish_node_ids.push(node_id);
|
||||
}
|
||||
tc.commit_backup(node_id, device_value.copy(), comp_xfers)
|
||||
.expect("live test node");
|
||||
tc.mark_write_through_pending(publish_node_ids, /* ack_id = */ node_id)
|
||||
.expect("live test nodes");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_back_swa_publish_leaves_an_ancestor_pending_under_another_ack_alone() {
|
||||
// c(Full unbacked) -> b(Full backed) -> t1(Full backed)
|
||||
// -> d(Full unbacked) -> t2(Full backed)
|
||||
// Every node holds a device-only SWA value inside the window.
|
||||
let mut tc = UnifiedTreeCore::new(
|
||||
CacheInitParams {
|
||||
is_write_back: true,
|
||||
enable_hicache: true,
|
||||
has_swa_host_pool: true,
|
||||
swa_sliding_window_size: Some(8),
|
||||
..Default::default()
|
||||
},
|
||||
vec![FULL, SWA],
|
||||
);
|
||||
let (key_t1, key_t2) = (vec![1, 2, 3, 4], vec![1, 2, 5, 6]);
|
||||
tc.insert(&insert_params(&key_t1, &[10, 11, 12, 13]));
|
||||
tc.insert(&insert_params(&key_t2, &[10, 11, 14, 15]));
|
||||
tc.insert(&insert_params(&vec![1, 2, 3], &[10, 11, 12]));
|
||||
tc.insert(&insert_params(&vec![1, 2, 5], &[10, 11, 14]));
|
||||
let child = |parent, page: &[i64]| {
|
||||
tc.arena
|
||||
.child_on_page(parent, /* extra_key = */ None, page)
|
||||
.expect("child on page")
|
||||
};
|
||||
let c_idx = child(tc.arena.root(), &[1]);
|
||||
let b_idx = child(c_idx, &[3]);
|
||||
let t1_idx = child(b_idx, &[4]);
|
||||
let d_idx = child(c_idx, &[5]);
|
||||
let t2_idx = child(d_idx, &[6]);
|
||||
let [c, b, t1, d, t2] = [c_idx, b_idx, t1_idx, d_idx, t2_idx].map(|idx| tc.arena.node(idx).id);
|
||||
for idx in [c_idx, b_idx, t1_idx, d_idx, t2_idx] {
|
||||
let full = tc.arena.node(idx).device_value(FULL).copy();
|
||||
tc.set_component_device_value(tc.arena.node(idx).id, SWA, full.copy())
|
||||
.expect("live test node");
|
||||
if [b_idx, t1_idx, t2_idx].contains(&idx) {
|
||||
tc.arena.set_host_value(idx, FULL, full);
|
||||
}
|
||||
}
|
||||
|
||||
// Two requests in one batch finish at t1 and t2; neither ack drains.
|
||||
for (key, target) in [(&key_t1, t1), (&key_t2, t2)] {
|
||||
let result = tc.insert(&insert_params(key, &[20, 21, 22, 23]));
|
||||
let backups: Vec<&BackupKV> = result
|
||||
.cache_actions
|
||||
.iter()
|
||||
.filter_map(|action| match action {
|
||||
CacheAction::BackupKV(backup) => Some(backup),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(backups.len(), 1);
|
||||
assert_eq!(backups[0].node_ids, vec![target]);
|
||||
execute_backup_kv_for_test(&mut tc, backups[0]);
|
||||
}
|
||||
|
||||
let pending = |node_id| {
|
||||
tc.arena
|
||||
.node(tc.arena.resolve(node_id).expect("live test node"))
|
||||
.write_through_pending_id
|
||||
};
|
||||
for (node_id, ack_id) in [(c, t1), (b, t1), (t1, t1), (d, t2), (t2, t2)] {
|
||||
assert_eq!(pending(node_id), Some(ack_id));
|
||||
}
|
||||
// Write-back defers the unbacked Full prefix to eviction.
|
||||
assert!(
|
||||
!tc.arena
|
||||
.node(tc.arena.resolve(c).expect("live test node"))
|
||||
.backuped()
|
||||
);
|
||||
assert!(
|
||||
!tc.arena
|
||||
.node(tc.arena.resolve(d).expect("live test node"))
|
||||
.backuped()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backup_kv_action_stops_at_an_externally_stored_or_pending_ancestor() {
|
||||
let mut tc = core();
|
||||
|
||||
@@ -1833,7 +1833,8 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
&& self.needs_incremental_component_backup_(target_node_id)
|
||||
}
|
||||
|
||||
/// Refresh the LRUs and append terminal backup actions.
|
||||
/// Refresh the LRUs and append the insert backup: a new-leaf write-through,
|
||||
/// or an SWA window publish on an existing backed node.
|
||||
fn insert_tail_step_(&mut self, state: &mut InsertWalkState<K>) {
|
||||
let target_node_id = state
|
||||
.target_node_id
|
||||
@@ -1850,10 +1851,8 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
}
|
||||
|
||||
if self.should_backup_after_insert_(state, target_node_id) {
|
||||
let backup = self.build_backup_kv_action_(
|
||||
self.arena.node(target_node_id),
|
||||
/* write_back = */ false,
|
||||
);
|
||||
let backup =
|
||||
self.build_backup_kv_action_(self.arena.node(target_node_id), self.is_write_back);
|
||||
state.pending_actions.push(CacheAction::BackupKV(backup));
|
||||
}
|
||||
}
|
||||
@@ -3611,7 +3610,8 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
Ok(order)
|
||||
}
|
||||
|
||||
/// Build the backup action for a node and its not-yet-persisted ancestors.
|
||||
/// Build the backup action for a node; write-through also chains its
|
||||
/// unbacked ancestors.
|
||||
pub fn build_backup_kv_action_(&self, node: &Node<K>, write_back: bool) -> BackupKV {
|
||||
let mut chain = vec![node.id];
|
||||
if !write_back {
|
||||
|
||||
@@ -205,3 +205,6 @@ codegen-units = 1
|
||||
[profile.dev-opt]
|
||||
inherits = "dev"
|
||||
opt-level = 1
|
||||
|
||||
[patch.crates-io]
|
||||
openai-protocol = { path = "vendor/openai-protocol" }
|
||||
|
||||
@@ -378,6 +378,12 @@ impl RouterConfigBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
/// G9i pool-aware admission control + fast-P routing config (PD mode).
|
||||
pub fn g9i(mut self, g9i: Option<crate::routers::http::g9i::G9iConfig>) -> Self {
|
||||
self.config.g9i = g9i;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn model_path<S: Into<String>>(mut self, path: S) -> Self {
|
||||
self.config.model_path = Some(path.into());
|
||||
self
|
||||
|
||||
@@ -99,6 +99,9 @@ pub struct RouterConfig {
|
||||
/// Enable WASM support
|
||||
#[serde(default)]
|
||||
pub enable_wasm: bool,
|
||||
/// G9i pool-aware admission control + fast-P routing (PD mode, HTTP only).
|
||||
#[serde(default)]
|
||||
pub g9i: Option<crate::routers::http::g9i::G9iConfig>,
|
||||
}
|
||||
|
||||
/// Tokenizer cache configuration
|
||||
@@ -551,6 +554,7 @@ impl Default for RouterConfig {
|
||||
ca_certificates: vec![],
|
||||
mcp_config: None,
|
||||
enable_wasm: false,
|
||||
g9i: None,
|
||||
server_cert: None,
|
||||
server_key: None,
|
||||
}
|
||||
|
||||
@@ -219,6 +219,71 @@ struct CliArgs {
|
||||
#[arg(long, default_value_t = 30, help_heading = "PD Disaggregation")]
|
||||
worker_startup_check_interval: u64,
|
||||
|
||||
// ==================== G9i: Pool Admission + Fast-P Routing ====================
|
||||
/// Enable G9i pool-aware admission control + fast-P routing (PD HTTP mode)
|
||||
#[arg(long, default_value_t = false, help_heading = "G9i Pool Admission")]
|
||||
g9i_enabled: bool,
|
||||
|
||||
/// Fast-P pool member base URLs (repeatable; router-side role, engine unaware)
|
||||
#[arg(long, action = ArgAction::Append, help_heading = "G9i Pool Admission")]
|
||||
g9i_fast_prefill_url: Vec<String>,
|
||||
|
||||
/// Small-prefill cutoff on NEW tokens (total - prefix hit). Tentative; tune by bench.
|
||||
#[arg(long, default_value_t = 4096, help_heading = "G9i Pool Admission")]
|
||||
g9i_max_new_tokens: u64,
|
||||
|
||||
/// 429 when the least-wait fast-P queue estimate exceeds this many seconds. Tentative.
|
||||
#[arg(long, default_value_t = 5.0, help_heading = "G9i Pool Admission")]
|
||||
g9i_fast_p_queue_reject_secs: f64,
|
||||
|
||||
/// D-pool admission ceiling: sum of in-processing requests across decode nodes. Tentative.
|
||||
#[arg(long, default_value_t = 256, help_heading = "G9i Pool Admission")]
|
||||
g9i_decode_max_running: u64,
|
||||
|
||||
/// 429 when the least-wait REGULAR pool queue estimate exceeds this many seconds. Tentative 10.0; 0=off.
|
||||
#[arg(long, default_value_t = 10.0, help_heading = "G9i Pool Admission")]
|
||||
g9i_regular_p_queue_reject_secs: f64,
|
||||
|
||||
/// Drain-rate floor (tok/s) for queue-wait estimates (div-by-zero / cold-start guard)
|
||||
#[arg(long, default_value_t = 1000.0, help_heading = "G9i Pool Admission")]
|
||||
g9i_drain_rate_floor: f64,
|
||||
|
||||
/// Snapshot poll interval (seconds)
|
||||
#[arg(long, default_value_t = 1, help_heading = "G9i Pool Admission")]
|
||||
g9i_poll_interval_secs: u64,
|
||||
|
||||
/// Snapshot TTL — stale nodes leave pool decisions (seconds)
|
||||
#[arg(long, default_value_t = 5, help_heading = "G9i Pool Admission")]
|
||||
g9i_snapshot_ttl_secs: u64,
|
||||
|
||||
/// No fresh D-pool snapshot: "allow" (default, fail-open + warn) or "reject"
|
||||
#[arg(long, default_value = "allow", value_parser = ["allow", "reject"], help_heading = "G9i Pool Admission")]
|
||||
g9i_d_no_snapshot: String,
|
||||
|
||||
/// /query_cache_hit probe timeout per request (ms)
|
||||
#[arg(long, default_value_t = 2000, help_heading = "G9i Pool Admission")]
|
||||
g9i_cache_hit_query_timeout_ms: u64,
|
||||
|
||||
/// Also least-wait route the regular P pool (default: keep configured policy)
|
||||
#[arg(long, default_value_t = false, help_heading = "G9i Pool Admission")]
|
||||
g9i_regular_pool_least_wait: bool,
|
||||
|
||||
/// Sliding window for TTFT p95/p99 statistics publication (seconds; observability only, never gates)
|
||||
#[arg(long, default_value_t = 300, help_heading = "G9i Pool Admission")]
|
||||
g9i_ttft_window_s: u64,
|
||||
|
||||
/// G9i v2 shadow mode: prefix-chain bookkeeping + shadow classifier + queue model (observability only, zero routing effect; requires --g9i-enabled)
|
||||
#[arg(long, default_value_t = false, help_heading = "G9i Pool Admission")]
|
||||
g9i_shadow_enabled: bool,
|
||||
|
||||
/// Shadow prefix-table capacity in entries (LRU + TTL evicted)
|
||||
#[arg(long, default_value_t = 100_000, help_heading = "G9i Pool Admission")]
|
||||
g9i_shadow_table_max_entries: usize,
|
||||
|
||||
/// Shadow prefix-table entry TTL (seconds)
|
||||
#[arg(long, default_value_t = 86400, help_heading = "G9i Pool Admission")]
|
||||
g9i_shadow_table_ttl_secs: u64,
|
||||
|
||||
// ==================== Service Discovery (Kubernetes) ====================
|
||||
/// Enable Kubernetes service discovery
|
||||
#[arg(
|
||||
@@ -1070,11 +1135,39 @@ impl CliArgs {
|
||||
.circuit_breaker(!self.disable_circuit_breaker)
|
||||
.enable_wasm(self.enable_wasm)
|
||||
.igw(self.enable_igw)
|
||||
.g9i(self.build_g9i_config()?)
|
||||
.maybe_server_cert_and_key(self.tls_cert_path.as_ref(), self.tls_key_path.as_ref());
|
||||
|
||||
builder.build()
|
||||
}
|
||||
|
||||
/// G9i pool admission + fast-P routing config; None when the feature is off.
|
||||
fn build_g9i_config(&self) -> ConfigResult<Option<smg::routers::http::g9i::G9iConfig>> {
|
||||
if !self.g9i_enabled && self.g9i_fast_prefill_url.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
let config = smg::routers::http::g9i::G9iConfig {
|
||||
enabled: self.g9i_enabled,
|
||||
fast_prefill_urls: self.g9i_fast_prefill_url.clone(),
|
||||
max_new_tokens: self.g9i_max_new_tokens,
|
||||
fast_p_queue_reject_secs: self.g9i_fast_p_queue_reject_secs,
|
||||
regular_p_queue_reject_secs: self.g9i_regular_p_queue_reject_secs,
|
||||
decode_max_running: self.g9i_decode_max_running,
|
||||
drain_rate_floor: self.g9i_drain_rate_floor,
|
||||
poll_interval_secs: self.g9i_poll_interval_secs,
|
||||
snapshot_ttl_secs: self.g9i_snapshot_ttl_secs,
|
||||
d_no_snapshot: self.g9i_d_no_snapshot.clone(),
|
||||
cache_hit_query_timeout_ms: self.g9i_cache_hit_query_timeout_ms,
|
||||
regular_pool_least_wait: self.g9i_regular_pool_least_wait,
|
||||
ttft_window_s: self.g9i_ttft_window_s,
|
||||
shadow_enabled: self.g9i_shadow_enabled,
|
||||
shadow_table_max_entries: self.g9i_shadow_table_max_entries,
|
||||
shadow_table_ttl_secs: self.g9i_shadow_table_ttl_secs,
|
||||
};
|
||||
config.validate().map_err(|reason| ConfigError::ValidationFailed { reason })?;
|
||||
Ok(Some(config))
|
||||
}
|
||||
|
||||
fn to_server_config(&self, router_config: RouterConfig) -> ServerConfig {
|
||||
let service_discovery_config = if self.service_discovery {
|
||||
// Get router discovery config from router_config.discovery if available
|
||||
|
||||
@@ -301,11 +301,12 @@ impl HarmonyBuilder {
|
||||
.reasoning
|
||||
.as_ref()
|
||||
.and_then(|r| r.effort.as_ref())
|
||||
.map(|effort| match effort {
|
||||
ResponsesReasoningEffort::High => ReasoningEffort::High,
|
||||
ResponsesReasoningEffort::Medium => ReasoningEffort::Medium,
|
||||
ResponsesReasoningEffort::Low => ReasoningEffort::Low,
|
||||
ResponsesReasoningEffort::Minimal => ReasoningEffort::Low,
|
||||
.and_then(|effort| match effort {
|
||||
ResponsesReasoningEffort::None => None,
|
||||
ResponsesReasoningEffort::High => Some(ReasoningEffort::High),
|
||||
ResponsesReasoningEffort::Medium => Some(ReasoningEffort::Medium),
|
||||
ResponsesReasoningEffort::Low => Some(ReasoningEffort::Low),
|
||||
ResponsesReasoningEffort::Minimal => Some(ReasoningEffort::Low),
|
||||
});
|
||||
|
||||
self.build_system_message(reasoning_effort, with_custom_tools)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,675 @@
|
||||
//! G9i v2 P1 shadow mode — T1: prefix hash chain (pure function layer).
|
||||
//!
|
||||
//! Design source: `experiments/g9i/g9i-v2-design-20261013.md` §3.2/§4.1,
|
||||
//! content-handling constraints C4/C10. This module is **read-only with
|
||||
//! respect to routing**: it computes hashes, it never touches dispatch.
|
||||
//!
|
||||
//! Why a chain: a chat body with `m` messages has exactly `m` prefix
|
||||
//! boundaries, and the session bookkeeping table ("prefix → last serving
|
||||
//! node + exact token count") needs a stable key per boundary. Rolling the
|
||||
//! hash forward per message gives all `m` keys in one pass; lookup then
|
||||
//! probes from deep to shallow and takes the deepest hit. In practice a new
|
||||
//! turn is `previous body + assistant reply + new user message`, so hits
|
||||
//! usually land at `h[m-2]`.
|
||||
//!
|
||||
//! seed = xxh3(canonical_json({model, chat_template_kwargs, tools_hash, system}))
|
||||
//! h[0] = seed
|
||||
//! h[k] = xxh3(h[k-1] LE bytes ‖ len(role) ‖ role ‖ tagged content
|
||||
//! ‖ tagged canonical(tool_calls) ‖ tagged meta) k = 1..=m
|
||||
//!
|
||||
//! Encoding decisions (why, not what):
|
||||
//! * Streaming `Xxh3` instead of concatenating into a `Vec`: chat bodies
|
||||
//! with base64 images are routinely hundreds of KB; copying them per
|
||||
//! chain step would be pure waste.
|
||||
//! * Every variable-length field is length-prefixed (u64 LE) and every
|
||||
//! content piece carries a type tag. Without that, `["ab","c"]` and
|
||||
//! `["a","bc"]` (or role/content, part/part splits) would hash
|
||||
//! identically — silent false cache hits.
|
||||
//! * Missing role defaults to "user" (OpenAI clients routinely omit it);
|
||||
//! missing `model` / `chat_template_kwargs` / `tools` / system message
|
||||
//! serialize as JSON `null` in the seed document — a fixed placeholder
|
||||
//! that cannot collide with any real value because `null` is not a
|
||||
//! string/object the field could otherwise hold.
|
||||
//! * Byte-level semantics for multimodal (C4): `image_url` URLs are
|
||||
//! hashed as the raw string — `http(s):` URLs as-is, `data:` URLs
|
||||
//! including the base64 payload, **never fetched, never decoded**.
|
||||
//! Same image at different URLs hashes differently; that is correct —
|
||||
//! the rendered prompt bytes differ.
|
||||
//! * `is_unclassifiable` and `chain_hashes` share one traversal
|
||||
//! (`walk_messages`) so the "weird content" judgement can never drift
|
||||
//! between the two call sites (C10). Unknown part types still
|
||||
//! contribute a deterministic tag to the chain — the chain stays
|
||||
//! well-defined, the caller just must not trust it for table lookups
|
||||
//! when `is_unclassifiable` is true.
|
||||
//! * `content: null` / missing renders as empty content in every chat
|
||||
//! template, so it hashes exactly like `""` — NOT unclassifiable.
|
||||
//! Flagging it would disenfranchise tool-use conversations (assistant
|
||||
//! turns with `content: null` + `tool_calls`), our main traffic shape.
|
||||
//! Genuinely malformed content (number/object/bool) keeps the
|
||||
//! Empty + unclassifiable treatment.
|
||||
//! * `tool_calls` are rendered into the prompt by the chat template, so
|
||||
//! they MUST enter the chain (as canonical JSON, key-order
|
||||
//! insensitive) — otherwise two conversations with different tool
|
||||
//! calls could share a hash: a false hit in the dangerous direction.
|
||||
//! `reasoning_content` is deliberately excluded: templates never
|
||||
//! render it, so it cannot change prompt bytes.
|
||||
//! * `tool_call_id` (on `role:"tool"` results) and `name` (any role)
|
||||
//! are rendered by most templates, so they enter the chain as one
|
||||
//! canonical meta object. The META segment is unconditional — a
|
||||
//! message without these fields contributes `TAG_META ‖ len=0`,
|
||||
//! keeping the formula single-path (no presence/absence ambiguity).
|
||||
|
||||
use serde_json::Value;
|
||||
use xxhash_rust::xxh3::{xxh3_64, Xxh3};
|
||||
|
||||
/// Part-type tags mixed in before each content piece. Terminated with `\0`
|
||||
/// so tags are self-delimiting; content bytes are length-prefixed on top.
|
||||
const TAG_TEXT: &[u8] = b"text\0";
|
||||
const TAG_IMAGE_URL: &[u8] = b"image_url\0";
|
||||
const TAG_UNKNOWN: &[u8] = b"unknown\0";
|
||||
const TAG_TOOL_CALLS: &[u8] = b"tool_calls\0";
|
||||
const TAG_META: &[u8] = b"meta\0";
|
||||
|
||||
/// Chain seed: folds every body field that changes the rendered prompt but
|
||||
/// lives outside `messages` into one u64.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub struct ChainSeed {
|
||||
pub value: u64,
|
||||
}
|
||||
|
||||
impl ChainSeed {
|
||||
/// Build the seed from a chat request body.
|
||||
///
|
||||
/// Inputs: `model`, `chat_template_kwargs`, `tools` (folded to a single
|
||||
/// hash of its canonical form), and the first `role=="system"` message's
|
||||
/// string content. Missing fields become JSON `null` in the seed
|
||||
/// document (fixed placeholder, see module docs). Canonicalization
|
||||
/// sorts object keys recursively and emits no whitespace, so semantically
|
||||
/// equal bodies from different clients produce the same seed.
|
||||
///
|
||||
/// Note: a system message is *also* hashed as `messages[0]` inside the
|
||||
/// chain — deliberately. The chain must stay 1:1 with `messages`
|
||||
/// indices (the lookup table keys are `h[k]` by position), and the
|
||||
/// redundancy only costs one extra mix of already-known bytes.
|
||||
pub fn from_body(body: &Value) -> ChainSeed {
|
||||
let system_content: Option<&str> = body
|
||||
.get("messages")
|
||||
.and_then(Value::as_array)
|
||||
.and_then(|msgs| {
|
||||
msgs.iter()
|
||||
.find(|m| m.get("role").and_then(Value::as_str) == Some("system"))
|
||||
})
|
||||
.and_then(|m| m.get("content"))
|
||||
.and_then(Value::as_str);
|
||||
|
||||
let tools_hash: Option<u64> = body
|
||||
.get("tools")
|
||||
.map(|t| xxh3_64(canonical_json(t).as_bytes()));
|
||||
|
||||
let seed_doc = serde_json::json!({
|
||||
"model": body.get("model").cloned().unwrap_or(Value::Null),
|
||||
"chat_template_kwargs": body
|
||||
.get("chat_template_kwargs")
|
||||
.cloned()
|
||||
.unwrap_or(Value::Null),
|
||||
"tools_hash": tools_hash,
|
||||
"system": system_content,
|
||||
});
|
||||
ChainSeed {
|
||||
value: xxh3_64(canonical_json(&seed_doc).as_bytes()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute the chain hashes `h[1..=m]`, one per message, in order.
|
||||
///
|
||||
/// `messages` missing / not an array / empty → empty `Vec`. The caller
|
||||
/// treats an empty chain as "cannot classify" (same fall-back path as
|
||||
/// `is_unclassifiable`); we never panic on malformed bodies because this
|
||||
/// sits on the hot request path.
|
||||
///
|
||||
/// The returned chain is deterministic even when `is_unclassifiable(body)`
|
||||
/// is true — unknown parts contribute their tag — but such chains are only
|
||||
/// suitable for logging/shadow comparison, not for bookkeeping lookups.
|
||||
pub fn chain_hashes(seed: &ChainSeed, body: &Value) -> Vec<u64> {
|
||||
let (msgs, _) = walk_messages(body);
|
||||
let mut chain = Vec::with_capacity(msgs.len());
|
||||
let mut prev = seed.value;
|
||||
for msg in &msgs {
|
||||
let mut h = Xxh3::new();
|
||||
h.update(&prev.to_le_bytes());
|
||||
|
||||
let role = msg.role.as_bytes();
|
||||
h.update(&(role.len() as u64).to_le_bytes());
|
||||
h.update(role);
|
||||
|
||||
match &msg.content {
|
||||
ContentView::Str(s) => {
|
||||
update_tagged(&mut h, TAG_TEXT, s.as_bytes());
|
||||
}
|
||||
ContentView::Parts(parts) => {
|
||||
for part in parts {
|
||||
match part {
|
||||
PartView::Text(t) => update_tagged(&mut h, TAG_TEXT, t.as_bytes()),
|
||||
PartView::ImageUrl(u) => {
|
||||
update_tagged(&mut h, TAG_IMAGE_URL, u.as_bytes())
|
||||
}
|
||||
PartView::Unknown => h.update(TAG_UNKNOWN),
|
||||
}
|
||||
}
|
||||
}
|
||||
ContentView::Empty => {}
|
||||
}
|
||||
|
||||
// Third segment: tool_calls are rendered into prompt bytes by the
|
||||
// chat template, so they must be in the chain.
|
||||
if let Some(tc) = &msg.tool_calls {
|
||||
update_tagged(&mut h, TAG_TOOL_CALLS, tc.as_bytes());
|
||||
}
|
||||
|
||||
// Fourth segment (unconditional): name / tool_call_id meta. Empty
|
||||
// meta hashes as TAG_META ‖ len=0 — one formula for all messages.
|
||||
update_tagged(&mut h, TAG_META, msg.extra.as_bytes());
|
||||
|
||||
prev = h.digest();
|
||||
chain.push(prev);
|
||||
}
|
||||
chain
|
||||
}
|
||||
|
||||
/// Whether the body contains content this module has no hashing semantics
|
||||
/// for: unknown part types (`audio_url`, `video_url`, `file`, ...),
|
||||
/// malformed parts, unsupported `image_url` schemes, or genuinely
|
||||
/// malformed message content (number/object/bool). Null/missing content
|
||||
/// and `tool_calls`-carrying assistant turns return false — both have
|
||||
/// well-defined rendering semantics. See `walk_messages` — this is the
|
||||
/// same traversal, so the answer cannot disagree with what `chain_hashes`
|
||||
/// actually hashed.
|
||||
pub fn is_unclassifiable(body: &Value) -> bool {
|
||||
walk_messages(body).1
|
||||
}
|
||||
|
||||
/// Mix `tag ‖ len(bytes) ‖ bytes` into the hasher. Length prefix kills
|
||||
/// concatenation ambiguity between adjacent pieces.
|
||||
fn update_tagged(h: &mut Xxh3, tag: &[u8], bytes: &[u8]) {
|
||||
h.update(tag);
|
||||
h.update(&(bytes.len() as u64).to_le_bytes());
|
||||
h.update(bytes);
|
||||
}
|
||||
|
||||
/// One message, flattened to just what the hash needs. Borrows from the
|
||||
/// body — zero copies on the hot path, except `tool_calls` / `extra`
|
||||
/// whose canonical forms must be owned.
|
||||
struct MessageView<'a> {
|
||||
role: &'a str,
|
||||
content: ContentView<'a>,
|
||||
/// Canonical JSON of `tool_calls` when the field exists and is an
|
||||
/// array (a non-array shape can't be rendered by the template either,
|
||||
/// so it's treated as absent).
|
||||
tool_calls: Option<String>,
|
||||
/// Canonical JSON of the meta object {"name"?, "tool_call_id"?}, or
|
||||
/// empty when neither field applies. See `walk_messages` for which
|
||||
/// fields qualify per role.
|
||||
extra: String,
|
||||
}
|
||||
|
||||
enum ContentView<'a> {
|
||||
Str(&'a str),
|
||||
Parts(Vec<PartView<'a>>),
|
||||
/// Genuinely malformed content (number/object/bool): contributes
|
||||
/// nothing, flagged unclassifiable. Null/missing content never lands
|
||||
/// here — it maps to `Str("")`.
|
||||
Empty,
|
||||
}
|
||||
|
||||
enum PartView<'a> {
|
||||
Text(&'a str),
|
||||
ImageUrl(&'a str),
|
||||
/// Unrecognized or malformed part: deterministic placeholder.
|
||||
Unknown,
|
||||
}
|
||||
|
||||
/// Single traversal shared by `chain_hashes` and `is_unclassifiable`
|
||||
/// (C10: one source of truth for "what can we hash").
|
||||
fn walk_messages(body: &Value) -> (Vec<MessageView<'_>>, bool) {
|
||||
let Some(msgs) = body.get("messages").and_then(Value::as_array) else {
|
||||
return (Vec::new(), false);
|
||||
};
|
||||
let mut unclassifiable = false;
|
||||
let mut out = Vec::with_capacity(msgs.len());
|
||||
for m in msgs {
|
||||
let role = m.get("role").and_then(Value::as_str).unwrap_or("user");
|
||||
// `reasoning_content` is deliberately NOT hashed: chat templates do
|
||||
// not render it, so it cannot change prompt bytes — mixing it in
|
||||
// would only manufacture misses.
|
||||
let tool_calls = match m.get("tool_calls") {
|
||||
Some(v @ Value::Array(_)) => Some(canonical_json(v)),
|
||||
_ => None,
|
||||
};
|
||||
// Meta segment: `name` (any role) and `tool_call_id` (tool results
|
||||
// only) are rendered by most templates -> they must be in the
|
||||
// chain. String and non-string values alike go through the
|
||||
// canonical serializer, so wire-shape differences can't collide.
|
||||
// Explicit null counts as absent (templates render it as nothing).
|
||||
let mut meta = serde_json::Map::new();
|
||||
if let Some(v) = m.get("name").filter(|v| !v.is_null()) {
|
||||
meta.insert("name".to_string(), v.clone());
|
||||
}
|
||||
if role == "tool" {
|
||||
if let Some(v) = m.get("tool_call_id").filter(|v| !v.is_null()) {
|
||||
meta.insert("tool_call_id".to_string(), v.clone());
|
||||
}
|
||||
}
|
||||
let extra = if meta.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
canonical_json(&Value::Object(meta))
|
||||
};
|
||||
let content = match m.get("content") {
|
||||
Some(Value::String(s)) => ContentView::Str(s.as_str()),
|
||||
Some(Value::Array(parts)) => {
|
||||
let mut views = Vec::with_capacity(parts.len());
|
||||
for p in parts {
|
||||
match classify_part(p) {
|
||||
Some(view) => views.push(view),
|
||||
None => {
|
||||
unclassifiable = true;
|
||||
views.push(PartView::Unknown);
|
||||
}
|
||||
}
|
||||
}
|
||||
ContentView::Parts(views)
|
||||
}
|
||||
// null / absent renders as empty content in every chat
|
||||
// template — hash exactly like "" and stay classifiable. This
|
||||
// is the tool-use conversation shape (assistant turn with
|
||||
// content:null + tool_calls), our main traffic.
|
||||
None | Some(Value::Null) => ContentView::Str(""),
|
||||
// number/object/bool content: genuinely malformed, no defined
|
||||
// rendering semantics — fail closed.
|
||||
_ => {
|
||||
unclassifiable = true;
|
||||
ContentView::Empty
|
||||
}
|
||||
};
|
||||
out.push(MessageView {
|
||||
role,
|
||||
content,
|
||||
tool_calls,
|
||||
extra,
|
||||
});
|
||||
}
|
||||
(out, unclassifiable)
|
||||
}
|
||||
|
||||
/// Classify one content part. `None` = no hashing semantics for this part
|
||||
/// (unknown type, malformed shape, or an `image_url` scheme outside
|
||||
/// http/https/data — e.g. `file://` would tempt a local read we must never
|
||||
/// do).
|
||||
fn classify_part(p: &Value) -> Option<PartView<'_>> {
|
||||
let ty = p.get("type").and_then(Value::as_str)?;
|
||||
match ty {
|
||||
"text" => p
|
||||
.get("text")
|
||||
.and_then(Value::as_str)
|
||||
.map(PartView::Text),
|
||||
"image_url" => {
|
||||
// Both wire shapes exist in the wild:
|
||||
// {"image_url":{"url":...}} and {"image_url":"..."}.
|
||||
let url = match p.get("image_url") {
|
||||
Some(Value::String(s)) => Some(s.as_str()),
|
||||
Some(v @ Value::Object(_)) => v.get("url").and_then(Value::as_str),
|
||||
_ => None,
|
||||
}?;
|
||||
if url.starts_with("http://") || url.starts_with("https://") || url.starts_with("data:")
|
||||
{
|
||||
Some(PartView::ImageUrl(url))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Canonical JSON: object keys sorted recursively, no whitespace. Hand-rolled
|
||||
/// (no new crate) because we only need it for seed inputs, which are small —
|
||||
/// `tools` is hashed from its canonical form, and the seed doc itself is
|
||||
/// four fields.
|
||||
fn canonical_json(v: &Value) -> String {
|
||||
let mut out = String::new();
|
||||
write_canonical(v, &mut out);
|
||||
out
|
||||
}
|
||||
|
||||
fn write_canonical(v: &Value, out: &mut String) {
|
||||
match v {
|
||||
Value::Object(map) => {
|
||||
out.push('{');
|
||||
let mut keys: Vec<&String> = map.keys().collect();
|
||||
keys.sort();
|
||||
for (i, k) in keys.iter().enumerate() {
|
||||
if i > 0 {
|
||||
out.push(',');
|
||||
}
|
||||
// Serializing a &str Value is infallible; avoid unwrap anyway.
|
||||
if let Ok(k_json) = serde_json::to_string(k) {
|
||||
out.push_str(&k_json);
|
||||
}
|
||||
out.push(':');
|
||||
write_canonical(&map[*k], out);
|
||||
}
|
||||
out.push('}');
|
||||
}
|
||||
Value::Array(arr) => {
|
||||
out.push('[');
|
||||
for (i, item) in arr.iter().enumerate() {
|
||||
if i > 0 {
|
||||
out.push(',');
|
||||
}
|
||||
write_canonical(item, out);
|
||||
}
|
||||
out.push(']');
|
||||
}
|
||||
other => {
|
||||
if let Ok(s) = serde_json::to_string(other) {
|
||||
out.push_str(&s);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
fn text_body(msgs: Vec<Value>) -> Value {
|
||||
json!({"model": "dsv41-flash", "messages": msgs})
|
||||
}
|
||||
|
||||
fn user(t: &str) -> Value {
|
||||
json!({"role": "user", "content": t})
|
||||
}
|
||||
|
||||
fn assistant(t: &str) -> Value {
|
||||
json!({"role": "assistant", "content": t})
|
||||
}
|
||||
|
||||
fn chain(body: &Value) -> Vec<u64> {
|
||||
chain_hashes(&ChainSeed::from_body(body), body)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_prefix_different_tail() {
|
||||
let a = text_body(vec![user("hello"), assistant("hi"), user("tail A")]);
|
||||
let b = text_body(vec![user("hello"), assistant("hi"), user("tail B")]);
|
||||
let ca = chain(&a);
|
||||
let cb = chain(&b);
|
||||
assert_eq!(ca.len(), 3);
|
||||
assert_eq!(cb.len(), 3);
|
||||
assert_eq!(ca[..2], cb[..2], "shared prefix must hash identically");
|
||||
assert_ne!(ca[2], cb[2], "different tail must diverge");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn forked_conversation_shares_prefix() {
|
||||
let base = vec![user("q1"), assistant("a1"), user("q2"), assistant("a2")];
|
||||
let mut fork_x = base.clone();
|
||||
fork_x.push(user("branch X"));
|
||||
let mut fork_y = base;
|
||||
fork_y.push(user("branch Y"));
|
||||
let bx = text_body(fork_x);
|
||||
let by = text_body(fork_y);
|
||||
let cx = chain(&bx);
|
||||
let cy = chain(&by);
|
||||
assert_eq!(cx[..4], cy[..4]);
|
||||
assert_ne!(cx[4], cy[4]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn editing_history_rewrites_from_that_point_on() {
|
||||
let a = text_body(vec![user("q1"), assistant("old answer"), user("q2")]);
|
||||
let b = text_body(vec![user("q1"), assistant("edited answer"), user("q2")]);
|
||||
let ca = chain(&a);
|
||||
let cb = chain(&b);
|
||||
assert_eq!(ca[0], cb[0], "before the edit: unchanged");
|
||||
assert_ne!(ca[1], cb[1], "at the edit: changed");
|
||||
assert_ne!(ca[2], cb[2], "after the edit: still changed even with identical text");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multimodal_byte_level_semantics() {
|
||||
let mk = |url: &str| {
|
||||
text_body(vec![json!({
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "what is in this image?"},
|
||||
{"type": "image_url", "image_url": {"url": url}}
|
||||
]
|
||||
})])
|
||||
};
|
||||
// Same image content, different URLs -> different chain (the rendered
|
||||
// prompt bytes differ; this is the correct byte-level behaviour).
|
||||
let http_a = mk("https://example.com/a.png");
|
||||
let http_b = mk("https://example.com/b.png");
|
||||
assert_ne!(chain(&http_a), chain(&http_b));
|
||||
|
||||
// Identical base64 payload -> identical chain (hit).
|
||||
let data_1 = mk("data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==");
|
||||
let data_2 = mk("data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==");
|
||||
assert_eq!(chain(&data_1), chain(&data_2));
|
||||
|
||||
// base64 vs http form of the "same" image -> different (bytes differ).
|
||||
assert_ne!(chain(&data_1), chain(&http_a));
|
||||
|
||||
// Wire-shape variant {"image_url":"..."} hashes like the object form.
|
||||
let flat = text_body(vec![json!({
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "what is in this image?"},
|
||||
{"type": "image_url", "image_url": "https://example.com/a.png"}
|
||||
]
|
||||
})]);
|
||||
assert_eq!(chain(&flat), chain(&http_a));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn seed_fields_change_whole_chain() {
|
||||
let msgs = || vec![user("same"), assistant("same")];
|
||||
let base = text_body(msgs());
|
||||
let other_model = json!({"model": "other-model", "messages": msgs()});
|
||||
let with_kwargs = json!({
|
||||
"model": "dsv41-flash",
|
||||
"chat_template_kwargs": {"thinking": true},
|
||||
"messages": msgs()
|
||||
});
|
||||
let with_tools = json!({
|
||||
"model": "dsv41-flash",
|
||||
"tools": [{"type": "function", "function": {"name": "f"}}],
|
||||
"messages": msgs()
|
||||
});
|
||||
|
||||
let c_base = chain(&base);
|
||||
assert_ne!(ChainSeed::from_body(&base), ChainSeed::from_body(&other_model));
|
||||
assert_ne!(chain(&other_model), c_base);
|
||||
assert_ne!(chain(&with_kwargs), c_base);
|
||||
assert_ne!(chain(&with_tools), c_base);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn seed_canonicalization_is_key_order_insensitive() {
|
||||
let a = json!({"model": "m", "chat_template_kwargs": {"x": 1, "y": 2}, "messages": [user("hi")]});
|
||||
let b = json!({"messages": [user("hi")], "chat_template_kwargs": {"y": 2, "x": 1}, "model": "m"});
|
||||
assert_eq!(ChainSeed::from_body(&a), ChainSeed::from_body(&b));
|
||||
assert_eq!(chain(&a), chain(&b));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_or_malformed_messages_give_empty_chain() {
|
||||
let no_messages = json!({"model": "m"});
|
||||
let empty_messages = json!({"model": "m", "messages": []});
|
||||
let wrong_type = json!({"model": "m", "messages": "not-an-array"});
|
||||
for b in [&no_messages, &empty_messages, &wrong_type] {
|
||||
assert!(chain(b).is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_part_types_flag_unclassifiable() {
|
||||
let audio = text_body(vec![json!({
|
||||
"role": "user",
|
||||
"content": [{"type": "audio_url", "audio_url": {"url": "https://x/a.mp3"}}]
|
||||
})]);
|
||||
assert!(is_unclassifiable(&audio));
|
||||
// Chain is still full-length and deterministic — just not to be trusted.
|
||||
assert_eq!(chain(&audio).len(), 1);
|
||||
assert_eq!(chain(&audio), chain(&audio));
|
||||
|
||||
let file_scheme = text_body(vec![json!({
|
||||
"role": "user",
|
||||
"content": [{"type": "image_url", "image_url": {"url": "file:///etc/passwd"}}]
|
||||
})]);
|
||||
assert!(is_unclassifiable(&file_scheme));
|
||||
|
||||
let plain = text_body(vec![user("hello"), assistant("hi")]);
|
||||
assert!(!is_unclassifiable(&plain));
|
||||
|
||||
let multimodal_ok = text_body(vec![json!({
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "look"},
|
||||
{"type": "image_url", "image_url": {"url": "https://x/a.png"}}
|
||||
]
|
||||
})]);
|
||||
assert!(!is_unclassifiable(&multimodal_ok));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_role_defaults_to_user() {
|
||||
let implicit = text_body(vec![json!({"content": "hi"})]);
|
||||
let explicit = text_body(vec![user("hi")]);
|
||||
assert_eq!(chain(&implicit), chain(&explicit));
|
||||
assert!(!is_unclassifiable(&implicit));
|
||||
}
|
||||
|
||||
fn tool_call(name: &str, args: &str) -> Value {
|
||||
json!([{
|
||||
"id": format!("call_{}", name),
|
||||
"type": "function",
|
||||
"function": {"name": name, "arguments": args}
|
||||
}])
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tool_calls_enter_the_chain() {
|
||||
let mk = |tc: Value| {
|
||||
text_body(vec![
|
||||
user("weather in Shanghai?"),
|
||||
json!({"role": "assistant", "content": null, "tool_calls": tc}),
|
||||
user("and tomorrow?"),
|
||||
])
|
||||
};
|
||||
let a = mk(tool_call("get_weather", "{\"city\":\"sh\"}"));
|
||||
let b = mk(tool_call("get_weather", "{\"city\":\"bj\"}"));
|
||||
let c = mk(tool_call("get_weather", "{\"city\":\"sh\"}"));
|
||||
let (ca, cb, cc) = (chain(&a), chain(&b), chain(&c));
|
||||
assert_eq!(ca[0], cb[0], "before the tool call: unchanged");
|
||||
assert_ne!(ca[1], cb[1], "different tool_calls must diverge");
|
||||
assert_ne!(ca[2], cb[2], "and stay diverged afterwards");
|
||||
assert_eq!(ca, cc, "identical tool_calls must hit");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn content_null_with_tool_calls_is_classifiable() {
|
||||
let tc = tool_call("get_weather", "{}");
|
||||
let null_content = text_body(vec![json!({
|
||||
"role": "assistant", "content": null, "tool_calls": tc.clone()
|
||||
})]);
|
||||
let empty_content = text_body(vec![json!({
|
||||
"role": "assistant", "content": "", "tool_calls": tc
|
||||
})]);
|
||||
assert!(!is_unclassifiable(&null_content));
|
||||
assert_eq!(
|
||||
chain(&null_content),
|
||||
chain(&empty_content),
|
||||
"content:null renders as empty -> must hash like content:\"\""
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn content_null_without_tool_calls_is_empty() {
|
||||
let null_content = text_body(vec![json!({"role": "assistant", "content": null})]);
|
||||
let missing_content = text_body(vec![json!({"role": "assistant"})]);
|
||||
let empty_content = text_body(vec![assistant("")]);
|
||||
assert!(!is_unclassifiable(&null_content));
|
||||
assert!(!is_unclassifiable(&missing_content));
|
||||
assert_eq!(chain(&null_content), chain(&empty_content));
|
||||
assert_eq!(chain(&missing_content), chain(&empty_content));
|
||||
|
||||
// Genuinely malformed content types keep the fail-closed treatment.
|
||||
let malformed = text_body(vec![json!({"role": "assistant", "content": 42})]);
|
||||
assert!(is_unclassifiable(&malformed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tool_calls_canonicalization_key_order_insensitive() {
|
||||
let a = text_body(vec![json!({
|
||||
"role": "assistant",
|
||||
"content": null,
|
||||
"tool_calls": [{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "f", "arguments": "{}"}
|
||||
}]
|
||||
})]);
|
||||
let b = text_body(vec![json!({
|
||||
"role": "assistant",
|
||||
"content": null,
|
||||
"tool_calls": [{
|
||||
"function": {"arguments": "{}", "name": "f"},
|
||||
"type": "function",
|
||||
"id": "call_1"
|
||||
}]
|
||||
})]);
|
||||
assert_eq!(chain(&a), chain(&b));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tool_call_id_enters_chain() {
|
||||
let mk = |id: &str| {
|
||||
text_body(vec![
|
||||
user("weather in Shanghai?"),
|
||||
json!({
|
||||
"role": "assistant",
|
||||
"content": null,
|
||||
"tool_calls": tool_call("get_weather", "{}")
|
||||
}),
|
||||
json!({"role": "tool", "tool_call_id": id, "content": "sunny"}),
|
||||
user("thanks"),
|
||||
])
|
||||
};
|
||||
let a = mk("call_1");
|
||||
let b = mk("call_2");
|
||||
let c = mk("call_1");
|
||||
let (ca, cb, cc) = (chain(&a), chain(&b), chain(&c));
|
||||
assert_eq!(ca[..2], cb[..2], "before the tool result: unchanged");
|
||||
assert_ne!(ca[2], cb[2], "different tool_call_id must diverge");
|
||||
assert_ne!(ca[3], cb[3], "and stay diverged afterwards");
|
||||
assert_eq!(ca, cc, "identical tool_call_id must hit");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn name_field_enters_chain() {
|
||||
let a = text_body(vec![json!({"role": "user", "name": "alice", "content": "hi"})]);
|
||||
let b = text_body(vec![json!({"role": "user", "name": "bob", "content": "hi"})]);
|
||||
let c = text_body(vec![json!({"role": "user", "name": "alice", "content": "hi"})]);
|
||||
assert_ne!(chain(&a), chain(&b), "different name must diverge");
|
||||
assert_eq!(chain(&a), chain(&c), "identical name must hit");
|
||||
|
||||
// Absent name and empty meta stay equivalent to a plain message.
|
||||
let plain = text_body(vec![user("hi")]);
|
||||
let no_name = text_body(vec![json!({"role": "user", "content": "hi"})]);
|
||||
assert_eq!(chain(&plain), chain(&no_name));
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,5 +1,8 @@
|
||||
//! HTTP router implementations
|
||||
|
||||
pub mod g9i;
|
||||
pub mod g9i_prefix_chain;
|
||||
pub mod g9i_shadow;
|
||||
pub mod pd_router;
|
||||
pub mod pd_types;
|
||||
pub mod router;
|
||||
|
||||
@@ -18,6 +18,7 @@ use serde_json::{json, Value};
|
||||
use tokio_stream::wrappers::UnboundedReceiverStream;
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
use super::g9i_shadow::{ShadowRequestContext, ShadowResponseContext};
|
||||
use super::pd_types::api_path;
|
||||
use crate::{
|
||||
config::types::RetryConfig,
|
||||
@@ -58,6 +59,8 @@ pub struct PDRouter {
|
||||
pub retry_config: RetryConfig,
|
||||
pub api_key: Option<String>,
|
||||
pub enable_igw: bool,
|
||||
/// G9i pool admission + fast-P routing state; None = feature off.
|
||||
pub g9i: Option<Arc<super::g9i::G9iState>>,
|
||||
}
|
||||
|
||||
struct PreparedWorkerRequest<'a> {
|
||||
@@ -182,6 +185,14 @@ impl PDRouter {
|
||||
}
|
||||
|
||||
pub async fn new(ctx: &Arc<crate::app_context::AppContext>) -> Result<Self, String> {
|
||||
let g9i = match &ctx.router_config.g9i {
|
||||
Some(config) if config.enabled => {
|
||||
let state = Arc::new(super::g9i::G9iState::new(config.clone()));
|
||||
state.start_monitor(Arc::clone(&ctx.worker_registry), ctx.client.clone());
|
||||
Some(state)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
Ok(PDRouter {
|
||||
worker_registry: Arc::clone(&ctx.worker_registry),
|
||||
policy_registry: Arc::clone(&ctx.policy_registry),
|
||||
@@ -189,6 +200,7 @@ impl PDRouter {
|
||||
retry_config: ctx.router_config.effective_retry_config(),
|
||||
api_key: ctx.router_config.api_key.clone(),
|
||||
enable_igw: ctx.router_config.enable_igw,
|
||||
g9i,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -390,6 +402,90 @@ impl PDRouter {
|
||||
// Clone request once outside the retry loop, then use Arc to share across attempts
|
||||
// This avoids O(retries) clones by sharing the same data
|
||||
let shared_request = Arc::new(original_request.clone());
|
||||
|
||||
// G9i admission (router-side, zero engine changes). Computed once per
|
||||
// request, outside the retry loop: our own 429s must never be
|
||||
// retried. `ttft_path` labels the TTFT histogram dimension for this
|
||||
// request's eventual first-token sample. `shadow_ctx` is the G9i v2
|
||||
// shadow bookkeeping carrier (T2): Some only for chat requests with
|
||||
// a classifiable prefix chain while shadow mode is on; it never
|
||||
// influences routing.
|
||||
let (g9i_prefill_override, g9i_ttft_path, shadow_ctx): (
|
||||
Option<String>,
|
||||
&'static str,
|
||||
Option<ShadowRequestContext>,
|
||||
) = if let Some(g9i) = &self.g9i {
|
||||
if let Err(info) = g9i.check_decode_pool(&self.worker_registry).await {
|
||||
metrics::counter!("smg_g9i_rejects_total", "reason" => info.reason)
|
||||
.increment(1);
|
||||
return super::g9i::reject_response(&info);
|
||||
}
|
||||
if context.route == "/v1/chat/completions" {
|
||||
match serde_json::to_value(original_request) {
|
||||
Ok(body) => {
|
||||
// Shadow T2: compute the prefix chain once, here,
|
||||
// alongside the v1 classification. Skipped entirely
|
||||
// when shadow mode is off.
|
||||
let shadow_ctx = if g9i.shadow_enabled() {
|
||||
ShadowRequestContext::from_body(&body, context.is_stream)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
match g9i
|
||||
.classify_prefill(&self.worker_registry, &self.client, &body)
|
||||
.await
|
||||
{
|
||||
(plan, path, probe_new_work) => {
|
||||
// Shadow T3: shadow-classify alongside the v1
|
||||
// verdict and record agreement/divergence
|
||||
// counters. Routing still follows the v1 plan
|
||||
// alone; the decision only rides the context
|
||||
// into dispatch for queue bookkeeping.
|
||||
let shadow_ctx = match shadow_ctx {
|
||||
Some(mut ctx) => {
|
||||
if let Some(decision) = g9i.shadow_classify(&ctx) {
|
||||
super::g9i_shadow::record_agreement(
|
||||
path,
|
||||
probe_new_work,
|
||||
&ctx,
|
||||
&decision,
|
||||
);
|
||||
ctx.decision = Some(decision);
|
||||
}
|
||||
Some(ctx)
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
match plan.into_override() {
|
||||
Ok(override_url) => (override_url, path, shadow_ctx),
|
||||
Err(info) => {
|
||||
metrics::counter!(
|
||||
"smg_g9i_rejects_total",
|
||||
"reason" => info.reason
|
||||
)
|
||||
.increment(1);
|
||||
return super::g9i::reject_response(&info);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(_) => (None, "regular", None),
|
||||
}
|
||||
} else {
|
||||
// Non-chat routes skip classification but still land on
|
||||
// the regular pool — the regular queue gate applies.
|
||||
if let Err(info) = g9i.check_regular_queue(&self.worker_registry) {
|
||||
metrics::counter!("smg_g9i_rejects_total", "reason" => info.reason)
|
||||
.increment(1);
|
||||
return super::g9i::reject_response(&info);
|
||||
}
|
||||
(None, "non_chat", None)
|
||||
}
|
||||
} else {
|
||||
(None, "untracked", None)
|
||||
};
|
||||
|
||||
let response = RetryExecutor::execute_response_with_retry(
|
||||
&self.retry_config,
|
||||
{
|
||||
@@ -397,6 +493,8 @@ impl PDRouter {
|
||||
// Clone Arc (cheap reference count increment) instead of cloning the entire request
|
||||
let shared_request = Arc::clone(&shared_request);
|
||||
let context = context.clone();
|
||||
let g9i_prefill_override = g9i_prefill_override.clone();
|
||||
let shadow_ctx = shadow_ctx.clone();
|
||||
async move {
|
||||
let (prefill, decode) = match self
|
||||
.select_pd_pair(
|
||||
@@ -412,6 +510,38 @@ impl PDRouter {
|
||||
}
|
||||
};
|
||||
|
||||
// G9i least-wait override: route onto the classified
|
||||
// fast-P node instead of the policy pick.
|
||||
let (prefill, decode) =
|
||||
if let Some(override_url) = g9i_prefill_override.as_ref() {
|
||||
let matched = self
|
||||
.worker_registry
|
||||
.get_prefill_workers()
|
||||
.into_iter()
|
||||
.find(|w| {
|
||||
w.url().trim_end_matches('/') == override_url.as_str()
|
||||
&& w.is_available()
|
||||
});
|
||||
match matched {
|
||||
Some(w) => {
|
||||
debug!(
|
||||
"G9i: overriding prefill selection with {}",
|
||||
override_url
|
||||
);
|
||||
(w, decode)
|
||||
}
|
||||
None => {
|
||||
warn!(
|
||||
"G9i: override target {} unavailable, keeping policy pick",
|
||||
override_url
|
||||
);
|
||||
(prefill, decode)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
(prefill, decode)
|
||||
};
|
||||
|
||||
debug!(
|
||||
"PD retry attempt {} using prefill={} decode={}",
|
||||
attempt,
|
||||
@@ -428,6 +558,15 @@ impl PDRouter {
|
||||
json_request["stream"] = Value::Bool(context.is_stream);
|
||||
}
|
||||
|
||||
// G9i shadow T2: inject the usage / cache-details
|
||||
// flags the client didn't set (chat + shadow mode
|
||||
// only; `shadow_ctx` is None otherwise). Paired with
|
||||
// response-side stripping: we inject exactly what we
|
||||
// later strip, so clients see no change.
|
||||
if let Some(sctx) = &shadow_ctx {
|
||||
sctx.inject(&mut json_request);
|
||||
}
|
||||
|
||||
json_request = match Self::inject_bootstrap_into_value(
|
||||
json_request,
|
||||
prefill.as_ref(),
|
||||
@@ -446,6 +585,8 @@ impl PDRouter {
|
||||
Arc::clone(&prefill),
|
||||
Arc::clone(&decode),
|
||||
start_time,
|
||||
g9i_ttft_path,
|
||||
shadow_ctx,
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -575,6 +716,9 @@ impl PDRouter {
|
||||
Some(response_headers),
|
||||
prefill,
|
||||
decode,
|
||||
None,
|
||||
// Error envelope: no usage will ever arrive — no shadow hook.
|
||||
None,
|
||||
)
|
||||
} else {
|
||||
// Handle non-streaming error response
|
||||
@@ -649,6 +793,7 @@ impl PDRouter {
|
||||
}
|
||||
|
||||
// Internal method that performs the actual dual dispatch (without retry logic)
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn execute_dual_dispatch_internal(
|
||||
&self,
|
||||
headers: Option<&HeaderMap>,
|
||||
@@ -656,7 +801,11 @@ impl PDRouter {
|
||||
context: PDRequestContext<'_>,
|
||||
prefill: Arc<dyn Worker>,
|
||||
decode: Arc<dyn Worker>,
|
||||
_start_time: Instant,
|
||||
start_time: Instant,
|
||||
g9i_path: &'static str,
|
||||
// G9i shadow T2: prefix chain + injection flags; None = shadow off
|
||||
// or unclassifiable request — zero behavior change.
|
||||
shadow_ctx: Option<ShadowRequestContext>,
|
||||
) -> Response {
|
||||
// For non-streaming: use guard for automatic load management
|
||||
// For streaming: load will be managed in create_streaming_response
|
||||
@@ -868,6 +1017,23 @@ impl PDRouter {
|
||||
|
||||
let response_headers = header_utils::preserve_response_headers(res.headers());
|
||||
|
||||
// Shadow T2: hand the chain + the ACTUAL prefill landing
|
||||
// node (post-override) to the stream wrapper, which
|
||||
// harvests the usage event and strips injected fields.
|
||||
let shadow = shadow_ctx.map(|ctx| {
|
||||
ShadowResponseContext::from_request(
|
||||
ctx,
|
||||
super::g9i::normalize_url(prefill.url()),
|
||||
)
|
||||
});
|
||||
// Shadow T3: admit into the queue model keyed by the
|
||||
// ACTUAL landing node; the stream wrapper pairs this with
|
||||
// exactly one complete (usage recorded) or release
|
||||
// (abort/disconnect/error) before teardown.
|
||||
if let (Some(g9i), Some(s)) = (&self.g9i, &shadow) {
|
||||
g9i.shadow_admit(s);
|
||||
}
|
||||
|
||||
self.create_streaming_response(
|
||||
res.bytes_stream(),
|
||||
status,
|
||||
@@ -876,15 +1042,31 @@ impl PDRouter {
|
||||
Some(response_headers),
|
||||
prefill,
|
||||
decode,
|
||||
Some((start_time, g9i_path)),
|
||||
shadow,
|
||||
)
|
||||
} else {
|
||||
// Non-streaming response
|
||||
let shadow = shadow_ctx.map(|ctx| {
|
||||
ShadowResponseContext::from_request(
|
||||
ctx,
|
||||
super::g9i::normalize_url(prefill.url()),
|
||||
)
|
||||
});
|
||||
// Shadow T3: admit into the queue model keyed by the
|
||||
// ACTUAL landing node; paired with complete/release in
|
||||
// shadow_record_non_stream (or the read-failure paths
|
||||
// below / in process_non_streaming_response).
|
||||
if let (Some(g9i), Some(s)) = (&self.g9i, &shadow) {
|
||||
g9i.shadow_admit(s);
|
||||
}
|
||||
if context.return_logprob {
|
||||
self.process_non_streaming_response(
|
||||
res,
|
||||
status,
|
||||
context.return_logprob,
|
||||
prefill_body,
|
||||
shadow,
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
@@ -894,6 +1076,8 @@ impl PDRouter {
|
||||
|
||||
match res.bytes().await {
|
||||
Ok(decode_body) => {
|
||||
let decode_body =
|
||||
self.shadow_record_non_stream(decode_body, status, &shadow);
|
||||
let mut response = Response::new(Body::from(decode_body));
|
||||
*response.status_mut() = status;
|
||||
*response.headers_mut() = response_headers;
|
||||
@@ -901,6 +1085,11 @@ impl PDRouter {
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Failed to read decode response: {}", e);
|
||||
// Shadow T3: the admitted dispatch produced no
|
||||
// readable body — release the queue slot.
|
||||
if let (Some(g9i), Some(s)) = (&self.g9i, &shadow) {
|
||||
g9i.shadow_release(s);
|
||||
}
|
||||
error::internal_error(
|
||||
"read_response_failed",
|
||||
"Failed to read response",
|
||||
@@ -1116,11 +1305,39 @@ impl PDRouter {
|
||||
headers: Option<HeaderMap>,
|
||||
prefill: Arc<dyn Worker>,
|
||||
decode: Arc<dyn Worker>,
|
||||
// TTFT sampling hook: Some((dispatch-entry instant, path label)) for
|
||||
// real upstream streams; None for synthetic error envelopes (their
|
||||
// "first chunk" is not a token — sampling them would poison the
|
||||
// window with near-zero error samples) and for tests.
|
||||
ttft: Option<(Instant, &'static str)>,
|
||||
// G9i shadow T2 hook: Some only for shadowable chat requests on a
|
||||
// success-status stream; None on error envelopes and tests.
|
||||
shadow: Option<ShadowResponseContext>,
|
||||
) -> Response {
|
||||
use crate::core::AttachedBody;
|
||||
|
||||
let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
|
||||
|
||||
// G9i TTFT sample: record on the first upstream chunk — that moment is
|
||||
// the request's time-to-first-token as seen by the gateway (dispatch
|
||||
// entry -> first token). Non-streaming requests have no observable
|
||||
// first-token moment and are not sampled.
|
||||
let g9i_ttft = self.g9i.clone().zip(ttft);
|
||||
let mut ttft_recorded = false;
|
||||
|
||||
// G9i shadow T2: zip the state handle with the per-request context
|
||||
// (chain + node + strip flags) for the stream loop below.
|
||||
let g9i_shadow = self.g9i.clone().zip(shadow);
|
||||
// Shadow T3 queue bookkeeping state. `ttft` is Option<(Instant,
|
||||
// &'static str)> which is Copy, so zip() above left it usable:
|
||||
// mirror the dispatch-entry instant so the first chunk can also feed
|
||||
// the shadow TTFT calibration. `shadow_done` latches once usage is
|
||||
// recorded; teardown releases the admission only when no completion
|
||||
// happened (abort / client disconnect / upstream error).
|
||||
let ttft_start = ttft.map(|(t, _)| t);
|
||||
let mut shadow_ttft_secs: Option<f64> = None;
|
||||
let mut shadow_done = false;
|
||||
|
||||
// Uses select! to race stream.next() against tx.closed() so that
|
||||
// when the client disconnects the upstream HTTP connection is dropped
|
||||
// promptly, allowing the engine to abort the request.
|
||||
@@ -1149,8 +1366,54 @@ impl PDRouter {
|
||||
chunk_result = tracked.next() => {
|
||||
match chunk_result {
|
||||
Some(Ok(chunk)) => {
|
||||
// G9i shadow T2: harvest the usage event for
|
||||
// prefix-table bookkeeping and strip exactly
|
||||
// the fields we injected on the way in.
|
||||
// Chunks without a usage field pass through
|
||||
// byte-identically. A chunk reduced to nothing
|
||||
// (usage-only event we must hide from the
|
||||
// client) is dropped entirely.
|
||||
let chunk = if let Some((g9i, sctx)) = &g9i_shadow {
|
||||
let (usage, forward) =
|
||||
super::g9i_shadow::process_sse_chunk(
|
||||
&chunk,
|
||||
sctx.strip_usage,
|
||||
sctx.strip_cached_details,
|
||||
);
|
||||
if let Some(json) = usage {
|
||||
// Shadow T3: exactly one completion
|
||||
// per admission — the latch keeps a
|
||||
// (malformed) repeat usage event from
|
||||
// double-completing the queue slot.
|
||||
if !shadow_done {
|
||||
g9i.shadow_record(sctx, &json, shadow_ttft_secs);
|
||||
shadow_done = true;
|
||||
}
|
||||
}
|
||||
match forward {
|
||||
Some(bytes) => bytes,
|
||||
None => continue,
|
||||
}
|
||||
} else {
|
||||
chunk
|
||||
};
|
||||
|
||||
let is_done = memmem::find(&chunk, b"data: [DONE]").is_some();
|
||||
|
||||
if !ttft_recorded {
|
||||
ttft_recorded = true;
|
||||
if let Some((g9i, (start_time, g9i_path))) = &g9i_ttft {
|
||||
g9i.record_ttft(
|
||||
g9i_path,
|
||||
start_time.elapsed().as_secs_f64(),
|
||||
);
|
||||
}
|
||||
// Shadow T3: the same first-token moment
|
||||
// calibrates est_TTFT (est − actual).
|
||||
shadow_ttft_secs =
|
||||
ttft_start.map(|t| t.elapsed().as_secs_f64());
|
||||
}
|
||||
|
||||
let result = if return_logprob && prefill_logprobs.is_some() {
|
||||
Self::merge_streaming_logprobs(prefill_logprobs.clone(), &chunk)
|
||||
.unwrap_or(chunk)
|
||||
@@ -1199,6 +1462,15 @@ impl PDRouter {
|
||||
}
|
||||
}
|
||||
}
|
||||
// Shadow T3: the stream ended without a recorded usage event
|
||||
// (abort, client disconnect, or upstream error) — release the
|
||||
// queue admission so outstanding stays conserved. No-op when the
|
||||
// usage event already completed it.
|
||||
if !shadow_done {
|
||||
if let Some((g9i, sctx)) = &g9i_shadow {
|
||||
g9i.shadow_release(sctx);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let stream = UnboundedReceiverStream::new(rx);
|
||||
@@ -1226,16 +1498,26 @@ impl PDRouter {
|
||||
status: StatusCode,
|
||||
return_logprob: bool,
|
||||
prefill_body: Option<bytes::Bytes>,
|
||||
shadow: Option<ShadowResponseContext>,
|
||||
) -> Response {
|
||||
let response = res.bytes().await;
|
||||
let decode_body = match response {
|
||||
Ok(decode_body) => decode_body,
|
||||
Err(e) => {
|
||||
error!("Failed to read decode response: {}", e);
|
||||
// Shadow T3: the admitted dispatch produced no readable
|
||||
// body — release the queue slot.
|
||||
if let (Some(g9i), Some(s)) = (&self.g9i, &shadow) {
|
||||
g9i.shadow_release(s);
|
||||
}
|
||||
return error::internal_error("read_response_failed", "Failed to read response");
|
||||
}
|
||||
};
|
||||
|
||||
// Shadow T2: record usage + strip injected fields before any
|
||||
// logprob merging (merge touches choices only, usage is untouched).
|
||||
let decode_body = self.shadow_record_non_stream(decode_body, status, &shadow);
|
||||
|
||||
if !return_logprob {
|
||||
return (status, decode_body).into_response();
|
||||
}
|
||||
@@ -1265,6 +1547,47 @@ impl PDRouter {
|
||||
}
|
||||
}
|
||||
|
||||
/// G9i shadow bookkeeping for non-streaming responses: parse the body,
|
||||
/// record usage into the prefix table (T2), pair the T3 queue admission
|
||||
/// with exactly one complete (parseable body) or release (non-success /
|
||||
/// unparseable), and strip `cached_tokens_details` when (and only when)
|
||||
/// we injected the request flag ourselves.
|
||||
///
|
||||
/// Passthrough guarantees: shadow off / non-success status / unparseable
|
||||
/// body / nothing stripped -> the ORIGINAL bytes are returned untouched,
|
||||
/// so clients never see a re-serialized response unless we actually
|
||||
/// removed an injected field.
|
||||
fn shadow_record_non_stream(
|
||||
&self,
|
||||
body: bytes::Bytes,
|
||||
status: StatusCode,
|
||||
shadow: &Option<ShadowResponseContext>,
|
||||
) -> bytes::Bytes {
|
||||
let (Some(g9i), Some(ctx)) = (&self.g9i, shadow) else {
|
||||
return body;
|
||||
};
|
||||
// Only 200 responses with usage may write the table (design §3.3-3).
|
||||
// A non-success response still releases the T3 queue admission.
|
||||
if !status.is_success() {
|
||||
g9i.shadow_release(ctx);
|
||||
return body;
|
||||
}
|
||||
let (parsed, rewritten) =
|
||||
super::g9i_shadow::strip_and_extract(&body, ctx.strip_cached_details);
|
||||
match &parsed {
|
||||
// TTFT calibration is streaming-only (non-streaming has no
|
||||
// observable first-token moment) -> None.
|
||||
Some(json) => g9i.shadow_record(ctx, json, None),
|
||||
// Unparseable success body: nothing to record, but the dispatch
|
||||
// is over — release rather than leak the outstanding count.
|
||||
None => g9i.shadow_release(ctx),
|
||||
}
|
||||
match rewritten {
|
||||
Some(bytes) => bytes::Bytes::from(bytes),
|
||||
None => body,
|
||||
}
|
||||
}
|
||||
|
||||
// Helper to process prefill response and extract body if needed for logprobs
|
||||
async fn process_prefill_response(
|
||||
&self,
|
||||
@@ -1780,6 +2103,7 @@ mod tests {
|
||||
retry_config: RetryConfig::default(),
|
||||
api_key: Some("test_api_key".to_string()),
|
||||
enable_igw: false,
|
||||
g9i: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2048,6 +2372,8 @@ mod tests {
|
||||
None,
|
||||
prefill_ref.clone(),
|
||||
decode_ref.clone(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
|
||||
// Guards are now attached to response body, so load should be 1
|
||||
|
||||
@@ -321,6 +321,7 @@ mod pd_responses_routing_tests {
|
||||
retry_config: RetryConfig::default(),
|
||||
api_key: None,
|
||||
enable_igw: false,
|
||||
g9i: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
{"v":1}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"git": {
|
||||
"sha1": "7455eaee1da5212ae5e0fd33d80a903029d2f981"
|
||||
},
|
||||
"path_in_vcs": "protocols"
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
|
||||
#
|
||||
# When uploading crates to the registry Cargo will automatically
|
||||
# "normalize" Cargo.toml files for maximal compatibility
|
||||
# with all versions of Cargo and also rewrite `path` dependencies
|
||||
# to registry (e.g., crates.io) dependencies.
|
||||
#
|
||||
# If you are reading this file be aware that the original Cargo.toml
|
||||
# will likely look very different (and much more reasonable).
|
||||
# See Cargo.toml.orig for the original contents.
|
||||
|
||||
[package]
|
||||
edition = "2021"
|
||||
name = "openai-protocol"
|
||||
version = "1.0.0"
|
||||
authors = [
|
||||
"Simo Lin <linsimo.mark@gmail.com>",
|
||||
"Chang Su <mckvtl@gmail.com>",
|
||||
"Keyang Ru <rukeyang@gmail.com>",
|
||||
]
|
||||
build = false
|
||||
autolib = false
|
||||
autobins = false
|
||||
autoexamples = false
|
||||
autotests = false
|
||||
autobenches = false
|
||||
description = "OpenAI-compatible API protocol definitions and types"
|
||||
readme = false
|
||||
keywords = [
|
||||
"openai",
|
||||
"llm",
|
||||
"api",
|
||||
"protocol",
|
||||
"types",
|
||||
]
|
||||
categories = [
|
||||
"api-bindings",
|
||||
"data-structures",
|
||||
]
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/lightseekorg/smg"
|
||||
|
||||
[features]
|
||||
axum = ["dep:axum"]
|
||||
default = []
|
||||
|
||||
[lib]
|
||||
name = "openai_protocol"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[dependencies.axum]
|
||||
version = "0.8.6"
|
||||
optional = true
|
||||
|
||||
[dependencies.chrono]
|
||||
version = "0.4"
|
||||
|
||||
[dependencies.rand]
|
||||
version = "0.9.2"
|
||||
|
||||
[dependencies.serde]
|
||||
version = "1.0"
|
||||
features = ["derive"]
|
||||
|
||||
[dependencies.serde_json]
|
||||
version = "1.0"
|
||||
features = ["preserve_order"]
|
||||
|
||||
[dependencies.tokio]
|
||||
version = "1.42.0"
|
||||
|
||||
[dependencies.tracing]
|
||||
version = "0.1"
|
||||
|
||||
[dependencies.validator]
|
||||
version = "0.20.0"
|
||||
features = ["derive"]
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
[package]
|
||||
name = "openai-protocol"
|
||||
version = "1.0.0"
|
||||
edition = "2021"
|
||||
description = "OpenAI-compatible API protocol definitions and types"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/lightseekorg/smg"
|
||||
authors = [
|
||||
"Simo Lin <linsimo.mark@gmail.com>",
|
||||
"Chang Su <mckvtl@gmail.com>",
|
||||
"Keyang Ru <rukeyang@gmail.com>",
|
||||
]
|
||||
keywords = ["openai", "llm", "api", "protocol", "types"]
|
||||
categories = ["api-bindings", "data-structures"]
|
||||
|
||||
[features]
|
||||
default = []
|
||||
axum = ["dep:axum"]
|
||||
|
||||
[dependencies]
|
||||
axum = { workspace = true, optional = true }
|
||||
chrono.workspace = true
|
||||
rand.workspace = true
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true, features = ["preserve_order"] }
|
||||
tokio.workspace = true
|
||||
tracing.workspace = true
|
||||
# Only used by protocols
|
||||
validator = { version = "0.20.0", features = ["derive"] }
|
||||
@@ -0,0 +1,7 @@
|
||||
//! Builders for Chat Completion API response types
|
||||
|
||||
pub mod response;
|
||||
pub mod stream_response;
|
||||
|
||||
pub use response::ChatCompletionResponseBuilder;
|
||||
pub use stream_response::ChatCompletionStreamResponseBuilder;
|
||||
@@ -0,0 +1,218 @@
|
||||
//! Builder for ChatCompletionResponse
|
||||
//!
|
||||
//! Provides an ergonomic fluent API for constructing chat completion responses.
|
||||
|
||||
use crate::{chat::*, common::Usage};
|
||||
|
||||
/// Builder for ChatCompletionResponse
|
||||
///
|
||||
/// Provides a fluent interface for constructing chat completion responses with sensible defaults.
|
||||
#[must_use = "Builder does nothing until .build() is called"]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ChatCompletionResponseBuilder {
|
||||
id: String,
|
||||
object: String,
|
||||
created: u64,
|
||||
model: String,
|
||||
choices: Vec<ChatChoice>,
|
||||
usage: Option<Usage>,
|
||||
system_fingerprint: Option<String>,
|
||||
}
|
||||
|
||||
impl ChatCompletionResponseBuilder {
|
||||
/// Create a new builder with required fields
|
||||
///
|
||||
/// # Arguments
|
||||
/// - `id`: Completion ID (e.g., "chatcmpl_abc123")
|
||||
/// - `model`: Model name used for generation
|
||||
pub fn new(id: impl Into<String>, model: impl Into<String>) -> Self {
|
||||
Self {
|
||||
id: id.into(),
|
||||
object: "chat.completion".to_string(),
|
||||
created: chrono::Utc::now().timestamp() as u64,
|
||||
model: model.into(),
|
||||
choices: Vec::new(),
|
||||
usage: None,
|
||||
system_fingerprint: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Copy common fields from a ChatCompletionRequest
|
||||
///
|
||||
/// This populates the model field from the request.
|
||||
pub fn copy_from_request(mut self, request: &ChatCompletionRequest) -> Self {
|
||||
self.model = request.model.clone();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the object type (default: "chat.completion")
|
||||
pub fn object(mut self, object: impl Into<String>) -> Self {
|
||||
self.object = object.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the creation timestamp (default: current time)
|
||||
pub fn created(mut self, timestamp: u64) -> Self {
|
||||
self.created = timestamp;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the choices
|
||||
pub fn choices(mut self, choices: Vec<ChatChoice>) -> Self {
|
||||
self.choices = choices;
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a single choice
|
||||
pub fn add_choice(mut self, choice: ChatChoice) -> Self {
|
||||
self.choices.push(choice);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set usage information
|
||||
pub fn usage(mut self, usage: Usage) -> Self {
|
||||
self.usage = Some(usage);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set usage if provided (handles Option)
|
||||
pub fn maybe_usage(mut self, usage: Option<Usage>) -> Self {
|
||||
if let Some(u) = usage {
|
||||
self.usage = Some(u);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Set system fingerprint if provided (handles Option)
|
||||
pub fn maybe_system_fingerprint(mut self, fingerprint: Option<impl Into<String>>) -> Self {
|
||||
if let Some(fp) = fingerprint {
|
||||
self.system_fingerprint = Some(fp.into());
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Build the ChatCompletionResponse
|
||||
pub fn build(self) -> ChatCompletionResponse {
|
||||
ChatCompletionResponse {
|
||||
id: self.id,
|
||||
object: self.object,
|
||||
created: self.created,
|
||||
model: self.model,
|
||||
choices: self.choices,
|
||||
usage: self.usage,
|
||||
system_fingerprint: self.system_fingerprint,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tests
|
||||
// ============================================================================
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_build_minimal() {
|
||||
let response = ChatCompletionResponse::builder("chatcmpl_123", "gpt-4").build();
|
||||
|
||||
assert_eq!(response.id, "chatcmpl_123");
|
||||
assert_eq!(response.model, "gpt-4");
|
||||
assert_eq!(response.object, "chat.completion");
|
||||
assert!(response.choices.is_empty());
|
||||
assert!(response.usage.is_none());
|
||||
assert!(response.system_fingerprint.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_complete() {
|
||||
let choice = ChatChoice {
|
||||
index: 0,
|
||||
message: ChatCompletionMessage {
|
||||
role: "assistant".to_string(),
|
||||
content: Some("Hello!".to_string()),
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: Some("stop".to_string()),
|
||||
matched_stop: None,
|
||||
hidden_states: None,
|
||||
};
|
||||
|
||||
let usage = Usage {
|
||||
prompt_tokens: 10,
|
||||
completion_tokens: 20,
|
||||
total_tokens: 30,
|
||||
completion_tokens_details: None,
|
||||
};
|
||||
|
||||
let response = ChatCompletionResponse::builder("chatcmpl_456", "gpt-4")
|
||||
.choices(vec![choice.clone()])
|
||||
.maybe_usage(Some(usage))
|
||||
.maybe_system_fingerprint(Some("fp_123abc"))
|
||||
.build();
|
||||
|
||||
assert_eq!(response.id, "chatcmpl_456");
|
||||
assert_eq!(response.choices.len(), 1);
|
||||
assert_eq!(response.choices[0].index, 0);
|
||||
assert!(response.usage.is_some());
|
||||
assert_eq!(response.system_fingerprint.as_ref().unwrap(), "fp_123abc");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_multiple_choices() {
|
||||
let choice1 = ChatChoice {
|
||||
index: 0,
|
||||
message: ChatCompletionMessage {
|
||||
role: "assistant".to_string(),
|
||||
content: Some("Option 1".to_string()),
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: Some("stop".to_string()),
|
||||
matched_stop: None,
|
||||
hidden_states: None,
|
||||
};
|
||||
|
||||
let choice2 = ChatChoice {
|
||||
index: 1,
|
||||
message: ChatCompletionMessage {
|
||||
role: "assistant".to_string(),
|
||||
content: Some("Option 2".to_string()),
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: Some("stop".to_string()),
|
||||
matched_stop: None,
|
||||
hidden_states: None,
|
||||
};
|
||||
|
||||
let response = ChatCompletionResponse::builder("chatcmpl_789", "gpt-4")
|
||||
.add_choice(choice1)
|
||||
.add_choice(choice2)
|
||||
.build();
|
||||
|
||||
assert_eq!(response.choices.len(), 2);
|
||||
assert_eq!(response.choices[0].index, 0);
|
||||
assert_eq!(response.choices[1].index, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_copy_from_request() {
|
||||
let request = ChatCompletionRequest {
|
||||
messages: vec![],
|
||||
model: "gpt-3.5-turbo".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let response = ChatCompletionResponse::builder("chatcmpl_101", "gpt-4")
|
||||
.copy_from_request(&request)
|
||||
.build();
|
||||
|
||||
assert_eq!(response.model, "gpt-3.5-turbo"); // Copied from request
|
||||
}
|
||||
}
|
||||
+421
@@ -0,0 +1,421 @@
|
||||
//! Builder for ChatCompletionStreamResponse
|
||||
//!
|
||||
//! Provides an ergonomic fluent API for constructing streaming chat completion responses.
|
||||
|
||||
use std::borrow::Cow;
|
||||
|
||||
use crate::{
|
||||
chat::*,
|
||||
common::{FunctionCallDelta, ToolCallDelta, Usage},
|
||||
};
|
||||
|
||||
/// Builder for ChatCompletionStreamResponse
|
||||
///
|
||||
/// Provides a fluent interface for constructing streaming chat completion chunks with sensible defaults.
|
||||
#[must_use = "Builder does nothing until .build() is called"]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ChatCompletionStreamResponseBuilder {
|
||||
id: String,
|
||||
object: String,
|
||||
created: u64,
|
||||
model: String,
|
||||
choices: Vec<ChatStreamChoice>,
|
||||
usage: Option<Usage>,
|
||||
system_fingerprint: Option<String>,
|
||||
}
|
||||
|
||||
impl ChatCompletionStreamResponseBuilder {
|
||||
/// Create a new builder with required fields
|
||||
///
|
||||
/// # Arguments
|
||||
/// - `id`: Completion ID (e.g., "chatcmpl_abc123")
|
||||
/// - `model`: Model name used for generation
|
||||
pub fn new(id: impl Into<String>, model: impl Into<String>) -> Self {
|
||||
Self {
|
||||
id: id.into(),
|
||||
object: "chat.completion.chunk".to_string(),
|
||||
created: chrono::Utc::now().timestamp() as u64,
|
||||
model: model.into(),
|
||||
choices: Vec::new(),
|
||||
usage: None,
|
||||
system_fingerprint: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Copy common fields from a ChatCompletionRequest
|
||||
///
|
||||
/// This populates the model field from the request.
|
||||
pub fn copy_from_request(mut self, request: &ChatCompletionRequest) -> Self {
|
||||
self.model = request.model.clone();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the object type (default: "chat.completion.chunk")
|
||||
pub fn object(mut self, object: impl Into<String>) -> Self {
|
||||
self.object = object.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the creation timestamp (default: current time)
|
||||
pub fn created(mut self, timestamp: u64) -> Self {
|
||||
self.created = timestamp;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the choices
|
||||
pub fn choices(mut self, choices: Vec<ChatStreamChoice>) -> Self {
|
||||
self.choices = choices;
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a single choice (delta)
|
||||
pub fn add_choice(mut self, choice: ChatStreamChoice) -> Self {
|
||||
self.choices.push(choice);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set usage information (typically sent in final chunk)
|
||||
pub fn usage(mut self, usage: Usage) -> Self {
|
||||
self.usage = Some(usage);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set system fingerprint if provided (handles Option)
|
||||
pub fn maybe_system_fingerprint(mut self, fingerprint: Option<impl Into<String>>) -> Self {
|
||||
if let Some(fp) = fingerprint {
|
||||
self.system_fingerprint = Some(fp.into());
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Set usage if provided (handles Option)
|
||||
pub fn maybe_usage(mut self, usage: Option<Usage>) -> Self {
|
||||
if let Some(u) = usage {
|
||||
self.usage = Some(u);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice delta that sets `role` and `content`
|
||||
pub fn add_choice_content(
|
||||
mut self,
|
||||
index: u32,
|
||||
role: impl Into<String>,
|
||||
content: impl Into<String>,
|
||||
) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some(role.into()),
|
||||
content: Some(content.into()),
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice delta that sets `role`, `content`, and `logprobs`
|
||||
pub fn add_choice_content_with_logprobs(
|
||||
mut self,
|
||||
index: u32,
|
||||
role: impl Into<String>,
|
||||
content: impl Into<String>,
|
||||
logprobs: Option<crate::common::ChatLogProbs>,
|
||||
) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some(role.into()),
|
||||
content: Some(content.into()),
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice delta that only sets `role`
|
||||
pub fn add_choice_role(mut self, index: u32, role: impl Into<String>) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some(role.into()),
|
||||
content: None,
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice delta that appends a tool-call *arguments delta*
|
||||
/// Uses `Cow` so you can pass `&str` or `String` without extra clones
|
||||
pub fn add_choice_tool_args(
|
||||
mut self,
|
||||
index: u32,
|
||||
args_delta: impl Into<Cow<'static, str>>,
|
||||
) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some("assistant".to_string()),
|
||||
content: None,
|
||||
tool_calls: Some(vec![ToolCallDelta {
|
||||
index: 0,
|
||||
id: None,
|
||||
tool_type: None,
|
||||
function: Some(FunctionCallDelta {
|
||||
name: None,
|
||||
arguments: Some(args_delta.into().into_owned()),
|
||||
}),
|
||||
}]),
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice delta that sets reasoning content (for models that stream reasoning)
|
||||
pub fn add_choice_reasoning(mut self, index: u32, reasoning: impl Into<String>) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some("assistant".to_string()),
|
||||
content: None,
|
||||
tool_calls: None,
|
||||
reasoning_content: Some(reasoning.into()),
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice delta for tool call with function name and ID
|
||||
pub fn add_choice_tool_name(
|
||||
mut self,
|
||||
index: u32,
|
||||
tool_call_id: impl Into<String>,
|
||||
function_name: impl Into<String>,
|
||||
) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some("assistant".to_string()),
|
||||
content: None,
|
||||
tool_calls: Some(vec![ToolCallDelta {
|
||||
index: 0,
|
||||
id: Some(tool_call_id.into()),
|
||||
tool_type: Some("function".to_string()),
|
||||
function: Some(FunctionCallDelta {
|
||||
name: Some(function_name.into()),
|
||||
arguments: None,
|
||||
}),
|
||||
}]),
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice delta with a pre-constructed ToolCallDelta
|
||||
/// Useful when you already have a ToolCallDelta object to emit
|
||||
pub fn add_choice_tool_call_delta(
|
||||
mut self,
|
||||
index: u32,
|
||||
tool_call_delta: ToolCallDelta,
|
||||
) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some("assistant".to_string()),
|
||||
content: None,
|
||||
tool_calls: Some(vec![tool_call_delta]),
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice with finish_reason (final chunk)
|
||||
/// This is used for the last chunk in a stream to signal completion
|
||||
pub fn add_choice_finish_reason(
|
||||
mut self,
|
||||
index: u32,
|
||||
finish_reason: impl Into<String>,
|
||||
matched_stop: Option<serde_json::Value>,
|
||||
) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: None,
|
||||
content: None,
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: Some(finish_reason.into()),
|
||||
matched_stop,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Build the ChatCompletionStreamResponse
|
||||
pub fn build(self) -> ChatCompletionStreamResponse {
|
||||
ChatCompletionStreamResponse {
|
||||
id: self.id,
|
||||
object: self.object,
|
||||
created: self.created,
|
||||
model: self.model,
|
||||
system_fingerprint: self.system_fingerprint,
|
||||
choices: self.choices,
|
||||
usage: self.usage,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tests
|
||||
// ============================================================================
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_build_minimal() {
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_123", "gpt-4").build();
|
||||
|
||||
assert_eq!(chunk.id, "chatcmpl_123");
|
||||
assert_eq!(chunk.model, "gpt-4");
|
||||
assert_eq!(chunk.object, "chat.completion.chunk");
|
||||
assert!(chunk.choices.is_empty());
|
||||
assert!(chunk.usage.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_with_content_delta() {
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_456", "gpt-4")
|
||||
.add_choice_content(0, "assistant", "Hello")
|
||||
.build();
|
||||
|
||||
assert_eq!(chunk.choices.len(), 1);
|
||||
assert_eq!(chunk.choices[0].index, 0);
|
||||
assert_eq!(chunk.choices[0].delta.content.as_ref().unwrap(), "Hello");
|
||||
assert_eq!(chunk.choices[0].delta.role.as_ref().unwrap(), "assistant");
|
||||
assert!(chunk.choices[0].finish_reason.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_with_role_delta() {
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_789", "gpt-4")
|
||||
.add_choice_role(0, "assistant")
|
||||
.build();
|
||||
|
||||
assert_eq!(chunk.choices.len(), 1);
|
||||
assert_eq!(chunk.choices[0].delta.role.as_ref().unwrap(), "assistant");
|
||||
assert!(chunk.choices[0].delta.content.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_with_finish_reason() {
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_101", "gpt-4")
|
||||
.add_choice_finish_reason(0, "stop", None)
|
||||
.build();
|
||||
|
||||
assert_eq!(chunk.choices.len(), 1);
|
||||
assert_eq!(chunk.choices[0].finish_reason.as_ref().unwrap(), "stop");
|
||||
assert!(chunk.choices[0].delta.content.is_none());
|
||||
assert!(chunk.choices[0].delta.role.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multiple_deltas() {
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_202", "gpt-4")
|
||||
.add_choice_role(0, "assistant")
|
||||
.add_choice_content(0, "assistant", "Hello")
|
||||
.add_choice_content(0, "assistant", " world")
|
||||
.add_choice_finish_reason(0, "stop", None)
|
||||
.build();
|
||||
|
||||
assert_eq!(chunk.choices.len(), 4); // role + 2 content + finish
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_with_usage() {
|
||||
let usage = Usage {
|
||||
prompt_tokens: 10,
|
||||
completion_tokens: 20,
|
||||
total_tokens: 30,
|
||||
completion_tokens_details: None,
|
||||
};
|
||||
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_303", "gpt-4")
|
||||
.add_choice_finish_reason(0, "stop", None)
|
||||
.usage(usage)
|
||||
.build();
|
||||
|
||||
assert!(chunk.usage.is_some());
|
||||
assert_eq!(chunk.usage.as_ref().unwrap().total_tokens, 30);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_copy_from_request() {
|
||||
let request = ChatCompletionRequest {
|
||||
messages: vec![],
|
||||
model: "gpt-3.5-turbo".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_404", "gpt-4")
|
||||
.copy_from_request(&request)
|
||||
.add_choice_content(0, "assistant", "test")
|
||||
.build();
|
||||
|
||||
assert_eq!(chunk.model, "gpt-3.5-turbo"); // Copied from request
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_choice_explicit() {
|
||||
let choice = ChatStreamChoice {
|
||||
index: 0,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some("assistant".to_string()),
|
||||
content: Some("Hello".to_string()),
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
};
|
||||
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_505", "gpt-4")
|
||||
.add_choice(choice)
|
||||
.build();
|
||||
|
||||
assert_eq!(chunk.choices.len(), 1);
|
||||
assert_eq!(chunk.choices[0].delta.role.as_ref().unwrap(), "assistant");
|
||||
assert_eq!(chunk.choices[0].delta.content.as_ref().unwrap(), "Hello");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
//! Builder patterns for protocol response types
|
||||
//!
|
||||
//! This module provides ergonomic builders for response types with many optional fields.
|
||||
//! Builders help avoid telescoping constructors and make construction intent clear.
|
||||
//!
|
||||
//! # Organization
|
||||
//!
|
||||
//! Builders are organized by API:
|
||||
//! - `chat/` - Chat Completion API builders (response, stream_response)
|
||||
//! - `responses/` - Responses API builder (response)
|
||||
//!
|
||||
//! # Optional Fields
|
||||
//!
|
||||
//! For optional fields, builders provide `maybe_*` methods that handle `Option<T>` directly:
|
||||
//! ```ignore
|
||||
//! builder
|
||||
//! .field(value)
|
||||
//! .maybe_optional_field(optional_value) // Accepts Option<T>
|
||||
//! .build()
|
||||
//! ```
|
||||
|
||||
pub mod chat;
|
||||
pub mod responses;
|
||||
|
||||
// Re-export all builders for convenient access
|
||||
pub use chat::{ChatCompletionResponseBuilder, ChatCompletionStreamResponseBuilder};
|
||||
pub use responses::ResponsesResponseBuilder;
|
||||
@@ -0,0 +1,5 @@
|
||||
//! Builders for Responses API response types
|
||||
|
||||
pub mod response;
|
||||
|
||||
pub use response::ResponsesResponseBuilder;
|
||||
@@ -0,0 +1,411 @@
|
||||
//! Builder for ResponsesResponse
|
||||
//!
|
||||
//! Provides an ergonomic fluent API for constructing ResponsesResponse instances.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::responses::*;
|
||||
|
||||
/// Builder for ResponsesResponse
|
||||
///
|
||||
/// Provides a fluent interface for constructing responses with sensible defaults.
|
||||
#[must_use = "Builder does nothing until .build() is called"]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ResponsesResponseBuilder {
|
||||
id: String,
|
||||
object: String,
|
||||
created_at: i64,
|
||||
status: ResponseStatus,
|
||||
error: Option<Value>,
|
||||
incomplete_details: Option<Value>,
|
||||
instructions: Option<String>,
|
||||
max_output_tokens: Option<u32>,
|
||||
model: String,
|
||||
output: Vec<ResponseOutputItem>,
|
||||
parallel_tool_calls: bool,
|
||||
previous_response_id: Option<String>,
|
||||
reasoning: Option<ReasoningInfo>,
|
||||
store: bool,
|
||||
temperature: Option<f32>,
|
||||
text: Option<TextConfig>,
|
||||
tool_choice: String,
|
||||
tools: Vec<ResponseTool>,
|
||||
top_p: Option<f32>,
|
||||
truncation: Option<String>,
|
||||
usage: Option<ResponsesUsage>,
|
||||
user: Option<String>,
|
||||
safety_identifier: Option<String>,
|
||||
metadata: HashMap<String, Value>,
|
||||
}
|
||||
|
||||
impl ResponsesResponseBuilder {
|
||||
/// Create a new builder with required fields
|
||||
///
|
||||
/// # Arguments
|
||||
/// - `id`: Response ID (e.g., "resp_abc123")
|
||||
/// - `model`: Model name used for generation
|
||||
pub fn new(id: impl Into<String>, model: impl Into<String>) -> Self {
|
||||
Self {
|
||||
id: id.into(),
|
||||
object: "response".to_string(),
|
||||
created_at: chrono::Utc::now().timestamp(),
|
||||
status: ResponseStatus::InProgress,
|
||||
error: None,
|
||||
incomplete_details: None,
|
||||
instructions: None,
|
||||
max_output_tokens: None,
|
||||
model: model.into(),
|
||||
output: Vec::new(),
|
||||
parallel_tool_calls: true,
|
||||
previous_response_id: None,
|
||||
reasoning: None,
|
||||
store: true,
|
||||
temperature: None,
|
||||
text: None,
|
||||
tool_choice: "auto".to_string(),
|
||||
tools: Vec::new(),
|
||||
top_p: None,
|
||||
truncation: None,
|
||||
usage: None,
|
||||
user: None,
|
||||
safety_identifier: None,
|
||||
metadata: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Copy common fields from a ResponsesRequest
|
||||
///
|
||||
/// This populates fields like instructions, max_output_tokens, temperature, etc.
|
||||
/// from the original request, making it easy to construct a response that mirrors
|
||||
/// the request parameters.
|
||||
///
|
||||
/// Note: `safety_identifier` is intentionally NOT copied as it is for content moderation
|
||||
/// and should be set independently from the request's `user` field (which is for billing/tracking).
|
||||
pub fn copy_from_request(mut self, request: &ResponsesRequest) -> Self {
|
||||
self.instructions = request.instructions.clone();
|
||||
self.max_output_tokens = request.max_output_tokens;
|
||||
self.parallel_tool_calls = request.parallel_tool_calls.unwrap_or(true);
|
||||
self.previous_response_id = request.previous_response_id.clone();
|
||||
self.store = request.store.unwrap_or(true);
|
||||
self.temperature = request.temperature;
|
||||
self.tool_choice = if let Some(ref tc) = request.tool_choice {
|
||||
serde_json::to_string(tc).unwrap_or_else(|_| "auto".to_string())
|
||||
} else {
|
||||
"auto".to_string()
|
||||
};
|
||||
self.tools = request.tools.clone().unwrap_or_default();
|
||||
self.top_p = request.top_p;
|
||||
self.user = request.user.clone();
|
||||
self.metadata = request.metadata.clone().unwrap_or_default();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the object type (default: "response")
|
||||
pub fn object(mut self, object: impl Into<String>) -> Self {
|
||||
self.object = object.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the creation timestamp (default: current time)
|
||||
pub fn created_at(mut self, timestamp: i64) -> Self {
|
||||
self.created_at = timestamp;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the response status
|
||||
pub fn status(mut self, status: ResponseStatus) -> Self {
|
||||
self.status = status;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set error information (if status is failed)
|
||||
pub fn error(mut self, error: Value) -> Self {
|
||||
self.error = Some(error);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set incomplete details (if response was truncated)
|
||||
pub fn incomplete_details(mut self, details: Value) -> Self {
|
||||
self.incomplete_details = Some(details);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set system instructions
|
||||
pub fn instructions(mut self, instructions: impl Into<String>) -> Self {
|
||||
self.instructions = Some(instructions.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Set max output tokens
|
||||
pub fn max_output_tokens(mut self, tokens: u32) -> Self {
|
||||
self.max_output_tokens = Some(tokens);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set output items
|
||||
pub fn output(mut self, output: Vec<ResponseOutputItem>) -> Self {
|
||||
self.output = output;
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a single output item
|
||||
pub fn add_output(mut self, item: ResponseOutputItem) -> Self {
|
||||
self.output.push(item);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set whether parallel tool calls are enabled
|
||||
pub fn parallel_tool_calls(mut self, enabled: bool) -> Self {
|
||||
self.parallel_tool_calls = enabled;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set previous response ID (if continuation)
|
||||
pub fn previous_response_id(mut self, id: impl Into<String>) -> Self {
|
||||
self.previous_response_id = Some(id.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Set reasoning information
|
||||
pub fn reasoning(mut self, reasoning: ReasoningInfo) -> Self {
|
||||
self.reasoning = Some(reasoning);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set whether the response is stored
|
||||
pub fn store(mut self, store: bool) -> Self {
|
||||
self.store = store;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set temperature setting
|
||||
pub fn temperature(mut self, temperature: f32) -> Self {
|
||||
self.temperature = Some(temperature);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set text format settings if provided (handles Option)
|
||||
pub fn maybe_text(mut self, text: Option<TextConfig>) -> Self {
|
||||
if let Some(t) = text {
|
||||
self.text = Some(t);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Set tool choice setting
|
||||
pub fn tool_choice(mut self, tool_choice: impl Into<String>) -> Self {
|
||||
self.tool_choice = tool_choice.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set available tools
|
||||
pub fn tools(mut self, tools: Vec<ResponseTool>) -> Self {
|
||||
self.tools = tools;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set top-p setting
|
||||
pub fn top_p(mut self, top_p: f32) -> Self {
|
||||
self.top_p = Some(top_p);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set truncation strategy
|
||||
pub fn truncation(mut self, truncation: impl Into<String>) -> Self {
|
||||
self.truncation = Some(truncation.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Set usage information
|
||||
pub fn usage(mut self, usage: ResponsesUsage) -> Self {
|
||||
self.usage = Some(usage);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set usage if provided (handles Option)
|
||||
pub fn maybe_usage(mut self, usage: Option<ResponsesUsage>) -> Self {
|
||||
if let Some(u) = usage {
|
||||
self.usage = Some(u);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Copy from request if provided (handles Option)
|
||||
pub fn maybe_copy_from_request(mut self, request: Option<&ResponsesRequest>) -> Self {
|
||||
if let Some(req) = request {
|
||||
self = self.copy_from_request(req);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Set user identifier
|
||||
pub fn user(mut self, user: impl Into<String>) -> Self {
|
||||
self.user = Some(user.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Set safety identifier
|
||||
pub fn safety_identifier(mut self, identifier: impl Into<String>) -> Self {
|
||||
self.safety_identifier = Some(identifier.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Set metadata
|
||||
pub fn metadata(mut self, metadata: HashMap<String, Value>) -> Self {
|
||||
self.metadata = metadata;
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a single metadata entry
|
||||
pub fn add_metadata(mut self, key: impl Into<String>, value: Value) -> Self {
|
||||
self.metadata.insert(key.into(), value);
|
||||
self
|
||||
}
|
||||
|
||||
/// Build the ResponsesResponse
|
||||
pub fn build(self) -> ResponsesResponse {
|
||||
ResponsesResponse {
|
||||
id: self.id,
|
||||
object: self.object,
|
||||
created_at: self.created_at,
|
||||
status: self.status,
|
||||
error: self.error,
|
||||
incomplete_details: self.incomplete_details,
|
||||
instructions: self.instructions,
|
||||
max_output_tokens: self.max_output_tokens,
|
||||
model: self.model,
|
||||
output: self.output,
|
||||
parallel_tool_calls: self.parallel_tool_calls,
|
||||
previous_response_id: self.previous_response_id,
|
||||
reasoning: self.reasoning,
|
||||
store: self.store,
|
||||
temperature: self.temperature,
|
||||
text: self.text,
|
||||
tool_choice: self.tool_choice,
|
||||
tools: self.tools,
|
||||
top_p: self.top_p,
|
||||
truncation: self.truncation,
|
||||
usage: self.usage,
|
||||
user: self.user,
|
||||
safety_identifier: self.safety_identifier,
|
||||
metadata: self.metadata,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tests
|
||||
// ============================================================================
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_build_minimal() {
|
||||
let response = ResponsesResponse::builder("resp_123", "gpt-4").build();
|
||||
|
||||
assert_eq!(response.id, "resp_123");
|
||||
assert_eq!(response.model, "gpt-4");
|
||||
assert_eq!(response.object, "response");
|
||||
assert_eq!(response.status, ResponseStatus::InProgress);
|
||||
assert!(response.output.is_empty());
|
||||
assert!(response.parallel_tool_calls);
|
||||
assert!(response.store);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_complete() {
|
||||
let response = ResponsesResponse::builder("resp_123", "gpt-4")
|
||||
.status(ResponseStatus::Completed)
|
||||
.instructions("You are a helpful assistant")
|
||||
.max_output_tokens(1000)
|
||||
.temperature(0.7)
|
||||
.top_p(0.9)
|
||||
.parallel_tool_calls(false)
|
||||
.store(false)
|
||||
.build();
|
||||
|
||||
assert_eq!(response.status, ResponseStatus::Completed);
|
||||
assert_eq!(
|
||||
response.instructions.as_ref().unwrap(),
|
||||
"You are a helpful assistant"
|
||||
);
|
||||
assert_eq!(response.max_output_tokens, Some(1000));
|
||||
assert_eq!(response.temperature, Some(0.7));
|
||||
assert_eq!(response.top_p, Some(0.9));
|
||||
assert!(!response.parallel_tool_calls);
|
||||
assert!(!response.store);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_copy_from_request() {
|
||||
let request = ResponsesRequest {
|
||||
model: "gpt-4".to_string(),
|
||||
input: ResponseInput::Text("test".to_string()),
|
||||
instructions: Some("Be helpful".to_string()),
|
||||
max_output_tokens: Some(500),
|
||||
temperature: Some(0.8),
|
||||
top_p: Some(0.95),
|
||||
parallel_tool_calls: Some(false),
|
||||
store: Some(false),
|
||||
user: Some("user_123".to_string()),
|
||||
metadata: Some(HashMap::from([(
|
||||
"key".to_string(),
|
||||
serde_json::json!("value"),
|
||||
)])),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let response = ResponsesResponse::builder("resp_456", "gpt-4")
|
||||
.copy_from_request(&request)
|
||||
.status(ResponseStatus::Completed)
|
||||
.build();
|
||||
|
||||
assert_eq!(response.instructions.as_ref().unwrap(), "Be helpful");
|
||||
assert_eq!(response.max_output_tokens, Some(500));
|
||||
assert_eq!(response.temperature, Some(0.8));
|
||||
assert_eq!(response.top_p, Some(0.95));
|
||||
assert!(!response.parallel_tool_calls);
|
||||
assert!(!response.store);
|
||||
assert_eq!(response.user.as_ref().unwrap(), "user_123");
|
||||
assert_eq!(
|
||||
response.metadata.get("key").unwrap(),
|
||||
&serde_json::json!("value")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_output_items() {
|
||||
let response = ResponsesResponse::builder("resp_789", "gpt-4")
|
||||
.add_output(ResponseOutputItem::Message {
|
||||
id: "msg_1".to_string(),
|
||||
role: "assistant".to_string(),
|
||||
content: vec![],
|
||||
status: "completed".to_string(),
|
||||
})
|
||||
.add_output(ResponseOutputItem::Message {
|
||||
id: "msg_2".to_string(),
|
||||
role: "assistant".to_string(),
|
||||
content: vec![],
|
||||
status: "completed".to_string(),
|
||||
})
|
||||
.build();
|
||||
|
||||
assert_eq!(response.output.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_metadata() {
|
||||
let response = ResponsesResponse::builder("resp_101", "gpt-4")
|
||||
.add_metadata("key1", serde_json::json!("value1"))
|
||||
.add_metadata("key2", serde_json::json!(42))
|
||||
.build();
|
||||
|
||||
assert_eq!(response.metadata.len(), 2);
|
||||
assert_eq!(response.metadata.get("key1").unwrap(), "value1");
|
||||
assert_eq!(response.metadata.get("key2").unwrap(), 42);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,799 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use validator::Validate;
|
||||
|
||||
use super::{
|
||||
common::{
|
||||
default_model, default_true, validate_stop, ChatLogProbs, ContentPart, Function,
|
||||
FunctionCall, FunctionChoice, GenerationRequest, ResponseFormat, StreamOptions,
|
||||
StringOrArray, Tool, ToolCall, ToolCallDelta, ToolChoice, ToolChoiceValue, ToolReference,
|
||||
Usage,
|
||||
},
|
||||
sampling_params::{validate_top_k_value, validate_top_p_value},
|
||||
};
|
||||
use crate::{
|
||||
builders::{ChatCompletionResponseBuilder, ChatCompletionStreamResponseBuilder},
|
||||
validated::Normalizable,
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// Chat Messages
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(tag = "role")]
|
||||
pub enum ChatMessage {
|
||||
#[serde(rename = "system")]
|
||||
System {
|
||||
content: MessageContent,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
name: Option<String>,
|
||||
},
|
||||
#[serde(rename = "user")]
|
||||
User {
|
||||
content: MessageContent,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
name: Option<String>,
|
||||
},
|
||||
#[serde(rename = "assistant")]
|
||||
Assistant {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
content: Option<MessageContent>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
name: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
tool_calls: Option<Vec<ToolCall>>,
|
||||
/// Reasoning content for O1-style models (SGLang extension)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
reasoning_content: Option<String>,
|
||||
},
|
||||
#[serde(rename = "tool")]
|
||||
Tool {
|
||||
content: MessageContent,
|
||||
tool_call_id: String,
|
||||
},
|
||||
#[serde(rename = "function")]
|
||||
Function { content: String, name: String },
|
||||
#[serde(rename = "developer")]
|
||||
Developer {
|
||||
content: MessageContent,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
tools: Option<Vec<Tool>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
name: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
|
||||
#[serde(untagged)]
|
||||
pub enum MessageContent {
|
||||
Text(String),
|
||||
Parts(Vec<ContentPart>),
|
||||
}
|
||||
|
||||
impl MessageContent {
|
||||
/// Returns the text content, cloning only when necessary.
|
||||
/// For simple text, returns a clone of the string.
|
||||
/// For parts, concatenates text parts with spaces.
|
||||
/// Optimized to avoid intermediate Vec allocation.
|
||||
pub fn to_simple_string(&self) -> String {
|
||||
match self {
|
||||
MessageContent::Text(text) => text.clone(),
|
||||
MessageContent::Parts(parts) => {
|
||||
// Use fold to build string directly without intermediate Vec allocation
|
||||
let mut result = String::new();
|
||||
let mut first = true;
|
||||
for part in parts {
|
||||
if let ContentPart::Text { text } = part {
|
||||
if !first {
|
||||
result.push(' ');
|
||||
}
|
||||
result.push_str(text);
|
||||
first = false;
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Appends text content directly to a buffer, avoiding intermediate allocations.
|
||||
/// Returns true if any content was appended.
|
||||
#[inline]
|
||||
pub fn append_text_to(&self, buffer: &mut String) -> bool {
|
||||
match self {
|
||||
MessageContent::Text(text) => {
|
||||
if !text.is_empty() {
|
||||
buffer.push_str(text);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
MessageContent::Parts(parts) => {
|
||||
let mut appended = false;
|
||||
for part in parts {
|
||||
if let ContentPart::Text { text } = part {
|
||||
if !text.is_empty() {
|
||||
if appended {
|
||||
buffer.push(' ');
|
||||
}
|
||||
buffer.push_str(text);
|
||||
appended = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
appended
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns true if this content contains any non-empty text.
|
||||
#[inline]
|
||||
pub fn has_text(&self) -> bool {
|
||||
match self {
|
||||
MessageContent::Text(text) => !text.is_empty(),
|
||||
MessageContent::Parts(parts) => parts
|
||||
.iter()
|
||||
.any(|part| matches!(part, ContentPart::Text { text } if !text.is_empty())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Chat Completion Request
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, Default, Validate)]
|
||||
#[validate(schema(function = "validate_chat_cross_parameters"))]
|
||||
pub struct ChatCompletionRequest {
|
||||
/// A list of messages comprising the conversation so far
|
||||
#[validate(custom(function = "validate_messages"))]
|
||||
pub messages: Vec<ChatMessage>,
|
||||
|
||||
/// ID of the model to use
|
||||
#[serde(default = "default_model")]
|
||||
pub model: String,
|
||||
|
||||
/// Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = -2.0, max = 2.0))]
|
||||
pub frequency_penalty: Option<f32>,
|
||||
|
||||
/// Deprecated: Replaced by tool_choice
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[deprecated(note = "Use tool_choice instead")]
|
||||
pub function_call: Option<FunctionCall>,
|
||||
|
||||
/// Deprecated: Replaced by tools
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[deprecated(note = "Use tools instead")]
|
||||
pub functions: Option<Vec<Function>>,
|
||||
|
||||
/// Modify the likelihood of specified tokens appearing in the completion
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logit_bias: Option<HashMap<String, f32>>,
|
||||
|
||||
/// Whether to return log probabilities of the output tokens
|
||||
#[serde(default)]
|
||||
pub logprobs: bool,
|
||||
|
||||
/// Deprecated: Replaced by max_completion_tokens
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[deprecated(note = "Use max_completion_tokens instead")]
|
||||
#[validate(range(min = 1))]
|
||||
pub max_tokens: Option<u32>,
|
||||
|
||||
/// An upper bound for the number of tokens that can be generated for a completion
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 1))]
|
||||
pub max_completion_tokens: Option<u32>,
|
||||
|
||||
/// Developer-defined tags and values used for filtering completions in the dashboard
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub metadata: Option<HashMap<String, String>>,
|
||||
|
||||
/// Output types that you would like the model to generate for this request
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub modalities: Option<Vec<String>>,
|
||||
|
||||
/// How many chat completion choices to generate for each input message
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 1, max = 10))]
|
||||
pub n: Option<u32>,
|
||||
|
||||
/// Whether to enable parallel function calling during tool use
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub parallel_tool_calls: Option<bool>,
|
||||
|
||||
/// Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = -2.0, max = 2.0))]
|
||||
pub presence_penalty: Option<f32>,
|
||||
|
||||
/// Cache key for prompts (beta feature)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub prompt_cache_key: Option<String>,
|
||||
|
||||
/// Effort level for reasoning models (low, medium, high)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub reasoning_effort: Option<String>,
|
||||
|
||||
/// An object specifying the format that the model must output
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub response_format: Option<ResponseFormat>,
|
||||
|
||||
/// Safety identifier for content moderation
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub safety_identifier: Option<String>,
|
||||
|
||||
/// Deprecated: This feature is in Legacy mode
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[deprecated(note = "This feature is in Legacy mode")]
|
||||
pub seed: Option<i64>,
|
||||
|
||||
/// The service tier to use for this request
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub service_tier: Option<String>,
|
||||
|
||||
/// Up to 4 sequences where the API will stop generating further tokens
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(custom(function = "validate_stop"))]
|
||||
pub stop: Option<StringOrArray>,
|
||||
|
||||
/// If set, partial message deltas will be sent
|
||||
#[serde(default)]
|
||||
pub stream: bool,
|
||||
|
||||
/// Options for streaming response
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stream_options: Option<StreamOptions>,
|
||||
|
||||
/// What sampling temperature to use, between 0 and 2
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 0.0, max = 2.0))]
|
||||
pub temperature: Option<f32>,
|
||||
|
||||
/// Controls which (if any) tool is called by the model
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tool_choice: Option<ToolChoice>,
|
||||
|
||||
/// A list of tools the model may call
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tools: Option<Vec<Tool>>,
|
||||
|
||||
/// An integer between 0 and 20 specifying the number of most likely tokens to return
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 0, max = 20))]
|
||||
pub top_logprobs: Option<u32>,
|
||||
|
||||
/// An alternative to sampling with temperature
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(custom(function = "validate_top_p_value"))]
|
||||
pub top_p: Option<f32>,
|
||||
|
||||
/// Verbosity level for debugging
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub verbosity: Option<i32>,
|
||||
|
||||
// =============================================================================
|
||||
// Engine-Specific Sampling Parameters
|
||||
// =============================================================================
|
||||
// These parameters are extensions beyond the OpenAI API specification and
|
||||
// control model generation behavior in engine-specific ways.
|
||||
// =============================================================================
|
||||
/// Top-k sampling parameter (-1 to disable)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(custom(function = "validate_top_k_value"))]
|
||||
pub top_k: Option<i32>,
|
||||
|
||||
/// Min-p nucleus sampling parameter
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 0.0, max = 1.0))]
|
||||
pub min_p: Option<f32>,
|
||||
|
||||
/// Minimum number of tokens to generate
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 1))]
|
||||
pub min_tokens: Option<u32>,
|
||||
|
||||
/// Repetition penalty for reducing repetitive text
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 0.0, max = 2.0))]
|
||||
pub repetition_penalty: Option<f32>,
|
||||
|
||||
/// Regex constraint for output generation
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub regex: Option<String>,
|
||||
|
||||
/// EBNF grammar constraint for structured output
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub ebnf: Option<String>,
|
||||
|
||||
/// Specific token IDs to use as stop conditions
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stop_token_ids: Option<Vec<u32>>,
|
||||
|
||||
/// Skip trimming stop tokens from output
|
||||
#[serde(default)]
|
||||
pub no_stop_trim: bool,
|
||||
|
||||
/// Ignore end-of-sequence tokens during generation
|
||||
#[serde(default)]
|
||||
pub ignore_eos: bool,
|
||||
|
||||
/// Continue generating from final assistant message
|
||||
#[serde(default)]
|
||||
pub continue_final_message: bool,
|
||||
|
||||
/// Skip special tokens during detokenization
|
||||
#[serde(default = "default_true")]
|
||||
pub skip_special_tokens: bool,
|
||||
|
||||
/// Path to LoRA adapter(s) for model customization
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub lora_path: Option<String>,
|
||||
|
||||
/// Session parameters for continual prompting
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub session_params: Option<HashMap<String, Value>>,
|
||||
|
||||
/// Separate reasoning content from final answer (O1-style models)
|
||||
#[serde(default = "default_true")]
|
||||
pub separate_reasoning: bool,
|
||||
|
||||
/// Stream reasoning tokens during generation
|
||||
#[serde(default = "default_true")]
|
||||
pub stream_reasoning: bool,
|
||||
|
||||
/// Chat template kwargs
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub chat_template_kwargs: Option<HashMap<String, Value>>,
|
||||
|
||||
/// Return model hidden states
|
||||
#[serde(default)]
|
||||
pub return_hidden_states: bool,
|
||||
|
||||
/// Random seed for sampling for deterministic outputs
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub sampling_seed: Option<u64>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Validation Functions
|
||||
// ============================================================================
|
||||
|
||||
/// Validates messages array is not empty and has valid content
|
||||
fn validate_messages(messages: &[ChatMessage]) -> Result<(), validator::ValidationError> {
|
||||
if messages.is_empty() {
|
||||
return Err(validator::ValidationError::new("messages cannot be empty"));
|
||||
}
|
||||
|
||||
for msg in messages.iter() {
|
||||
if let ChatMessage::User { content, .. } = msg {
|
||||
match content {
|
||||
MessageContent::Text(text) if text.is_empty() => {
|
||||
return Err(validator::ValidationError::new(
|
||||
"message content cannot be empty",
|
||||
));
|
||||
}
|
||||
MessageContent::Parts(parts) if parts.is_empty() => {
|
||||
return Err(validator::ValidationError::new(
|
||||
"message content parts cannot be empty",
|
||||
));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Schema-level validation for cross-field dependencies
|
||||
fn validate_chat_cross_parameters(
|
||||
req: &ChatCompletionRequest,
|
||||
) -> Result<(), validator::ValidationError> {
|
||||
// 1. Validate logprobs dependency
|
||||
if req.top_logprobs.is_some() && !req.logprobs {
|
||||
let mut e = validator::ValidationError::new("top_logprobs_requires_logprobs");
|
||||
e.message = Some("top_logprobs is only allowed when logprobs is enabled".into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// 2. Validate stream_options dependency
|
||||
if req.stream_options.is_some() && !req.stream {
|
||||
let mut e = validator::ValidationError::new("stream_options_requires_stream");
|
||||
e.message =
|
||||
Some("The 'stream_options' parameter is only allowed when 'stream' is enabled".into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// 3. Validate token limits - min <= max
|
||||
if let (Some(min), Some(max)) = (req.min_tokens, req.max_completion_tokens) {
|
||||
if min > max {
|
||||
let mut e = validator::ValidationError::new("min_tokens_exceeds_max");
|
||||
e.message = Some("min_tokens cannot exceed max_tokens/max_completion_tokens".into());
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Validate structured output conflicts
|
||||
let has_json_format = matches!(
|
||||
req.response_format,
|
||||
Some(ResponseFormat::JsonObject | ResponseFormat::JsonSchema { .. })
|
||||
);
|
||||
|
||||
if has_json_format && req.regex.is_some() {
|
||||
let mut e = validator::ValidationError::new("regex_conflicts_with_json");
|
||||
e.message = Some("cannot use regex constraint with JSON response format".into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
if has_json_format && req.ebnf.is_some() {
|
||||
let mut e = validator::ValidationError::new("ebnf_conflicts_with_json");
|
||||
e.message = Some("cannot use EBNF constraint with JSON response format".into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// 5. Validate mutually exclusive structured output constraints
|
||||
let constraint_count = [
|
||||
req.regex.is_some(),
|
||||
req.ebnf.is_some(),
|
||||
matches!(req.response_format, Some(ResponseFormat::JsonSchema { .. })),
|
||||
]
|
||||
.iter()
|
||||
.filter(|&&x| x)
|
||||
.count();
|
||||
|
||||
if constraint_count > 1 {
|
||||
let mut e = validator::ValidationError::new("multiple_constraints");
|
||||
e.message = Some("only one structured output constraint (regex, ebnf, or json_schema) can be active at a time".into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// 6. Validate response format JSON schema name
|
||||
if let Some(ResponseFormat::JsonSchema { json_schema }) = &req.response_format {
|
||||
if json_schema.name.is_empty() {
|
||||
let mut e = validator::ValidationError::new("json_schema_name_empty");
|
||||
e.message = Some("JSON schema name cannot be empty".into());
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
|
||||
// 7. Validate tool_choice requires tools (except for "none")
|
||||
if let Some(ref tool_choice) = req.tool_choice {
|
||||
let has_tools = req.tools.as_ref().is_some_and(|t| !t.is_empty());
|
||||
|
||||
// Check if tool_choice is anything other than "none"
|
||||
let is_some_choice = !matches!(tool_choice, ToolChoice::Value(ToolChoiceValue::None));
|
||||
|
||||
if is_some_choice && !has_tools {
|
||||
let mut e = validator::ValidationError::new("tool_choice_requires_tools");
|
||||
e.message = Some("Invalid value for 'tool_choice': 'tool_choice' is only allowed when 'tools' are specified.".into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// Additional validation when tools are present
|
||||
if has_tools {
|
||||
let tools = req.tools.as_ref().unwrap();
|
||||
|
||||
match tool_choice {
|
||||
ToolChoice::Function { function, .. } => {
|
||||
// Validate that the specified function name exists in tools
|
||||
let function_exists = tools.iter().any(|tool| {
|
||||
tool.tool_type == "function" && tool.function.name == function.name
|
||||
});
|
||||
|
||||
if !function_exists {
|
||||
let mut e =
|
||||
validator::ValidationError::new("tool_choice_function_not_found");
|
||||
e.message = Some(
|
||||
format!(
|
||||
"Invalid value for 'tool_choice': function '{}' not found in 'tools'.",
|
||||
function.name
|
||||
)
|
||||
.into(),
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
ToolChoice::AllowedTools {
|
||||
mode,
|
||||
tools: allowed_tools,
|
||||
..
|
||||
} => {
|
||||
// Validate mode is "auto" or "required"
|
||||
if mode != "auto" && mode != "required" {
|
||||
let mut e = validator::ValidationError::new("tool_choice_invalid_mode");
|
||||
e.message = Some(format!(
|
||||
"Invalid value for 'tool_choice.mode': must be 'auto' or 'required', got '{}'.",
|
||||
mode
|
||||
).into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// Validate that all ToolReferences are Function type (Chat API only supports function tools)
|
||||
for tool_ref in allowed_tools {
|
||||
match tool_ref {
|
||||
ToolReference::Function { name } => {
|
||||
// Validate that the function exists in tools array
|
||||
let tool_exists = tools.iter().any(|tool| {
|
||||
tool.tool_type == "function" && tool.function.name == *name
|
||||
});
|
||||
|
||||
if !tool_exists {
|
||||
let mut e = validator::ValidationError::new(
|
||||
"tool_choice_tool_not_found",
|
||||
);
|
||||
e.message = Some(
|
||||
format!(
|
||||
"Invalid value for 'tool_choice.tools': tool '{}' not found in 'tools'.",
|
||||
name
|
||||
)
|
||||
.into(),
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
// Chat Completion API only supports function tools in tool_choice
|
||||
let mut e = validator::ValidationError::new(
|
||||
"tool_choice_invalid_tool_type",
|
||||
);
|
||||
e.message = Some(
|
||||
format!(
|
||||
"Invalid value for 'tool_choice.tools': Chat Completion API only supports function tools, got '{}'.",
|
||||
tool_ref.identifier()
|
||||
)
|
||||
.into(),
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Normalizable Implementation
|
||||
// ============================================================================
|
||||
|
||||
impl Normalizable for ChatCompletionRequest {
|
||||
/// Normalize the request by applying migrations and defaults:
|
||||
/// 1. Migrate deprecated fields to their replacements
|
||||
/// 2. Clear deprecated fields and log warnings
|
||||
/// 3. Apply OpenAI defaults for tool_choice
|
||||
fn normalize(&mut self) {
|
||||
// Migrate deprecated max_tokens → max_completion_tokens
|
||||
#[allow(deprecated)]
|
||||
if self.max_completion_tokens.is_none() && self.max_tokens.is_some() {
|
||||
self.max_completion_tokens = self.max_tokens;
|
||||
self.max_tokens = None; // Clear deprecated field
|
||||
}
|
||||
|
||||
// Migrate deprecated functions → tools
|
||||
#[allow(deprecated)]
|
||||
if self.tools.is_none() && self.functions.is_some() {
|
||||
tracing::warn!("functions is deprecated, use tools instead");
|
||||
self.tools = self.functions.as_ref().map(|functions| {
|
||||
functions
|
||||
.iter()
|
||||
.map(|func| Tool {
|
||||
tool_type: "function".to_string(),
|
||||
function: func.clone(),
|
||||
})
|
||||
.collect()
|
||||
});
|
||||
self.functions = None; // Clear deprecated field
|
||||
}
|
||||
|
||||
// Migrate deprecated function_call → tool_choice
|
||||
#[allow(deprecated)]
|
||||
if self.tool_choice.is_none() && self.function_call.is_some() {
|
||||
tracing::warn!("function_call is deprecated, use tool_choice instead");
|
||||
self.tool_choice = self.function_call.as_ref().map(|fc| match fc {
|
||||
FunctionCall::None => ToolChoice::Value(ToolChoiceValue::None),
|
||||
FunctionCall::Auto => ToolChoice::Value(ToolChoiceValue::Auto),
|
||||
FunctionCall::Function { name } => ToolChoice::Function {
|
||||
tool_type: "function".to_string(),
|
||||
function: FunctionChoice { name: name.clone() },
|
||||
},
|
||||
});
|
||||
self.function_call = None; // Clear deprecated field
|
||||
}
|
||||
|
||||
// Apply tool_choice defaults
|
||||
if self.tool_choice.is_none() {
|
||||
if let Some(tools) = &self.tools {
|
||||
let choice_value = if !tools.is_empty() {
|
||||
ToolChoiceValue::Auto
|
||||
} else {
|
||||
ToolChoiceValue::None
|
||||
};
|
||||
self.tool_choice = Some(ToolChoice::Value(choice_value));
|
||||
}
|
||||
// If tools is None, leave tool_choice as None (don't set it)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// GenerationRequest Trait Implementation
|
||||
// ============================================================================
|
||||
|
||||
impl GenerationRequest for ChatCompletionRequest {
|
||||
fn is_stream(&self) -> bool {
|
||||
self.stream
|
||||
}
|
||||
|
||||
fn get_model(&self) -> Option<&str> {
|
||||
Some(&self.model)
|
||||
}
|
||||
|
||||
fn extract_text_for_routing(&self) -> String {
|
||||
// Extract text from messages for routing decisions
|
||||
// Use a single buffer to avoid intermediate Vec<String> allocations
|
||||
let mut buffer = String::new();
|
||||
let mut has_content = false;
|
||||
|
||||
for msg in &self.messages {
|
||||
match msg {
|
||||
ChatMessage::System { content, .. }
|
||||
| ChatMessage::User { content, .. }
|
||||
| ChatMessage::Tool { content, .. }
|
||||
| ChatMessage::Developer { content, .. } => {
|
||||
if has_content && content.has_text() {
|
||||
buffer.push(' ');
|
||||
}
|
||||
if content.append_text_to(&mut buffer) {
|
||||
has_content = true;
|
||||
}
|
||||
}
|
||||
ChatMessage::Assistant {
|
||||
content,
|
||||
reasoning_content,
|
||||
..
|
||||
} => {
|
||||
// Append main content
|
||||
if let Some(c) = content {
|
||||
if has_content && c.has_text() {
|
||||
buffer.push(' ');
|
||||
}
|
||||
if c.append_text_to(&mut buffer) {
|
||||
has_content = true;
|
||||
}
|
||||
}
|
||||
// Append reasoning content
|
||||
if let Some(reasoning) = reasoning_content {
|
||||
if !reasoning.is_empty() {
|
||||
if has_content {
|
||||
buffer.push(' ');
|
||||
}
|
||||
buffer.push_str(reasoning);
|
||||
has_content = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
ChatMessage::Function { content, .. } => {
|
||||
if !content.is_empty() {
|
||||
if has_content {
|
||||
buffer.push(' ');
|
||||
}
|
||||
buffer.push_str(content);
|
||||
has_content = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
buffer
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Response Types
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ChatCompletionResponse {
|
||||
pub id: String,
|
||||
pub object: String, // "chat.completion"
|
||||
pub created: u64,
|
||||
pub model: String,
|
||||
pub choices: Vec<ChatChoice>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub usage: Option<Usage>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub system_fingerprint: Option<String>,
|
||||
}
|
||||
|
||||
impl ChatCompletionResponse {
|
||||
/// Create a new builder for ChatCompletionResponse
|
||||
pub fn builder(
|
||||
id: impl Into<String>,
|
||||
model: impl Into<String>,
|
||||
) -> ChatCompletionResponseBuilder {
|
||||
ChatCompletionResponseBuilder::new(id, model)
|
||||
}
|
||||
}
|
||||
|
||||
/// Response message structure for ChatCompletionResponse (different from request ChatMessage)
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ChatCompletionMessage {
|
||||
pub role: String, // Always "assistant" for responses
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub content: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tool_calls: Option<Vec<ToolCall>>,
|
||||
pub reasoning_content: Option<String>,
|
||||
// Note: function_call is deprecated and not included
|
||||
// Note: refusal, annotations, audio are not added yet
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ChatChoice {
|
||||
pub index: u32,
|
||||
pub message: ChatCompletionMessage,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logprobs: Option<ChatLogProbs>,
|
||||
pub finish_reason: Option<String>, // "stop", "length", "tool_calls", "content_filter", "function_call"
|
||||
/// Information about which stop condition was matched
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub matched_stop: Option<Value>, // Can be string or integer
|
||||
/// Hidden states from the model (SGLang extension)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub hidden_states: Option<Vec<f32>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ChatCompletionStreamResponse {
|
||||
pub id: String,
|
||||
pub object: String, // "chat.completion.chunk"
|
||||
pub created: u64,
|
||||
pub model: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub system_fingerprint: Option<String>,
|
||||
pub choices: Vec<ChatStreamChoice>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub usage: Option<Usage>,
|
||||
}
|
||||
|
||||
impl ChatCompletionStreamResponse {
|
||||
/// Create a new builder for ChatCompletionStreamResponse
|
||||
pub fn builder(
|
||||
id: impl Into<String>,
|
||||
model: impl Into<String>,
|
||||
) -> ChatCompletionStreamResponseBuilder {
|
||||
ChatCompletionStreamResponseBuilder::new(id, model)
|
||||
}
|
||||
}
|
||||
|
||||
/// Delta structure for streaming chat completion responses
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ChatMessageDelta {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub role: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub content: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tool_calls: Option<Vec<ToolCallDelta>>,
|
||||
pub reasoning_content: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ChatStreamChoice {
|
||||
pub index: u32,
|
||||
pub delta: ChatMessageDelta,
|
||||
pub logprobs: Option<ChatLogProbs>,
|
||||
pub finish_reason: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub matched_stop: Option<Value>,
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
//! Classify API protocol definitions.
|
||||
//!
|
||||
//! This module defines the request and response types for the `/v1/classify` API,
|
||||
//! which is compatible with vLLM's classification endpoint.
|
||||
//!
|
||||
//! Classification reuses the embedding backend - the scheduler returns logits as
|
||||
//! "embeddings", and the classify layer applies softmax + label mapping.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::common::{GenerationRequest, UsageInfo};
|
||||
|
||||
// ============================================================================
|
||||
// Classify API
|
||||
// ============================================================================
|
||||
|
||||
/// Classification request - compatible with vLLM's /v1/classify API
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ClassifyRequest {
|
||||
/// ID of the model to use
|
||||
pub model: String,
|
||||
|
||||
/// Input can be a string, array of strings, or token IDs
|
||||
/// - Single string: "text to classify"
|
||||
/// - Array of strings: ["text1", "text2"]
|
||||
/// - Token IDs: [1, 2, 3] (advanced usage)
|
||||
pub input: Value,
|
||||
|
||||
/// Optional user identifier
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub user: Option<String>,
|
||||
|
||||
/// SGLang extension: request id for tracking
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub rid: Option<String>,
|
||||
|
||||
/// SGLang extension: request priority
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub priority: Option<i32>,
|
||||
|
||||
/// SGLang extension: enable/disable logging of metrics
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub log_metrics: Option<bool>,
|
||||
}
|
||||
|
||||
impl GenerationRequest for ClassifyRequest {
|
||||
fn is_stream(&self) -> bool {
|
||||
false // Classification is always non-streaming
|
||||
}
|
||||
|
||||
fn get_model(&self) -> Option<&str> {
|
||||
Some(&self.model)
|
||||
}
|
||||
|
||||
fn extract_text_for_routing(&self) -> String {
|
||||
match &self.input {
|
||||
Value::String(s) => s.clone(),
|
||||
Value::Array(arr) => arr
|
||||
.iter()
|
||||
.filter_map(|v| v.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(" "),
|
||||
_ => String::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Classify Response
|
||||
// ============================================================================
|
||||
|
||||
/// Single classification result
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ClassifyData {
|
||||
/// Index of this result (for batch requests)
|
||||
pub index: u32,
|
||||
/// Predicted class label (from id2label mapping)
|
||||
pub label: String,
|
||||
/// Probability distribution over all classes (softmax of logits)
|
||||
pub probs: Vec<f32>,
|
||||
/// Number of classes
|
||||
pub num_classes: u32,
|
||||
}
|
||||
|
||||
/// Classification response
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ClassifyResponse {
|
||||
/// Unique request ID (format: "classify-{uuid}")
|
||||
pub id: String,
|
||||
/// Always "list"
|
||||
pub object: String,
|
||||
/// Unix timestamp (seconds since epoch)
|
||||
pub created: u64,
|
||||
/// Model name
|
||||
pub model: String,
|
||||
/// Classification results (one per input in batch)
|
||||
pub data: Vec<ClassifyData>,
|
||||
/// Token usage info
|
||||
pub usage: UsageInfo,
|
||||
}
|
||||
|
||||
impl ClassifyResponse {
|
||||
/// Create a new ClassifyResponse with the given data
|
||||
pub fn new(
|
||||
id: String,
|
||||
model: String,
|
||||
created: u64,
|
||||
data: Vec<ClassifyData>,
|
||||
usage: UsageInfo,
|
||||
) -> Self {
|
||||
Self {
|
||||
id,
|
||||
object: "list".to_string(),
|
||||
created,
|
||||
model,
|
||||
data,
|
||||
usage,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,524 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use validator;
|
||||
|
||||
use super::UNKNOWN_MODEL_ID;
|
||||
|
||||
// ============================================================================
|
||||
// Default value helpers
|
||||
// ============================================================================
|
||||
|
||||
/// Default model value when not specified
|
||||
pub(crate) fn default_model() -> String {
|
||||
UNKNOWN_MODEL_ID.to_string()
|
||||
}
|
||||
|
||||
/// Helper function for serde default value (returns true)
|
||||
pub fn default_true() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// GenerationRequest Trait
|
||||
// ============================================================================
|
||||
|
||||
/// Trait for unified access to generation request properties
|
||||
/// Implemented by ChatCompletionRequest, CompletionRequest, GenerateRequest,
|
||||
/// EmbeddingRequest, RerankRequest, and ResponsesRequest
|
||||
pub trait GenerationRequest: Send + Sync {
|
||||
/// Check if the request is for streaming
|
||||
fn is_stream(&self) -> bool;
|
||||
|
||||
/// Get the model name if specified
|
||||
fn get_model(&self) -> Option<&str>;
|
||||
|
||||
/// Extract text content for routing decisions
|
||||
fn extract_text_for_routing(&self) -> String;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// String/Array Utilities
|
||||
// ============================================================================
|
||||
|
||||
/// A type that can be either a single string or an array of strings
|
||||
#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum StringOrArray {
|
||||
String(String),
|
||||
Array(Vec<String>),
|
||||
}
|
||||
|
||||
impl StringOrArray {
|
||||
/// Get the number of items in the StringOrArray
|
||||
pub fn len(&self) -> usize {
|
||||
match self {
|
||||
StringOrArray::String(_) => 1,
|
||||
StringOrArray::Array(arr) => arr.len(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if the StringOrArray is empty
|
||||
pub fn is_empty(&self) -> bool {
|
||||
match self {
|
||||
StringOrArray::String(s) => s.is_empty(),
|
||||
StringOrArray::Array(arr) => arr.is_empty(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert to a vector of strings (clones the data)
|
||||
pub fn to_vec(&self) -> Vec<String> {
|
||||
match self {
|
||||
StringOrArray::String(s) => vec![s.clone()],
|
||||
StringOrArray::Array(arr) => arr.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns an iterator over string references without cloning.
|
||||
/// Use this instead of `to_vec()` when you only need to iterate.
|
||||
pub fn iter(&self) -> StringOrArrayIter<'_> {
|
||||
StringOrArrayIter {
|
||||
inner: self,
|
||||
index: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the first string, or None if empty
|
||||
pub fn first(&self) -> Option<&str> {
|
||||
match self {
|
||||
StringOrArray::String(s) => {
|
||||
if s.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(s)
|
||||
}
|
||||
}
|
||||
StringOrArray::Array(arr) => arr.first().map(|s| s.as_str()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Iterator over StringOrArray that yields string references without cloning
|
||||
pub struct StringOrArrayIter<'a> {
|
||||
inner: &'a StringOrArray,
|
||||
index: usize,
|
||||
}
|
||||
|
||||
impl<'a> Iterator for StringOrArrayIter<'a> {
|
||||
type Item = &'a str;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
match self.inner {
|
||||
StringOrArray::String(s) => {
|
||||
if self.index == 0 {
|
||||
self.index = 1;
|
||||
Some(s.as_str())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
StringOrArray::Array(arr) => {
|
||||
if self.index < arr.len() {
|
||||
let item = &arr[self.index];
|
||||
self.index += 1;
|
||||
Some(item.as_str())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn size_hint(&self) -> (usize, Option<usize>) {
|
||||
let remaining = match self.inner {
|
||||
StringOrArray::String(_) => 1 - self.index,
|
||||
StringOrArray::Array(arr) => arr.len() - self.index,
|
||||
};
|
||||
(remaining, Some(remaining))
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> ExactSizeIterator for StringOrArrayIter<'a> {}
|
||||
|
||||
/// Validates stop sequences (max 4, non-empty strings)
|
||||
/// Used by both ChatCompletionRequest and ResponsesRequest
|
||||
pub fn validate_stop(stop: &StringOrArray) -> Result<(), validator::ValidationError> {
|
||||
match stop {
|
||||
StringOrArray::String(s) => {
|
||||
if s.is_empty() {
|
||||
return Err(validator::ValidationError::new(
|
||||
"stop sequences cannot be empty",
|
||||
));
|
||||
}
|
||||
}
|
||||
StringOrArray::Array(arr) => {
|
||||
if arr.len() > 4 {
|
||||
return Err(validator::ValidationError::new(
|
||||
"maximum 4 stop sequences allowed",
|
||||
));
|
||||
}
|
||||
for s in arr {
|
||||
if s.is_empty() {
|
||||
return Err(validator::ValidationError::new(
|
||||
"stop sequences cannot be empty",
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Content Parts (for multimodal messages)
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
|
||||
#[serde(tag = "type")]
|
||||
pub enum ContentPart {
|
||||
#[serde(rename = "text")]
|
||||
Text { text: String },
|
||||
#[serde(rename = "image_url")]
|
||||
ImageUrl { image_url: ImageUrl },
|
||||
#[serde(rename = "video_url")]
|
||||
VideoUrl { video_url: VideoUrl },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
|
||||
pub struct ImageUrl {
|
||||
pub url: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub detail: Option<String>, // "auto", "low", or "high"
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
|
||||
pub struct VideoUrl {
|
||||
pub url: String,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Response Format (for structured outputs)
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(tag = "type")]
|
||||
pub enum ResponseFormat {
|
||||
#[serde(rename = "text")]
|
||||
Text,
|
||||
#[serde(rename = "json_object")]
|
||||
JsonObject,
|
||||
#[serde(rename = "json_schema")]
|
||||
JsonSchema { json_schema: JsonSchemaFormat },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct JsonSchemaFormat {
|
||||
pub name: String,
|
||||
pub schema: Value,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub strict: Option<bool>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Streaming
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct StreamOptions {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub include_usage: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ToolCallDelta {
|
||||
pub index: u32,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub id: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[serde(rename = "type")]
|
||||
pub tool_type: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub function: Option<FunctionCallDelta>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct FunctionCallDelta {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub name: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub arguments: Option<String>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tools and Function Calling
|
||||
// ============================================================================
|
||||
|
||||
/// Tool choice value for simple string options
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ToolChoiceValue {
|
||||
Auto,
|
||||
Required,
|
||||
None,
|
||||
}
|
||||
|
||||
/// Tool choice for both Chat Completion and Responses APIs
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum ToolChoice {
|
||||
Value(ToolChoiceValue),
|
||||
Function {
|
||||
#[serde(rename = "type")]
|
||||
tool_type: String, // "function"
|
||||
function: FunctionChoice,
|
||||
},
|
||||
AllowedTools {
|
||||
#[serde(rename = "type")]
|
||||
tool_type: String, // "allowed_tools"
|
||||
mode: String, // "auto" | "required" TODO: need validation
|
||||
tools: Vec<ToolReference>,
|
||||
},
|
||||
}
|
||||
|
||||
impl Default for ToolChoice {
|
||||
fn default() -> Self {
|
||||
Self::Value(ToolChoiceValue::Auto)
|
||||
}
|
||||
}
|
||||
|
||||
impl ToolChoice {
|
||||
/// Serialize tool_choice to string for ResponsesResponse
|
||||
///
|
||||
/// Returns the JSON-serialized tool_choice or "auto" as default
|
||||
pub fn serialize_to_string(tool_choice: &Option<ToolChoice>) -> String {
|
||||
tool_choice
|
||||
.as_ref()
|
||||
.map(|tc| serde_json::to_string(tc).unwrap_or_else(|_| "auto".to_string()))
|
||||
.unwrap_or_else(|| "auto".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// Function choice specification for ToolChoice::Function
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct FunctionChoice {
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
/// Tool reference for ToolChoice::AllowedTools
|
||||
///
|
||||
/// Represents a reference to a specific tool in the allowed_tools array.
|
||||
/// Different tool types have different required fields.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(tag = "type")]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ToolReference {
|
||||
/// Reference to a function tool
|
||||
#[serde(rename = "function")]
|
||||
Function { name: String },
|
||||
|
||||
/// Reference to an MCP tool
|
||||
#[serde(rename = "mcp")]
|
||||
Mcp {
|
||||
server_label: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
name: Option<String>,
|
||||
},
|
||||
|
||||
/// File search hosted tool
|
||||
#[serde(rename = "file_search")]
|
||||
FileSearch,
|
||||
|
||||
/// Web search preview hosted tool
|
||||
#[serde(rename = "web_search_preview")]
|
||||
WebSearchPreview,
|
||||
|
||||
/// Computer use preview hosted tool
|
||||
#[serde(rename = "computer_use_preview")]
|
||||
ComputerUsePreview,
|
||||
|
||||
/// Code interpreter hosted tool
|
||||
#[serde(rename = "code_interpreter")]
|
||||
CodeInterpreter,
|
||||
|
||||
/// Image generation hosted tool
|
||||
#[serde(rename = "image_generation")]
|
||||
ImageGeneration,
|
||||
}
|
||||
|
||||
impl ToolReference {
|
||||
/// Get a unique identifier for this tool reference
|
||||
pub fn identifier(&self) -> String {
|
||||
match self {
|
||||
ToolReference::Function { name } => format!("function:{}", name),
|
||||
ToolReference::Mcp { server_label, name } => {
|
||||
if let Some(n) = name {
|
||||
format!("mcp:{}:{}", server_label, n)
|
||||
} else {
|
||||
format!("mcp:{}", server_label)
|
||||
}
|
||||
}
|
||||
ToolReference::FileSearch => "file_search".to_string(),
|
||||
ToolReference::WebSearchPreview => "web_search_preview".to_string(),
|
||||
ToolReference::ComputerUsePreview => "computer_use_preview".to_string(),
|
||||
ToolReference::CodeInterpreter => "code_interpreter".to_string(),
|
||||
ToolReference::ImageGeneration => "image_generation".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the tool name if this is a function tool
|
||||
pub fn function_name(&self) -> Option<&str> {
|
||||
match self {
|
||||
ToolReference::Function { name } => Some(name.as_str()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct Tool {
|
||||
#[serde(rename = "type")]
|
||||
pub tool_type: String, // "function"
|
||||
pub function: Function,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct Function {
|
||||
pub name: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub description: Option<String>,
|
||||
pub parameters: Value, // JSON Schema
|
||||
/// Whether to enable strict schema adherence (OpenAI structured outputs)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub strict: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ToolCall {
|
||||
pub id: String,
|
||||
#[serde(rename = "type")]
|
||||
pub tool_type: String, // "function"
|
||||
pub function: FunctionCallResponse,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum FunctionCall {
|
||||
None,
|
||||
Auto,
|
||||
Function { name: String },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct FunctionCallResponse {
|
||||
pub name: String,
|
||||
#[serde(default)]
|
||||
pub arguments: Option<String>, // JSON string
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Usage and Logging
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct Usage {
|
||||
pub prompt_tokens: u32,
|
||||
pub completion_tokens: u32,
|
||||
pub total_tokens: u32,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub completion_tokens_details: Option<CompletionTokensDetails>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct CompletionTokensDetails {
|
||||
pub reasoning_tokens: Option<u32>,
|
||||
}
|
||||
|
||||
/// Usage information (used by rerank and other endpoints)
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct UsageInfo {
|
||||
pub prompt_tokens: u32,
|
||||
pub completion_tokens: u32,
|
||||
pub total_tokens: u32,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub reasoning_tokens: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub prompt_tokens_details: Option<PromptTokenUsageInfo>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct PromptTokenUsageInfo {
|
||||
pub cached_tokens: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct LogProbs {
|
||||
pub tokens: Vec<String>,
|
||||
pub token_logprobs: Vec<Option<f32>>,
|
||||
pub top_logprobs: Vec<Option<HashMap<String, f32>>>,
|
||||
pub text_offset: Vec<u32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum ChatLogProbs {
|
||||
Detailed {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
content: Option<Vec<ChatLogProbsContent>>,
|
||||
},
|
||||
Raw(Value),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ChatLogProbsContent {
|
||||
pub token: String,
|
||||
pub logprob: f32,
|
||||
pub bytes: Option<Vec<u8>>,
|
||||
pub top_logprobs: Vec<TopLogProb>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct TopLogProb {
|
||||
pub token: String,
|
||||
pub logprob: f32,
|
||||
pub bytes: Option<Vec<u8>>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Error Types
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ErrorResponse {
|
||||
pub error: ErrorDetail,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ErrorDetail {
|
||||
pub message: String,
|
||||
#[serde(rename = "type")]
|
||||
pub error_type: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub param: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub code: Option<String>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Input Types
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum InputIds {
|
||||
Single(Vec<i32>),
|
||||
Batch(Vec<Vec<i32>>),
|
||||
}
|
||||
|
||||
/// LoRA adapter path - can be single path or batch of paths (SGLang extension)
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum LoRAPath {
|
||||
Single(Option<String>),
|
||||
Batch(Vec<Option<String>>),
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use super::common::*;
|
||||
|
||||
// ============================================================================
|
||||
// Completions API (v1/completions) - DEPRECATED but still supported
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct CompletionRequest {
|
||||
/// ID of the model to use (required for OpenAI, optional for some implementations, such as SGLang)
|
||||
pub model: String,
|
||||
|
||||
/// The prompt(s) to generate completions for
|
||||
pub prompt: StringOrArray,
|
||||
|
||||
/// The suffix that comes after a completion of inserted text
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub suffix: Option<String>,
|
||||
|
||||
/// The maximum number of tokens to generate
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub max_tokens: Option<u32>,
|
||||
|
||||
/// What sampling temperature to use, between 0 and 2
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub temperature: Option<f32>,
|
||||
|
||||
/// An alternative to sampling with temperature (nucleus sampling)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub top_p: Option<f32>,
|
||||
|
||||
/// How many completions to generate for each prompt
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub n: Option<u32>,
|
||||
|
||||
/// Whether to stream back partial progress
|
||||
#[serde(default)]
|
||||
pub stream: bool,
|
||||
|
||||
/// Options for streaming response
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stream_options: Option<StreamOptions>,
|
||||
|
||||
/// Include the log probabilities on the logprobs most likely tokens
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logprobs: Option<u32>,
|
||||
|
||||
/// Echo back the prompt in addition to the completion
|
||||
#[serde(default)]
|
||||
pub echo: bool,
|
||||
|
||||
/// Up to 4 sequences where the API will stop generating further tokens
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stop: Option<StringOrArray>,
|
||||
|
||||
/// Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub presence_penalty: Option<f32>,
|
||||
|
||||
/// Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub frequency_penalty: Option<f32>,
|
||||
|
||||
/// Generates best_of completions server-side and returns the "best"
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub best_of: Option<u32>,
|
||||
|
||||
/// Modify the likelihood of specified tokens appearing in the completion
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logit_bias: Option<HashMap<String, f32>>,
|
||||
|
||||
/// A unique identifier representing your end-user
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub user: Option<String>,
|
||||
|
||||
/// If specified, our system will make a best effort to sample deterministically
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub seed: Option<i64>,
|
||||
|
||||
// -------- Engine Specific Sampling Parameters --------
|
||||
/// Top-k sampling parameter (-1 to disable)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub top_k: Option<i32>,
|
||||
|
||||
/// Min-p nucleus sampling parameter
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub min_p: Option<f32>,
|
||||
|
||||
/// Minimum number of tokens to generate
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub min_tokens: Option<u32>,
|
||||
|
||||
/// Repetition penalty for reducing repetitive text
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub repetition_penalty: Option<f32>,
|
||||
|
||||
/// Regex constraint for output generation
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub regex: Option<String>,
|
||||
|
||||
/// EBNF grammar constraint for structured output
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub ebnf: Option<String>,
|
||||
|
||||
/// JSON schema constraint for structured output
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub json_schema: Option<String>,
|
||||
|
||||
/// Specific token IDs to use as stop conditions
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stop_token_ids: Option<Vec<u32>>,
|
||||
|
||||
/// Skip trimming stop tokens from output
|
||||
#[serde(default)]
|
||||
pub no_stop_trim: bool,
|
||||
|
||||
/// Ignore end-of-sequence tokens during generation
|
||||
#[serde(default)]
|
||||
pub ignore_eos: bool,
|
||||
|
||||
/// Skip special tokens during detokenization
|
||||
#[serde(default = "default_true")]
|
||||
pub skip_special_tokens: bool,
|
||||
|
||||
/// Path to LoRA adapter(s) for model customization
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub lora_path: Option<String>,
|
||||
|
||||
/// Session parameters for continual prompting
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub session_params: Option<HashMap<String, Value>>,
|
||||
|
||||
/// Return model hidden states
|
||||
#[serde(default)]
|
||||
pub return_hidden_states: bool,
|
||||
|
||||
/// Sampling seed for deterministic outputs
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub sampling_seed: Option<u64>,
|
||||
|
||||
/// Additional fields including bootstrap info for PD routing
|
||||
#[serde(flatten)]
|
||||
pub other: Map<String, Value>,
|
||||
}
|
||||
|
||||
impl GenerationRequest for CompletionRequest {
|
||||
fn is_stream(&self) -> bool {
|
||||
self.stream
|
||||
}
|
||||
|
||||
fn get_model(&self) -> Option<&str> {
|
||||
Some(&self.model)
|
||||
}
|
||||
|
||||
fn extract_text_for_routing(&self) -> String {
|
||||
match &self.prompt {
|
||||
StringOrArray::String(s) => s.clone(),
|
||||
StringOrArray::Array(v) => v.join(" "),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Response Types
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct CompletionResponse {
|
||||
pub id: String,
|
||||
pub object: String, // "text_completion"
|
||||
pub created: u64,
|
||||
pub model: String,
|
||||
pub choices: Vec<CompletionChoice>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub usage: Option<Usage>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub system_fingerprint: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct CompletionChoice {
|
||||
pub text: String,
|
||||
pub index: u32,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logprobs: Option<LogProbs>,
|
||||
pub finish_reason: Option<String>, // "stop", "length", "content_filter", etc.
|
||||
/// Information about which stop condition was matched
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub matched_stop: Option<Value>, // Can be string or integer
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct CompletionStreamResponse {
|
||||
pub id: String,
|
||||
pub object: String, // "text_completion"
|
||||
pub created: u64,
|
||||
pub choices: Vec<CompletionStreamChoice>,
|
||||
pub model: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub system_fingerprint: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct CompletionStreamChoice {
|
||||
pub text: String,
|
||||
pub index: u32,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logprobs: Option<LogProbs>,
|
||||
pub finish_reason: Option<String>,
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::common::{GenerationRequest, UsageInfo};
|
||||
|
||||
// ============================================================================
|
||||
// Embedding API
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct EmbeddingRequest {
|
||||
/// ID of the model to use
|
||||
pub model: String,
|
||||
|
||||
/// Input can be a string, array of strings, tokens, or batch inputs
|
||||
pub input: Value,
|
||||
|
||||
/// Optional encoding format (e.g., "float", "base64")
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub encoding_format: Option<String>,
|
||||
|
||||
/// Optional user identifier
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub user: Option<String>,
|
||||
|
||||
/// Optional number of dimensions for the embedding
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub dimensions: Option<u32>,
|
||||
|
||||
/// SGLang extension: request id for tracking
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub rid: Option<String>,
|
||||
|
||||
/// SGLang extension: enable/disable logging of metrics for this request
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub log_metrics: Option<bool>,
|
||||
}
|
||||
|
||||
impl GenerationRequest for EmbeddingRequest {
|
||||
fn is_stream(&self) -> bool {
|
||||
// Embeddings are non-streaming
|
||||
false
|
||||
}
|
||||
|
||||
fn get_model(&self) -> Option<&str> {
|
||||
Some(&self.model)
|
||||
}
|
||||
|
||||
fn extract_text_for_routing(&self) -> String {
|
||||
// Best effort: extract text content for routing decisions
|
||||
match &self.input {
|
||||
Value::String(s) => s.clone(),
|
||||
Value::Array(arr) => arr
|
||||
.iter()
|
||||
.filter_map(|v| v.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(" "),
|
||||
_ => String::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct EmbeddingObject {
|
||||
pub object: String, // "embedding"
|
||||
pub embedding: Vec<f32>,
|
||||
pub index: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct EmbeddingResponse {
|
||||
pub object: String, // "list"
|
||||
pub data: Vec<EmbeddingObject>,
|
||||
pub model: String,
|
||||
pub usage: UsageInfo,
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
use std::fmt;
|
||||
|
||||
/// Response lifecycle events
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum ResponseEvent {
|
||||
Created,
|
||||
InProgress,
|
||||
Completed,
|
||||
}
|
||||
|
||||
impl ResponseEvent {
|
||||
pub const CREATED: &'static str = "response.created";
|
||||
pub const IN_PROGRESS: &'static str = "response.in_progress";
|
||||
pub const COMPLETED: &'static str = "response.completed";
|
||||
|
||||
pub const fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Created => Self::CREATED,
|
||||
Self::InProgress => Self::IN_PROGRESS,
|
||||
Self::Completed => Self::COMPLETED,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ResponseEvent {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// Output item events for streaming
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum OutputItemEvent {
|
||||
Added,
|
||||
Done,
|
||||
Delta,
|
||||
}
|
||||
|
||||
impl OutputItemEvent {
|
||||
pub const ADDED: &'static str = "response.output_item.added";
|
||||
pub const DONE: &'static str = "response.output_item.done";
|
||||
pub const DELTA: &'static str = "response.output_item.delta";
|
||||
|
||||
pub const fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Added => Self::ADDED,
|
||||
Self::Done => Self::DONE,
|
||||
Self::Delta => Self::DELTA,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for OutputItemEvent {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// Function call argument streaming events
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum FunctionCallEvent {
|
||||
ArgumentsDelta,
|
||||
ArgumentsDone,
|
||||
}
|
||||
|
||||
impl FunctionCallEvent {
|
||||
pub const ARGUMENTS_DELTA: &'static str = "response.function_call_arguments.delta";
|
||||
pub const ARGUMENTS_DONE: &'static str = "response.function_call_arguments.done";
|
||||
|
||||
pub const fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::ArgumentsDelta => Self::ARGUMENTS_DELTA,
|
||||
Self::ArgumentsDone => Self::ARGUMENTS_DONE,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for FunctionCallEvent {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// Content part streaming events
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum ContentPartEvent {
|
||||
Added,
|
||||
Done,
|
||||
}
|
||||
|
||||
impl ContentPartEvent {
|
||||
pub const ADDED: &'static str = "response.content_part.added";
|
||||
pub const DONE: &'static str = "response.content_part.done";
|
||||
|
||||
pub const fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Added => Self::ADDED,
|
||||
Self::Done => Self::DONE,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ContentPartEvent {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// Output text streaming events
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum OutputTextEvent {
|
||||
Delta,
|
||||
Done,
|
||||
}
|
||||
|
||||
impl OutputTextEvent {
|
||||
pub const DELTA: &'static str = "response.output_text.delta";
|
||||
pub const DONE: &'static str = "response.output_text.done";
|
||||
|
||||
pub const fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Delta => Self::DELTA,
|
||||
Self::Done => Self::DONE,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for OutputTextEvent {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// MCP Events
|
||||
// ============================================================================
|
||||
|
||||
/// MCP (Model Context Protocol) call events
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum McpEvent {
|
||||
CallArgumentsDelta,
|
||||
CallArgumentsDone,
|
||||
CallInProgress,
|
||||
CallCompleted,
|
||||
CallFailed,
|
||||
ListToolsInProgress,
|
||||
ListToolsCompleted,
|
||||
}
|
||||
|
||||
impl McpEvent {
|
||||
pub const CALL_ARGUMENTS_DELTA: &'static str = "response.mcp_call_arguments.delta";
|
||||
pub const CALL_ARGUMENTS_DONE: &'static str = "response.mcp_call_arguments.done";
|
||||
pub const CALL_IN_PROGRESS: &'static str = "response.mcp_call.in_progress";
|
||||
pub const CALL_COMPLETED: &'static str = "response.mcp_call.completed";
|
||||
pub const CALL_FAILED: &'static str = "response.mcp_call.failed";
|
||||
pub const LIST_TOOLS_IN_PROGRESS: &'static str = "response.mcp_list_tools.in_progress";
|
||||
pub const LIST_TOOLS_COMPLETED: &'static str = "response.mcp_list_tools.completed";
|
||||
|
||||
pub const fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::CallArgumentsDelta => Self::CALL_ARGUMENTS_DELTA,
|
||||
Self::CallArgumentsDone => Self::CALL_ARGUMENTS_DONE,
|
||||
Self::CallInProgress => Self::CALL_IN_PROGRESS,
|
||||
Self::CallCompleted => Self::CALL_COMPLETED,
|
||||
Self::CallFailed => Self::CALL_FAILED,
|
||||
Self::ListToolsInProgress => Self::LIST_TOOLS_IN_PROGRESS,
|
||||
Self::ListToolsCompleted => Self::LIST_TOOLS_COMPLETED,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for McpEvent {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// Item type discriminators used in output items
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum ItemType {
|
||||
FunctionCall,
|
||||
FunctionToolCall,
|
||||
McpCall,
|
||||
Function,
|
||||
McpListTools,
|
||||
}
|
||||
|
||||
impl ItemType {
|
||||
pub const FUNCTION_CALL: &'static str = "function_call";
|
||||
pub const FUNCTION_TOOL_CALL: &'static str = "function_tool_call";
|
||||
pub const MCP_CALL: &'static str = "mcp_call";
|
||||
pub const FUNCTION: &'static str = "function";
|
||||
pub const MCP_LIST_TOOLS: &'static str = "mcp_list_tools";
|
||||
|
||||
pub const fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::FunctionCall => Self::FUNCTION_CALL,
|
||||
Self::FunctionToolCall => Self::FUNCTION_TOOL_CALL,
|
||||
Self::McpCall => Self::MCP_CALL,
|
||||
Self::Function => Self::FUNCTION,
|
||||
Self::McpListTools => Self::MCP_LIST_TOOLS,
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if this is a function call variant (FunctionCall or FunctionToolCall)
|
||||
pub const fn is_function_call(&self) -> bool {
|
||||
matches!(self, Self::FunctionCall | Self::FunctionToolCall)
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ItemType {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if an event type string matches any response lifecycle event
|
||||
pub fn is_response_event(event_type: &str) -> bool {
|
||||
matches!(
|
||||
event_type,
|
||||
ResponseEvent::CREATED | ResponseEvent::IN_PROGRESS | ResponseEvent::COMPLETED
|
||||
)
|
||||
}
|
||||
|
||||
/// Check if an item type string is a function call variant
|
||||
pub fn is_function_call_type(item_type: &str) -> bool {
|
||||
item_type == ItemType::FUNCTION_CALL || item_type == ItemType::FUNCTION_TOOL_CALL
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use validator::Validate;
|
||||
|
||||
use super::{
|
||||
common::{default_true, GenerationRequest, InputIds},
|
||||
sampling_params::SamplingParams,
|
||||
};
|
||||
use crate::validated::Normalizable;
|
||||
|
||||
// ============================================================================
|
||||
// SGLang Generate API (native format)
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, Validate)]
|
||||
#[validate(schema(function = "validate_generate_request"))]
|
||||
pub struct GenerateRequest {
|
||||
/// Text input - SGLang native format
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub text: Option<String>,
|
||||
|
||||
pub model: Option<String>,
|
||||
|
||||
/// Input IDs for tokenized input
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub input_ids: Option<InputIds>,
|
||||
|
||||
/// Input embeddings for direct embedding input
|
||||
/// Can be a 2D array (single request) or 3D array (batch of requests)
|
||||
/// Placeholder for future use
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub input_embeds: Option<Value>,
|
||||
|
||||
/// Image input data
|
||||
/// Can be an image instance, file name, URL, or base64 encoded string
|
||||
/// Supports single images, lists of images, or nested lists for batch processing
|
||||
/// Placeholder for future use
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub image_data: Option<Value>,
|
||||
|
||||
/// Video input data
|
||||
/// Can be a file name, URL, or base64 encoded string
|
||||
/// Supports single videos, lists of videos, or nested lists for batch processing
|
||||
/// Placeholder for future use
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub video_data: Option<Value>,
|
||||
|
||||
/// Audio input data
|
||||
/// Can be a file name, URL, or base64 encoded string
|
||||
/// Supports single audio files, lists of audio, or nested lists for batch processing
|
||||
/// Placeholder for future use
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub audio_data: Option<Value>,
|
||||
|
||||
/// Sampling parameters (sglang style)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub sampling_params: Option<SamplingParams>,
|
||||
|
||||
/// Whether to return logprobs
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub return_logprob: Option<bool>,
|
||||
|
||||
/// If return logprobs, the start location in the prompt for returning logprobs.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logprob_start_len: Option<i32>,
|
||||
|
||||
/// If return logprobs, the number of top logprobs to return at each position.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub top_logprobs_num: Option<i32>,
|
||||
|
||||
/// If return logprobs, the token ids to return logprob for.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub token_ids_logprob: Option<Vec<u32>>,
|
||||
|
||||
/// Whether to detokenize tokens in text in the returned logprobs.
|
||||
#[serde(default)]
|
||||
pub return_text_in_logprobs: bool,
|
||||
|
||||
/// Whether to stream the response
|
||||
#[serde(default)]
|
||||
pub stream: bool,
|
||||
|
||||
/// Whether to log metrics for this request (e.g. health_generate calls do not log metrics)
|
||||
#[serde(default = "default_true")]
|
||||
pub log_metrics: bool,
|
||||
|
||||
/// Return model hidden states
|
||||
#[serde(default)]
|
||||
pub return_hidden_states: bool,
|
||||
|
||||
/// The modalities of the image data [image, multi-images, video]
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub modalities: Option<Vec<String>>,
|
||||
|
||||
/// Session parameters for continual prompting
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub session_params: Option<HashMap<String, Value>>,
|
||||
|
||||
/// Path to LoRA adapter(s) for model customization
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub lora_path: Option<String>,
|
||||
|
||||
/// LoRA adapter ID (if pre-loaded)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub lora_id: Option<String>,
|
||||
|
||||
/// Custom logit processor for advanced sampling control. Must be a serialized instance
|
||||
/// of `CustomLogitProcessor` in python/sglang/srt/sampling/custom_logit_processor.py
|
||||
/// Use the processor's `to_str()` method to generate the serialized string.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub custom_logit_processor: Option<String>,
|
||||
|
||||
/// For disaggregated inference
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub bootstrap_host: Option<String>,
|
||||
|
||||
/// For disaggregated inference
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub bootstrap_port: Option<i32>,
|
||||
|
||||
/// For disaggregated inference
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub bootstrap_room: Option<i32>,
|
||||
|
||||
/// For disaggregated inference
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub bootstrap_pair_key: Option<String>,
|
||||
|
||||
/// Data parallel rank routing
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub data_parallel_rank: Option<i32>,
|
||||
|
||||
/// Background response
|
||||
#[serde(default)]
|
||||
pub background: bool,
|
||||
|
||||
/// Conversation ID for tracking
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub conversation_id: Option<String>,
|
||||
|
||||
/// Priority for the request
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub priority: Option<i32>,
|
||||
|
||||
/// Extra key for classifying the request (e.g. cache_salt)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub extra_key: Option<String>,
|
||||
|
||||
/// Whether to disallow logging for this request (e.g. due to ZDR)
|
||||
#[serde(default)]
|
||||
pub no_logs: bool,
|
||||
|
||||
/// Custom metric labels
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub custom_labels: Option<HashMap<String, String>>,
|
||||
|
||||
/// Whether to return bytes for image generation
|
||||
#[serde(default)]
|
||||
pub return_bytes: bool,
|
||||
|
||||
/// Whether to return entropy
|
||||
#[serde(default)]
|
||||
pub return_entropy: bool,
|
||||
|
||||
/// Request ID for tracking (inherited from BaseReq in Python)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub rid: Option<String>,
|
||||
}
|
||||
|
||||
impl Normalizable for GenerateRequest {
|
||||
// Use default no-op implementation - no normalization needed for GenerateRequest
|
||||
}
|
||||
|
||||
/// Validation function for GenerateRequest - ensure exactly one input type is provided
|
||||
fn validate_generate_request(req: &GenerateRequest) -> Result<(), validator::ValidationError> {
|
||||
// Exactly one of text or input_ids must be provided
|
||||
// Note: input_embeds not yet supported in Rust implementation
|
||||
let has_text = req.text.is_some();
|
||||
let has_input_ids = req.input_ids.is_some();
|
||||
|
||||
let count = [has_text, has_input_ids].iter().filter(|&&x| x).count();
|
||||
|
||||
if count == 0 {
|
||||
return Err(validator::ValidationError::new(
|
||||
"Either text or input_ids should be provided.",
|
||||
));
|
||||
}
|
||||
|
||||
if count > 1 {
|
||||
return Err(validator::ValidationError::new(
|
||||
"Either text or input_ids should be provided.",
|
||||
));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl GenerationRequest for GenerateRequest {
|
||||
fn is_stream(&self) -> bool {
|
||||
self.stream
|
||||
}
|
||||
|
||||
fn get_model(&self) -> Option<&str> {
|
||||
// Generate requests have an optional model field
|
||||
if let Some(s) = &self.model {
|
||||
Some(s.as_str())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_text_for_routing(&self) -> String {
|
||||
// Check fields in priority order: text, input_ids
|
||||
if let Some(ref text) = self.text {
|
||||
return text.clone();
|
||||
}
|
||||
|
||||
if let Some(ref input_ids) = self.input_ids {
|
||||
return match input_ids {
|
||||
InputIds::Single(ids) => ids
|
||||
.iter()
|
||||
.map(|&id| id.to_string())
|
||||
.collect::<Vec<String>>()
|
||||
.join(" "),
|
||||
InputIds::Batch(batches) => batches
|
||||
.iter()
|
||||
.flat_map(|batch| batch.iter().map(|&id| id.to_string()))
|
||||
.collect::<Vec<String>>()
|
||||
.join(" "),
|
||||
};
|
||||
}
|
||||
|
||||
// No text input found
|
||||
String::new()
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// SGLang Generate Response Types
|
||||
// ============================================================================
|
||||
|
||||
/// SGLang generate response (single completion or array for n>1)
|
||||
///
|
||||
/// Format for n=1:
|
||||
/// ```json
|
||||
/// {
|
||||
/// "text": "...",
|
||||
/// "output_ids": [...],
|
||||
/// "meta_info": { ... }
|
||||
/// }
|
||||
/// ```
|
||||
///
|
||||
/// Format for n>1:
|
||||
/// ```json
|
||||
/// [
|
||||
/// {"text": "...", "output_ids": [...], "meta_info": {...}},
|
||||
/// {"text": "...", "output_ids": [...], "meta_info": {...}}
|
||||
/// ]
|
||||
/// ```
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct GenerateResponse {
|
||||
pub text: String,
|
||||
pub output_ids: Vec<u32>,
|
||||
pub meta_info: GenerateMetaInfo,
|
||||
}
|
||||
|
||||
/// Metadata for a single generate completion
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct GenerateMetaInfo {
|
||||
pub id: String,
|
||||
pub finish_reason: GenerateFinishReason,
|
||||
pub prompt_tokens: u32,
|
||||
pub weight_version: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub input_token_logprobs: Option<Vec<Vec<Option<f64>>>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub output_token_logprobs: Option<Vec<Vec<Option<f64>>>>,
|
||||
pub completion_tokens: u32,
|
||||
pub cached_tokens: u32,
|
||||
pub e2e_latency: f64,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub matched_stop: Option<Value>,
|
||||
}
|
||||
|
||||
/// Finish reason for generate endpoint
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "lowercase")]
|
||||
pub enum GenerateFinishReason {
|
||||
Length {
|
||||
length: u32,
|
||||
},
|
||||
Stop,
|
||||
#[serde(untagged)]
|
||||
Other(Value),
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
// Protocol definitions and validation for various LLM APIs
|
||||
// This module provides a structured approach to handling different API protocols
|
||||
|
||||
/// Default model identifier used when no model is specified.
|
||||
///
|
||||
/// This constant should be used instead of hardcoded "unknown" strings
|
||||
/// throughout the codebase for consistency.
|
||||
pub const UNKNOWN_MODEL_ID: &str = "unknown";
|
||||
|
||||
pub mod builders;
|
||||
pub mod chat;
|
||||
pub mod classify;
|
||||
pub mod common;
|
||||
pub mod completion;
|
||||
pub mod embedding;
|
||||
pub mod event_types;
|
||||
pub mod generate;
|
||||
pub mod messages;
|
||||
pub mod parser;
|
||||
pub mod rerank;
|
||||
pub mod responses;
|
||||
pub mod sampling_params;
|
||||
pub mod tokenize;
|
||||
pub mod validated;
|
||||
pub mod worker_spec;
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user