Compare commits
64
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e7dc175e94 | ||
|
|
6612af9e0e | ||
|
|
71c6d8bc55 | ||
|
|
dbc661602e | ||
|
|
054dcee896 | ||
|
|
73a25267ac | ||
|
|
bcad628d30 | ||
|
|
baba18bc7d | ||
|
|
0afcb334a4 | ||
|
|
33e6381b7d | ||
|
|
9c4ff32562 | ||
|
|
7d5759b2d9 | ||
|
|
ac11e9162d | ||
|
|
84fc632088 | ||
|
|
8d9bce30a7 | ||
|
|
5f2ffa70b8 | ||
|
|
f4ef648b16 | ||
|
|
6469cf4197 | ||
|
|
3f8751c6ed | ||
|
|
e4f1310d6f | ||
|
|
a42ab6fce0 | ||
|
|
fa8ad18b5c | ||
|
|
7f1edbba80 | ||
|
|
99c317b131 | ||
|
|
2b2ede8211 | ||
|
|
b4dbcebbec | ||
|
|
30ba5818e7 | ||
|
|
c9bfb24bcc | ||
|
|
3d109d50be | ||
|
|
4eabf5e228 | ||
|
|
7d67d57804 | ||
|
|
491989284f | ||
|
|
53ea8fb2a7 | ||
|
|
efdbfd2cc3 | ||
|
|
8342b89e85 | ||
|
|
c92938d718 | ||
|
|
7578cdd218 | ||
|
|
21f7ce37dd | ||
|
|
236686454c | ||
|
|
1f6ca2cf68 | ||
|
|
bde1fcbf51 | ||
|
|
c4f3e3be33 | ||
|
|
9ef65b51c2 | ||
|
|
6ba34507f1 | ||
|
|
93d610c1fd | ||
|
|
4d0efcc921 | ||
|
|
f71fafa684 | ||
|
|
1068926cd6 | ||
|
|
48fb49baef | ||
|
|
2372cdc06e | ||
|
|
7b815cc98b | ||
|
|
d06c5dde42 | ||
|
|
8d0ee5e51e | ||
|
|
75758cb0a0 | ||
|
|
60e80ac7d1 | ||
|
|
4ef799aaa6 | ||
|
|
1be762368b | ||
|
|
f577b61b52 | ||
|
|
4792b61332 | ||
|
|
62f1ed099e | ||
|
|
bfd9bb2e03 | ||
|
|
6deac260bf | ||
|
|
842ad7c4d4 | ||
|
|
b605d2d370 |
@@ -8,4 +8,17 @@ FROM lmsysorg/sglang:dev-dsv41
|
||||
# checkout 含 .git,setuptools-scm 可打戳(main 系分支 version 显示 dev 属正常)
|
||||
COPY . /sgl-workspace/sglang
|
||||
|
||||
# Rust gateway(sgl-model-gateway)编译,与 b300 离线版 deploy/b300-build/Dockerfile 同构。
|
||||
# 基底自带 rustup stable,RUSTUP_TOOLCHAIN=stable 绕过 rust-toolchain.toml 的 1.90 钉版;
|
||||
# python 扩展必须显式 -p,产物重命名为 sglang_router_rs.abi3.so 覆盖 python 包内同文件。
|
||||
# 海外 runner 直连 crates.io 无网络问题;cache mount 海外环境同样无害。
|
||||
RUN --mount=type=cache,target=/cargo-home \
|
||||
--mount=type=cache,target=/cargo-target \
|
||||
cd /sgl-workspace/sglang/sgl-model-gateway \
|
||||
&& CARGO_HOME=/cargo-home CARGO_TARGET_DIR=/cargo-target RUSTUP_TOOLCHAIN=stable \
|
||||
cargo build --release -p sgl-model-gateway -p sgl-model-gateway-python \
|
||||
&& cp /cargo-target/release/sgl-model-gateway /opt/sglang/bin/sgl-model-gateway \
|
||||
&& cp /cargo-target/release/libsglang_router_rs.so \
|
||||
/opt/sglang/lib/python3.12/site-packages/sglang_router/sglang_router_rs.abi3.so
|
||||
|
||||
RUN pip install --no-cache-dir -e /sgl-workspace/sglang/python
|
||||
|
||||
@@ -1,9 +1,19 @@
|
||||
# 优化版(2026-09-24):解决清华源限流下每次构建重下 torch 的问题。
|
||||
# 优化版(2026-10-08 v3):torch/nvidia 全家桶永远复用基底,全量路径零重下。
|
||||
# 要点:
|
||||
# 1. pyproject build-system.requires 含 torch==2.13.0,pip 隔离构建环境每次都重下 ~6GB。
|
||||
# 基底镜像已装 torch 2.13 → 用 --no-build-isolation 复用,构建环境零下载。
|
||||
# 2. 基底缺 setuptools-scm(版本打戳要用),单层预装 + pip cache mount,只下载一次。
|
||||
# 3. RUN 的 pip cache mount 持久化在宿主机,依赖解析命中的包不再重复下载。
|
||||
# 2. 老全量路径让 pip 解析全部依赖:bare `torchvision` 等会让解析器弃用基底已装的
|
||||
# 本地版本(torch 2.13.0+cu130),改用 PyPI 的 torch==2.13.0,连带重下整套 nvidia
|
||||
# 依赖(9/23 实测 nvidia-nccl-cu13 等 6GB+,限流下 2h40m);且基底 NCCL 是
|
||||
# 刻意装高的 2.30.7(torch 元数据钉 ==2.29.7),任何全量解析都会把它降级回去。
|
||||
# → v3 不再让 pip 碰重型依赖:check_deps.py 在构建期对照 pyproject 与基底
|
||||
# 已装版本,把增量分三类——重型(torch*/nvidia-*/cuda-*)永不进解析、
|
||||
# 不满足即让构建失败并提示决策(显式失败好过静默重装);
|
||||
# kernel wheel(sglang-kernel/flashinfer*/sgl-deep-gemm)要跟随源码版本,
|
||||
# 钉版变动时以 pip --no-deps 单独装(防止顺藤摸瓜解析 torch);
|
||||
# 其余轻量依赖只装未满足的增量。
|
||||
# 3. 基底缺 setuptools-scm(版本打戳要用),单层预装 + pip cache mount,只下载一次。
|
||||
# 4. RUN 的 pip cache mount 持久化在宿主机,轻量依赖增量不再重复下载。
|
||||
FROM uhub.service.ucloud.cn/umirror/sglang:dev-dsv41
|
||||
|
||||
ARG PIP_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
@@ -13,9 +23,55 @@ ENV PIP_INDEX_URL=${PIP_INDEX}
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install "setuptools>=61" "setuptools-scm>=8" "setuptools-rust>=1.11" wheel
|
||||
|
||||
# 依赖增量计算脚本(对照 pyproject 与基底已装版本,产出轻量/kernel 增量清单)
|
||||
# 注意 build context 是 /data/ymk 且 .dockerignore 排除了 build/,只能从 sglang/ 树内 COPY
|
||||
COPY sglang/deploy/b300-build/check_deps.py /tmp/check_deps.py
|
||||
|
||||
# 整个源码树(含 .git,用于 setuptools-scm 版本打戳)
|
||||
COPY sglang/ /sgl-workspace/sglang/
|
||||
|
||||
# editable 安装,全量依赖解析(pyproject.toml 变化时走这个)
|
||||
# Rust gateway(sgl-model-gateway)编译。基底自带 rustup stable(实测 cargo 1.98.1),
|
||||
# RUSTUP_TOOLCHAIN=stable 绕过仓库 rust-toolchain.toml 钉的 1.90(避免重复下载工具链);
|
||||
# b300 直连 crates.io(稀疏索引 + static.crates.io 均通),无需任何镜像源。
|
||||
# cache mount 持久化 registry/target:冷编译 ~15min(二进制 ~11min + python 扩展 ~4min),
|
||||
# 增量重建秒~分钟级。python 扩展必须显式 -p(workspace 根包默认不编译扩展),
|
||||
# 产物 libsglang_router_rs.so 需重命名为 sglang_router_rs.abi3.so 覆盖 python 包内同文件
|
||||
# (与 H20 线上部署方式一致)。2026-10-09 b300 实测验证。
|
||||
RUN --mount=type=cache,target=/cargo-home \
|
||||
--mount=type=cache,target=/cargo-target \
|
||||
cd /sgl-workspace/sglang/sgl-model-gateway \
|
||||
&& CARGO_HOME=/cargo-home CARGO_TARGET_DIR=/cargo-target RUSTUP_TOOLCHAIN=stable \
|
||||
cargo build --release -p sgl-model-gateway -p sgl-model-gateway-python \
|
||||
&& cp /cargo-target/release/sgl-model-gateway /opt/sglang/bin/sgl-model-gateway \
|
||||
&& cp /cargo-target/release/libsglang_router_rs.so \
|
||||
/opt/sglang/lib/python3.12/site-packages/sglang_router/sglang_router_rs.abi3.so
|
||||
|
||||
# 计算依赖增量(重型 torch/nvidia/cuda 永不进入清单;不满足则 check_deps 非零、构建失败)
|
||||
RUN python /tmp/check_deps.py /sgl-workspace/sglang/python/pyproject.toml \
|
||||
/tmp/missing_light.txt /tmp/missing_kernel.txt \
|
||||
&& echo "== kernel wheel delta (pip --no-deps) ==" && cat /tmp/missing_kernel.txt \
|
||||
&& echo "== lightweight dep delta ==" && cat /tmp/missing_light.txt
|
||||
|
||||
# kernel wheel 增量(如 sglang-kernel 升版):--no-deps 防止 pip 顺藤摸瓜解析 torch
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install --no-build-isolation -e /sgl-workspace/sglang/python
|
||||
if grep -q '[^[:space:]]' /tmp/missing_kernel.txt; then \
|
||||
pip install --no-deps -r /tmp/missing_kernel.txt; \
|
||||
else \
|
||||
echo "== no kernel wheel delta"; \
|
||||
fi
|
||||
|
||||
# 轻量依赖增量(通常为空或几个小包),重型全家桶原样复用基底。
|
||||
# ⚠️ 必须 --no-deps:即使目标只是 xgrammar 这种小包,只要依赖图里出现 torch,
|
||||
# pip 就会校验 torch+cu130 元数据钉死的 nvidia-nccl-cu13==2.29.7,发现基底装的是
|
||||
# 2.30.7 便"好心"下载 206MB 降级(10-08 v3 首跑实测踩中)。基底的传递依赖本来就
|
||||
# 齐全,增量小包的二级依赖也一样;若有漏网,b300 冒烟会在 import 阶段显式暴露。
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
if grep -q '[^[:space:]]' /tmp/missing_light.txt; then \
|
||||
pip install --no-deps -r /tmp/missing_light.txt; \
|
||||
else \
|
||||
echo "== no lightweight delta, skipping dependency install"; \
|
||||
fi
|
||||
|
||||
# editable 安装 sglang 本体,--no-deps 保证 pip 完全不碰依赖图
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install --no-deps --no-build-isolation -e /sgl-workspace/sglang/python
|
||||
|
||||
@@ -1,15 +1,65 @@
|
||||
# fast 路径:python/pyproject.toml 未变化时使用(build.sh 自动判断)。
|
||||
# --no-deps 跳过全部依赖解析(连 index 元数据请求都省掉),纯源码迭代秒级完成。
|
||||
# 依赖有变化时必须走 Dockerfile 全量路径(build.sh 按 pyproject sha256 自动切换)。
|
||||
# fast 路径(2026-10-08 v3.1):python/pyproject.toml 未变化时使用(build.sh 自动判断)。
|
||||
# 与 Dockerfile 全量路径同构:同样跑 check_deps.py 增量安装。
|
||||
# 原因:全量构建若装过 kernel wheel 增量(如 sglang-kernel 0.4.7),随后 pyproject 未变的
|
||||
# fast 构建若只 --no-deps,会静默退回基底的旧 kernel(0.4.6.post1)——镜像内容漂移。
|
||||
# check_deps 本地比对 ~0.5s,增量命中 pip cache 秒级,成本可忽略。
|
||||
FROM uhub.service.ucloud.cn/umirror/sglang:dev-dsv41
|
||||
|
||||
ARG PIP_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
ENV PIP_INDEX_URL=${PIP_INDEX}
|
||||
|
||||
# 构建后端(pyproject build-system.requires,torch 除外——基底已有)
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install "setuptools>=61" "setuptools-scm>=8" "setuptools-rust>=1.11" wheel
|
||||
|
||||
# 依赖增量计算脚本(对照 pyproject 与基底已装版本,产出轻量/kernel 增量清单)
|
||||
# 注意 build context 是 /data/ymk 且 .dockerignore 排除了 build/,只能从 sglang/ 树内 COPY
|
||||
COPY sglang/deploy/b300-build/check_deps.py /tmp/check_deps.py
|
||||
|
||||
# 整个源码树(含 .git,用于 setuptools-scm 版本打戳)
|
||||
COPY sglang/ /sgl-workspace/sglang/
|
||||
|
||||
# Rust gateway(sgl-model-gateway)编译。基底自带 rustup stable(实测 cargo 1.98.1),
|
||||
# RUSTUP_TOOLCHAIN=stable 绕过仓库 rust-toolchain.toml 钉的 1.90(避免重复下载工具链);
|
||||
# b300 直连 crates.io(稀疏索引 + static.crates.io 均通),无需任何镜像源。
|
||||
# cache mount 持久化 registry/target:冷编译 ~15min,增量重建秒~分钟级。
|
||||
# python 扩展必须显式 -p(workspace 根包默认不编译扩展),产物 libsglang_router_rs.so
|
||||
# 需重命名为 sglang_router_rs.abi3.so 覆盖 python 包内同文件(与 H20 线上部署方式一致)。
|
||||
RUN --mount=type=cache,target=/cargo-home \
|
||||
--mount=type=cache,target=/cargo-target \
|
||||
cd /sgl-workspace/sglang/sgl-model-gateway \
|
||||
&& CARGO_HOME=/cargo-home CARGO_TARGET_DIR=/cargo-target RUSTUP_TOOLCHAIN=stable \
|
||||
cargo build --release -p sgl-model-gateway -p sgl-model-gateway-python \
|
||||
&& cp /cargo-target/release/sgl-model-gateway /opt/sglang/bin/sgl-model-gateway \
|
||||
&& cp /cargo-target/release/libsglang_router_rs.so \
|
||||
/opt/sglang/lib/python3.12/site-packages/sglang_router/sglang_router_rs.abi3.so
|
||||
|
||||
# 计算依赖增量(重型 torch/nvidia/cuda 永不进入清单;不满足则 check_deps 非零、构建失败)
|
||||
RUN python /tmp/check_deps.py /sgl-workspace/sglang/python/pyproject.toml \
|
||||
/tmp/missing_light.txt /tmp/missing_kernel.txt \
|
||||
&& echo "== kernel wheel delta (pip --no-deps) ==" && cat /tmp/missing_kernel.txt \
|
||||
&& echo "== lightweight dep delta ==" && cat /tmp/missing_light.txt
|
||||
|
||||
# kernel wheel 增量(如 sglang-kernel 升版):--no-deps 防止 pip 顺藤摸瓜解析 torch
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
if grep -q '[^[:space:]]' /tmp/missing_kernel.txt; then \
|
||||
pip install --no-deps -r /tmp/missing_kernel.txt; \
|
||||
else \
|
||||
echo "== no kernel wheel delta"; \
|
||||
fi
|
||||
|
||||
# 轻量依赖增量(通常为空或几个小包),重型全家桶原样复用基底。
|
||||
# ⚠️ 必须 --no-deps:即使目标只是 xgrammar 这种小包,只要依赖图里出现 torch,
|
||||
# pip 就会校验 torch+cu130 元数据钉死的 nvidia-nccl-cu13==2.29.7,发现基底装的是
|
||||
# 2.30.7 便"好心"下载 206MB 降级(10-08 v3 首跑实测踩中)。基底的传递依赖本来就
|
||||
# 齐全,增量小包的二级依赖也一样;若有漏网,b300 冒烟会在 import 阶段显式暴露。
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
if grep -q '[^[:space:]]' /tmp/missing_light.txt; then \
|
||||
pip install --no-deps -r /tmp/missing_light.txt; \
|
||||
else \
|
||||
echo "== no lightweight delta, skipping dependency install"; \
|
||||
fi
|
||||
|
||||
# editable 安装 sglang 本体,--no-deps 保证 pip 完全不碰依赖图
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install --no-deps --no-build-isolation -e /sgl-workspace/sglang/python
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compute dependency deltas for a b300 sglang image build, in three classes.
|
||||
|
||||
Reads pyproject.toml, evaluates every requirement against the *installed*
|
||||
distributions in the base image, and writes two requirements files:
|
||||
|
||||
argv[2] (light): unsatisfied lightweight requirements → `pip install -r`
|
||||
argv[3] (kernel): unsatisfied kernel wheels (sglang-kernel, flashinfer*,
|
||||
sgl-deep-gemm) → `pip install --no-deps -r`
|
||||
These track the sglang source being built and are meant
|
||||
to be upgraded from the index when pyproject bumps them;
|
||||
--no-deps keeps pip from walking torch's metadata.
|
||||
|
||||
The PyTorch stack (torch/torchvision/torchaudio/torchcodec/
|
||||
torch_memory_saver), the nvidia-* CUDA family and cuda-* packages are NEVER
|
||||
emitted. If one of them is missing or its installed version no longer
|
||||
satisfies the pinned specifier (e.g. upstream bumps nvidia-cutlass-dsl), the
|
||||
script fails the build loudly instead of letting pip download ~6GB or
|
||||
"fix" the base image's deliberately newer nvidia-nccl-cu13 2.30.7 back to
|
||||
torch's metadata pin ==2.29.7 (any pip full-resolution would do both —
|
||||
2026-09-23/24 builds did, throttled to 2h40m).
|
||||
"""
|
||||
|
||||
import sys
|
||||
import tomllib
|
||||
from importlib import metadata as md
|
||||
|
||||
from packaging.requirements import Requirement
|
||||
|
||||
# Never resolved by pip: reused from the base image or the build fails.
|
||||
HEAVY_PREFIXES = ("torch", "nvidia-", "cuda-")
|
||||
# Kernel wheels: upgraded via pip --no-deps when pyproject pins move.
|
||||
KERNEL_PREFIXES = ("sglang-kernel", "flashinfer", "sgl-deep-gemm")
|
||||
|
||||
|
||||
def norm(name: str) -> str:
|
||||
return name.lower().replace("_", "-")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
pyproject, light_out, kernel_out = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
with open(pyproject, "rb") as f:
|
||||
deps = tomllib.load(f)["project"]["dependencies"]
|
||||
|
||||
light: list[str] = []
|
||||
kernel: list[str] = []
|
||||
heavy_unsat: list[str] = []
|
||||
for dep in deps:
|
||||
req = Requirement(dep)
|
||||
if req.marker is not None and not req.marker.evaluate():
|
||||
continue
|
||||
try:
|
||||
installed = md.version(req.name)
|
||||
except md.PackageNotFoundError:
|
||||
installed = None
|
||||
satisfied = installed is not None and req.specifier.contains(
|
||||
installed, prereleases=True
|
||||
)
|
||||
if satisfied:
|
||||
continue
|
||||
name = norm(req.name)
|
||||
if name.startswith(HEAVY_PREFIXES):
|
||||
heavy_unsat.append(f"{dep} (installed: {installed})")
|
||||
elif name.startswith(KERNEL_PREFIXES):
|
||||
kernel.append(dep)
|
||||
else:
|
||||
light.append(dep)
|
||||
|
||||
if heavy_unsat:
|
||||
print("=" * 70, file=sys.stderr)
|
||||
print("FATAL: heavy base-image deps no longer satisfy pyproject:", file=sys.stderr)
|
||||
for item in heavy_unsat:
|
||||
print(f" {item}", file=sys.stderr)
|
||||
print(
|
||||
"Refusing to let pip resolve these (would re-download ~6GB and/or\n"
|
||||
"downgrade NCCL). Decide explicitly: bump the base image, or adjust\n"
|
||||
"the pin in pyproject.toml.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
with open(light_out, "w") as f:
|
||||
f.write("\n".join(light) + ("\n" if light else ""))
|
||||
with open(kernel_out, "w") as f:
|
||||
f.write("\n".join(kernel) + ("\n" if kernel else ""))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,27 @@
|
||||
cd /data/ymk/ds41
|
||||
HDR="Authorization=Bearer deepseek-v4-flash-0731-key"
|
||||
PORT=30040
|
||||
MODEL=/data/models/DeepSeek-V4.1-Flash
|
||||
|
||||
# 一律 --warmup-requests 0:默认 warmup 会把 prompt#0 种进 L1/L2 污染 L3 测量(已踩过)
|
||||
run() {
|
||||
echo "===== [$(date +%H:%M:%S)] RUN: $1 (np=$2) ====="
|
||||
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--warmup-requests 0 --num-prompts "$2" --max-concurrency "$2" --seed "$3" --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR" 2>&1 | grep -E 'Input token throughput|Mean TTFT|P99 TTFT|Successful'
|
||||
sleep 2
|
||||
}
|
||||
flush() { curl -sf -m 10 -X POST http://127.0.0.1:$PORT/flush_cache > /dev/null && echo "[$(date +%H:%M:%S)] flushed"; sleep 3; }
|
||||
|
||||
echo "===== PRIME COLD mc8(冷基线 + 写入 L3) ====="
|
||||
run PRIME_COLD 8 1541391
|
||||
flush
|
||||
run MC1_L3 1 1541391
|
||||
flush
|
||||
run MC2_L3 2 1541391
|
||||
flush
|
||||
run MC4_L3 4 1541391
|
||||
flush
|
||||
run MC8_L3 8 1541391
|
||||
echo "===== [$(date +%H:%M:%S)] ALL DONE ====="
|
||||
@@ -0,0 +1,204 @@
|
||||
#!/bin/bash
|
||||
# bench_suite_v2.sh — DeepSeek-V4.1-Flash on b300-01,2026-10-09 新版基线方法
|
||||
#
|
||||
# 相对 v1 的方法变化:
|
||||
# 1. 服务刚起数据不准 → 每个测试模式自带 warmup 轮(结果丢弃,标记 WARM)
|
||||
# 2. prefill 冷/热双轮:冷轮写入 hicache(L1/L3),同 seed 热轮测纯 cache-hit TPS
|
||||
# 3. decode 阶梯并发:步长 32,上限 = cuda-graph-max-bs-decode 的 80%;
|
||||
# 每档先 warm(灌 L1)再测,prefill 全 cache hit;
|
||||
# 报告 TPOT + 整实例输出 TPS + 推算实际并发(= 输出TPS × TPOT / 1000)
|
||||
# 4. l3 模式(3p1d 经 router):大语料溢出 L1 后重放,验证 L3 命中
|
||||
#
|
||||
# 在宿主机执行,bench 客户端通过 docker exec 进容器跑(需要 /data 挂载)。
|
||||
#
|
||||
# 用法:
|
||||
# bench_suite_v2.sh warmup <container> <port>
|
||||
# bench_suite_v2.sh prefill <tag> <container> <port>
|
||||
# bench_suite_v2.sh decode <tag> <container> <port> [cg_max_bs=256]
|
||||
# bench_suite_v2.sh l3 <tag> <container> <port> [num_prompts=24]
|
||||
# bench_suite_v2.sh report <log>
|
||||
# 产物: /data/ymk/ds41/bench_v2_<tag>.log
|
||||
set -u
|
||||
|
||||
BASE=/data/ymk/ds41
|
||||
MODEL=${MODEL:-/data/models/DeepSeek-V4.1-Flash}
|
||||
HDR="Authorization=Bearer deepseek-v4-flash-0731-key"
|
||||
|
||||
usage() { sed -n 2,20p "$0" >&2; exit 1; }
|
||||
|
||||
# $1=container $2=port $3=tag -> 生成可执行脚本路径并 echo
|
||||
emit_common() {
|
||||
cat <<EOF
|
||||
cd $BASE
|
||||
HDR="$HDR"
|
||||
PORT=$2
|
||||
run() {
|
||||
local name="\$1"; shift
|
||||
echo "===== [\$(date +%H:%M:%S)] START: \$name ====="
|
||||
python3 -m sglang.bench_serving "\$@"
|
||||
echo "===== [\$(date +%H:%M:%S)] END(exit=\$?): \$name ====="
|
||||
sleep 5
|
||||
}
|
||||
EOF
|
||||
}
|
||||
|
||||
# ---------------- warmup ----------------
|
||||
cmd_warmup() {
|
||||
local c=${1:?container} port=${2:?port}
|
||||
local gen=$BASE/.bench_v2_warmup.sh
|
||||
{ emit_common "$c" "$port"; cat <<'EOF'
|
||||
run WARMUP_a --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
|
||||
--dataset-name random-ids --random-input-len 4096 --random-output-len 16 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 777001 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
run WARMUP_b --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
|
||||
--dataset-name random-ids --random-input-len 16000 --random-output-len 1 \
|
||||
--num-prompts 2 --max-concurrency 1 --seed 777002 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
echo "===== [$(date +%H:%M:%S)] ALL DONE ====="
|
||||
EOF
|
||||
} | sed "s|MODELPH|$MODEL|g" > "$gen"
|
||||
chmod +x "$gen"
|
||||
docker exec "$c" bash "$gen"
|
||||
}
|
||||
|
||||
# ---------------- prefill(冷/热双轮) ----------------
|
||||
cmd_prefill() {
|
||||
local tag=${1:?tag} c=${2:?container} port=${3:?port}
|
||||
local p=$(date +%H%M%S) gen=$BASE/.bench_v2_gen_${tag}.sh
|
||||
{ emit_common "$c" "$port"
|
||||
# warmup(丢弃)
|
||||
cat <<'EOF'
|
||||
run WARMUP --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
|
||||
--dataset-name random-ids --random-input-len 4096 --random-output-len 16 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 777000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
EOF
|
||||
# 冷轮
|
||||
for stage in "bs1_16000 16000 1 4 1" "bs1_160000 160000 1 4 2" "bs16_4096 4096 16 32 3"; do
|
||||
set -- $stage
|
||||
cat <<EOF
|
||||
run COLD_$1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len $2 --random-output-len 1 \\
|
||||
--num-prompts $4 --max-concurrency $3 --seed ${p}$5 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
done
|
||||
# 热轮(同 seed → cache hit)
|
||||
for stage in "bs1_16000 16000 1 4 1" "bs1_160000 160000 1 4 2" "bs16_4096 4096 16 32 3"; do
|
||||
set -- $stage
|
||||
cat <<EOF
|
||||
run HIT_$1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len $2 --random-output-len 1 \\
|
||||
--num-prompts $4 --max-concurrency $3 --seed ${p}$5 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
done
|
||||
echo 'echo "===== [$(date +%H:%M:%S)] ALL DONE ====="'
|
||||
} | sed "s|MODELPH|$MODEL|g" > "$gen"
|
||||
chmod +x "$gen"
|
||||
nohup docker exec "$c" bash "$gen" > "$BASE/bench_v2_${tag}.log" 2>&1 &
|
||||
echo "launched pid=$! seeds=${p}1..3 log=$BASE/bench_v2_${tag}.log"
|
||||
}
|
||||
|
||||
# ---------------- decode 阶梯 ----------------
|
||||
cmd_decode() {
|
||||
local tag=${1:?tag} c=${2:?container} port=${3:?port} cg=${4:-256}
|
||||
local maxc=$(( cg * 80 / 100 / 32 * 32 )) # 80% 向下取整到 32 的倍数
|
||||
local p=$(date +%H%M%S) gen=$BASE/.bench_v2_gen_${tag}.sh
|
||||
echo "ladder: 32..$maxc step 32 (cg_max_bs=$cg)"
|
||||
{ emit_common "$c" "$port"
|
||||
cat <<'EOF'
|
||||
run WARMUP --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 128 \
|
||||
--num-prompts 16 --max-concurrency 16 --seed 777000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
EOF
|
||||
# bs1 延迟地板(warm + measure)
|
||||
cat <<EOF
|
||||
run WARM_bs1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
|
||||
--num-prompts 4 --max-concurrency 1 --seed ${p}01 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
run DEC_bs1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
|
||||
--num-prompts 4 --max-concurrency 1 --seed ${p}01 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
local cc
|
||||
for (( cc=32; cc<=maxc; cc+=32 )); do
|
||||
local np=$(( cc * 2 ))
|
||||
cat <<EOF
|
||||
run WARM_c${cc} --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
|
||||
--num-prompts $np --max-concurrency $cc --seed ${p}1${cc} --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
run DEC_c${cc} --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
|
||||
--num-prompts $np --max-concurrency $cc --seed ${p}1${cc} --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
done
|
||||
echo 'echo "===== [$(date +%H:%M:%S)] ALL DONE ====="'
|
||||
} | sed "s|MODELPH|$MODEL|g" > "$gen"
|
||||
chmod +x "$gen"
|
||||
nohup docker exec "$c" bash "$gen" > "$BASE/bench_v2_${tag}.log" 2>&1 &
|
||||
echo "launched pid=$! log=$BASE/bench_v2_${tag}.log"
|
||||
}
|
||||
|
||||
# ---------------- L3 语料溢出(3p1d 经 router) ----------------
|
||||
cmd_l3() {
|
||||
local tag=${1:?tag} c=${2:?container} port=${3:?port} np=${4:-48}
|
||||
local p=$(date +%H%M%S) gen=$BASE/.bench_v2_gen_${tag}.sh
|
||||
{ emit_common "$c" "$port"
|
||||
cat <<EOF
|
||||
run WARMUP --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 4096 --random-output-len 16 \\
|
||||
--num-prompts 8 --max-concurrency 8 --seed 777000 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
run L3_COLD_160k --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \\
|
||||
--num-prompts $np --max-concurrency 8 --seed ${p}1 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
echo "===== [\$(date +%H:%M:%S)] DRAIN: sleep 120 等 L2→L3 备份排空 ====="
|
||||
sleep 120
|
||||
run L3_HIT_160k --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \\
|
||||
--num-prompts $np --max-concurrency 8 --seed ${p}1 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
echo 'echo "===== [$(date +%H:%M:%S)] ALL DONE ====="'
|
||||
} > "$gen"
|
||||
chmod +x "$gen"
|
||||
nohup docker exec "$c" bash "$gen" > "$BASE/bench_v2_${tag}.log" 2>&1 &
|
||||
echo "launched pid=$! corpus=${np}x160k log=$BASE/bench_v2_${tag}.log"
|
||||
}
|
||||
|
||||
# ---------------- report ----------------
|
||||
cmd_report() {
|
||||
local log=${1:?log required}
|
||||
awk '
|
||||
/START:/ { name=$4; sub(/:$/, "", name); tps=""; itps=""; tpot=""; ttft=""; acc="" }
|
||||
/Input token throughput/ { itps=$NF }
|
||||
/Output token throughput/ { tps=$NF }
|
||||
/Mean TPOT/ { tpot=$NF }
|
||||
/Mean TTFT/ { ttft=$NF }
|
||||
/[Aa]ccept/ { acc=$0; gsub(/^ +/, "", acc) }
|
||||
/END\(exit/ {
|
||||
line=sprintf("%-22s in_tps=%-10s out_tps=%-10s tpot_ms=%-8s ttft_ms=%-8s", name, itps, tps, tpot, ttft)
|
||||
if (tps != "" && tpot != "" && tpot+0 > 0)
|
||||
line=line sprintf(" eff_concurrency=%.1f", tps * tpot / 1000)
|
||||
print line
|
||||
if (acc != "") print " " acc
|
||||
}
|
||||
' "$log"
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
warmup) shift; cmd_warmup "$@" ;;
|
||||
prefill) shift; cmd_prefill "$@" ;;
|
||||
decode) shift; cmd_decode "$@" ;;
|
||||
l3) shift; cmd_l3 "$@" ;;
|
||||
report) shift; cmd_report "$@" ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
@@ -0,0 +1,61 @@
|
||||
cd /data/ymk/ds41
|
||||
HDR="Authorization=Bearer deepseek-v4-flash-0731-key"
|
||||
PORT=30040
|
||||
MODEL=/data/models/DeepSeek-V4.1-Flash
|
||||
|
||||
run() {
|
||||
local name="$1"; shift
|
||||
echo "===== [$(date +%H:%M:%S)] START: $name ====="
|
||||
python3 -m sglang.bench_serving "$@" 2>&1 | grep -vE '^Namespace|warn'
|
||||
echo "===== [$(date +%H:%M:%S)] END(exit=$?): $name ====="
|
||||
sleep 3
|
||||
}
|
||||
|
||||
r160() { # $1=name $2=seed $3=num_prompts $4=mc
|
||||
run "$1" --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--num-prompts "$3" --max-concurrency "$4" --seed "$2" --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
}
|
||||
|
||||
# WARMUP
|
||||
run WARMUP --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 4096 --random-output-len 16 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 777000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
|
||||
# R0 纯计算极限(集合 A 首次出现,L1/L2/L3 全空)
|
||||
r160 R0_COMPUTE_COLD 1541391 8 8
|
||||
# R1 L1 极限(A 全量在 L1 设备池:1.28M < 1.6M)
|
||||
r160 R1_L1_HIT 1541391 8 8
|
||||
# R2 填充集 B(12×160k=1.92M > L1 1.6M),把 A 挤出 L1 到 L2
|
||||
r160 R2_FILLER_EVICT 999000 12 8
|
||||
# R3 L2 极限(A 已被挤出 L1,从 L2 host 池 load-back)
|
||||
r160 R3_L2_HIT 1541391 8 8
|
||||
# R4 flush L1+L2(L3 保留),走 L3 prefetch
|
||||
echo "===== [$(date +%H:%M:%S)] FLUSH L1+L2 ====="
|
||||
curl -sf -m 10 -X POST http://127.0.0.1:$PORT/flush_cache && echo "flush ok"
|
||||
sleep 5
|
||||
r160 R4_L3_HIT 1541391 8 8
|
||||
# R5 嫁接后 L2 复核
|
||||
r160 R5_L2_AFTER_GRAFT 1541391 8 8
|
||||
|
||||
# R6 混合负载:新集合 C(纯计算,4×160k mc4)+ 集合 A(命中,8×160k mc8)并发打
|
||||
echo "===== [$(date +%H:%M:%S)] START: R6_MIX_PARALLEL ====="
|
||||
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--num-prompts 4 --max-concurrency 4 --seed 555000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR" > /data/ymk/ds41/.tier_r6_coldclient.log 2>&1 &
|
||||
COLD_PID=$!
|
||||
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 1541391 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR" > /data/ymk/ds41/.tier_r6_hitclient.log 2>&1 &
|
||||
HIT_PID=$!
|
||||
wait $COLD_PID $HIT_PID
|
||||
echo "===== [$(date +%H:%M:%S)] END: R6_MIX_PARALLEL ====="
|
||||
echo "--- R6 cold client (compute) ---"
|
||||
grep -E 'Input token throughput|Mean TTFT|Request throughput' /data/ymk/ds41/.tier_r6_coldclient.log
|
||||
echo "--- R6 hit client (cache) ---"
|
||||
grep -E 'Input token throughput|Mean TTFT|Request throughput' /data/ymk/ds41/.tier_r6_hitclient.log
|
||||
echo "===== [$(date +%H:%M:%S)] ALL DONE ====="
|
||||
@@ -0,0 +1,27 @@
|
||||
cd /data/ymk/ds41
|
||||
HDR="Authorization=Bearer deepseek-v4-flash-0731-key"
|
||||
PORT=30040
|
||||
MODEL=/data/models/DeepSeek-V4.1-Flash
|
||||
|
||||
echo "===== [$(date +%H:%M:%S)] FLUSH L1+L2(A 只剩 L3) ====="
|
||||
curl -sf -m 10 -X POST http://127.0.0.1:$PORT/flush_cache && echo "flush ok"
|
||||
sleep 5
|
||||
|
||||
echo "===== [$(date +%H:%M:%S)] START: R7_MIX_L3_PLUS_COMPUTE ====="
|
||||
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--num-prompts 4 --max-concurrency 4 --seed 666000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR" > /data/ymk/ds41/.tier_r7_coldclient.log 2>&1 &
|
||||
COLD_PID=$!
|
||||
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 1541391 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR" > /data/ymk/ds41/.tier_r7_hitclient.log 2>&1 &
|
||||
HIT_PID=$!
|
||||
wait $COLD_PID $HIT_PID
|
||||
echo "===== [$(date +%H:%M:%S)] END: R7 ====="
|
||||
echo "--- R7 cold client (compute, 4x160k mc4) ---"
|
||||
grep -E 'Input token throughput|Mean TTFT|Request throughput' /data/ymk/ds41/.tier_r7_coldclient.log
|
||||
echo "--- R7 hit client (L3, 8x160k mc8) ---"
|
||||
grep -E 'Input token throughput|Mean TTFT|Request throughput' /data/ymk/ds41/.tier_r7_hitclient.log
|
||||
echo "===== [$(date +%H:%M:%S)] ALL DONE ====="
|
||||
@@ -28,6 +28,7 @@ x-p-environment: &p-environment
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
# 【L3 dummy client 变体 2026-10-09 深夜,定型=shm+best_effort】sglang→standalone_storage,数据面经本机 mooncake_client(50052 IPC+进程映射),不碰 TCP 传输引擎
|
||||
# 实测 24x160k 热/冷:shm+best_effort +26.4%(唯一正收益);wait_complete 因 all-or-nothing discard 等待白等反而 -17.7%。TCP 传输引擎两组皆负。
|
||||
# 【L3 cache-hit 测试变体 2026-10-09】基于 dockerserve-3p1d-otlp.yml,仅改 P:
|
||||
# --max-total-tokens 800000(L1 缩到 ~0.8M token)+ --hicache-ratio 1.0(L2=L1=0.8M token。⚠️ 教训:ratio 0.05 时 L2 太小,build_storage_backup_spec 拿不到 host 页会静默跳过 L3 写入,热轮无数据可中;V4 hicache 不支持 --hicache-size)
|
||||
# 使 ~2.6M token/P 的语料(48×160k)溢出 L1+L2(1.6M),热轮强制走 L3 mooncake 命中。
|
||||
# 【2026-10-10 L2 放大】hicache-ratio 1.0→4.0(L2=3.2M token/P,宿主可用内存 ~1.9TB 充裕):
|
||||
# #38348 fail-soft 移植后 bench 显示收益被容量掩盖——24x160k 下 host_capacity 拒绝
|
||||
# 0.75M~2.97M token/P(L2 装不下,prefetch 未发起即被拒退回重算)。放大后验证:
|
||||
# ①热轮请求 L3/L2 命中不被容量拒绝;②L3 hit 嫁接进 L2 树后在 L2 长期留存。
|
||||
# 与主模板同容器名/端口,用前必须先 down 主栈。D 与 router 不变。
|
||||
# 原注释: 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram + OTLP trace(2026-09-27)
|
||||
# 基于 dockerserve-3p1d-soak.yml 增加: --enable-trace --trace-modules request,mooncake
|
||||
# --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name ds41-3p1d-<实例名>
|
||||
# 依赖: b300-01 k8s cw jaeger-trace(jaegertracing/all-in-one:1.76.0,badger 6h TTL,hostNetwork 4317)
|
||||
# 安全性: BatchSpanProcessor 导出失败只丢 span;Jaeger 宕机不影响推理服务
|
||||
# 调优旋钮(默认未开): SGLANG_TRACE_ASYNC=1(异步 span 创建), SGLANG_TRACE_LEVEL(默认 3)
|
||||
# router(ds41-router-3p1d,rust)无 OTLP 支持,不产生 span,追踪从各 sglang HTTP 层开始
|
||||
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-baba18bc7d-local-20261010-2310
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python:/sgl-workspace/sglang/python
|
||||
- /data/ymk/ds41/fake-cargo:/root/.cargo/bin/cargo:ro
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
RUSTUP_TOOLCHAIN: "1.92-x86_64-unknown-linux-gnu"
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store-dummy.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 4.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p1
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 4.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p2
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 4.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p3
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
# G9 P0 验证(2026-10-11):D 单独升镜像拿 /query_cache_hit 修复(71c6d8bc5),
|
||||
# 并开 decode radix + hicache L3 客户端(复用 dummy store,只读查询)。
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-6612af9e0-local-20261011-0740
|
||||
container_name: ds41-3p1d-d1
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
RUSTUP_TOOLCHAIN: "1.92-x86_64-unknown-linux-gnu"
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store-dummy.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
--disaggregation-decode-enable-radix-cache
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-d1
|
||||
@@ -0,0 +1,239 @@
|
||||
# 【G9 P0 验证栈:真实 store 3p1d 2026-10-11】基于 dockerserve-3p1d-otlp.yml。
|
||||
# 用途:验证 P0 改动(/v1/loads 排队 token + drain rate 字段、P 侧 /query_cache_hit)。
|
||||
# 与主模板同容器名/端口,用前必须先 down 主栈;router(ds41-router-3p1d)独立不在本文件。
|
||||
# 2026-10-11 改造要点(vs 原 l3test):
|
||||
# - 镜像升 6612af9e0(含 G9 P0 + /query_cache_hit KeyError 修复)+ scheduler.py
|
||||
# 热补丁(L3 查询门限从 enable_decode_hicache 放宽到 enable_hicache_storage,
|
||||
# 否则 P 节点永远查不到 L3)。
|
||||
# - 每实例独立 MOONCAKE_LOCAL_HOSTNAME(a/b/c/d),extra_hosts 互配全部 rank 别名
|
||||
# (真实 store 跨实例副本写要求对端 rank 主机名可解析,见 realstore-x2 注释;
|
||||
# rank 命名规则 {hostname}-d{dp}t{tp}p{pp}c{attn_cp},commit 84fc632088;
|
||||
# D 为 dp2 dp-attention(attn_tp=1),后缀推断为 -d0t0p0c0/-d1t1p0c0,未实证)。
|
||||
# - ⚠️ D 侧 cache 方案实证否决(2026-10-11):SWA 模型下
|
||||
# --disaggregation-decode-enable-radix-cache 先要 SGLANG_ENABLE_UNIFIED_RADIX_TREE=1,
|
||||
# 开了又报 "supports only device-resident cache and is incompatible with
|
||||
# --enable-hierarchical-cache"(kv_cache_builder.py 硬 ValueError)——D 挂不了 L3。
|
||||
# 结论:/query_cache_hit 放 P(L3 全局统一,P 有完整 hicache),D 保持生产配置。
|
||||
# - D mem-fraction 0.80:曾因崩溃 d1 泄漏 ~61GiB 临时降 0.75,全栈 down 掉旧容器后
|
||||
# 驱动即回收(GPU 全零),已恢复 0.80。D max-running 保持 256(512 时 SWA 池挤爆 KV 池)。
|
||||
# P 侧保持 l3test 原状:--max-total-tokens 800000 + ratio 1.0(L1/L2 各 ~0.8M token),
|
||||
# write_through 保证 prefill 后 L3 有数据,供 /query_cache_hit 对拍。
|
||||
# 原注释: 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram + OTLP trace(2026-09-27)
|
||||
# 依赖: mooncake-master 容器(50051,起栈前 restart 清旧 segment 注册);
|
||||
# b300-01 k8s cw jaeger-trace(4317,导出失败只丢 span 不影响服务)
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-6612af9e0-local-20261011-0740
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-rank-hosts: &rank-hosts
|
||||
# mooncake rank 主机名别名:P(tp2/cp2 interleave)-d0t0p0c0/-d0t1p0c1;
|
||||
# D(dp2 dp-attention)-d0t0p0c0/-d1t1p0c0(推断,起栈后核对 master 注册名)
|
||||
- "localhost-a-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-a-d0t1p0c1:127.0.0.1"
|
||||
- "localhost-b-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-b-d0t1p0c1:127.0.0.1"
|
||||
- "localhost-c-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-c-d0t1p0c1:127.0.0.1"
|
||||
- "localhost-d-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-d-d1t1p0c0:127.0.0.1"
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p1
|
||||
extra_hosts: *rank-hosts
|
||||
environment:
|
||||
<<: *p-environment
|
||||
MOONCAKE_LOCAL_HOSTNAME: localhost-a
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p1
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p2
|
||||
extra_hosts: *rank-hosts
|
||||
environment:
|
||||
<<: *p-environment
|
||||
MOONCAKE_LOCAL_HOSTNAME: localhost-b
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p2
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p3
|
||||
extra_hosts: *rank-hosts
|
||||
environment:
|
||||
<<: *p-environment
|
||||
MOONCAKE_LOCAL_HOSTNAME: localhost-c
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p3
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-d1
|
||||
extra_hosts: *rank-hosts
|
||||
# ⚠️ D 保持生产配置(2026-10-11 实证):SWA 模型下
|
||||
# --disaggregation-decode-enable-radix-cache 与 --enable-hierarchical-cache 互斥
|
||||
# (kv_cache_builder.py 硬 ValueError,"supports only device-resident cache"),
|
||||
# D 无法挂 L3 —— /query_cache_hit 改由 P 节点回答(L3 全局统一,P 有完整 hicache)。
|
||||
# MOONCAKE_LOCAL_HOSTNAME 保留备用(无害)。
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
MOONCAKE_LOCAL_HOSTNAME: localhost-d
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-d1
|
||||
@@ -35,6 +35,7 @@ x-p-environment: &p-environment
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
@@ -35,6 +35,7 @@ x-p-environment: &p-environment
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ x-p-environment: &p-environment
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ x-p-environment: &p-environment
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
container_name: ds41-sglang-b4
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-92632a60b-20260922-2100
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
container_name: ds41-cpdspark-nr
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
# 单独 D 节点基线(非 PD role):3p1d 的 d1 配置去掉 disaggregation,直接对外服务
|
||||
# 用途:测单 D 节点 decode 基线(阶梯并发 32..80%*cuda-graph-max-bs-decode,
|
||||
# 报 TPOT / 实际并发 / 整实例输出 TPS;prefill 走 L1 radix cache-hit)
|
||||
# GPU 6,7;端口 30001;无 hicache 依赖(L1 即可支撑重复 prompt 命中)
|
||||
# 2026-10-09 建:配合 bench_suite_v2.sh decode
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
d1s:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-d1-standalone
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
@@ -0,0 +1,61 @@
|
||||
# 单独 P 节点基线(非 PD role):3p1d 的 p1 配置去掉 disaggregation,直接对外服务
|
||||
# 用途:测单 P 节点 prefill 基线吞吐(冷)+ hicache cache-hit TPS(热)
|
||||
# GPU 0,1;端口 30000;依赖 mooncake-master(hicache L3,127.0.0.1:50051)
|
||||
# 2026-10-09 建:配合 bench_suite_v2.sh prefill
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
p1s:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-p1-standalone
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
@@ -22,6 +22,7 @@ services:
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
- SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1
|
||||
- SGLANG_HICACHE_MOONCAKE_CONFIG_PATH=/data/ymk/ds41/mooncake-store.json
|
||||
- SGLANG_HICACHE_PREFETCH_IO_THREADS=4
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
|
||||
@@ -18,6 +18,7 @@ services:
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
- SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1
|
||||
- SGLANG_HICACHE_MOONCAKE_CONFIG_PATH=/data/ymk/ds41/mooncake-store.json
|
||||
- SGLANG_HICACHE_PREFETCH_IO_THREADS=4
|
||||
- SGLANG_DISAGGREGATION_QUEUE_SIZE=16
|
||||
- SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=32
|
||||
volumes:
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
# 【跨 P 实例 L3 读写验证 2026-10-10】dockerserve-tier-cp2-realstore.yml 的第二实例:
|
||||
# 与 ds41-tier-t2(A:GPU 0,1,端口 30040)同时跑,共用 mooncake master 50051。
|
||||
# 关键差异:
|
||||
# - GPU 2,3,端口 30041,容器名 ds41-tier-t3
|
||||
# - MOONCAKE_LOCAL_HOSTNAME=localhost-b:mooncake 客户端 hostname 前缀按实例区分
|
||||
# (rank 后缀 -d0t0p0c0/-d0t1p0c1 由 84fc632088 拼在后面的),否则两实例 rank 同名
|
||||
# → master preferred-segment 分不出段归属 → 副本落错进程段 → 跨进程回退 TCP(实例内)
|
||||
# 且跨实例段地址冲突。extra_hosts 别名同步加 -b 前缀(P2P handshake 要求可解析,
|
||||
# ac11e9162d)。
|
||||
# 验证路径:A prime 写 L3 → flush B 的 L1/L2 → B 读同 seed prompts = 跨实例 TCP P2P 命中。
|
||||
x-common: &common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
shm_size: 32gb
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python:/sgl-workspace/sglang/python
|
||||
- /data/ymk/ds41/fake-cargo:/root/.cargo/bin/cargo:ro
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
t3:
|
||||
<<: *common
|
||||
container_name: ds41-tier-t3
|
||||
extra_hosts:
|
||||
- "localhost-b-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-b-d0t1p0c1:127.0.0.1"
|
||||
# 跨实例 P2P:对端实例(realstore compose 的 t2)rank 主机名也要能解析
|
||||
- "localhost-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-d0t1p0c1:127.0.0.1"
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: 1
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_TIMING: 1
|
||||
MOONCAKE_LOCAL_HOSTNAME: localhost-b
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: 4
|
||||
CUDA_VISIBLE_DEVICES: 2,3
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--chunked-prefill-size 32768
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30041
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 4.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--max-total-tokens 1600000
|
||||
@@ -0,0 +1,73 @@
|
||||
# 【L3 真实 store A/B 2026-10-10】同 dockerserve-tier-cp2.yml,唯一差异:
|
||||
# L3 从 dummy standalone(mooncake-store-dummy.json,50052 守护进程)
|
||||
# 切到真实 mooncake store(mooncake-store.json:master 50051 + P2PHANDSHAKE + tcp,
|
||||
# 客户端注册 300GB 自身内存段)——与 k8s 线上 ds41-3p1d 的 L3 模式一致。
|
||||
# 目的:区分 "sglang L3 路径本身慢" 还是 "dummy 后端 per-key 路径慢"(mc 扫描 A/B)。
|
||||
x-common: &common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
shm_size: 32gb
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python:/sgl-workspace/sglang/python
|
||||
- /data/ymk/ds41/fake-cargo:/root/.cargo/bin/cargo:ro
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
t2:
|
||||
<<: *common
|
||||
container_name: ds41-tier-t2
|
||||
extra_hosts:
|
||||
- "localhost-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-d0t1p0c1:127.0.0.1"
|
||||
# 跨实例 P2P:对端实例(x2 compose 的 t3)rank 主机名也要能解析,
|
||||
# 否则 master 把副本分配到对端段后本端 open segment 失败(TRANSFER_FAIL)
|
||||
- "localhost-b-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-b-d0t1p0c1:127.0.0.1"
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: 1
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_TIMING: 1
|
||||
# L3 预取传输并行 lane 数(crc32(rid) 分 lane,每 lane 独立 gloo 组;
|
||||
# 实例内所有 rank 必须一致)。2026-10-10 A/B 见 l3-hicache-deep-dive 文档。
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: 4
|
||||
CUDA_VISIBLE_DEVICES: 0,1
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--chunked-prefill-size 32768
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30040
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 4.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--max-total-tokens 1600000
|
||||
@@ -0,0 +1,65 @@
|
||||
# 【prefill 分层极限测试 2026-10-10】单实例 cp2(tp2+ep2+prefill-cp interleave)+ dspark b5
|
||||
# + hicache L2(ratio 4.0=6.4M token) + L3(shm dummy) + best_effort
|
||||
# 目的:测 纯计算 / L1 / L2 / L3 四档 prefill 极限吞吐 + 混合负载异步聚合验证
|
||||
# L1=1.6M token(装得下 8×160k=1.28M 工作集);L2 6.4M;L3 shm 64GB
|
||||
# GPU 0,1(用前停 p3d1 栈);端口 30040;共用 mooncake-store(50052)/master(50051)
|
||||
x-common: &common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
shm_size: 32gb
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python:/sgl-workspace/sglang/python
|
||||
- /data/ymk/ds41/fake-cargo:/root/.cargo/bin/cargo:ro
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
t2:
|
||||
<<: *common
|
||||
container_name: ds41-tier-t2
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: 1
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store-dummy.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: 4
|
||||
SGLANG_HICACHE_TIMING: 1
|
||||
SGLANG_HICACHE_STORAGE_BATCH_SIZE: 512
|
||||
CUDA_VISIBLE_DEVICES: 0,1
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--chunked-prefill-size 32768
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30040
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 4.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--max-total-tokens 1600000
|
||||
@@ -0,0 +1,454 @@
|
||||
# ds41-3p1d: DeepSeek-V4.1-Flash 3P1D PD 分离(cs32k + OTLP trace),RBG 版
|
||||
# 拓扑: 3×prefill(tp2+ep2+cp2, cs32k, dspark, hicache L3) + 1×decode(tp2+ep2+dp2, cg256 mr320, dspark,
|
||||
# load-balance-method=total_requests 按请求数分流) + mooncake-master(hicache L3 store) + router(mini-lb PD)
|
||||
# 全部钉 b300-01(单机 8 卡,hostNetwork + hostPID——mooncake NVLink-intra 跨 Pod CUDA IPC 必需,
|
||||
# 等同 docker compose 的 pid: host;参考 AGENTS.md「部署四要素」)
|
||||
# 镜像已侧载 containerd: docker.io/ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831(IfNotPresent)
|
||||
# 参数来源: deploy/b300-ds41/dockerserve-3p1d-otlp-cs32k.yml(docker compose 版)
|
||||
# 对外: 仅 ClusterIP Service ds41-3p1d:50002(router 用普通 Pod 网络,不占宿主机端口,
|
||||
# 天然不对宿主机/外网暴露);new-api-gw 用集群 DNS http://ds41-3p1d:50002/v1 访问
|
||||
# trace: OTLP → 127.0.0.1:4317(同节点 jaeger-trace cw,badger 6h 轮窗)
|
||||
apiVersion: workloads.x-k8s.io/v1alpha2
|
||||
kind: RoleBasedGroup
|
||||
metadata:
|
||||
name: ds41-3p1d
|
||||
labels:
|
||||
app: ds41-3p1d
|
||||
sglang-model: ds-v41-flash
|
||||
spec:
|
||||
roleTemplates:
|
||||
- name: prefill-base
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
sglang-model: ds-v41-flash
|
||||
sglang-role: prefill
|
||||
spec:
|
||||
hostNetwork: true
|
||||
hostPID: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: b300-01
|
||||
containers:
|
||||
- name: sglang
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec sglang serve \
|
||||
--trust-remote-code \
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash \
|
||||
--tp 2 --ep-size 2 \
|
||||
--enable-prefill-cp --cp-strategy interleave \
|
||||
--chunked-prefill-size 32768 --max-prefill-tokens 32768 \
|
||||
--mem-fraction-static 0.75 \
|
||||
--attention-backend dsv4 \
|
||||
--moe-runner-backend flashinfer_mxfp4 \
|
||||
--cuda-graph-max-bs-decode 32 \
|
||||
--reasoning-parser auto --tool-call-parser auto \
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' \
|
||||
--max-running-requests 256 \
|
||||
--prefill-max-requests 8 \
|
||||
--tokenizer-worker-num 8 \
|
||||
--host 0.0.0.0 --port "$SGLANG_PORT" \
|
||||
--enable-cache-report --enable-metrics \
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5 \
|
||||
--enable-hierarchical-cache \
|
||||
--hicache-ratio 2 \
|
||||
--hicache-mem-layout page_first_direct \
|
||||
--hicache-io-backend direct \
|
||||
--hicache-write-policy write_through \
|
||||
--hicache-storage-backend mooncake \
|
||||
--hicache-storage-prefetch-policy wait_complete \
|
||||
--hicache-size 0 \
|
||||
--disaggregation-mode prefill \
|
||||
--disaggregation-transfer-backend mooncake \
|
||||
--disaggregation-bootstrap-port "$SGLANG_BOOTSTRAP_PORT" \
|
||||
--enable-trace \
|
||||
--trace-modules request,mooncake \
|
||||
--otlp-traces-endpoint 127.0.0.1:4317 \
|
||||
--otlp-service-name "$OTLP_NAME"
|
||||
env:
|
||||
- name: SGLANG_TORCH_PROFILER_DIR
|
||||
value: /data/ymk/prof/torch
|
||||
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||
value: "1"
|
||||
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||
value: static
|
||||
- name: MC_INTRANODE_NVLINK
|
||||
value: "true"
|
||||
- name: MC_INTRA_NVLINK
|
||||
value: "true"
|
||||
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||
value: "1"
|
||||
- name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH
|
||||
value: /data/ymk/ds41/mooncake-store.json
|
||||
- name: SGLANG_HICACHE_PREFETCH_IO_THREADS
|
||||
value: "4"
|
||||
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||
value: "16"
|
||||
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||
value: "32"
|
||||
- name: TZ
|
||||
value: Asia/Shanghai
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: metrics
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 90
|
||||
resources:
|
||||
requests:
|
||||
cpu: "32"
|
||||
memory: 220Gi
|
||||
nvidia.com/gpu: "2"
|
||||
limits:
|
||||
cpu: "96"
|
||||
memory: 950Gi
|
||||
nvidia.com/gpu: "2"
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
- name: sglang-cache
|
||||
mountPath: /root/.cache/sglang
|
||||
- name: dshm
|
||||
mountPath: /dev/shm
|
||||
volumes:
|
||||
- name: data
|
||||
hostPath:
|
||||
path: /data
|
||||
type: Directory
|
||||
- name: sglang-cache
|
||||
hostPath:
|
||||
path: /data/ymk/cache/sglang
|
||||
type: Directory
|
||||
- name: dshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 32Gi
|
||||
- name: decode-base
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
sglang-model: ds-v41-flash
|
||||
sglang-role: decode
|
||||
spec:
|
||||
hostNetwork: true
|
||||
hostPID: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: b300-01
|
||||
containers:
|
||||
- name: sglang
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec sglang serve \
|
||||
--trust-remote-code \
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash \
|
||||
--tp 2 --ep-size 2 --dp-size 2 \
|
||||
--enable-dp-attention --enable-dp-lm-head \
|
||||
--mem-fraction-static 0.83 \
|
||||
--attention-backend dsv4 \
|
||||
--moe-runner-backend flashinfer_mxfp4 \
|
||||
--cuda-graph-max-bs-decode 256 \
|
||||
--load-balance-method total_requests \
|
||||
--reasoning-parser auto --tool-call-parser auto \
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' \
|
||||
--max-running-requests 320 \
|
||||
--tokenizer-worker-num 8 \
|
||||
--host 0.0.0.0 --port 30030 \
|
||||
--enable-cache-report --enable-metrics \
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5 \
|
||||
--disaggregation-mode decode \
|
||||
--disaggregation-transfer-backend mooncake \
|
||||
--disaggregation-bootstrap-port 9001 \
|
||||
--enable-trace \
|
||||
--trace-modules request,mooncake \
|
||||
--otlp-traces-endpoint 127.0.0.1:4317 \
|
||||
--otlp-service-name ds41-3p1d-d1
|
||||
env:
|
||||
- name: SGLANG_TORCH_PROFILER_DIR
|
||||
value: /data/ymk/prof/torch
|
||||
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||
value: "1"
|
||||
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||
value: static
|
||||
- name: MC_INTRANODE_NVLINK
|
||||
value: "true"
|
||||
- name: MC_INTRA_NVLINK
|
||||
value: "true"
|
||||
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||
value: "1"
|
||||
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||
value: "16"
|
||||
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||
value: "32"
|
||||
- name: TZ
|
||||
value: Asia/Shanghai
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 30030
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: metrics
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 90
|
||||
resources:
|
||||
requests:
|
||||
cpu: "32"
|
||||
memory: 100Gi
|
||||
nvidia.com/gpu: "2"
|
||||
limits:
|
||||
cpu: "96"
|
||||
memory: 600Gi
|
||||
nvidia.com/gpu: "2"
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
- name: sglang-cache
|
||||
mountPath: /root/.cache/sglang
|
||||
- name: dshm
|
||||
mountPath: /dev/shm
|
||||
volumes:
|
||||
- name: data
|
||||
hostPath:
|
||||
path: /data
|
||||
type: Directory
|
||||
- name: sglang-cache
|
||||
hostPath:
|
||||
path: /data/ymk/cache/sglang
|
||||
type: Directory
|
||||
- name: dshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 32Gi
|
||||
roles:
|
||||
- name: mooncake-master
|
||||
replicas: 1
|
||||
standalonePattern:
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-mooncake-master
|
||||
spec:
|
||||
hostNetwork: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: b300-01
|
||||
containers:
|
||||
- name: mooncake-master
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec /opt/sglang/bin/mooncake_master \
|
||||
--port 50051 \
|
||||
--enable_http_metadata_server=true \
|
||||
--http_metadata_server_port=18080 \
|
||||
--eviction_high_watermark_ratio=0.90
|
||||
readinessProbe:
|
||||
tcpSocket:
|
||||
port: 50051
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
failureThreshold: 60
|
||||
resources:
|
||||
requests:
|
||||
cpu: "1"
|
||||
memory: 8Gi
|
||||
limits:
|
||||
cpu: "4"
|
||||
memory: 64Gi
|
||||
- name: p1
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- mooncake-master
|
||||
standalonePattern:
|
||||
templateRef:
|
||||
name: prefill-base
|
||||
patch:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-p1
|
||||
spec:
|
||||
containers:
|
||||
- name: sglang
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: "0,1"
|
||||
- name: SGLANG_PORT
|
||||
value: "30000"
|
||||
- name: SGLANG_BOOTSTRAP_PORT
|
||||
value: "8998"
|
||||
- name: OTLP_NAME
|
||||
value: ds41-3p1d-p1
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 30000
|
||||
- name: p2
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- mooncake-master
|
||||
standalonePattern:
|
||||
templateRef:
|
||||
name: prefill-base
|
||||
patch:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-p2
|
||||
spec:
|
||||
containers:
|
||||
- name: sglang
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: "2,3"
|
||||
- name: SGLANG_PORT
|
||||
value: "30010"
|
||||
- name: SGLANG_BOOTSTRAP_PORT
|
||||
value: "8999"
|
||||
- name: OTLP_NAME
|
||||
value: ds41-3p1d-p2
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 30010
|
||||
- name: p3
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- mooncake-master
|
||||
standalonePattern:
|
||||
templateRef:
|
||||
name: prefill-base
|
||||
patch:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-p3
|
||||
spec:
|
||||
containers:
|
||||
- name: sglang
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: "4,5"
|
||||
- name: SGLANG_PORT
|
||||
value: "30020"
|
||||
- name: SGLANG_BOOTSTRAP_PORT
|
||||
value: "9000"
|
||||
- name: OTLP_NAME
|
||||
value: ds41-3p1d-p3
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 30020
|
||||
- name: d1
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- mooncake-master
|
||||
standalonePattern:
|
||||
templateRef:
|
||||
name: decode-base
|
||||
patch:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-d1
|
||||
spec:
|
||||
containers:
|
||||
- name: sglang
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: "6,7"
|
||||
- name: router
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- p1
|
||||
- p2
|
||||
- p3
|
||||
- d1
|
||||
standalonePattern:
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-router
|
||||
spec:
|
||||
# 入口不走 hostNetwork:普通 Pod 网络 + ClusterIP,天然不对宿主机/外网暴露。
|
||||
# P/D 地址用节点内网 IP(hostNetwork 监听在 192.168.1.243 上,Calico 可达)。
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: b300-01
|
||||
containers:
|
||||
- name: router
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec sglang-router launch \
|
||||
--mini-lb --pd-disaggregation \
|
||||
--host 0.0.0.0 --port 50002 \
|
||||
--prefill http://192.168.1.243:30000 8998 \
|
||||
--prefill http://192.168.1.243:30010 8999 \
|
||||
--prefill http://192.168.1.243:30020 9000 \
|
||||
--decode http://192.168.1.243:30030
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 50002
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 30
|
||||
resources:
|
||||
requests:
|
||||
cpu: "1"
|
||||
memory: 4Gi
|
||||
limits:
|
||||
cpu: "8"
|
||||
memory: 16Gi
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: ds41-3p1d
|
||||
labels:
|
||||
app: ds41-3p1d
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
rbg.workloads.x-k8s.io/group-name: ds41-3p1d
|
||||
rbg.workloads.x-k8s.io/role-name: router
|
||||
ports:
|
||||
- name: http
|
||||
port: 50002
|
||||
targetPort: 50002
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PodMonitor
|
||||
metadata:
|
||||
name: ds41-3p1d
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
sglang-model: ds-v41-flash
|
||||
podMetricsEndpoints:
|
||||
- interval: 15s
|
||||
path: /metrics
|
||||
port: metrics
|
||||
@@ -0,0 +1,2 @@
|
||||
#!/bin/sh
|
||||
exit 1
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"standalone_storage": true,
|
||||
"client_server_address": "127.0.0.1:50052"
|
||||
}
|
||||
@@ -18,6 +18,8 @@
|
||||
|
||||
namespace sglang {
|
||||
|
||||
using device::math::fast_mod_div_u32_t;
|
||||
|
||||
/// \brief Ratio-2 decode compressor: pair-pool, RMSNorm and the main-KV write.
|
||||
///
|
||||
/// `kv_input` and `kv_state` rows are `2 * kHeadDim` floats, kv then score.
|
||||
@@ -34,8 +36,7 @@ struct Compress2DecodeParams {
|
||||
const int64_t* __restrict__ req; // [num_tokens], req_pool_idx per token
|
||||
const void* __restrict__ raw_out_loc; // [num_tokens] LocT, the FULL slot; 0 marks a padded row
|
||||
uint8_t* __restrict__ kvcache; // [npages, kPageBytes] uint8
|
||||
/// Positions per request slot in the pair-state ring.
|
||||
uint32_t ring_size;
|
||||
fast_mod_div_u32_t ring_size; // Positions per request slot in the pair-state ring.
|
||||
float eps;
|
||||
};
|
||||
|
||||
@@ -43,21 +44,32 @@ struct Compress2DecodeParams {
|
||||
/// At (512, 64) it also keeps the nope/rope split warp-aligned, as the fp8 amax reduction requires.
|
||||
constexpr uint32_t kC2VecSize = 2;
|
||||
|
||||
/// \brief grid = num_tokens, block = kHeadDim / kC2VecSize.
|
||||
/// \brief Where a completing row finds its partner, and who parks a pending row.
|
||||
enum class C2Mode {
|
||||
/// One row per request: the partner comes from the ring, a pending row parks there.
|
||||
Decode,
|
||||
/// `draft_len` consecutive positions per request: `blockIdx.x` is the position inside the
|
||||
/// block, `blockIdx.y` the request, and every row but the first takes its partner from the
|
||||
/// previous `kv_input` row instead of the ring.
|
||||
Verify,
|
||||
/// Arbitrary rows, paired as `_low_ratio_pair_partners` pairs them: the partner is the previous
|
||||
/// row when it is the same request's previous position, else the ring. A pending row does
|
||||
/// nothing here; `c2_prefill_store_kernel` parks the ring rows after this kernel read them.
|
||||
Prefill,
|
||||
};
|
||||
|
||||
/// \brief grid = num_tokens (Decode, Prefill) or `draft_len x batch` (Verify),
|
||||
/// block = kHeadDim / kC2VecSize.
|
||||
///
|
||||
/// An odd position completes a group with its even predecessor; an even one parks in the state.
|
||||
///
|
||||
/// Under target-verify, `draft_len` consecutive positions per request: `blockIdx.x` is the
|
||||
/// position inside the block, `blockIdx.y` the request, and every row but the first takes its
|
||||
/// partner from the previous `kv_input` row instead of the ring.
|
||||
///
|
||||
/// The three reductions have different widths and are not interchangeable: the RMSNorm
|
||||
/// statistic spans the row, an fp8 store scale 64 elements, an fp4 block 16.
|
||||
///
|
||||
/// kLayout is the cache's page format: V4 and V41 store the fake-quantized value; V41_FP4
|
||||
/// stores the e2m1 codes and their e4m3 scales directly, so the fp4 rounding happens once.
|
||||
template <
|
||||
bool kVerify,
|
||||
C2Mode kMode,
|
||||
int64_t kHeadDim,
|
||||
int64_t kRopeDim,
|
||||
int32_t kPageBits,
|
||||
@@ -91,16 +103,21 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
|
||||
|
||||
const auto tx = threadIdx.x;
|
||||
// Verify gives each request a CTA column; decode a flat grid of one row each.
|
||||
const auto row = kVerify ? blockIdx.y * gridDim.x + blockIdx.x : blockIdx.x;
|
||||
const auto row = kMode == C2Mode::Verify ? blockIdx.y * gridDim.x + blockIdx.x : blockIdx.x;
|
||||
PDLWaitPrimary<kUsePDL>();
|
||||
// Slots fit in int32 whatever width the scheduler hands them in.
|
||||
const auto raw_out_loc = static_cast<int32_t>(static_cast<const LocT*>(params.raw_out_loc)[row]);
|
||||
// CUDA graph padding is a completely inert row: do not read schedule, input,
|
||||
// state, or RoPE data, and do not publish output or update the cache.
|
||||
if (raw_out_loc == 0) return PDLTriggerSecondary<kUsePDL>();
|
||||
const auto pos = static_cast<const PosT*>(params.positions)[row];
|
||||
// A completing row reads the slot left by `pos - 1`;
|
||||
// a pending row writes its own slot, so reads and writes stay disjoint.
|
||||
const auto* loc_ptr = static_cast<const LocT*>(params.raw_out_loc);
|
||||
const auto* pos_ptr = static_cast<const PosT*>(params.positions);
|
||||
const auto raw_out_loc = loc_ptr[row];
|
||||
// Prefill rows are many, so a CUDA graph padding row exits before any load.
|
||||
// Decode and verify rows are few: every row issues its loads to keep more in flight.
|
||||
if constexpr (kMode == C2Mode::Prefill) {
|
||||
if (raw_out_loc == 0) return PDLTriggerSecondary<kUsePDL>();
|
||||
}
|
||||
const auto pos = pos_ptr[row];
|
||||
// Prefill leaves a pending row's latent unwritten; its index-K slot is -1, so nothing reads it.
|
||||
if constexpr (kMode == C2Mode::Prefill) {
|
||||
if ((pos & 1) == 0) return PDLTriggerSecondary<kUsePDL>();
|
||||
}
|
||||
const auto rid = params.req[row];
|
||||
|
||||
fp32_vec_t kv_new, score_new;
|
||||
@@ -108,21 +125,32 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
|
||||
score_new.load(params.kv_input + row * kStride, tx + kCTASize);
|
||||
|
||||
const auto ring = static_cast<int64_t>(rid) * params.ring_size;
|
||||
const auto read_row = ring + (pos - 1 + params.ring_size) % params.ring_size;
|
||||
const auto write_row = ring + pos % params.ring_size;
|
||||
const auto write_offset = static_cast<uint32_t>(pos) % params.ring_size;
|
||||
const auto read_offset = write_offset == 0 ? params.ring_size - 1 : write_offset - 1;
|
||||
const auto write_row = ring + write_offset;
|
||||
const auto read_row = ring + read_offset;
|
||||
|
||||
fp32_vec_t kv_old, score_old;
|
||||
const float* partner = params.kv_state + read_row * kStride;
|
||||
if constexpr (kVerify) {
|
||||
if constexpr (kMode == C2Mode::Verify) {
|
||||
if (blockIdx.x != 0) partner = params.kv_input + static_cast<int64_t>(row - 1) * kStride;
|
||||
}
|
||||
if constexpr (kMode == C2Mode::Prefill) {
|
||||
if (row > 0 && params.req[row - 1] == rid && pos_ptr[row - 1] == pos - 1 && loc_ptr[row - 1] != 0) {
|
||||
partner = params.kv_input + static_cast<int64_t>(row - 1) * kStride;
|
||||
}
|
||||
}
|
||||
kv_old.load(partner, tx);
|
||||
score_old.load(partner, tx + kCTASize);
|
||||
|
||||
if ((pos & 1) == 0) {
|
||||
kv_new.store(params.kv_state + write_row * kStride, tx);
|
||||
score_new.store(params.kv_state + write_row * kStride, tx + kCTASize);
|
||||
return PDLTriggerSecondary<kUsePDL>();
|
||||
if constexpr (kMode != C2Mode::Prefill) {
|
||||
// CUDA graph padding case
|
||||
if (raw_out_loc == 0) return PDLTriggerSecondary<kUsePDL>();
|
||||
if ((pos & 1) == 0) {
|
||||
kv_new.store(params.kv_state + write_row * kStride, tx);
|
||||
score_new.store(params.kv_state + write_row * kStride, tx + kCTASize);
|
||||
return PDLTriggerSecondary<kUsePDL>();
|
||||
}
|
||||
}
|
||||
|
||||
constexpr uint32_t kNumWarps = kCTASize / kWarpThreads;
|
||||
@@ -203,9 +231,9 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
|
||||
}
|
||||
}
|
||||
|
||||
const int32_t out_loc = static_cast<uint32_t>(raw_out_loc) >> 1;
|
||||
if constexpr (kLayout == KVLayout::V41_FP4) {
|
||||
// The fp4 cache takes the rotated bf16 value as is: its row quantizer is the fake quant, minus the dequant.
|
||||
const int32_t out_loc = raw_out_loc >> 1;
|
||||
const auto kv_row = Paged::row(params.kvcache, out_loc);
|
||||
return deepseek_v4::v41::store_row<kLayout>(kv_row.data, kv_row.scale, tx, staged);
|
||||
}
|
||||
@@ -228,7 +256,6 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
|
||||
}
|
||||
|
||||
// `raw_out_loc / ratio`; ratio 2 makes it a shift.
|
||||
const int32_t out_loc = raw_out_loc >> 1;
|
||||
const auto kv_row = Paged::row(params.kvcache, out_loc);
|
||||
|
||||
if constexpr (kLayout == KVLayout::V41) {
|
||||
@@ -264,21 +291,63 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
|
||||
}
|
||||
}
|
||||
|
||||
/// \brief Prefill ring write-back, launched after `flash_c2_decode_kernel<C2Mode::Prefill>`.
|
||||
///
|
||||
/// Each request's last `ring_size` rows park their `| kv | score |` at `req * ring_size +
|
||||
/// pos % ring_size` for a later step's first partner. It cannot run inside the pooling kernel:
|
||||
/// a chunk whose first row is odd and whose last row is even reads and writes the same ring
|
||||
/// slot. grid = `ring_size x num_reqs`, CTA (k, i) parks the k-th row from the end of request i,
|
||||
/// one float4 per thread; those rows are consecutive positions, so their slots are distinct.
|
||||
template <int64_t kHeadDim, typename PosT, typename LocT, bool kUsePDL>
|
||||
__global__ __launch_bounds__(kHeadDim / 2) void c2_prefill_store_kernel(
|
||||
const Compress2DecodeParams params,
|
||||
const int32_t* __restrict__ extend_start_loc,
|
||||
const int32_t* __restrict__ extend_seq_lens) {
|
||||
using namespace device;
|
||||
constexpr int64_t kStride = kHeadDim * 2;
|
||||
using vec_t = AlignedVector<float, 4>;
|
||||
static_assert(kStride == 4 * (kHeadDim / 2));
|
||||
|
||||
// The pooling kernel reads the ring rows this one overwrites.
|
||||
PDLWaitPrimary<kUsePDL>();
|
||||
const auto bx = static_cast<int32_t>(blockIdx.x);
|
||||
const auto len = extend_seq_lens[blockIdx.y];
|
||||
const auto start_loc = extend_start_loc[blockIdx.y];
|
||||
if (bx >= len) return;
|
||||
const auto row = start_loc + (len - 1 - bx);
|
||||
const auto rid = params.req[row];
|
||||
const auto out = static_cast<const LocT*>(params.raw_out_loc)[row];
|
||||
const auto pos = static_cast<uint32_t>(static_cast<const PosT*>(params.positions)[row]);
|
||||
const auto dst_row = rid * params.ring_size + pos % params.ring_size;
|
||||
if (out == 0) return;
|
||||
vec_t buf;
|
||||
buf.load(params.kv_input + row * kStride, threadIdx.x);
|
||||
PDLTriggerSecondary<kUsePDL>();
|
||||
buf.store(params.kv_state + dst_row * kStride, threadIdx.x);
|
||||
}
|
||||
|
||||
template <int64_t kHeadDim, int64_t kRopeDim, uint32_t kPageSize, deepseek_v4::KVLayout kLayout, bool kUsePDL>
|
||||
struct FlashCompress2Kernel {
|
||||
static constexpr uint32_t kBlockSize = kHeadDim / kC2VecSize;
|
||||
static constexpr int32_t kPageBits = std::bit_width(kPageSize) - 1;
|
||||
static constexpr int64_t kPageBytes = deepseek_v4::kv_page_bytes<kLayout>(kPageSize);
|
||||
static_assert(kLayout != deepseek_v4::KVLayout::V4 || kPageBytes == host::div_ceil(584ll * kPageSize, 576) * 576);
|
||||
template <bool kVerify, typename PosT, typename LocT>
|
||||
template <C2Mode kMode, typename PosT, typename LocT>
|
||||
static constexpr auto kernel =
|
||||
flash_c2_decode_kernel<kVerify, kHeadDim, kRopeDim, kPageBits, PosT, LocT, kLayout, kUsePDL>;
|
||||
flash_c2_decode_kernel<kMode, kHeadDim, kRopeDim, kPageBits, PosT, LocT, kLayout, kUsePDL>;
|
||||
template <typename PosT, typename LocT>
|
||||
static constexpr auto store_kernel = c2_prefill_store_kernel<kHeadDim, PosT, LocT, kUsePDL>;
|
||||
|
||||
/// \brief The (`positions`, `raw_out_loc`) dtype pair, resolved at run time.
|
||||
template <bool kVerify>
|
||||
template <C2Mode kMode>
|
||||
static auto select(const bool pos_i32, const bool loc_i32) {
|
||||
if (pos_i32) return loc_i32 ? kernel<kVerify, int32_t, int32_t> : kernel<kVerify, int32_t, int64_t>;
|
||||
return loc_i32 ? kernel<kVerify, int64_t, int32_t> : kernel<kVerify, int64_t, int64_t>;
|
||||
if (pos_i32) return loc_i32 ? kernel<kMode, int32_t, int32_t> : kernel<kMode, int32_t, int64_t>;
|
||||
return loc_i32 ? kernel<kMode, int64_t, int32_t> : kernel<kMode, int64_t, int64_t>;
|
||||
}
|
||||
|
||||
static auto select_store(const bool pos_i32, const bool loc_i32) {
|
||||
if (pos_i32) return loc_i32 ? store_kernel<int32_t, int32_t> : store_kernel<int32_t, int64_t>;
|
||||
return loc_i32 ? store_kernel<int64_t, int32_t> : store_kernel<int64_t, int64_t>;
|
||||
}
|
||||
|
||||
// The sum of squares reduces through a fixed-size shared array, so the CTA must be whole warps.
|
||||
@@ -302,7 +371,7 @@ struct FlashCompress2Kernel {
|
||||
const tvm::ffi::TensorView kvcache,
|
||||
const int64_t ring_size,
|
||||
const int64_t draft_len) {
|
||||
launch(
|
||||
launch<false>(
|
||||
kv_input,
|
||||
kv_state,
|
||||
kv_output,
|
||||
@@ -317,7 +386,62 @@ struct FlashCompress2Kernel {
|
||||
draft_len);
|
||||
}
|
||||
|
||||
/// \brief The extend-batch pooling kernel, then the ring write-back.
|
||||
///
|
||||
/// Rows may mix requests, prefix offsets and padding; a pending (even) row's `kv_output`
|
||||
/// row is left unwritten. `extend_start_loc` / `extend_seq_lens` are int32 `[num_reqs]`:
|
||||
/// request i owns rows `[start[i], start[i] + len[i])`; zero-length requests are skipped.
|
||||
static void run_prefill_fusion(
|
||||
const tvm::ffi::TensorView kv_input,
|
||||
const tvm::ffi::TensorView kv_state,
|
||||
const tvm::ffi::TensorView kv_output,
|
||||
const tvm::ffi::TensorView norm_weight,
|
||||
const tvm::ffi::TensorView positions,
|
||||
const tvm::ffi::TensorView req,
|
||||
const tvm::ffi::TensorView raw_out_loc,
|
||||
const float eps,
|
||||
const tvm::ffi::TensorView freqs_cis,
|
||||
const tvm::ffi::TensorView kvcache,
|
||||
const int64_t ring_size,
|
||||
const tvm::ffi::TensorView extend_start_loc,
|
||||
const tvm::ffi::TensorView extend_seq_lens) {
|
||||
using namespace host;
|
||||
auto B = SymbolicSize{"num_reqs"};
|
||||
auto device_ = SymbolicDevice{};
|
||||
device_.set_options<kDLGPU>();
|
||||
TensorMatcher({B}).with_dtype<int32_t>().with_device(device_).verify(extend_start_loc);
|
||||
TensorMatcher({B}).with_dtype<int32_t>().with_device(device_).verify(extend_seq_lens);
|
||||
const auto requests = StoreRequests{
|
||||
.extend_start_loc = static_cast<const int32_t*>(extend_start_loc.data_ptr()),
|
||||
.extend_seq_lens = static_cast<const int32_t*>(extend_seq_lens.data_ptr()),
|
||||
.num_reqs = static_cast<uint32_t>(B.unwrap()),
|
||||
};
|
||||
launch<true>(
|
||||
kv_input,
|
||||
kv_state,
|
||||
kv_output,
|
||||
norm_weight,
|
||||
positions,
|
||||
req,
|
||||
raw_out_loc,
|
||||
eps,
|
||||
ring_size,
|
||||
freqs_cis,
|
||||
kvcache,
|
||||
1,
|
||||
&requests);
|
||||
}
|
||||
|
||||
private:
|
||||
struct StoreRequests {
|
||||
const int32_t* extend_start_loc;
|
||||
const int32_t* extend_seq_lens;
|
||||
uint32_t num_reqs;
|
||||
};
|
||||
|
||||
/// \tparam kPrefill `C2Mode::Prefill` plus the ring write-back of `requests`; otherwise
|
||||
/// Decode, or Verify when `draft_len > 1`.
|
||||
template <bool kPrefill>
|
||||
static void launch(
|
||||
const tvm::ffi::TensorView kv_input,
|
||||
const tvm::ffi::TensorView kv_state,
|
||||
@@ -330,7 +454,8 @@ struct FlashCompress2Kernel {
|
||||
const int64_t ring_size,
|
||||
const tvm::ffi::TensorView freqs_cis,
|
||||
const tvm::ffi::TensorView kvcache,
|
||||
const int64_t draft_len) {
|
||||
const int64_t draft_len,
|
||||
const StoreRequests* requests = nullptr) {
|
||||
using namespace host;
|
||||
|
||||
auto N = SymbolicSize{"num_tokens"};
|
||||
@@ -358,7 +483,7 @@ struct FlashCompress2Kernel {
|
||||
|
||||
const auto num_tokens = static_cast<uint32_t>(N.unwrap());
|
||||
if (num_tokens == 0) return;
|
||||
const auto is_verify = draft_len > 1;
|
||||
const auto is_verify = !kPrefill && draft_len > 1;
|
||||
CHECK_HOST(ring_size > 0 && draft_len >= 1);
|
||||
CHECK_HOST(!is_verify || num_tokens % draft_len == 0);
|
||||
CHECK_HOST(!is_verify || ring_size > draft_len)
|
||||
@@ -373,19 +498,28 @@ struct FlashCompress2Kernel {
|
||||
.req = static_cast<const int64_t*>(req.data_ptr()),
|
||||
.raw_out_loc = raw_out_loc.data_ptr(),
|
||||
.kvcache = static_cast<uint8_t*>(kvcache.data_ptr()),
|
||||
.ring_size = static_cast<uint32_t>(ring_size),
|
||||
.ring_size = static_cast<fast_mod_div_u32_t>(ring_size),
|
||||
.eps = eps,
|
||||
};
|
||||
// `LaunchKernel` is move-only, so each arm builds its own.
|
||||
const auto pos_i32 = pos_dtype.is_type<int32_t>();
|
||||
const auto loc_i32 = loc_dtype.is_type<int32_t>();
|
||||
if (is_verify) {
|
||||
if constexpr (kPrefill) {
|
||||
const auto k = select<C2Mode::Prefill>(pos_i32, loc_i32);
|
||||
LaunchKernel(num_tokens, kBlockSize, device_.unwrap()) //
|
||||
.enable_pdl(kUsePDL)(k, params);
|
||||
if (requests->num_reqs == 0) return;
|
||||
const auto store = select_store(pos_i32, loc_i32);
|
||||
const auto grid = dim3{static_cast<uint32_t>(ring_size), requests->num_reqs};
|
||||
LaunchKernel(grid, kHeadDim / 2, device_.unwrap()) //
|
||||
.enable_pdl(kUsePDL)(store, params, requests->extend_start_loc, requests->extend_seq_lens);
|
||||
} else if (is_verify) {
|
||||
const auto block = static_cast<uint32_t>(draft_len);
|
||||
const auto k = select<true>(pos_i32, loc_i32);
|
||||
const auto k = select<C2Mode::Verify>(pos_i32, loc_i32);
|
||||
LaunchKernel(dim3{block, num_tokens / block}, kBlockSize, device_.unwrap()) //
|
||||
.enable_pdl(kUsePDL)(k, params);
|
||||
} else {
|
||||
const auto k = select<false>(pos_i32, loc_i32);
|
||||
const auto k = select<C2Mode::Decode>(pos_i32, loc_i32);
|
||||
LaunchKernel(num_tokens, kBlockSize, device_.unwrap()) //
|
||||
.enable_pdl(kUsePDL)(k, params);
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
#include <sgl_kernel/tensor.h>
|
||||
#include <sgl_kernel/utils.h>
|
||||
|
||||
#include <sgl_kernel/math.cuh>
|
||||
#include <sgl_kernel/runtime.cuh>
|
||||
#include <sgl_kernel/type.cuh>
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
@@ -21,31 +22,7 @@ namespace sglang {
|
||||
|
||||
using device::distributed::PushWorkSpace;
|
||||
using device::distributed::Semaphore;
|
||||
|
||||
// Runtime uint32 division as a multiply-high and a shift (round-up magic,
|
||||
// exact below 2^31); cuda::fast_mod_div needs a newer CCCL than CUDA 13 bundles.
|
||||
struct fast_mod_div_u32_t {
|
||||
uint32_t divisor;
|
||||
uint32_t magic;
|
||||
uint32_t shift;
|
||||
|
||||
__host__ explicit fast_mod_div_u32_t(uint32_t d) : divisor(d), magic(0), shift(0) {
|
||||
if (d > 1) {
|
||||
const uint32_t log2_ceil = 32 - std::countl_zero(d - 1);
|
||||
const uint32_t p = 31 + log2_ceil;
|
||||
magic = static_cast<uint32_t>(((uint64_t{1} << p) + d - 1) / d);
|
||||
shift = p - 32;
|
||||
}
|
||||
}
|
||||
|
||||
__device__ friend uint32_t operator/(uint32_t n, const fast_mod_div_u32_t& fd) {
|
||||
return fd.divisor == 1 ? n : __umulhi(n, fd.magic) >> fd.shift;
|
||||
}
|
||||
|
||||
__device__ friend uint32_t operator%(uint32_t n, const fast_mod_div_u32_t& fd) {
|
||||
return n - (n / fd) * fd.divisor;
|
||||
}
|
||||
};
|
||||
using device::math::fast_mod_div_u32_t;
|
||||
|
||||
template <uint32_t kWorldSize>
|
||||
struct NVLinkCommPushParams {
|
||||
@@ -168,8 +145,7 @@ PUSH_KERNEL void nvlink_push_kernel(const __grid_constant__ NVLinkCommPushParams
|
||||
ptx::st_multimem_16B(vec, dst_ptr_mc, vid);
|
||||
}
|
||||
} else /* reduce-scatter only */ {
|
||||
const auto token_id = vid / params.vecs_per_token_div;
|
||||
const auto offset = vid % params.vecs_per_token_div;
|
||||
const auto [token_id, offset] = math::div(vid, params.vecs_per_token_div);
|
||||
// Both by a compile-time constant, so this is a mask and a shift.
|
||||
const auto dst_rank = token_id % kWorldSize;
|
||||
const auto dst_token_id = token_id / kWorldSize;
|
||||
|
||||
@@ -112,6 +112,53 @@ SGL_DEVICE float fma_f32_bf16(bf16_t a, float b, float acc) {
|
||||
return fmaf(cast<fp32_t>(a), b, acc);
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief Fast uint32 division and modulo by a compile-time constant divisor.
|
||||
*
|
||||
* This struct is only exact when the divisor is less than `2^31`. Larger dividends
|
||||
* will lead to incorrect results. Be careful when using this struct.
|
||||
*/
|
||||
struct fast_mod_div_u32_t {
|
||||
public:
|
||||
explicit constexpr fast_mod_div_u32_t(uint32_t d) : m_divisor(d) {
|
||||
if (d > 1) {
|
||||
const uint32_t log2_ceil = 32 - std::countl_zero(d - 1);
|
||||
const uint32_t p = 31 + log2_ceil;
|
||||
m_magic = static_cast<uint32_t>(((uint64_t{1} << p) + d - 1) / d);
|
||||
m_shift = p - 32;
|
||||
} else {
|
||||
m_magic = 0;
|
||||
m_shift = 0;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
SGL_DEVICE friend uint32_t operator/(T n, const fast_mod_div_u32_t& d) {
|
||||
static_assert(std::is_unsigned_v<T> && sizeof(T) <= sizeof(uint32_t));
|
||||
return d.m_divisor == 1 ? n : __umulhi(n, d.m_magic) >> d.m_shift;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
SGL_DEVICE friend uint32_t operator%(T n, const fast_mod_div_u32_t& d) {
|
||||
static_assert(std::is_unsigned_v<T> && sizeof(T) <= sizeof(uint32_t));
|
||||
return n - (n / d) * d.m_divisor;
|
||||
}
|
||||
|
||||
constexpr operator uint32_t() const {
|
||||
return m_divisor;
|
||||
}
|
||||
|
||||
private:
|
||||
uint32_t m_divisor;
|
||||
uint32_t m_magic;
|
||||
uint32_t m_shift;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
SGL_DEVICE uint2 div(T n, const fast_mod_div_u32_t& d) {
|
||||
return {n / d, n % d};
|
||||
}
|
||||
|
||||
} // namespace device::math
|
||||
|
||||
} // namespace sglang
|
||||
|
||||
@@ -273,6 +273,67 @@ def store_fp4_index_k_cache(
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _gather_fp4_index_k_kernel(
|
||||
cache,
|
||||
cache_stride,
|
||||
slots,
|
||||
payload,
|
||||
scales,
|
||||
n,
|
||||
page_size: tl.constexpr,
|
||||
BLOCK_ROWS: tl.constexpr,
|
||||
):
|
||||
# int32 view of a page: page_size * 16 payload words, then page_size scale words.
|
||||
rows = tl.program_id(0) * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS)
|
||||
valid = rows < n
|
||||
slot = tl.load(slots + rows, valid, 0).to(tl.int64)
|
||||
page = slot // page_size
|
||||
offset = slot - page * page_size
|
||||
page_base = cache + page * cache_stride
|
||||
words = tl.arange(0, 16)
|
||||
payload_words = tl.load(
|
||||
page_base[:, None] + offset[:, None] * 16 + words[None, :], valid[:, None]
|
||||
)
|
||||
tl.store(
|
||||
payload + rows[:, None].to(tl.int64) * 16 + words[None, :],
|
||||
payload_words,
|
||||
valid[:, None],
|
||||
)
|
||||
tl.store(scales + rows, tl.load(page_base + page_size * 16 + offset, valid), valid)
|
||||
|
||||
|
||||
def gather_fp4_index_k(
|
||||
cache: torch.Tensor, slots: torch.Tensor, *, page_size: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Packed fp4 rows at ``slots`` of the index-K ``cache`` (``[pages, page_size *
|
||||
68]`` uint8, ``store_fp4_index_k_cache``'s layout): (payload int8 ``[n, 64]``,
|
||||
ue8m0 scales packed int32 ``[n]``), the layout ``quantize_fp4_indexer_tensor``
|
||||
returns."""
|
||||
assert cache.shape[1] == page_size * INDEX_K_SLOT_BYTES and cache.stride(1) == 1
|
||||
# The kernel addresses slots as `slots + rows`, so a strided view would be read
|
||||
# as if its elements were packed.
|
||||
assert slots.is_contiguous()
|
||||
words = cache.view(torch.int32)
|
||||
n = slots.shape[0]
|
||||
payload = torch.empty((n, 16), device=cache.device, dtype=torch.int32)
|
||||
scales = torch.empty((n,), device=cache.device, dtype=torch.int32)
|
||||
if n > 0:
|
||||
block_rows = 64
|
||||
_gather_fp4_index_k_kernel[(triton.cdiv(n, block_rows),)](
|
||||
words,
|
||||
words.stride(0),
|
||||
slots,
|
||||
payload,
|
||||
scales,
|
||||
n,
|
||||
page_size,
|
||||
BLOCK_ROWS=block_rows,
|
||||
num_warps=4,
|
||||
)
|
||||
return payload.view(torch.int8), scales
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _index_k_rope_pack_kernel(
|
||||
X,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Fused ratio-1 and ratio-2 decode compressors: RMSNorm, RoPE and the FlashMLA
|
||||
cache write in one launch.
|
||||
"""Fused ratio-1 and ratio-2 compressors: RMSNorm, RoPE and the FlashMLA cache
|
||||
write in one launch. The ratio-1 kernel is per token, so decode and extend share
|
||||
it; ratio 2 has a decode/verify entry point and an extend one.
|
||||
|
||||
Ratio 1 takes the bf16 ``wkv`` projection as is: RoPE uses the token's own
|
||||
position and the compressed slot equals the FULL slot. Ratio 2 pair-pools the
|
||||
@@ -17,6 +18,7 @@ import torch
|
||||
from sglang.kernels.jit.utils import (
|
||||
cache_once,
|
||||
is_arch_support_pdl,
|
||||
is_hip_runtime,
|
||||
load_jit,
|
||||
make_cpp_args,
|
||||
)
|
||||
@@ -119,11 +121,12 @@ def _jit_c2_module(
|
||||
is_arch_support_pdl(),
|
||||
)
|
||||
return load_jit(
|
||||
make_name("c2_decode"),
|
||||
make_name("c2"),
|
||||
*args,
|
||||
cuda_files=["deepseek_v4/c2.cuh"],
|
||||
cuda_wrappers=[
|
||||
("decode_fusion", f"FlashCompress2Kernel<{args}>::run_decode_fusion"),
|
||||
("prefill_fusion", f"FlashCompress2Kernel<{args}>::run_prefill_fusion"),
|
||||
],
|
||||
)
|
||||
|
||||
@@ -149,6 +152,12 @@ def c2_decode_norm_rope_store(
|
||||
|
||||
The cache store uses ``raw_out_loc // 2`` as its slot.
|
||||
|
||||
A padded graph row is marked by ``raw_out_loc == 0`` and writes nothing, but
|
||||
decode and verify rows issue their ring load before that check, so a padded
|
||||
row's ``req`` and ``positions`` must still address an in-bounds ring slot.
|
||||
The decode graph buffers zero both tails (``PaddingPolicy.ZERO``); a runner
|
||||
that left stale request indices there would read out of the ring pool.
|
||||
|
||||
:param freqs_cis: ``[max_pos, rope_dim]`` fp32, real/imag interleaved --
|
||||
``torch.view_as_real(freqs).flatten(-2)``. Indexed
|
||||
in-kernel at ``positions - 1``, the position the latent
|
||||
@@ -181,3 +190,60 @@ def c2_decode_norm_rope_store(
|
||||
draft_len,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def c2_prefill_norm_rope_store(
|
||||
kv_input: torch.Tensor,
|
||||
kv_state: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
req: torch.Tensor,
|
||||
raw_out_loc: torch.Tensor,
|
||||
eps: float,
|
||||
freqs_cis: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
extend_start_loc: torch.Tensor,
|
||||
extend_seq_lens: torch.Tensor,
|
||||
*,
|
||||
page_size: int,
|
||||
ring_size: int,
|
||||
layout: Union[KVLayout, str] = KVLayout.V4,
|
||||
out: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
""":func:`c2_decode_norm_rope_store` for an extend batch, then the ring write-back.
|
||||
|
||||
Rows may mix requests, prefix offsets and CUDA graph padding (``raw_out_loc
|
||||
== 0``). An odd position pairs with the previous row when that row is the
|
||||
same request's previous position, else with its ring slot. Afterwards each
|
||||
request's last ``ring_size`` rows are saved to the ring, which a later step
|
||||
reads for its first partner.
|
||||
|
||||
:param extend_start_loc: int32 ``[num_reqs]``, each request's first row.
|
||||
:param extend_seq_lens: int32 ``[num_reqs]``, each request's row count.
|
||||
:return: ``out``, ``[num_tokens, head_dim]`` bf16. Only the rows of odd
|
||||
positions are written; the rest are left as they were.
|
||||
"""
|
||||
num_tokens, fused_dim = kv_input.shape
|
||||
head_dim = fused_dim // 2
|
||||
if out is None:
|
||||
out = kv_input.new_empty((num_tokens, head_dim), dtype=torch.bfloat16)
|
||||
|
||||
layout = KVLayout.parse(layout)
|
||||
module = _jit_c2_module(head_dim, freqs_cis.shape[-1], page_size, layout)
|
||||
module.prefill_fusion(
|
||||
kv_input,
|
||||
kv_state,
|
||||
out,
|
||||
norm_weight,
|
||||
positions,
|
||||
req,
|
||||
raw_out_loc,
|
||||
eps,
|
||||
freqs_cis,
|
||||
# HIP's fp8_e4m3_t is uint8_t, so the kernel matches the fp8 pool as bytes
|
||||
k_cache.view(torch.uint8) if is_hip_runtime() else k_cache,
|
||||
ring_size,
|
||||
extend_start_loc,
|
||||
extend_seq_lens,
|
||||
)
|
||||
return out
|
||||
|
||||
@@ -36,7 +36,7 @@ def _build_swa_token_ids_kernel(
|
||||
tl.store(out_ptr + out_off + i, swa_id)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["top_k"])
|
||||
@triton.jit(do_not_specialize=["num_tokens", "num_reqs", "top_k"])
|
||||
def _combine_topk_swa_indices_kernel(
|
||||
combined_indices_ptr,
|
||||
combined_indices_stride,
|
||||
@@ -49,59 +49,76 @@ def _combine_topk_swa_indices_kernel(
|
||||
gather_lens_ptr,
|
||||
compressed_base_ptr,
|
||||
swa_base_ptr,
|
||||
num_tokens,
|
||||
num_reqs,
|
||||
top_k,
|
||||
COMPRESS_RATIO: tl.constexpr,
|
||||
WINDOW_SIZE: tl.constexpr,
|
||||
PADDED_TOP_K: tl.constexpr,
|
||||
WIDTH: tl.constexpr,
|
||||
PADDED_WIDTH: tl.constexpr,
|
||||
BLOCK_T: tl.constexpr,
|
||||
SEARCH_STEPS: tl.constexpr,
|
||||
):
|
||||
batch_idx = tl.program_id(0)
|
||||
worker_id = tl.program_id(1)
|
||||
num_workers = tl.num_programs(1)
|
||||
# BLOCK_T consecutive tokens per program, so the grid scales with tokens, not requests.
|
||||
token = tl.program_id(0) * BLOCK_T + tl.arange(0, BLOCK_T)
|
||||
valid = token < num_tokens
|
||||
|
||||
# query_start_loc may be a global tensor; rebase to chunk-local offsets
|
||||
# by subtracting the chunk's starting value.
|
||||
base = tl.load(query_start_loc_ptr)
|
||||
query_start = tl.load(query_start_loc_ptr + batch_idx) - base
|
||||
query_end = tl.load(query_start_loc_ptr + batch_idx + 1) - base
|
||||
seq_len = tl.load(seq_lens_ptr + batch_idx)
|
||||
gather_len = tl.load(gather_lens_ptr + batch_idx)
|
||||
compressed_base = tl.load(compressed_base_ptr + batch_idx)
|
||||
swa_base = tl.load(swa_base_ptr + batch_idx)
|
||||
# Each token's request: the last r with query_start_loc[r] - base <= token.
|
||||
lo = tl.zeros([BLOCK_T], dtype=tl.int32)
|
||||
hi = tl.full([BLOCK_T], 0, dtype=tl.int32) + num_reqs
|
||||
for _ in tl.static_range(SEARCH_STEPS):
|
||||
mid = (lo + hi + 1) // 2
|
||||
start = tl.load(query_start_loc_ptr + mid, mask=lo < hi, other=0) - base
|
||||
go_right = (lo < hi) & (start <= token)
|
||||
lo = tl.where(go_right, mid, lo)
|
||||
hi = tl.where((lo < hi) & ~go_right, mid - 1, hi)
|
||||
batch_idx = tl.minimum(lo, num_reqs - 1)
|
||||
# Rows past the last request's queries belong to no request: -1 rows, length 0.
|
||||
owned = valid & (token < tl.load(query_start_loc_ptr + num_reqs) - base)
|
||||
|
||||
pos = tl.load(query_pos_ptr + token, mask=owned, other=0)
|
||||
seq_len = tl.load(seq_lens_ptr + batch_idx, mask=owned, other=0)
|
||||
gather_len = tl.load(gather_lens_ptr + batch_idx, mask=owned, other=0)
|
||||
compressed_base = tl.load(compressed_base_ptr + batch_idx, mask=owned, other=0)
|
||||
swa_base = tl.load(swa_base_ptr + batch_idx, mask=owned, other=0)
|
||||
# SWA portion of the gathered buffer starts from position
|
||||
# (seq_len - gather_len), not 0. The +pos-gather_start formula maps a
|
||||
# query's window back into the workspace's SWA region.
|
||||
gather_start = seq_len - gather_len
|
||||
|
||||
for token_idx in range(query_start + worker_id, query_end, num_workers):
|
||||
pos = tl.load(query_pos_ptr + token_idx)
|
||||
# -1 entries inside the top-k span stay -1 (attention skips them).
|
||||
# top_k=0 disables the compressed portion for SWA-only layers.
|
||||
topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, top_k)
|
||||
swa_len = tl.minimum(pos + 1, WINDOW_SIZE)
|
||||
# -1 entries inside the top-k span stay -1 (attention skips them).
|
||||
# top_k=0 disables the compressed portion for SWA-only layers.
|
||||
topk_len = tl.where(owned, tl.minimum((pos + 1) // COMPRESS_RATIO, top_k), 0)
|
||||
swa_len = tl.where(owned, tl.minimum(pos + 1, WINDOW_SIZE), 0)
|
||||
|
||||
combined_row = token_idx.to(tl.int64) * combined_indices_stride
|
||||
topk_row = token_idx.to(tl.int64) * topk_indices_stride
|
||||
combined_row = token.to(tl.int64)[:, None] * combined_indices_stride
|
||||
topk_row = token.to(tl.int64)[:, None] * topk_indices_stride
|
||||
|
||||
offset = tl.arange(0, PADDED_TOP_K)
|
||||
mask = offset < topk_len
|
||||
topk_vals = tl.load(
|
||||
topk_indices_ptr + topk_row + offset,
|
||||
mask=mask,
|
||||
)
|
||||
tl.store(
|
||||
combined_indices_ptr + combined_row + offset,
|
||||
tl.where(topk_vals >= 0, topk_vals + compressed_base, -1),
|
||||
mask=mask,
|
||||
)
|
||||
# Whole rows, -1 padded, so the output needs no fill pass:
|
||||
# [top-k (-1 holes kept) | window | -1 up to WIDTH].
|
||||
offset = tl.arange(0, PADDED_WIDTH)[None, :]
|
||||
in_topk = offset < topk_len[:, None]
|
||||
topk_vals = tl.load(
|
||||
topk_indices_ptr + topk_row + offset, mask=valid[:, None] & in_topk, other=-1
|
||||
)
|
||||
# Workspace SWA index: swa_base[r] + (gather_offset_in_buffer).
|
||||
# For positions [pos - swa_len + 1, pos], the buffer offsets are
|
||||
# [pos - swa_len + 1 - gather_start, pos - gather_start].
|
||||
swa_start = swa_base + pos - swa_len + 1 - gather_start
|
||||
swa_vals = swa_start[:, None] + (offset - topk_len[:, None])
|
||||
in_swa = offset < (topk_len + swa_len)[:, None]
|
||||
vals = tl.where(
|
||||
in_topk,
|
||||
tl.where(topk_vals >= 0, topk_vals + compressed_base[:, None], -1),
|
||||
tl.where(in_swa, swa_vals, -1),
|
||||
)
|
||||
tl.store(
|
||||
combined_indices_ptr + combined_row + offset,
|
||||
vals,
|
||||
mask=valid[:, None] & (offset < WIDTH),
|
||||
)
|
||||
|
||||
offset = tl.arange(0, WINDOW_SIZE)
|
||||
# Workspace SWA index: swa_base[r] + (gather_offset_in_buffer).
|
||||
# For positions [pos - swa_len + 1, pos], the buffer offsets are
|
||||
# [pos - swa_len + 1 - gather_start, pos - gather_start].
|
||||
tl.store(
|
||||
combined_indices_ptr + combined_row + topk_len + offset,
|
||||
swa_base + offset + pos - swa_len + 1 - gather_start,
|
||||
mask=offset < swa_len,
|
||||
)
|
||||
|
||||
tl.store(combined_lens_ptr + token_idx, topk_len + swa_len)
|
||||
tl.store(combined_lens_ptr + token, topk_len + swa_len, mask=valid)
|
||||
|
||||
@@ -1882,6 +1882,34 @@ def hc_expand(x: torch.Tensor, n: int) -> torch.Tensor:
|
||||
return x.repeat(1, n)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _hc_broadcast_kernel(
|
||||
x_ptr, y_ptr, H, x_stride_m, HC: tl.constexpr, BLOCK_H: tl.constexpr
|
||||
):
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
offs = tl.program_id(1) * BLOCK_H + tl.arange(0, BLOCK_H)
|
||||
mask = offs < H
|
||||
x = tl.load(x_ptr + row * x_stride_m + offs, mask=mask)
|
||||
y_row = y_ptr + row * (HC * H)
|
||||
for k in tl.static_range(HC):
|
||||
tl.store(y_row + k * H + offs, x, mask=mask)
|
||||
|
||||
|
||||
def hc_broadcast(x: torch.Tensor, hc_mult: int) -> torch.Tensor:
|
||||
"""[T, H] -> [T, hc_mult, H], every copy equal to x: the initial mHC residual."""
|
||||
if not x.is_cuda or x.stride(-1) != 1:
|
||||
return x.unsqueeze(1).repeat(1, hc_mult, 1)
|
||||
num_tokens, hidden = x.shape
|
||||
y = x.new_empty(num_tokens, hc_mult, hidden)
|
||||
if num_tokens == 0:
|
||||
return y
|
||||
block_h = 1024
|
||||
_hc_broadcast_kernel[(num_tokens, triton.cdiv(hidden, block_h))](
|
||||
x, y, hidden, x.stride(0), HC=hc_mult, BLOCK_H=block_h, num_warps=4
|
||||
)
|
||||
return y
|
||||
|
||||
|
||||
def hc_contract(x: torch.Tensor, n: int) -> torch.Tensor:
|
||||
return x.unflatten(-1, (n, -1)).mean(dim=-2)
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ class TransferKVChunk:
|
||||
# Set when the staging worker first counts this chunk toward the per-room
|
||||
# outstanding count; stays set across re-enqueue on a watermark defer.
|
||||
staging_counted: bool = False
|
||||
# Mori early-send: CUDA event to synchronize before RDMA (optional).
|
||||
# Early-send: GPU event to synchronize before reading KV (optional).
|
||||
wait_event: Optional[object] = None
|
||||
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ from sglang.srt.managers.overlap_utils import RelayPayload
|
||||
from sglang.srt.mem_cache.common import maybe_cache_unfinished_req
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo
|
||||
from sglang.srt.utils.common import is_pin_memory_available
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -140,9 +141,14 @@ class ScheduleBatchDisaggregationDecodeMixin:
|
||||
error_message, HTTPStatus.INTERNAL_SERVER_ERROR
|
||||
)
|
||||
req.grammar.finished = req.finished()
|
||||
# Non-blocking H2D: with overlap this runs on the schedule stream after
|
||||
# its wait on the in-flight forward, so a blocking copy would stall the
|
||||
# host until that forward ends and leave the GPU idle.
|
||||
last_tokens_tensor = torch.tensor(
|
||||
last_tokens, dtype=torch.int64, device=self.device
|
||||
)
|
||||
last_tokens,
|
||||
dtype=torch.int64,
|
||||
pin_memory=is_pin_memory_available(self.device),
|
||||
).to(self.device, non_blocking=True)
|
||||
|
||||
spec_info = self.spec_algorithm.build_disagg_draft_input(
|
||||
self,
|
||||
|
||||
@@ -960,6 +960,8 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
dst_layer_ids: Optional[List[int]] = None,
|
||||
dst_device_data_indices: Optional[npt.NDArray[np.int32]] = None,
|
||||
dst_device_data_ptrs: Optional[set[int]] = None,
|
||||
dst_item_lens: list[int] | None = None,
|
||||
bootstrap_room: Optional[int] = None,
|
||||
) -> int:
|
||||
"""
|
||||
Generic KV cache transfer supporting both MHA and MLA architectures.
|
||||
@@ -1011,6 +1013,21 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
len(dst_data_ptrs),
|
||||
allow_positional_fallback=self.pp_size == 1,
|
||||
)
|
||||
if dst_item_lens is not None:
|
||||
for i, j in pairs:
|
||||
if item_lens[i] != dst_item_lens[j]:
|
||||
assert bootstrap_room is not None
|
||||
failure_reason = (
|
||||
f"{state_type} item length mismatch for paired "
|
||||
f"entries src[{i}]={item_lens[i]} "
|
||||
f"dst[{j}]={dst_item_lens[j]}"
|
||||
)
|
||||
logger.error(failure_reason)
|
||||
self.conclude_failure(
|
||||
bootstrap_room=bootstrap_room,
|
||||
failure_reason=failure_reason,
|
||||
)
|
||||
return -1
|
||||
layers_params = [
|
||||
(src_data_ptrs[i], dst_data_ptrs[j], item_lens[i]) for i, j in pairs
|
||||
]
|
||||
@@ -1020,6 +1037,24 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
src_data_ptrs, dst_data_ptrs, state_type
|
||||
)
|
||||
)
|
||||
if dst_item_lens is not None:
|
||||
_, mapped_dst_lens, _ = self.get_mla_kv_ptrs_with_pp(
|
||||
item_lens, dst_item_lens, state_type
|
||||
)
|
||||
for layer_id in range(layers_current_pp_stage):
|
||||
if item_lens[layer_id] != mapped_dst_lens[layer_id]:
|
||||
assert bootstrap_room is not None
|
||||
failure_reason = (
|
||||
f"{state_type} item length mismatch for positional "
|
||||
f"entry {layer_id}: prefill={item_lens[layer_id]} "
|
||||
f"decode={mapped_dst_lens[layer_id]}"
|
||||
)
|
||||
logger.error(failure_reason)
|
||||
self.conclude_failure(
|
||||
bootstrap_room=bootstrap_room,
|
||||
failure_reason=failure_reason,
|
||||
)
|
||||
return -1
|
||||
layers_params = [
|
||||
(
|
||||
src_kv_ptrs[layer_id],
|
||||
@@ -2110,9 +2145,13 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
),
|
||||
src_layer_ids=src_state_layer_ids,
|
||||
dst_layer_ids=dst_state_layer_ids,
|
||||
dst_item_lens=dst_item_lens,
|
||||
bootstrap_room=req.room,
|
||||
)
|
||||
or rc
|
||||
)
|
||||
if self.request_status.get(req.room, KVPoll.Failed) == KVPoll.Failed:
|
||||
return rc
|
||||
elif st in (StateType.MINIMAX_INDEX_K, StateType.MINIMAX_DENSE_KV):
|
||||
# Compacted layer lists require equal TP and PP=1 on both peers.
|
||||
if self.pp_size is not None and self.pp_size > 1:
|
||||
@@ -2376,6 +2415,19 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
self._staging_outstanding[kv_chunk.room] += 1
|
||||
kv_chunk.staging_counted = True
|
||||
|
||||
# Under overlap scheduling an early cached-prefix send can
|
||||
# precede completion of the previous prefill forward. Wait on
|
||||
# the worker, not the scheduler, before reading source KV.
|
||||
# A room that already failed skips the wait; the status check
|
||||
# below then drops the chunk, including on an abort during it.
|
||||
if kv_chunk.wait_event is not None:
|
||||
if (
|
||||
kv_chunk.room in self.request_status
|
||||
and self.check_status(kv_chunk.room) != KVPoll.Failed
|
||||
):
|
||||
kv_chunk.wait_event.synchronize()
|
||||
kv_chunk.wait_event = None
|
||||
|
||||
if (
|
||||
kv_chunk.room not in self.request_status
|
||||
or self.check_status(kv_chunk.room) == KVPoll.Failed
|
||||
@@ -2588,6 +2640,13 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
executor,
|
||||
target_rank_registration_info,
|
||||
)
|
||||
if (
|
||||
self.request_status.get(
|
||||
kv_chunk.room, KVPoll.Failed
|
||||
)
|
||||
== KVPoll.Failed
|
||||
):
|
||||
break
|
||||
if state_rc != 0:
|
||||
with self.session_lock:
|
||||
self.session_failures[
|
||||
@@ -2919,6 +2978,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
state_indices: Optional[List] = None,
|
||||
num_kv_tokens: Optional[int] = None,
|
||||
trace_ctx: Optional[Union[TraceReqContext, TraceNullContext]] = None,
|
||||
wait_event: Optional[object] = None,
|
||||
):
|
||||
assert self.disaggregation_mode == DisaggregationMode.PREFILL
|
||||
assert not is_last_chunk or (is_last_chunk and aux_index is not None)
|
||||
@@ -2958,6 +3018,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
state_indices=state_indices,
|
||||
num_kv_tokens=num_kv_tokens,
|
||||
trace_ctx=trace_ctx,
|
||||
wait_event=wait_event,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -3064,6 +3125,8 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
|
||||
if should_skip:
|
||||
return
|
||||
|
||||
wait_event = getattr(self, "_early_send_wait_event", None)
|
||||
self._early_send_wait_event = None
|
||||
if not is_last_chunk:
|
||||
self.kv_mgr.add_transfer_request(
|
||||
self.bootstrap_room,
|
||||
@@ -3072,6 +3135,7 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
|
||||
False,
|
||||
num_kv_tokens=num_kv_tokens,
|
||||
trace_ctx=self.trace_ctx.copy_for_thread(),
|
||||
wait_event=wait_event,
|
||||
)
|
||||
else:
|
||||
self.kv_mgr.add_transfer_request(
|
||||
@@ -3083,6 +3147,7 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
|
||||
state_indices=state_indices,
|
||||
num_kv_tokens=num_kv_tokens,
|
||||
trace_ctx=self.trace_ctx.copy_for_thread(),
|
||||
wait_event=wait_event,
|
||||
)
|
||||
self._record_transfer_indices(kv_indices, state_indices)
|
||||
|
||||
|
||||
@@ -851,7 +851,11 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
advance_logprob_pt(i, req)
|
||||
continue
|
||||
|
||||
maybe_cache_unfinished_req(req, self.tree_cache)
|
||||
self.checkpoint_disagg_prefill(req)
|
||||
# G9: the disagg-prefill path bypasses batch_result_processor's
|
||||
# prefill loop, so record the drain event here — final chunk
|
||||
# done, request heading to the inflight (KV transfer) queue.
|
||||
self.record_prefill_drain(req.seqlen)
|
||||
self.disagg_prefill_inflight_queue.append(req)
|
||||
if self.spec_algorithm.is_eagle() and draft_input is not None:
|
||||
req.output_topk_p = draft_input.topk_p[i]
|
||||
@@ -1208,7 +1212,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
chunked_req_to_exclude = set()
|
||||
if (req := self.chunked_req) is not None:
|
||||
chunked_req_to_exclude.add(req)
|
||||
maybe_cache_unfinished_req(req, self.tree_cache, chunked=True)
|
||||
self.checkpoint_disagg_prefill(req, chunked=True)
|
||||
|
||||
if not self.check_bootstrap(req):
|
||||
if is_aborted(req):
|
||||
@@ -1503,10 +1507,21 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
else:
|
||||
self.disagg_prefill_pending_chunk_rids.add(req.rid)
|
||||
|
||||
def checkpoint_disagg_prefill(
|
||||
self: Scheduler, req: Req, chunked: bool = False
|
||||
) -> None:
|
||||
"""Publish the request's computed prefix to the tree cache.
|
||||
|
||||
Name shim for the #42035-era call structure: this branch predates
|
||||
#41520's ``checkpoint_kv_cache`` / ``insert_req`` cache API, so the
|
||||
equivalent here is ``maybe_cache_unfinished_req``.
|
||||
"""
|
||||
maybe_cache_unfinished_req(req, self.tree_cache, chunked=chunked)
|
||||
|
||||
def optimistic_release_and_requeue(self: Scheduler, req: Req) -> None:
|
||||
"""Release KV cache and requeue an optimistic prefill request."""
|
||||
max_attempts = get_disagg().optimistic_prefill_attempts
|
||||
maybe_cache_unfinished_req(req, self.tree_cache)
|
||||
self.checkpoint_disagg_prefill(req)
|
||||
# The cached prefix is evictable once the KV is released. Its length
|
||||
# (capped at what a retry can match) seeds the retry's storage baseline,
|
||||
# so an evicted prefix is looked up in L3 once before it is recomputed.
|
||||
@@ -1519,10 +1534,11 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
)
|
||||
)
|
||||
self._release_aborted_request(req)
|
||||
# Mamba insertion donates the checkpoint and clears its sequence marker.
|
||||
release_kv_cache(
|
||||
req, self.tree_cache, is_insert=not self.tree_cache.supports_mamba()
|
||||
)
|
||||
# The checkpoint above (maybe_cache_unfinished_req) already handed the
|
||||
# prefill KV to the tree; the request is not finished, so the release
|
||||
# only frees the rest. (#42194 semantics; our tree predates #41520's
|
||||
# mamba-donation rework, mamba models are not run on this branch.)
|
||||
release_kv_cache(req, self.tree_cache, is_insert=False)
|
||||
req.reset_for_retract()
|
||||
req.output_ids = array("q")
|
||||
req.start_send_idx = 0
|
||||
|
||||
@@ -1776,6 +1776,18 @@ async def openai_v1_chat_completions(
|
||||
)
|
||||
|
||||
|
||||
@app.post("/query_cache_hit", dependencies=[Depends(validate_json_request)])
|
||||
async def query_cache_hit(request: ChatCompletionRequest, raw_request: Request):
|
||||
"""G9: probe prefix-cache hit (L1/L2 + L3 storage) for a chat request.
|
||||
|
||||
Deployed on decode engines; the PD router calls it before dispatch to
|
||||
decide conditional bypass. Accepts a normal chat body or `input_ids`.
|
||||
"""
|
||||
return await raw_request.app.state.openai_serving_chat.query_cache_hit(
|
||||
request, raw_request
|
||||
)
|
||||
|
||||
|
||||
@app.post(
|
||||
"/v1/embeddings",
|
||||
response_class=ORJSONResponse,
|
||||
|
||||
@@ -1140,6 +1140,73 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
return "text", processed_messages.prompt_ids
|
||||
return "input_ids", processed_messages.prompt_ids
|
||||
|
||||
async def query_cache_hit(
|
||||
self, request: ChatCompletionRequest, raw_request: Request = None
|
||||
) -> ORJSONResponse:
|
||||
"""G9 probe: report how much of this chat request's prompt is already
|
||||
cached (per-rank-best L1/L2 radix + shared L3 storage) without running
|
||||
any prefill. The router uses it for conditional PD-bypass decisions.
|
||||
|
||||
Multimodal requests are rejected (v1): their sentinel-token expansion
|
||||
costs ~10-100ms per image for only ~2% of traffic; the router skips
|
||||
them for bypass anyway.
|
||||
"""
|
||||
try:
|
||||
internal_req, _ = self._convert_to_internal_request(request, raw_request)
|
||||
except Exception as e:
|
||||
return ORJSONResponse(
|
||||
status_code=400,
|
||||
content={"error": f"failed to convert chat request: {e}"},
|
||||
)
|
||||
if internal_req.contains_mm_input():
|
||||
return ORJSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "multimodal requests are not supported by /query_cache_hit"},
|
||||
)
|
||||
try:
|
||||
# 只取 input_ids,不走 _tokenize_one_request——它要求 rid_to_state
|
||||
# 里已有 ReqState(正常生成路径在 tokenize 前注册),纯查询没有。
|
||||
# dsv41 系 chat encoding 直接渲染 prompt_ids;否则对渲染后的
|
||||
# text 走 tokenizer(_tokenize_texts 与真实路径同一入口)。
|
||||
if internal_req.input_ids is not None:
|
||||
input_ids = list(internal_req.input_ids)
|
||||
elif internal_req.text:
|
||||
input_ids, _ = await self.tokenizer_manager._tokenize_texts(
|
||||
internal_req.text, False
|
||||
)
|
||||
else:
|
||||
return ORJSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "request produced neither input_ids nor text"},
|
||||
)
|
||||
from sglang.srt.managers.io_struct import QueryCacheHitReqInput
|
||||
|
||||
result = await self.tokenizer_manager.query_cache_hit(
|
||||
QueryCacheHitReqInput(
|
||||
input_ids=list(input_ids),
|
||||
extra_key=getattr(internal_req, "extra_key", None),
|
||||
cache_salt=getattr(internal_req, "cache_salt", None),
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
return ORJSONResponse(
|
||||
status_code=500, content={"error": f"cache-hit query failed: {e}"}
|
||||
)
|
||||
if result.error is not None:
|
||||
return ORJSONResponse(status_code=500, content={"error": result.error})
|
||||
hit = result.l1_hit_tokens + result.l2_hit_tokens + result.l3_hit_tokens
|
||||
return ORJSONResponse(
|
||||
content={
|
||||
"total_tokens": result.total_tokens,
|
||||
"l1_hit_tokens": result.l1_hit_tokens,
|
||||
"l2_hit_tokens": result.l2_hit_tokens,
|
||||
"l3_hit_tokens": result.l3_hit_tokens,
|
||||
"hit_tokens": hit,
|
||||
"new_tokens": max(0, result.total_tokens - hit),
|
||||
"dp_rank": result.dp_rank,
|
||||
}
|
||||
)
|
||||
|
||||
def _convert_to_internal_request(
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
|
||||
@@ -758,6 +758,16 @@ class Envs:
|
||||
SGLANG_HICACHE_HF3FS_CONFIG_PATH = EnvStr(None)
|
||||
SGLANG_HICACHE_DECODE_OFFLOAD_STRIDE = EnvInt(None)
|
||||
SGLANG_HICACHE_SKIP_HOST_DUPLICATE_RECLAIM = EnvBool(False)
|
||||
# Per-stage timing logs for the L3 storage prefetch path (query / transfer /
|
||||
# ack reduce / drain / graft / load-back). Emits "HC_TIMING ..." lines.
|
||||
SGLANG_HICACHE_TIMING = EnvBool(False)
|
||||
# Pages per batched storage IO call (query / get / put chunking).
|
||||
SGLANG_HICACHE_STORAGE_BATCH_SIZE = EnvInt(128)
|
||||
# Number of parallel prefetch IO lanes (worker threads) per controller.
|
||||
# Operations are sharded by crc32(request_id) so every rank maps a request
|
||||
# to the same lane; each lane owns dedicated completion sync groups.
|
||||
# MUST be identical across all TP/CP/PP ranks of an instance.
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS = EnvInt(4)
|
||||
SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR = EnvStr(None)
|
||||
# File-backend LRU eviction (opt-in; sizes accept SI/IEC suffixes, "0" disables).
|
||||
SGLANG_HICACHE_FILE_BACKEND_MAX_SIZE = EnvStr(None)
|
||||
|
||||
@@ -206,6 +206,7 @@ def _create_flashmla_metadata():
|
||||
# only for that shape and go stale silently if FlashMLA retunes it.
|
||||
_FLASHMLA_SCHED_BLOCK_SIZE_N = 64
|
||||
_FLASHMLA_SCHED_FIXED_OVERHEAD = 5
|
||||
_FLASHMLA_SCHED_MAX_SHARED_BYTES = 48 << 10
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=None)
|
||||
@@ -242,6 +243,10 @@ def _maybe_precompute_flashmla_sched_meta(
|
||||
|
||||
b, s_q = q.shape[0], q.shape[1]
|
||||
num_sm_parts = max(_num_sms(q.device.index) // s_q, 1)
|
||||
# Skip when shared memory exceeds the limit.
|
||||
smem_bytes = 4 * (b * 5 + 1 + num_sm_parts * 8)
|
||||
if smem_bytes > _FLASHMLA_SCHED_MAX_SHARED_BYTES:
|
||||
return
|
||||
meta = torch.empty((num_sm_parts, META_INTS), dtype=torch.int32, device=q.device)
|
||||
num_splits = torch.empty((b + 1,), dtype=torch.int32, device=q.device)
|
||||
flashmla_sched_meta(
|
||||
@@ -2543,7 +2548,7 @@ class DeepseekV4AttnBackend(
|
||||
req_to_token=self.req_to_token,
|
||||
full_to_swa=self.token_to_kv_pool.full_to_swa_index_mapping,
|
||||
swa_window_size=SWA_WINDOW,
|
||||
swa_page_size=self.token_to_kv_pool.swa_kv_pool.page_size,
|
||||
swa_page_size=self.token_to_kv_pool.get_swa_key_page_size(),
|
||||
num_qo_tokens=num_qo_tokens,
|
||||
max_seq_len=max(seq_lens_cpu_list),
|
||||
total_swa=total_swa,
|
||||
@@ -2930,6 +2935,22 @@ class DeepseekV4AttnBackend(
|
||||
self._low_ratio_compress_fused(
|
||||
layer, x, req, pos, draft_len=self.speculative_num_draft_tokens
|
||||
)
|
||||
elif (
|
||||
layer.compressor.use_fused_compress
|
||||
and forward_batch.forward_mode.is_extend_without_speculative()
|
||||
and forward_batch.extend_start_loc is not None
|
||||
):
|
||||
# DP padding extends extend_seq_lens but not extend_start_loc, whose
|
||||
# length is the request count; padded rows belong to no request.
|
||||
start_loc = forward_batch.extend_start_loc
|
||||
seq_lens = forward_batch.extend_seq_lens[: start_loc.shape[0]]
|
||||
self._low_ratio_compress_fused(
|
||||
layer,
|
||||
x,
|
||||
req,
|
||||
pos,
|
||||
extend_offsets=(start_loc.to(torch.int32), seq_lens.to(torch.int32)),
|
||||
)
|
||||
else:
|
||||
self._low_ratio_compress_torch(
|
||||
layer,
|
||||
@@ -3001,13 +3022,18 @@ class DeepseekV4AttnBackend(
|
||||
fuse_index_store=_is_sm100_or_newer(),
|
||||
)
|
||||
|
||||
def _low_ratio_compress_fused(self, layer, x, req, pos, *, draft_len=1) -> None:
|
||||
def _low_ratio_compress_fused(
|
||||
self, layer, x, req, pos, *, draft_len=1, extend_offsets=None
|
||||
) -> None:
|
||||
"""Decode or target-verify; an extend batch passes ``extend_offsets``, the int32
|
||||
(first row, row count) of each request."""
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer_rope import (
|
||||
index_k_norm_rope_pack_store,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.low_ratio_compress import (
|
||||
c1_decode_norm_rope_store,
|
||||
c2_decode_norm_rope_store,
|
||||
c2_prefill_norm_rope_store,
|
||||
)
|
||||
|
||||
pool = self.token_to_kv_pool
|
||||
@@ -3040,7 +3066,7 @@ class DeepseekV4AttnBackend(
|
||||
# CompressStatePool stores each request's pending pairs in a position ring.
|
||||
# KVAndScore rows use | kv | score |, addressed as req * ring_size + pos % ring_size.
|
||||
state = pool.get_attention_compress_states(layer_id)
|
||||
latent = c2_decode_norm_rope_store(
|
||||
args = (
|
||||
compressor.project_fused(x),
|
||||
state.kv_score_buffer.kv_score,
|
||||
compressor.norm.weight.data,
|
||||
@@ -3050,11 +3076,23 @@ class DeepseekV4AttnBackend(
|
||||
compressor.norm.eps,
|
||||
freqs_cis,
|
||||
kv_cache,
|
||||
page_size=page_size,
|
||||
ring_size=state.ring_size,
|
||||
draft_len=draft_len,
|
||||
layout=kv_layout,
|
||||
)
|
||||
if extend_offsets is None:
|
||||
latent = c2_decode_norm_rope_store(
|
||||
*args,
|
||||
page_size=page_size,
|
||||
ring_size=state.ring_size,
|
||||
draft_len=draft_len,
|
||||
layout=kv_layout,
|
||||
)
|
||||
else:
|
||||
latent = c2_prefill_norm_rope_store(
|
||||
*args,
|
||||
*extend_offsets,
|
||||
page_size=page_size,
|
||||
ring_size=state.ring_size,
|
||||
layout=kv_layout,
|
||||
)
|
||||
out_loc = core.c2_out_loc
|
||||
|
||||
indexer = layer.indexer
|
||||
@@ -3760,7 +3798,7 @@ class DeepseekV4AttnBackend(
|
||||
compress_ratio
|
||||
)
|
||||
|
||||
swa_kv_page_size = token_to_kv_pool.swa_kv_pool.page_size
|
||||
swa_kv_page_size = token_to_kv_pool.get_swa_key_page_size()
|
||||
assert swa_k_cache.ndim == 2
|
||||
# The kernel detects each cache's format from the last dim of this
|
||||
# view: 584 (V4), 528 (V4.1 fp8) or 288 (V4.1 fp4, extra cache only).
|
||||
|
||||
@@ -1472,9 +1472,10 @@ class IndexerKPool(MultiPlatformOp):
|
||||
if metadata is None:
|
||||
return None
|
||||
|
||||
assert forward_batch.seq_lens_cpu is not None
|
||||
# Spec-v2 decode/verify batches may carry no CPU seq_lens mirror (the
|
||||
# backend opts out of the D2H); test emptiness on the device shape.
|
||||
mode = forward_batch.forward_mode
|
||||
if mode.is_idle() or len(forward_batch.seq_lens_cpu) == 0:
|
||||
if mode.is_idle() or forward_batch.seq_lens.shape[0] == 0:
|
||||
return torch.full(
|
||||
(x.shape[0], self.index_topk + self.index_kpool - 1),
|
||||
-1,
|
||||
|
||||
@@ -306,7 +306,8 @@ class DeepseekSparseAttnBackend(
|
||||
extend_dummy_seqs_capped_by_req_pool: bool = True
|
||||
# Decode/verify/draft graph replay rebuilds metadata from static buffers
|
||||
# (page-table width) and never reads seq_lens_cpu / seq_lens_sum; opt out of
|
||||
# the D2H sync. The eager fallback derives lengths from GPU seq_lens.
|
||||
# the D2H sync. The eager fallback derives lengths from GPU seq_lens, and the
|
||||
# KPool indexer reads the host mirror only for extend, which always has it.
|
||||
needs_cpu_seq_lens: bool = False
|
||||
# init_cuda_graph_state sizes this for every backend, but only the TRT-LLM
|
||||
# branch of __init__ allocates one.
|
||||
@@ -337,7 +338,6 @@ class DeepseekSparseAttnBackend(
|
||||
)
|
||||
self.dsa_index_topk = get_dsa_index_topk(hf_config)
|
||||
self.dsa_index_kpool = get_dsa_index_kpool(hf_config)
|
||||
self.needs_cpu_seq_lens = self.dsa_index_kpool > 1
|
||||
self._init_kpool_metadata_fusion()
|
||||
self.max_context_len = model_runner.model_config.context_len
|
||||
self.num_q_heads = (
|
||||
|
||||
@@ -136,8 +136,8 @@ def combine_topk_swa_indices(
|
||||
|
||||
Returns int32 indices [num_tokens, padded_topk_swa] and per-token scanned-prefix
|
||||
lengths, including -1 entries skipped by attention. Width is padded to 128.
|
||||
Preallocated out_indices must contain -1 outside the written prefix;
|
||||
reuse requires chunk-invariant scanned-prefix lengths. out_lens is overwritten.
|
||||
Rows are written whole and -1 padded, so preallocated out_indices / out_lens
|
||||
are overwritten entirely; rows past the last request are -1 with length 0.
|
||||
"""
|
||||
assert topk_indices.dtype == torch.int32
|
||||
assert query_start_loc.dtype == torch.int32
|
||||
@@ -155,9 +155,9 @@ def combine_topk_swa_indices(
|
||||
num_reqs = seq_lens.shape[0]
|
||||
combined_topk = combined_topk_width(topk, window_size)
|
||||
if out_indices is None:
|
||||
combined_indices = torch.full(
|
||||
# The kernel writes whole -1 padded rows.
|
||||
combined_indices = torch.empty(
|
||||
(num_tokens, combined_topk),
|
||||
-1,
|
||||
dtype=torch.int32,
|
||||
device=topk_indices.device,
|
||||
)
|
||||
@@ -166,7 +166,7 @@ def combine_topk_swa_indices(
|
||||
assert out_indices.dtype == torch.int32
|
||||
combined_indices = out_indices
|
||||
if out_lens is None:
|
||||
combined_lens = torch.zeros(
|
||||
combined_lens = torch.empty(
|
||||
num_tokens, dtype=torch.int32, device=topk_indices.device
|
||||
)
|
||||
else:
|
||||
@@ -174,8 +174,10 @@ def combine_topk_swa_indices(
|
||||
assert out_lens.dtype == torch.int32
|
||||
combined_lens = out_lens
|
||||
|
||||
NUM_WORKERS = 128
|
||||
_combine_topk_swa_indices_kernel[(num_reqs, NUM_WORKERS)](
|
||||
if num_tokens == 0:
|
||||
return combined_indices, combined_lens
|
||||
block_t = 2 # Measured on B200 with num_warps=2, 16384 tokens, bs 1 to 64.
|
||||
_combine_topk_swa_indices_kernel[(triton.cdiv(num_tokens, block_t),)](
|
||||
combined_indices,
|
||||
combined_indices.stride(0),
|
||||
combined_lens,
|
||||
@@ -187,10 +189,16 @@ def combine_topk_swa_indices(
|
||||
gather_lens,
|
||||
compressed_base,
|
||||
swa_base,
|
||||
num_tokens,
|
||||
num_reqs,
|
||||
top_k=topk,
|
||||
COMPRESS_RATIO=compress_ratio,
|
||||
WINDOW_SIZE=window_size,
|
||||
PADDED_TOP_K=triton.next_power_of_2(topk_indices.shape[-1]),
|
||||
WIDTH=combined_topk,
|
||||
PADDED_WIDTH=triton.next_power_of_2(combined_topk),
|
||||
BLOCK_T=block_t,
|
||||
SEARCH_STEPS=max(num_reqs, 1).bit_length(),
|
||||
num_warps=2,
|
||||
)
|
||||
return combined_indices, combined_lens
|
||||
|
||||
|
||||
@@ -239,6 +239,11 @@ class EngramHasher(nn.Module):
|
||||
self.image_token_id: Optional[int] = None
|
||||
self.history: Optional[torch.Tensor] = None
|
||||
self.pad_row = 0
|
||||
# typing only
|
||||
self.token_map: torch.Tensor
|
||||
self.multipliers: torch.Tensor
|
||||
self.primes: torch.Tensor
|
||||
self.offsets: torch.Tensor
|
||||
|
||||
def init_history(self, num_req_slots: int, device) -> None:
|
||||
"""Allocate oldest-first history with a spare row for graph padding."""
|
||||
|
||||
@@ -17,6 +17,7 @@ limitations under the License.
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import zlib
|
||||
from dataclasses import dataclass
|
||||
from queue import Empty, Queue
|
||||
from typing import TYPE_CHECKING, Callable, List, NamedTuple, Optional
|
||||
@@ -32,6 +33,17 @@ from sglang.srt.mem_cache.hicache_storage import (
|
||||
count_pool_hits,
|
||||
)
|
||||
|
||||
from sglang.srt.environ import envs as _envs
|
||||
|
||||
_HC_TIMING = None
|
||||
|
||||
|
||||
def _hc_timing_enabled() -> bool:
|
||||
global _HC_TIMING
|
||||
if _HC_TIMING is None:
|
||||
_HC_TIMING = _envs.SGLANG_HICACHE_TIMING.get()
|
||||
return _HC_TIMING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
||||
from sglang.srt.mem_cache.pool_host import HostKVCache
|
||||
@@ -306,7 +318,18 @@ class HiCacheController:
|
||||
self.attn_tp_group = attn_tp_group
|
||||
self.pp_group = pp_group
|
||||
self.prefetch_hits_sync_groups: List[torch.distributed.ProcessGroup] = []
|
||||
self.prefetch_completion_sync_groups: List[torch.distributed.ProcessGroup] = []
|
||||
# One set of completion sync groups per prefetch IO lane. Each lane's
|
||||
# worker/sync threads form a serial pipeline, so the collective
|
||||
# sequence on a lane's groups is identical across ranks as long as
|
||||
# lane assignment is deterministic (see dispatch_prefetch).
|
||||
self.prefetch_completion_sync_groups_by_lane: List[
|
||||
List[torch.distributed.ProcessGroup]
|
||||
] = []
|
||||
# Read once at construction; attach/reset reuse the stored value so
|
||||
# lane count never changes underneath live sync groups.
|
||||
self.num_prefetch_io_lanes = max(
|
||||
1, _envs.SGLANG_HICACHE_PREFETCH_IO_THREADS.get()
|
||||
)
|
||||
self.mem_pool_device_allocator = token_to_kv_pool_allocator
|
||||
mem_pool_device = token_to_kv_pool_allocator.get_kvcache()
|
||||
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool
|
||||
@@ -335,8 +358,8 @@ class HiCacheController:
|
||||
self.storage_stop_event = threading.Event()
|
||||
|
||||
# Storage control queues, (re)created whenever the storage threads start.
|
||||
self.prefetch_buffer: Optional[Queue[PrefetchOperation]] = None
|
||||
self.prefetch_sync_queue: Optional[Queue[PrefetchAck]] = None
|
||||
self.prefetch_io_queues: List[Queue[PrefetchOperation]] = []
|
||||
self.prefetch_sync_queues: List[Queue[PrefetchAck]] = []
|
||||
self.prefetch_hit_queue: Optional[Queue[StorageOperation]] = None
|
||||
self.ack_prefetch_queue = Queue[PrefetchAck]()
|
||||
self.ack_backup_queue: Optional[Queue[StorageOperation]] = None
|
||||
@@ -444,27 +467,39 @@ class HiCacheController:
|
||||
self.prefetch_thread = threading.Thread(
|
||||
target=self.prefetch_thread_func, daemon=True
|
||||
)
|
||||
self.prefetch_io_aux_thread = threading.Thread(
|
||||
target=self.prefetch_io_aux_func, daemon=True
|
||||
)
|
||||
self.prefetch_sync_thread = threading.Thread(
|
||||
target=self.prefetch_sync_thread_func, daemon=True
|
||||
)
|
||||
self.prefetch_io_aux_threads = [
|
||||
threading.Thread(
|
||||
target=self.prefetch_io_aux_func, args=(lane,), daemon=True
|
||||
)
|
||||
for lane in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
self.prefetch_sync_threads = [
|
||||
threading.Thread(
|
||||
target=self.prefetch_sync_thread_func, args=(lane,), daemon=True
|
||||
)
|
||||
for lane in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
self.backup_thread = threading.Thread(
|
||||
target=self.backup_thread_func, daemon=True
|
||||
)
|
||||
self.prefetch_queue = Queue()
|
||||
self.backup_queue = Queue()
|
||||
self.prefetch_buffer = Queue()
|
||||
self.prefetch_sync_queue = Queue()
|
||||
self.prefetch_io_queues = [
|
||||
Queue() for _ in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
self.prefetch_sync_queues = [
|
||||
Queue() for _ in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
self.prefetch_hit_queue = Queue()
|
||||
self.ack_prefetch_queue = Queue()
|
||||
self.ack_backup_queue = Queue()
|
||||
self.host_mem_release_queue = Queue()
|
||||
|
||||
self.prefetch_thread.start()
|
||||
self.prefetch_io_aux_thread.start()
|
||||
self.prefetch_sync_thread.start()
|
||||
for thread in self.prefetch_io_aux_threads:
|
||||
thread.start()
|
||||
for thread in self.prefetch_sync_threads:
|
||||
thread.start()
|
||||
self.backup_thread.start()
|
||||
|
||||
def _stop_storage_threads(self):
|
||||
@@ -485,10 +520,10 @@ class HiCacheController:
|
||||
self.prefetch_queue.put_nowait(None)
|
||||
if hasattr(self, "backup_queue"):
|
||||
self.backup_queue.put_nowait(None)
|
||||
if hasattr(self, "prefetch_buffer"):
|
||||
self.prefetch_buffer.put_nowait(None)
|
||||
if hasattr(self, "prefetch_sync_queue"):
|
||||
self.prefetch_sync_queue.put_nowait(None)
|
||||
for queue in getattr(self, "prefetch_io_queues", []) or []:
|
||||
queue.put_nowait(None)
|
||||
for queue in getattr(self, "prefetch_sync_queues", []) or []:
|
||||
queue.put_nowait(None)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -498,10 +533,8 @@ class HiCacheController:
|
||||
threads.append(self.prefetch_thread)
|
||||
if hasattr(self, "backup_thread"):
|
||||
threads.append(self.backup_thread)
|
||||
if hasattr(self, "prefetch_io_aux_thread"):
|
||||
threads.append(self.prefetch_io_aux_thread)
|
||||
if hasattr(self, "prefetch_sync_thread"):
|
||||
threads.append(self.prefetch_sync_thread)
|
||||
threads.extend(getattr(self, "prefetch_io_aux_threads", []) or [])
|
||||
threads.extend(getattr(self, "prefetch_sync_threads", []) or [])
|
||||
|
||||
for t in threads:
|
||||
try:
|
||||
@@ -585,7 +618,13 @@ class HiCacheController:
|
||||
# Use dedicated gloo groups so storage prefetch sync is isolated
|
||||
# from other collectives and consistent across CPxTP participants.
|
||||
self.prefetch_hits_sync_groups = self._create_sync_groups()
|
||||
self.prefetch_completion_sync_groups = self._create_sync_groups()
|
||||
# Each IO lane gets its own completion groups: per-lane collectives
|
||||
# stay ordered because a lane is a serial pipeline and dispatch is
|
||||
# deterministic (crc32(request_id) % num_prefetch_io_lanes).
|
||||
self.prefetch_completion_sync_groups_by_lane = [
|
||||
self._create_sync_groups()
|
||||
for _ in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
|
||||
# Select the get and set functions
|
||||
self.page_get_func = self._generic_page_get
|
||||
@@ -620,9 +659,10 @@ class HiCacheController:
|
||||
except Exception:
|
||||
pass
|
||||
self._destroy_sync_groups(self.prefetch_hits_sync_groups)
|
||||
self._destroy_sync_groups(self.prefetch_completion_sync_groups)
|
||||
for lane_groups in self.prefetch_completion_sync_groups_by_lane:
|
||||
self._destroy_sync_groups(lane_groups)
|
||||
self.prefetch_hits_sync_groups = []
|
||||
self.prefetch_completion_sync_groups = []
|
||||
self.prefetch_completion_sync_groups_by_lane = []
|
||||
try:
|
||||
if (
|
||||
hasattr(self, "storage_backend")
|
||||
@@ -660,11 +700,11 @@ class HiCacheController:
|
||||
raise RuntimeError("Stop storage threads failed; detach aborted.") from e
|
||||
|
||||
# Best-effort destroy process groups created for storage ops.
|
||||
self._destroy_sync_groups(
|
||||
self.prefetch_hits_sync_groups + self.prefetch_completion_sync_groups
|
||||
)
|
||||
self._destroy_sync_groups(self.prefetch_hits_sync_groups)
|
||||
for lane_groups in self.prefetch_completion_sync_groups_by_lane:
|
||||
self._destroy_sync_groups(lane_groups)
|
||||
self.prefetch_hits_sync_groups = []
|
||||
self.prefetch_completion_sync_groups = []
|
||||
self.prefetch_completion_sync_groups_by_lane = []
|
||||
|
||||
# Best-effort close (some backends rely on GC/destructor).
|
||||
try:
|
||||
@@ -758,13 +798,17 @@ class HiCacheController:
|
||||
self.ack_load_queue.clear()
|
||||
if self.enable_storage:
|
||||
self.prefetch_thread.join()
|
||||
self.prefetch_io_aux_thread.join()
|
||||
self.prefetch_sync_thread.join()
|
||||
for thread in self.prefetch_io_aux_threads:
|
||||
thread.join()
|
||||
for thread in self.prefetch_sync_threads:
|
||||
thread.join()
|
||||
self.backup_thread.join()
|
||||
self.prefetch_queue.queue.clear()
|
||||
self.backup_queue.queue.clear()
|
||||
self.prefetch_buffer.queue.clear()
|
||||
self.prefetch_sync_queue.queue.clear()
|
||||
for queue in self.prefetch_io_queues:
|
||||
queue.queue.clear()
|
||||
for queue in self.prefetch_sync_queues:
|
||||
queue.queue.clear()
|
||||
self.prefetch_hit_queue.queue.clear()
|
||||
self.ack_prefetch_queue.queue.clear()
|
||||
self.ack_backup_queue.queue.clear()
|
||||
@@ -777,18 +821,26 @@ class HiCacheController:
|
||||
self.prefetch_thread = threading.Thread(
|
||||
target=self.prefetch_thread_func, daemon=True
|
||||
)
|
||||
self.prefetch_io_aux_thread = threading.Thread(
|
||||
target=self.prefetch_io_aux_func, daemon=True
|
||||
)
|
||||
self.prefetch_sync_thread = threading.Thread(
|
||||
target=self.prefetch_sync_thread_func, daemon=True
|
||||
)
|
||||
self.prefetch_io_aux_threads = [
|
||||
threading.Thread(
|
||||
target=self.prefetch_io_aux_func, args=(lane,), daemon=True
|
||||
)
|
||||
for lane in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
self.prefetch_sync_threads = [
|
||||
threading.Thread(
|
||||
target=self.prefetch_sync_thread_func, args=(lane,), daemon=True
|
||||
)
|
||||
for lane in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
self.backup_thread = threading.Thread(
|
||||
target=self.backup_thread_func, daemon=True
|
||||
)
|
||||
self.prefetch_thread.start()
|
||||
self.prefetch_io_aux_thread.start()
|
||||
self.prefetch_sync_thread.start()
|
||||
for thread in self.prefetch_io_aux_threads:
|
||||
thread.start()
|
||||
for thread in self.prefetch_sync_threads:
|
||||
thread.start()
|
||||
self.backup_thread.start()
|
||||
|
||||
def has_inflight_device_transfers(self) -> bool:
|
||||
@@ -1075,7 +1127,23 @@ class HiCacheController:
|
||||
count += 1
|
||||
return count
|
||||
|
||||
def _page_transfer(self, operation: PrefetchOperation) -> int:
|
||||
def dispatch_prefetch(self, operation: PrefetchOperation) -> None:
|
||||
"""Route a prefetch operation to its IO lane.
|
||||
|
||||
Lane assignment must be deterministic and identical on every rank:
|
||||
each lane owns dedicated completion sync groups, and the collective
|
||||
sequence on a lane is the filtered dispatch order. crc32 (not the
|
||||
per-process salted builtin hash) keeps the mapping stable across
|
||||
processes. All chunks of one operation stay in one lane, preserving
|
||||
the per-operation ack order the sync thread reduces over.
|
||||
"""
|
||||
lane = (
|
||||
zlib.crc32(str(operation.request_id).encode())
|
||||
% self.num_prefetch_io_lanes
|
||||
)
|
||||
self.prefetch_io_queues[lane].put(operation)
|
||||
|
||||
def _page_transfer(self, operation: PrefetchOperation, sync_queue: Queue) -> int:
|
||||
# Transfer batch by batch
|
||||
prefix_keys = operation.prefix_keys
|
||||
kv_derived_transfers = [
|
||||
@@ -1087,8 +1155,8 @@ class HiCacheController:
|
||||
completed_pages = 0
|
||||
for i in range(0, len(operation.hash_value), STORAGE_BATCH_SIZE):
|
||||
# When an error is occurred, we should keep looping and produce the same number of
|
||||
# PrefetchAck as other ranks do, because prefetch_sync_thread (i.e. consumer of
|
||||
# prefetch_sync_queue) perform reduce on the results. This is so tricky.
|
||||
# PrefetchAck as other ranks do, because the prefetch sync thread (i.e. consumer of
|
||||
# the lane sync queue) perform reduce on the results. This is so tricky.
|
||||
if all_success and operation.is_terminated():
|
||||
all_success = False
|
||||
if all_success:
|
||||
@@ -1100,6 +1168,7 @@ class HiCacheController:
|
||||
# Get one batch token, and update the completed_tokens if succeed
|
||||
extra_info = HiCacheStorageExtraInfo(prefix_keys=prefix_keys)
|
||||
|
||||
_hc_tb = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
hit_pages = self._page_transfer_kv_batch(
|
||||
operation,
|
||||
batch_hashes,
|
||||
@@ -1107,6 +1176,14 @@ class HiCacheController:
|
||||
extra_info,
|
||||
kv_derived_transfers,
|
||||
)
|
||||
if _hc_timing_enabled():
|
||||
logger.info(
|
||||
"HC_TIMING stage=xfer_batch req=%s keys=%d ms=%.1f hit=%d",
|
||||
operation.request_id,
|
||||
len(batch_hashes),
|
||||
(time.monotonic() - _hc_tb) * 1e3,
|
||||
hit_pages,
|
||||
)
|
||||
# Check termination
|
||||
if hit_pages != len(batch_hashes):
|
||||
all_success = False
|
||||
@@ -1118,7 +1195,7 @@ class HiCacheController:
|
||||
completed_tokens=completed_pages * self.page_size,
|
||||
operation=operation,
|
||||
)
|
||||
self.prefetch_sync_queue.put(ack)
|
||||
sync_queue.put(ack)
|
||||
return completed_pages
|
||||
|
||||
def _page_transfer_kv_batch(
|
||||
@@ -1160,18 +1237,33 @@ class HiCacheController:
|
||||
# Clamp to minimal number of hits.
|
||||
return min([kv_hits, *sidecar_hits.values()])
|
||||
|
||||
def prefetch_io_aux_func(self):
|
||||
def prefetch_io_aux_func(self, lane: int = 0):
|
||||
"""
|
||||
Auxiliary function conducting IO operations for prefetching.
|
||||
|
||||
Each lane has its own worker thread and queues; an operation's whole
|
||||
chunk sequence stays on one lane so the sync thread sees it in order.
|
||||
"""
|
||||
io_queue = self.prefetch_io_queues[lane]
|
||||
sync_queue = self.prefetch_sync_queues[lane]
|
||||
while not self.storage_stop_event.is_set():
|
||||
try:
|
||||
operation = self.prefetch_buffer.get(block=True, timeout=1)
|
||||
operation = io_queue.get(block=True, timeout=1)
|
||||
if operation is None:
|
||||
continue
|
||||
self._page_transfer(operation)
|
||||
_hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
self._page_transfer(operation, sync_queue)
|
||||
if _hc_timing_enabled():
|
||||
logger.info(
|
||||
"HC_TIMING stage=xfer req=%s lane=%d wait_ms=%.1f xfer_ms=%.1f",
|
||||
operation.request_id,
|
||||
lane,
|
||||
(_hc_t0 - getattr(operation, "query_done_ts", operation.start_time))
|
||||
* 1e3,
|
||||
(time.monotonic() - _hc_t0) * 1e3,
|
||||
)
|
||||
|
||||
self.prefetch_sync_queue.put(
|
||||
sync_queue.put(
|
||||
PrefetchAck(
|
||||
rid=operation.request_id,
|
||||
completed_req=True,
|
||||
@@ -1216,7 +1308,16 @@ class HiCacheController:
|
||||
for start in range(0, len(page_hashes), STORAGE_BATCH_SIZE):
|
||||
batch_hashes = page_hashes[start : start + STORAGE_BATCH_SIZE]
|
||||
extra_info = HiCacheStorageExtraInfo(prefix_keys=prefix_keys)
|
||||
_hc_tb = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
hit_page_num = self.storage_backend.batch_exists(batch_hashes, extra_info)
|
||||
if _hc_timing_enabled():
|
||||
logger.info(
|
||||
"HC_TIMING stage=query_batch req=%s keys=%d ms=%.1f hit=%d",
|
||||
operation.request_id,
|
||||
len(batch_hashes),
|
||||
(time.monotonic() - _hc_tb) * 1e3,
|
||||
hit_page_num,
|
||||
)
|
||||
hash_value.extend(batch_hashes[:hit_page_num])
|
||||
storage_query_count += hit_page_num * self.page_size
|
||||
if hit_page_num < len(batch_hashes):
|
||||
@@ -1235,10 +1336,12 @@ class HiCacheController:
|
||||
operation = self.prefetch_queue.get(block=True, timeout=1)
|
||||
if operation is None:
|
||||
continue
|
||||
_hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
if operation.is_terminated():
|
||||
hash_value, storage_hit_count = [], 0
|
||||
else:
|
||||
hash_value, storage_hit_count = self._storage_hit_query(operation)
|
||||
_hc_t1 = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
storage_hit_count_tensor = torch.tensor(
|
||||
storage_hit_count, dtype=torch.int
|
||||
)
|
||||
@@ -1248,6 +1351,18 @@ class HiCacheController:
|
||||
self.prefetch_hits_sync_groups,
|
||||
)
|
||||
storage_hit_count = storage_hit_count_tensor.item()
|
||||
if _hc_timing_enabled():
|
||||
_hc_t2 = time.monotonic()
|
||||
operation.query_done_ts = _hc_t2
|
||||
logger.info(
|
||||
"HC_TIMING stage=query req=%s wait_ms=%.1f query_ms=%.1f "
|
||||
"hit_reduce_ms=%.1f hit_tokens=%d",
|
||||
operation.request_id,
|
||||
(_hc_t0 - operation.start_time) * 1e3,
|
||||
(_hc_t1 - _hc_t0) * 1e3,
|
||||
(_hc_t2 - _hc_t1) * 1e3,
|
||||
storage_hit_count,
|
||||
)
|
||||
|
||||
# Record the TP-synced hit count; the scheduler thread decides
|
||||
# at drain time whether to revoke (below threshold) or allocate.
|
||||
@@ -1329,28 +1444,41 @@ class HiCacheController:
|
||||
except Empty:
|
||||
continue
|
||||
|
||||
def prefetch_sync_thread_func(self):
|
||||
def prefetch_sync_thread_func(self, lane: int = 0):
|
||||
"""Synchronize prefetch results across all PP and TP ranks."""
|
||||
sync_queue = self.prefetch_sync_queues[lane]
|
||||
groups = self.prefetch_completion_sync_groups_by_lane[lane]
|
||||
while not self.storage_stop_event.is_set():
|
||||
try:
|
||||
ack = self.prefetch_sync_queue.get(block=True, timeout=1)
|
||||
ack = sync_queue.get(block=True, timeout=1)
|
||||
if ack is None:
|
||||
continue
|
||||
self._reduce_prefetch_ack(ack)
|
||||
self._reduce_prefetch_ack(ack, groups)
|
||||
if _hc_timing_enabled() and ack.completed_req:
|
||||
ack.operation.final_ack_enqueue_ts = time.monotonic()
|
||||
self.ack_prefetch_queue.put(ack)
|
||||
except Empty:
|
||||
continue
|
||||
|
||||
def _reduce_prefetch_ack(self, ack: PrefetchAck) -> None:
|
||||
def _reduce_prefetch_ack(
|
||||
self, ack: PrefetchAck, groups: List[torch.distributed.ProcessGroup]
|
||||
) -> None:
|
||||
"""Synchronize all ranks to agree on a PrefetchAck."""
|
||||
if ack.completed_tokens is not None:
|
||||
# Determine the minimal successful prefix of tokens.
|
||||
_hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
completed_tokens_tensor = torch.tensor(
|
||||
ack.completed_tokens, dtype=torch.int
|
||||
)
|
||||
self._all_reduce(
|
||||
completed_tokens_tensor,
|
||||
torch.distributed.ReduceOp.MIN,
|
||||
self.prefetch_completion_sync_groups,
|
||||
groups,
|
||||
)
|
||||
ack.completed_tokens = completed_tokens_tensor.item()
|
||||
if _hc_timing_enabled():
|
||||
logger.info(
|
||||
"HC_TIMING stage=ack_reduce req=%s ms=%.1f",
|
||||
ack.operation.request_id,
|
||||
(time.monotonic() - _hc_t0) * 1e3,
|
||||
)
|
||||
|
||||
@@ -2141,6 +2141,27 @@ class SetInternalStateReqOutput(BaseReq, kw_only=True):
|
||||
updated: bool
|
||||
|
||||
|
||||
class QueryCacheHitReqInput(BaseReq, kw_only=True):
|
||||
"""G9: read-only probe of prefix-cache hit for a token sequence.
|
||||
|
||||
Sent to every DP rank of a (decode) engine; each rank answers with its own
|
||||
L1/L2 radix match plus the shared L3 storage hit continuing from it.
|
||||
"""
|
||||
|
||||
input_ids: List[int]
|
||||
extra_key: Optional[str] = None
|
||||
cache_salt: Optional[str] = None
|
||||
|
||||
|
||||
class QueryCacheHitReqOutput(BaseReq, kw_only=True):
|
||||
total_tokens: int = 0
|
||||
l1_hit_tokens: int = 0
|
||||
l2_hit_tokens: int = 0
|
||||
l3_hit_tokens: int = 0
|
||||
dp_rank: int = 0
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
class ProfileReqType(Enum):
|
||||
START_PROFILE = 1
|
||||
STOP_PROFILE = 2
|
||||
|
||||
@@ -197,6 +197,12 @@ class LoadSnapshot(msgspec.Struct, omit_defaults=True):
|
||||
num_running_reqs: int = 0
|
||||
num_waiting_reqs: int = 0
|
||||
num_waiting_uncached_tokens: int = 0
|
||||
# G9 queue-time estimation (总输入长度口径, self-consistent numerator/denominator):
|
||||
# sum of total input lengths (req.seqlen) over all pending queues, and the
|
||||
# self-learned drain rate in the same unit — completed prefills' total
|
||||
# input tokens / wall second over a sliding window.
|
||||
queued_total_input_tokens: int = 0
|
||||
drain_rate_tokens_per_s: float = 0.0
|
||||
num_used_tokens: int = 0
|
||||
num_total_tokens: int = 0
|
||||
# num_total_tokens minus tokens still awaiting a KV transfer (equal to it
|
||||
|
||||
@@ -1034,12 +1034,12 @@ class Req(ReqDllmMixin):
|
||||
else self.origin_input_ids
|
||||
) # Before image padding
|
||||
# Each decode stage's output ids. Append-only by contract:
|
||||
# _refresh_fill_ids infers how many output tokens are already in
|
||||
# refresh_fill_ids infers how many output tokens are already in
|
||||
# full_untruncated_fill_ids from lengths alone, so in-place rewrites
|
||||
# that preserve length would silently corrupt fill_ids.
|
||||
self.output_ids = array("q")
|
||||
# Full untruncated sequence: origin + output (+ DLLM mask block).
|
||||
# Kept in sync by _refresh_fill_ids; admission only updates
|
||||
# Kept in sync by refresh_fill_ids; admission only updates
|
||||
# extend_range, never mutates this array's length.
|
||||
self.full_untruncated_fill_ids = array("q")
|
||||
self.extend_range: Optional[Range] = None
|
||||
@@ -1533,7 +1533,7 @@ class Req(ReqDllmMixin):
|
||||
def get_fill_ids(self) -> array:
|
||||
return self.full_untruncated_fill_ids[: self.extend_range.end]
|
||||
|
||||
def _refresh_fill_ids(self) -> None:
|
||||
def refresh_fill_ids(self) -> None:
|
||||
"""Keep full_untruncated_fill_ids == origin_input_ids + output_ids by
|
||||
appending only the new output tokens.
|
||||
|
||||
@@ -1562,7 +1562,7 @@ class Req(ReqDllmMixin):
|
||||
self._init_fill_ids_for_dllm()
|
||||
self.determine_dllm_phase()
|
||||
else:
|
||||
self._refresh_fill_ids()
|
||||
self.refresh_fill_ids()
|
||||
|
||||
input_len = len(self.full_untruncated_fill_ids)
|
||||
|
||||
@@ -3107,7 +3107,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
for req, prefix_len in zip(
|
||||
running_batch.reqs, running_prefix_lens, strict=True
|
||||
):
|
||||
req._refresh_fill_ids()
|
||||
req.refresh_fill_ids()
|
||||
req.set_extend_range(prefix_len, prefix_len + 1)
|
||||
|
||||
# Decode tokens of the running portion live in future_map.output_tokens_buf.
|
||||
@@ -3185,7 +3185,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
# cannot stand in for it. Rows past bs are a beam tail, not requests.
|
||||
seq_lens = self.seq_lens_cpu[:bs].tolist()
|
||||
for req, seq_len in zip(self.reqs, seq_lens, strict=True):
|
||||
req._refresh_fill_ids()
|
||||
req.refresh_fill_ids()
|
||||
# end runs one past full_untruncated_fill_ids while output_ids
|
||||
# trails; safe only while decoding_reqs suppresses cache_unfinished_req.
|
||||
req.set_extend_range(seq_len - 1, seq_len)
|
||||
|
||||
@@ -161,6 +161,8 @@ from sglang.srt.managers.io_struct import (
|
||||
PauseGenerationReqInput,
|
||||
PdRoleSwitchReqInput,
|
||||
ProfileReq,
|
||||
QueryCacheHitReqInput,
|
||||
QueryCacheHitReqOutput,
|
||||
ReleaseMemoryOccupationReqInput,
|
||||
RemoveExternalCorpusReqInput,
|
||||
RemoveExternalCorpusReqOutput,
|
||||
@@ -1806,6 +1808,7 @@ class Scheduler(
|
||||
(ShutdownReq, self.handle_shutdown),
|
||||
(GetInternalStateReq, self.get_internal_state),
|
||||
(SetInternalStateReq, self.set_internal_state),
|
||||
(QueryCacheHitReqInput, self.handle_query_cache_hit),
|
||||
(RpcReqInput, self.handle_rpc_request),
|
||||
(ExpertDistributionReq, self.expert_distribution_handle),
|
||||
(LoadLoRAAdapterReqInput, self.load_lora_adapter),
|
||||
@@ -2431,6 +2434,11 @@ class Scheduler(
|
||||
self._prev_step: Optional[Tuple[int, float, bool]] = None
|
||||
self._prev_prefill_end_ts: Optional[float] = None
|
||||
self._sched_idled = False
|
||||
# G9: sliding-window drain events (monotonic_ts, total_input_tokens) for
|
||||
# the queue-time estimator. Recorded once per request when its final
|
||||
# prefill chunk completes (batch_result_processor), in the same
|
||||
# total-input-length unit as LoadSnapshot.queued_total_input_tokens.
|
||||
self.prefill_drain_window: deque = deque()
|
||||
self.load_inquirer = SchedulerLoadInquirer(
|
||||
disaggregation_mode=self.disaggregation_mode,
|
||||
server_args=self.server_args,
|
||||
@@ -2469,8 +2477,35 @@ class Scheduler(
|
||||
),
|
||||
get_total_prefill_busy_us=lambda: self.total_prefill_busy_us,
|
||||
get_decode_moment_totals=lambda: self.decode_moment_totals,
|
||||
get_prefill_drain_rate=self.get_prefill_drain_rate,
|
||||
)
|
||||
|
||||
PREFILL_DRAIN_WINDOW_S = 60.0
|
||||
|
||||
def record_prefill_drain(self, total_input_tokens: int) -> None:
|
||||
"""G9: one drain event per request whose final prefill chunk completed.
|
||||
|
||||
Tokens are the request's total input length (seqlen), matching the
|
||||
queued_total_input_tokens numerator so the router's
|
||||
wait = queued_tokens / drain_rate is unit-consistent (决策②).
|
||||
"""
|
||||
now = time.monotonic()
|
||||
self.prefill_drain_window.append((now, total_input_tokens))
|
||||
cutoff = now - self.PREFILL_DRAIN_WINDOW_S
|
||||
while self.prefill_drain_window and self.prefill_drain_window[0][0] < cutoff:
|
||||
self.prefill_drain_window.popleft()
|
||||
|
||||
def get_prefill_drain_rate(self) -> float:
|
||||
"""Completed prefill input tokens per wall second over the window."""
|
||||
now = time.monotonic()
|
||||
cutoff = now - self.PREFILL_DRAIN_WINDOW_S
|
||||
while self.prefill_drain_window and self.prefill_drain_window[0][0] < cutoff:
|
||||
self.prefill_drain_window.popleft()
|
||||
if not self.prefill_drain_window:
|
||||
return 0.0
|
||||
span = max(now - self.prefill_drain_window[0][0], 1.0)
|
||||
return sum(tokens for _, tokens in self.prefill_drain_window) / span
|
||||
|
||||
def init_output_streamer(self) -> None:
|
||||
self.output_streamer = self.get_output_streamer_class()(
|
||||
send_to_detokenizer=self.ipc_channels.send_to_detokenizer,
|
||||
@@ -2521,6 +2556,7 @@ class Scheduler(
|
||||
output_streamer=self.output_streamer,
|
||||
beam_coordinator=self.beam_coordinator,
|
||||
abort_request=self.abort_request,
|
||||
record_prefill_drain=self.record_prefill_drain,
|
||||
)
|
||||
|
||||
def init_req_max_new_tokens(self, req):
|
||||
@@ -5215,6 +5251,86 @@ class Scheduler(
|
||||
success = False
|
||||
return success
|
||||
|
||||
def handle_query_cache_hit(self, recv_req: QueryCacheHitReqInput):
|
||||
"""G9: read-only prefix-hit probe (L1/L2 radix + L3 storage).
|
||||
|
||||
The router calls this (via /query_cache_hit on the decode engine) to
|
||||
decide conditional PD-bypass. Runs inside the scheduler event loop so
|
||||
radix/storage state is consistent, and reuses the exact admission-time
|
||||
match + L3 query logic (mirrors DecodeHiCachePreallocMixin
|
||||
._build_decode_prefix_match) without creating a Req or mutating
|
||||
request state. Node access metadata (LRU) may be touched; a hit found
|
||||
here can still be evicted before the real request arrives (inherent
|
||||
TOCTOU, acceptable for an estimate).
|
||||
"""
|
||||
from sglang.srt.mem_cache.base_prefix_cache import MatchPrefixParams
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
|
||||
total = len(recv_req.input_ids)
|
||||
# 树节点键是 array("q"),RadixKey.match_at 断言两侧类型一致——
|
||||
# 查询侧必须同样用 array,否则树非空时匹配直接炸(2026-10-11 实测)。
|
||||
from array import array
|
||||
|
||||
query_ids = array("q", recv_req.input_ids)
|
||||
try:
|
||||
match_result = self.tree_cache.match_prefix(
|
||||
MatchPrefixParams(
|
||||
key=RadixKey(
|
||||
token_ids=query_ids,
|
||||
extra_key=recv_req.extra_key,
|
||||
cache_salt=recv_req.cache_salt,
|
||||
)
|
||||
)
|
||||
)
|
||||
l1_hit = len(match_result.device_indices)
|
||||
l2_hit = getattr(match_result, "host_hit_length", 0) or 0
|
||||
l3_hit = 0
|
||||
last_host_node = getattr(match_result, "last_host_node", None)
|
||||
# L3 查询条件:本调度器挂了 hicache 存储后端即可(P 节点 HiCache
|
||||
# write_through/write_back 场景),不必是 decode-hicache 模式。
|
||||
# SWA 模型的 decode radix 与 hicache 互斥(kv_cache_builder 硬限制),
|
||||
# 若只认 enable_decode_hicache,P 节点永远查不到 L3。
|
||||
can_query_l3 = getattr(self, "enable_decode_hicache", False) or getattr(
|
||||
self, "enable_hicache_storage", False
|
||||
)
|
||||
if (
|
||||
can_query_l3
|
||||
and last_host_node is not None
|
||||
and hasattr(self.tree_cache, "query_storage_hit_length")
|
||||
and (
|
||||
self.tree_cache.is_backuped(last_host_node)
|
||||
or self.tree_cache.is_root(last_host_node)
|
||||
)
|
||||
):
|
||||
matched_len = l1_hit + l2_hit
|
||||
suffix_tokens = query_ids[matched_len:]
|
||||
last_hash = self.tree_cache.get_last_hash_value(last_host_node)
|
||||
prefix_keys = (
|
||||
self.tree_cache.get_prefix_hash_values(last_host_node)
|
||||
if getattr(
|
||||
self.tree_cache, "hicache_storage_pass_prefix_keys", False
|
||||
)
|
||||
else None
|
||||
)
|
||||
l3_hit = self.tree_cache.query_storage_hit_length(
|
||||
last_host_node,
|
||||
suffix_tokens,
|
||||
last_hash,
|
||||
prefix_keys,
|
||||
extra_key=recv_req.extra_key,
|
||||
cache_salt=recv_req.cache_salt,
|
||||
)
|
||||
return QueryCacheHitReqOutput(
|
||||
total_tokens=total,
|
||||
l1_hit_tokens=l1_hit,
|
||||
l2_hit_tokens=l2_hit,
|
||||
l3_hit_tokens=l3_hit,
|
||||
dp_rank=int(get_parallel().dp_rank or 0),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"handle_query_cache_hit failed: {e}")
|
||||
return QueryCacheHitReqOutput(total_tokens=total, error=str(e))
|
||||
|
||||
def get_internal_state(self, recv_req: GetInternalStateReq):
|
||||
# Resolved config (pristine server_args + post-publish overrides) so a
|
||||
# readback reflects values changed via /set_internal_state, not startup.
|
||||
|
||||
@@ -112,6 +112,8 @@ class SchedulerBatchResultProcessor:
|
||||
output_streamer: SchedulerOutputStreamer
|
||||
beam_coordinator: BeamCoordinator
|
||||
abort_request: Callable
|
||||
# G9: scheduler-side drain-event recorder for the queue-time estimator.
|
||||
record_prefill_drain: Callable
|
||||
|
||||
def process_batch_result_prebuilt(self, batch: ScheduleBatch):
|
||||
assert self.disaggregation_mode == DisaggregationMode.DECODE
|
||||
@@ -343,6 +345,9 @@ class SchedulerBatchResultProcessor:
|
||||
|
||||
if req.inflight_middle_chunks <= 0:
|
||||
req.time_stats.set_prefill_finished_time()
|
||||
# G9: exactly once per request (final chunk only) — feed the
|
||||
# queue-time estimator with the drained total input length.
|
||||
self.record_prefill_drain(req.seqlen)
|
||||
|
||||
if sampling_mask_finish_reason is not None:
|
||||
req.to_finish = sampling_mask_finish_reason
|
||||
|
||||
@@ -54,6 +54,7 @@ class SchedulerLoadInquirer:
|
||||
get_total_prefill_uncached_tokens: Callable
|
||||
get_total_prefill_busy_us: Callable
|
||||
get_decode_moment_totals: Callable
|
||||
get_prefill_drain_rate: Callable
|
||||
|
||||
def _get_num_pending_tokens(self, chunk_deduct: int = 0) -> int:
|
||||
"""Get the total number of tokens pending prefill.
|
||||
@@ -128,9 +129,10 @@ class SchedulerLoadInquirer:
|
||||
num_used_tokens, kv_token_usage = (
|
||||
self.pool_stats_observer.get_pool_stats().get_kv_token_stats()
|
||||
)
|
||||
num_total_tokens = num_used_tokens + sum(
|
||||
queued_total_input_tokens = sum(
|
||||
req.seqlen for queue in pending_token_queues for req in queue
|
||||
)
|
||||
num_total_tokens = num_used_tokens + queued_total_input_tokens
|
||||
num_active_tokens = max(0, num_total_tokens - awaiting_kv_tokens)
|
||||
|
||||
memory = None
|
||||
@@ -220,6 +222,8 @@ class SchedulerLoadInquirer:
|
||||
num_running_reqs=num_running_reqs,
|
||||
num_waiting_reqs=num_waiting_reqs,
|
||||
num_waiting_uncached_tokens=self.get_num_waiting_uncached_tokens(),
|
||||
queued_total_input_tokens=queued_total_input_tokens,
|
||||
drain_rate_tokens_per_s=round(self.get_prefill_drain_rate(), 2),
|
||||
num_used_tokens=num_used_tokens,
|
||||
num_total_tokens=num_total_tokens,
|
||||
num_active_tokens=num_active_tokens,
|
||||
|
||||
@@ -54,6 +54,8 @@ from sglang.srt.managers.io_struct import (
|
||||
ProfileReq,
|
||||
ProfileReqOutput,
|
||||
ProfileReqType,
|
||||
QueryCacheHitReqInput,
|
||||
QueryCacheHitReqOutput,
|
||||
ReleaseMemoryOccupationReqInput,
|
||||
ReleaseMemoryOccupationReqOutput,
|
||||
RemoveExternalCorpusReqInput,
|
||||
@@ -129,6 +131,7 @@ _COMMUNICATOR_SPECS = [
|
||||
("detach_hicache_storage", DetachHiCacheStorageReqOutput),
|
||||
("profile", ProfileReqOutput),
|
||||
("get_internal_state", GetInternalStateReqOutput),
|
||||
("query_cache_hit", QueryCacheHitReqOutput),
|
||||
("set_internal_state", SetInternalStateReqOutput),
|
||||
("expert_distribution", ExpertDistributionReqOutput),
|
||||
("update_lora_adapter", LoRAUpdateOutput),
|
||||
@@ -905,6 +908,37 @@ class TokenizerControlMixin:
|
||||
self.auto_create_handle_loop()
|
||||
return await self.dumper_control_communicator(obj)
|
||||
|
||||
async def query_cache_hit(
|
||||
self: TokenizerManager, obj: QueryCacheHitReqInput
|
||||
) -> QueryCacheHitReqOutput:
|
||||
"""G9: fan the probe out to all DP ranks and keep the best hit.
|
||||
|
||||
The decode node's DP controller routes new requests to any rank, so
|
||||
the bypass decision uses the best per-rank L1/L2 (L3 is shared); the
|
||||
real request still goes through normal dispatch afterwards.
|
||||
"""
|
||||
self.auto_create_handle_loop()
|
||||
results: List[QueryCacheHitReqOutput] = await self.query_cache_hit_communicator(
|
||||
obj
|
||||
)
|
||||
ok = [r for r in results if r.error is None]
|
||||
if not ok:
|
||||
msg = (
|
||||
"; ".join(r.error or "empty response" for r in results)
|
||||
or "no scheduler responded"
|
||||
)
|
||||
return QueryCacheHitReqOutput(total_tokens=len(obj.input_ids), error=msg)
|
||||
best = max(
|
||||
ok, key=lambda r: r.l1_hit_tokens + r.l2_hit_tokens + r.l3_hit_tokens
|
||||
)
|
||||
return QueryCacheHitReqOutput(
|
||||
total_tokens=best.total_tokens,
|
||||
l1_hit_tokens=best.l1_hit_tokens,
|
||||
l2_hit_tokens=best.l2_hit_tokens,
|
||||
l3_hit_tokens=best.l3_hit_tokens,
|
||||
dp_rank=best.dp_rank,
|
||||
)
|
||||
|
||||
async def get_loads(
|
||||
self: TokenizerManager,
|
||||
include: Optional[List[str]] = None,
|
||||
|
||||
@@ -866,3 +866,15 @@ class PureSWATokenToKVPoolAllocator(SWATokenToKVPoolAllocator):
|
||||
|
||||
def is_swa_req_ring(allocator) -> bool:
|
||||
return isinstance(allocator, SWATokenToKVPoolAllocator) and allocator.swa_req_ring
|
||||
|
||||
|
||||
def swa_kept_out_of_tree(allocator) -> bool:
|
||||
"""Whether radix-tree nodes do not own the SWA state: a per-request SWA ring,
|
||||
or the DeepSeek-V4.1 encoder-replay request window, which replay rebuilds on a
|
||||
prefix hit. SWA bookkeeping must not gate prefix matching for either."""
|
||||
if not isinstance(allocator, SWATokenToKVPoolAllocator):
|
||||
return False
|
||||
if allocator.swa_req_ring:
|
||||
return True
|
||||
kvcache = getattr(allocator, "_kvcache", None)
|
||||
return getattr(kvcache, "request_window", None) is not None
|
||||
|
||||
@@ -659,6 +659,12 @@ class DeepSeekV4IndexerPool(KVCache):
|
||||
from the page layout [page_size * 64 payload | page_size * 4 scale]."""
|
||||
assert self.use_fp4_indexer, "packed readback only applies to the fp4 layout"
|
||||
buf = self.index_k_with_scale_buffer[layer_id - self.start_layer]
|
||||
if buf.is_cuda:
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer import (
|
||||
gather_fp4_index_k,
|
||||
)
|
||||
|
||||
return gather_fp4_index_k(buf, slots, page_size=self.page_size)
|
||||
slots = slots.to(torch.int64)
|
||||
p = self.page_size
|
||||
page, off = (slots // p).unsqueeze(-1), slots % p
|
||||
@@ -1855,6 +1861,11 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
# swa_kv_pool is None under the request window and unified_kv.
|
||||
return self.kv_layout
|
||||
|
||||
def get_swa_key_page_size(self) -> int:
|
||||
if self.request_window is not None:
|
||||
return self.request_window.page_size
|
||||
return self.swa_kv_pool.page_size
|
||||
|
||||
def get_swa_key_bytes_per_token(self) -> int:
|
||||
"""Last dim of the ``(pages, page_size, 1, bytes)`` view the attention
|
||||
kernel detects the SWA cache's format from."""
|
||||
|
||||
@@ -91,7 +91,7 @@ def auto_size_hicache(
|
||||
return
|
||||
requested = get_memory().hicache_ratio
|
||||
device_bytes = _estimate_hicache_bytes(params, draft_plan)
|
||||
budget = int(host_memory_budget_bytes() * fraction)
|
||||
budget = int(host_memory_budget_bytes(auto_size=True) * fraction)
|
||||
ratio = min(requested, budget * (1 - _ALLOCATION_SLACK_FRACTION) / device_bytes)
|
||||
# One collective before any pool is built: PP stages own different pool
|
||||
# counts, so a per-pool collective could deadlock.
|
||||
|
||||
@@ -20,7 +20,7 @@ if TYPE_CHECKING:
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Max pages per batched storage IO call.
|
||||
STORAGE_BATCH_SIZE = 128
|
||||
STORAGE_BATCH_SIZE = envs.SGLANG_HICACHE_STORAGE_BATCH_SIZE.get()
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -653,7 +653,7 @@ class HiRadixCache(RadixCache):
|
||||
: alloc_len // self.page_size
|
||||
]
|
||||
operation.host_indices = host_indices
|
||||
cc.prefetch_buffer.put(operation)
|
||||
cc.dispatch_prefetch(operation)
|
||||
|
||||
def _drain_ack_prefetch():
|
||||
for ack in _drain_queue(cc.ack_prefetch_queue, n_ack_prefetch):
|
||||
|
||||
@@ -15,6 +15,11 @@ def _unescape_mount_path(value: str) -> str:
|
||||
return re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), value)
|
||||
|
||||
|
||||
def _memory_stat(directory: Path) -> dict[str, int]:
|
||||
lines = (directory / "memory.stat").read_text().splitlines()
|
||||
return {key: int(value) for key, value in map(str.split, lines)}
|
||||
|
||||
|
||||
def _cgroup_memory_headroom(proc_root: Path = Path("/proc")) -> int | None:
|
||||
memberships = {}
|
||||
try:
|
||||
@@ -65,6 +70,13 @@ def _cgroup_memory_headroom(proc_root: Path = Path("/proc")) -> int | None:
|
||||
usage_name = (
|
||||
"memory.current" if filesystem == "cgroup2" else "memory.usage_in_bytes"
|
||||
)
|
||||
# File LRU pages, as MemAvailable counts; shmem sits on the anon LRU.
|
||||
# v1's unprefixed counters omit descendants, unlike its usage.
|
||||
page_cache_keys = (
|
||||
("active_file", "inactive_file")
|
||||
if filesystem == "cgroup2"
|
||||
else ("total_active_file", "total_inactive_file")
|
||||
)
|
||||
while True:
|
||||
for name in limits:
|
||||
try:
|
||||
@@ -78,7 +90,10 @@ def _cgroup_memory_headroom(proc_root: Path = Path("/proc")) -> int | None:
|
||||
# Do not silently ignore an unreadable usage file for a known
|
||||
# limit: falling back to host RAM could overrun the container.
|
||||
usage = int((directory / usage_name).read_text())
|
||||
remaining = max(0, limit - usage)
|
||||
stat = _memory_stat(directory)
|
||||
page_cache = sum(stat[key] for key in page_cache_keys)
|
||||
# Charged page cache is reclaimed under the limit, not used.
|
||||
remaining = max(0, limit - max(0, usage - page_cache))
|
||||
headroom = remaining if headroom is None else min(headroom, remaining)
|
||||
if directory == mount:
|
||||
break
|
||||
@@ -90,10 +105,25 @@ def _cgroup_memory_headroom(proc_root: Path = Path("/proc")) -> int | None:
|
||||
return headroom
|
||||
|
||||
|
||||
def available_host_memory_bytes() -> int:
|
||||
"""Conservative allocatable RAM; charged file cache is not assumed reclaimable."""
|
||||
def available_host_memory_bytes(*, allow_cgroup_fallback: bool = False) -> int:
|
||||
"""Allocatable RAM, counting reclaimable page cache as free."""
|
||||
available = psutil.virtual_memory().available
|
||||
cgroup_headroom = _cgroup_memory_headroom()
|
||||
try:
|
||||
cgroup_headroom = _cgroup_memory_headroom()
|
||||
except (OSError, RuntimeError) as exc:
|
||||
if not allow_cgroup_fallback:
|
||||
raise RuntimeError(
|
||||
"Cannot automatically size HiCache host memory; "
|
||||
"set --hicache-size explicitly (GB per rank), leaving room "
|
||||
"for other memory use within the container limit."
|
||||
) from exc
|
||||
logger.warning(
|
||||
"Cannot determine cgroup memory headroom: %s. Checking explicitly "
|
||||
"sized HiCache pools against host available memory; ensure all "
|
||||
"local ranks' pools fit within the container memory limit.",
|
||||
exc,
|
||||
)
|
||||
return available
|
||||
if cgroup_headroom is not None:
|
||||
logger.info(
|
||||
"HiCache memory headroom: host %.1f GiB, cgroup %.1f GiB",
|
||||
|
||||
@@ -15,6 +15,7 @@ import torch
|
||||
|
||||
from sglang.srt.managers.cache_controller import (
|
||||
CacheOperation,
|
||||
_hc_timing_enabled,
|
||||
)
|
||||
from sglang.srt.managers.cache_controller import (
|
||||
HiCacheController as BaseHiCacheController,
|
||||
@@ -1005,7 +1006,7 @@ class HybridCacheController(BaseHiCacheController):
|
||||
if not self._allocate_pp_prefetch_buffers(ticket, operation):
|
||||
operation.host_indices = torch.empty(0, dtype=torch.int64)
|
||||
operation.mark_terminate()
|
||||
self.prefetch_buffer.put(operation)
|
||||
self.dispatch_prefetch(operation)
|
||||
except Exception:
|
||||
logger.exception("PP HiCache ticket processing failed.")
|
||||
if operation is None or not operation.hash_value:
|
||||
@@ -1015,7 +1016,7 @@ class HybridCacheController(BaseHiCacheController):
|
||||
operation.host_indices = torch.empty(0, dtype=torch.int64)
|
||||
operation.mark_terminate()
|
||||
# Preserve every KV/sidecar ACK even when local allocation fails.
|
||||
self.prefetch_buffer.put(operation)
|
||||
self.dispatch_prefetch(operation)
|
||||
finally:
|
||||
if is_source and objects:
|
||||
self.pp_prefetch_command_queue.task_done()
|
||||
@@ -1103,15 +1104,15 @@ class HybridCacheController(BaseHiCacheController):
|
||||
)
|
||||
return host_indices, device_indices, resolved_pool_transfers
|
||||
|
||||
def _page_transfer(self, operation: PrefetchOperation) -> bool:
|
||||
def _page_transfer(self, operation: PrefetchOperation, sync_queue: Queue) -> bool:
|
||||
# KV pools and KV-derived pools first — determines actual completed page count
|
||||
kv_completed_pages = super()._page_transfer(operation)
|
||||
kv_completed_pages = super()._page_transfer(operation, sync_queue)
|
||||
|
||||
# Read non-KV derived sidecar pool, e.g. SWA, Mamba.
|
||||
self._page_transfer_sidecar(operation, kv_completed_pages)
|
||||
self._page_transfer_sidecar(operation, kv_completed_pages, sync_queue)
|
||||
|
||||
def _page_transfer_sidecar(
|
||||
self, operation: PrefetchOperation, kv_completed_pages: int
|
||||
self, operation: PrefetchOperation, kv_completed_pages: int, sync_queue: Queue
|
||||
) -> None:
|
||||
if operation.pool_transfers is None:
|
||||
return
|
||||
@@ -1143,10 +1144,10 @@ class HybridCacheController(BaseHiCacheController):
|
||||
transfers_nonkv, extra_info=extra_info
|
||||
)
|
||||
pool_hits = count_pool_hits(results)
|
||||
# Emit PrefetchAck to prefetch_sync_queue, even the operation has been canceled by the
|
||||
# Emit PrefetchAck to the lane sync queue, even the operation has been canceled by the
|
||||
# scheduler thread. The prefetch sync thread expects the same number of PrefetchAck objects
|
||||
# to perform all_reduce.
|
||||
self.prefetch_sync_queue.put(
|
||||
sync_queue.put(
|
||||
PrefetchAck(
|
||||
rid=operation.request_id,
|
||||
operation=operation,
|
||||
@@ -1420,9 +1421,11 @@ class HybridCacheController(BaseHiCacheController):
|
||||
pool.device_indices = source.device_indices
|
||||
return extra_pools
|
||||
|
||||
def _reduce_prefetch_ack(self, ack: PrefetchAck) -> None:
|
||||
def _reduce_prefetch_ack(
|
||||
self, ack: PrefetchAck, groups: List[torch.distributed.ProcessGroup]
|
||||
) -> None:
|
||||
# Handle KV-derived pool.
|
||||
super()._reduce_prefetch_ack(ack)
|
||||
super()._reduce_prefetch_ack(ack, groups)
|
||||
|
||||
# Handle other sidecar pools, e.g. SWA, Mamba.
|
||||
if ack.pool_hits is not None:
|
||||
@@ -1435,20 +1438,24 @@ class HybridCacheController(BaseHiCacheController):
|
||||
self._all_reduce(
|
||||
packed,
|
||||
torch.distributed.ReduceOp.MIN,
|
||||
self.prefetch_completion_sync_groups,
|
||||
groups,
|
||||
)
|
||||
for i, pool in enumerate(PoolName):
|
||||
ack.pool_hits[pool.value] = packed[i].item()
|
||||
|
||||
def prefetch_sync_thread_func(self) -> None:
|
||||
def prefetch_sync_thread_func(self, lane: int = 0) -> None:
|
||||
"""Synchronize progressive ACKs and publish PP ticket completion."""
|
||||
sync_queue = self.prefetch_sync_queues[lane]
|
||||
groups = self.prefetch_completion_sync_groups_by_lane[lane]
|
||||
while not self.storage_stop_event.is_set():
|
||||
try:
|
||||
ack = self.prefetch_sync_queue.get(block=True, timeout=1)
|
||||
ack = sync_queue.get(block=True, timeout=1)
|
||||
if ack is None:
|
||||
continue
|
||||
self._reduce_prefetch_ack(ack)
|
||||
self._reduce_prefetch_ack(ack, groups)
|
||||
if not getattr(ack.operation, "is_pp_broadcast", False):
|
||||
if _hc_timing_enabled() and ack.completed_req:
|
||||
ack.operation.final_ack_enqueue_ts = time.monotonic()
|
||||
self.ack_prefetch_queue.put(ack)
|
||||
continue
|
||||
|
||||
|
||||
@@ -61,12 +61,16 @@ def ranks_per_host() -> int:
|
||||
return max(launch_world_size // get_parallel().nnodes, 1)
|
||||
|
||||
|
||||
def host_memory_budget_bytes(requested_bytes: int = 0) -> int:
|
||||
def host_memory_budget_bytes(
|
||||
requested_bytes: int = 0, *, auto_size: bool = False
|
||||
) -> int:
|
||||
"""Host RAM this rank may claim for a HiCache pool.
|
||||
|
||||
Bound machine availability by the visible cgroup limits before splitting
|
||||
among local ranks. Independent engines with separate container budgets
|
||||
therefore size against their own remaining allowance.
|
||||
Auto-sizing requires successful cgroup discovery; explicit pool sizing
|
||||
can fall back to checking host availability when discovery fails.
|
||||
|
||||
Inside host_memory_budget_scope, requested_bytes is booked against the
|
||||
snapshot when it fits; the allowance before booking is returned.
|
||||
@@ -77,7 +81,10 @@ def host_memory_budget_bytes(requested_bytes: int = 0) -> int:
|
||||
_host_memory_budget.set(available - requested_bytes)
|
||||
return available
|
||||
|
||||
free = available_host_memory_bytes() - HICACHE_HOST_MEMORY_RESERVE_BYTES
|
||||
free = (
|
||||
available_host_memory_bytes(allow_cgroup_fallback=not auto_size)
|
||||
- HICACHE_HOST_MEMORY_RESERVE_BYTES
|
||||
)
|
||||
return free // ranks_per_host()
|
||||
|
||||
|
||||
|
||||
@@ -460,7 +460,18 @@ class MooncakeStore(HiCacheStorage, MooncakeBaseStore):
|
||||
)
|
||||
device_name = ""
|
||||
if self.config.standalone_storage:
|
||||
if not isinstance(mem_pool.allocator, MooncakeHostTensorAllocator):
|
||||
# V4 unified cache hands us a LogicalHostPool anchor that owns no
|
||||
# physical buffers (kv_buffer=None, allocator=None); the real
|
||||
# buffers live in side pools registered later via
|
||||
# register_mem_host_pool_v2() -> register_buffer(ptr, size).
|
||||
# Only enforce the MooncakeHostTensorAllocator requirement when
|
||||
# the anchor actually owns memory.
|
||||
_anchor_owns_buffers = (
|
||||
getattr(mem_pool, "kv_buffer", None) is not None
|
||||
)
|
||||
if _anchor_owns_buffers and not isinstance(
|
||||
getattr(mem_pool, "allocator", None), MooncakeHostTensorAllocator
|
||||
):
|
||||
raise RuntimeError(
|
||||
"MooncakeStore with standalone_storage=True requires MooncakeHostTensorAllocator. "
|
||||
"Please set standalone_storage=False "
|
||||
@@ -504,6 +515,22 @@ class MooncakeStore(HiCacheStorage, MooncakeBaseStore):
|
||||
else:
|
||||
client_hostname = self.config.local_hostname
|
||||
transfer_engine = None
|
||||
# Each rank embeds its own Mooncake client process. Giving
|
||||
# every rank the same local_hostname makes the master's
|
||||
# preferred-segment allocation unable to tell their
|
||||
# segments apart, so replicas can land on a peer rank's
|
||||
# segment and the local-memcpy fast path (which requires
|
||||
# same-process endpoints) silently falls back to TCP.
|
||||
# Make the hostname unique per rank so own-writes land on
|
||||
# the own segment and reads can use local memcpy.
|
||||
if storage_config is not None:
|
||||
client_hostname = (
|
||||
f"{client_hostname}"
|
||||
f"-d{storage_config.dp_rank}"
|
||||
f"t{storage_config.tp_rank}"
|
||||
f"p{storage_config.pp_rank}"
|
||||
f"c{storage_config.attn_cp_rank}"
|
||||
)
|
||||
|
||||
setup_kwargs = {}
|
||||
if self.config.enable_ssd_offload:
|
||||
|
||||
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Callable, Optional, Sequence
|
||||
import torch
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.mem_cache.allocator.swa import is_swa_req_ring
|
||||
from sglang.srt.mem_cache.allocator.swa import swa_kept_out_of_tree
|
||||
from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
DecLockRefParams,
|
||||
EvictParams,
|
||||
@@ -323,9 +323,9 @@ class SWAComponent(TreeComponent):
|
||||
ct = self.component_type
|
||||
state = {"len": float("inf")}
|
||||
|
||||
# A per-request SWA ring is not stored in tree nodes, so its bookkeeping
|
||||
# must not gate prefix matching.
|
||||
swa_req_ring = is_swa_req_ring(self.cache.token_to_kv_pool_allocator)
|
||||
# SWA state that tree nodes do not own (a per-request ring, or a request
|
||||
# window rebuilt by replay) must not gate prefix matching.
|
||||
swa_out_of_tree = swa_kept_out_of_tree(self.cache.token_to_kv_pool_allocator)
|
||||
|
||||
def validator(node: UnifiedTreeNode) -> bool:
|
||||
cd = node.component_data[ct]
|
||||
@@ -333,7 +333,15 @@ class SWAComponent(TreeComponent):
|
||||
# — load_back will restore SWA from host before use.
|
||||
if cd.value is None and (match_device_only or cd.host_value is None):
|
||||
state["len"] = 0
|
||||
if swa_req_ring and (node.backuped or not node.evicted):
|
||||
if swa_out_of_tree and (node.backuped or not node.evicted):
|
||||
return True
|
||||
# Fail-soft: a backuped node whose SWA is absent on *both*
|
||||
# layers (device tombstone plus no L3 backup — the hole a
|
||||
# degraded prefetch can graft) still has Full KV on host.
|
||||
# Treat only that case as a valid match boundary — SWA will
|
||||
# be rebuilt from Full KV at load-back via
|
||||
# SWARebuild/RecoverSWAWithLockedFull.
|
||||
if cd.host_value is None and node.backuped:
|
||||
return True
|
||||
return False
|
||||
state["len"] += len(node.key)
|
||||
@@ -1080,7 +1088,12 @@ class SWAComponent(TreeComponent):
|
||||
cur is not self.tree_core.root_node and n_swa < self.sliding_window_size
|
||||
):
|
||||
cd = cur.component_data[ct]
|
||||
assert cd.host_value is not None or cd.value is not None
|
||||
# Fail-soft: a degraded prefetch may insert a backuped node
|
||||
# whose SWA was never backed up (tombstoned before backup).
|
||||
# Stop collecting — SWA will be rebuilt from Full KV at
|
||||
# load-back via SWARebuild/RecoverSWAWithLockedFull.
|
||||
if cd.host_value is None and cd.value is None:
|
||||
break
|
||||
if cd.value is not None:
|
||||
# device exists, skip it
|
||||
n_swa += len(cd.value)
|
||||
|
||||
@@ -1360,7 +1360,8 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
)
|
||||
|
||||
def _insert_tail_step(self, state: _InsertWalkState) -> None:
|
||||
"""Refresh the LRUs and append terminal backup actions."""
|
||||
"""Refresh the LRUs and append the insert backup: a new-leaf write-through,
|
||||
or an SWA window publish on an existing backed node."""
|
||||
if state.target_node is not self.root_node:
|
||||
for component in self.components:
|
||||
if component.component_type == BASE_COMPONENT_TYPE:
|
||||
@@ -1371,7 +1372,9 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
|
||||
if self._should_backup_after_insert(state):
|
||||
state.pending_actions.append(
|
||||
self._build_backup_kv_action(state.target_node)
|
||||
self._build_backup_kv_action(
|
||||
state.target_node, write_back=self.is_write_back
|
||||
)
|
||||
)
|
||||
|
||||
def _split_node(
|
||||
@@ -2350,7 +2353,8 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
def _build_backup_kv_action(
|
||||
self, node: UnifiedTreeNode, write_back: bool = False
|
||||
) -> BackupKV:
|
||||
"""Build the backup action for a node and its not-yet-persisted ancestors."""
|
||||
"""Build the backup action for a node; write-through also chains its
|
||||
unbacked ancestors."""
|
||||
chain = [node]
|
||||
if not write_back:
|
||||
ancestor = node.parent
|
||||
|
||||
@@ -12,7 +12,7 @@ import torch
|
||||
|
||||
from sglang.srt.distributed.communication_tags import P2PTag
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.managers.cache_controller import CacheOperation
|
||||
from sglang.srt.managers.cache_controller import CacheOperation, _hc_timing_enabled
|
||||
from sglang.srt.mem_cache.allocator.page_interleave import (
|
||||
page_interleave_shard_size,
|
||||
)
|
||||
@@ -48,6 +48,7 @@ from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
from sglang.srt.mem_cache.storage_prefetch import StoragePrefetchRetries
|
||||
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
|
||||
from sglang.srt.mem_cache.utils import get_hash_str
|
||||
from sglang.srt.mem_cache.unified_cache.cache_action import (
|
||||
BackupKV,
|
||||
CacheAction,
|
||||
@@ -1688,6 +1689,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
for comp in self._components_tuple
|
||||
}
|
||||
success = False
|
||||
_hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
try:
|
||||
success = self._load_back_transfers(
|
||||
node_id=node_id,
|
||||
@@ -1699,6 +1701,14 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
)
|
||||
return success
|
||||
finally:
|
||||
if _hc_timing_enabled():
|
||||
logger.info(
|
||||
"HC_TIMING stage=loadback req=%s node=%d ms=%.1f success=%s",
|
||||
getattr(req, "rid", None) if req is not None else "-",
|
||||
node_id,
|
||||
(time.monotonic() - _hc_t0) * 1e3,
|
||||
success,
|
||||
)
|
||||
for comp in self._components_tuple:
|
||||
comp.finalize_load_back(req, preps[comp.component_type], success)
|
||||
|
||||
@@ -1910,11 +1920,46 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
if len(prefetch_key) < self.prefetch_threshold:
|
||||
return 0
|
||||
|
||||
# DeepSeek V4 keeps no physical KV objects in storage (logical anchor),
|
||||
# so a plain batch_exists on KV keys always misses; the usable prefix is
|
||||
# decided by the side pools. Mirror prefetch_from_storage's transfer
|
||||
# construction so the query goes through batch_exists_v2. Both hooks are
|
||||
# pure sizing/descriptor builders: prepare_prefetch only computes
|
||||
# staging_tokens, and PREFETCH-phase transfers carry placeholder keys
|
||||
# whose length sizes the TRAILING_PAGES window for batch_exists_v2.
|
||||
comp_xfers: dict[ComponentType, list[PoolTransfer]] = {}
|
||||
for ct in self.tree_components:
|
||||
if ct == BASE_COMPONENT_TYPE:
|
||||
continue
|
||||
prep = self.components[ct].prepare_prefetch(
|
||||
last_host_node_id, prefetch_tokens=len(prefetch_key)
|
||||
)
|
||||
if prep.staging_tokens == 0:
|
||||
continue
|
||||
transfers = self.tree_core.build_hicache_transfers(
|
||||
ct,
|
||||
last_host_node_id,
|
||||
CacheTransferPhase.PREFETCH,
|
||||
token_ids=prefetch_key.token_ids,
|
||||
prefetch_tokens=len(prefetch_key),
|
||||
staging_tokens=prep.staging_tokens,
|
||||
last_hash=last_hash,
|
||||
)
|
||||
if transfers:
|
||||
comp_xfers[ct] = transfers
|
||||
kv_xfer = PoolTransfer(name=PoolName.KV, host_indices=None)
|
||||
sidecar_xfers = self._build_sidecar_transfers(
|
||||
CacheTransferPhase.PREFETCH, kv_xfer, comp_xfers
|
||||
)
|
||||
aux_xfers = [x for xfers in comp_xfers.values() for x in xfers]
|
||||
aux_xfers.extend(sidecar_xfers)
|
||||
|
||||
operation = PrefetchOperation(
|
||||
CacheRequestHandle("__storage_hit_query__", 0),
|
||||
prefetch_key,
|
||||
last_hash,
|
||||
prefix_keys,
|
||||
pool_transfers=aux_xfers or None,
|
||||
)
|
||||
_, storage_hit_count = self.cache_controller._storage_hit_query(operation)
|
||||
storage_hit_count_tensor = torch.tensor(storage_hit_count, dtype=torch.int)
|
||||
@@ -2005,6 +2050,15 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
self.storage_prefetch_retries.poll_miss(req_id, storage_hit_end)
|
||||
return
|
||||
|
||||
# Recompute last_hash from the full matched prefix so the hash chain
|
||||
# is correct regardless of which node was selected as anchor. The
|
||||
# anchor's hash_value may be at a different position than matched_len
|
||||
# when the anchor has no host_value (async backup not yet complete).
|
||||
if matched_prefix_tokens and len(matched_prefix_tokens) >= self.page_size:
|
||||
prefix_hashes = get_hash_str(matched_prefix_tokens, None, self.page_size)
|
||||
if prefix_hashes:
|
||||
last_hash = prefix_hashes[-1]
|
||||
|
||||
# Buffer mode holds no tree state during the fetch: buffers are
|
||||
# operation-owned, so the anchor needs no pin.
|
||||
anchor_lock_params = (
|
||||
@@ -2236,9 +2290,14 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
last_host_node_id,
|
||||
anchor_lock_params,
|
||||
prefetch_key,
|
||||
comp_xfers,
|
||||
):
|
||||
# Hybrid all-or-nothing check failed; result already discarded.
|
||||
return
|
||||
# The check may have degraded the result (fail-soft): re-read the
|
||||
# retained KV prefix length and hash chain off the operation.
|
||||
completed_tokens = operation.completed_tokens
|
||||
hash_value = operation.hash_value
|
||||
|
||||
allocated_tokens = len(host_indices)
|
||||
if completed_tokens < allocated_tokens:
|
||||
@@ -2262,12 +2321,15 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
)
|
||||
|
||||
fetched_key = prefetch_key[:completed_tokens]
|
||||
_hc_tg = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
insert_result = self.tree_core.insert_host(
|
||||
last_host_node_id,
|
||||
fetched_key,
|
||||
host_indices[:completed_tokens],
|
||||
hash_value[: completed_tokens // self.page_size],
|
||||
)
|
||||
if _hc_timing_enabled():
|
||||
operation.graft_ms = (time.monotonic() - _hc_tg) * 1e3
|
||||
|
||||
# Apply the host-insert walk's actions before the transfer commit.
|
||||
self._apply_cache_actions(insert_result.cache_actions)
|
||||
@@ -2323,6 +2385,16 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
loaded_from_storage,
|
||||
self.cache_controller.prefetch_tokens_occupied,
|
||||
)
|
||||
if _hc_timing_enabled():
|
||||
logger.info(
|
||||
"HC_TIMING stage=prefetch_e2e req=%s e2e_ms=%.1f graft_ms=%.1f "
|
||||
"completed=%d loaded=%d",
|
||||
request.rid,
|
||||
(time.monotonic() - operation.start_time) * 1e3,
|
||||
getattr(operation, "graft_ms", -1.0),
|
||||
completed_tokens,
|
||||
loaded_from_storage,
|
||||
)
|
||||
return
|
||||
|
||||
def _check_hybrid_prefetch_result(
|
||||
@@ -2335,6 +2407,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
last_host_node_id: NodeId,
|
||||
anchor_lock_params: DecLockRefParams,
|
||||
prefetch_key: RadixKey,
|
||||
comp_xfers: dict[ComponentType, list[PoolTransfer]],
|
||||
) -> bool:
|
||||
"""Decide the length of usable prefix.
|
||||
|
||||
@@ -2345,11 +2418,16 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
the Full KV pool and every sidecar. A partial prefix is still usable
|
||||
because the sidecar is page-aligned with KV and required for every page.
|
||||
* Everything else (SWA / Mamba components, mixed DeepSeekV4 stacks):
|
||||
*all-or-nothing*. Their pools only cover a window / tail and cannot be
|
||||
truncated page by page, so any shortfall discards the whole prefetch.
|
||||
*fail-soft degrade* when the shortfall is confined to pools with a
|
||||
cheap rebuild path (SWA — the missing window is rebuilt from Full KV
|
||||
at load-back): the fetched KV prefix is retained and only the failed
|
||||
pool is discarded. Falls back to *all-or-nothing* when nothing is
|
||||
usable or a non-rebuildable pool (Mamba) falls short, since those
|
||||
pools cannot be truncated page by page.
|
||||
|
||||
Returns true if prefetch success, or false when an all-or-nothing prefetch
|
||||
was discarded (the caller should then treat the prefetch as finished).
|
||||
Returns true if prefetch success (possibly degraded), or false when an
|
||||
all-or-nothing prefetch was discarded (the caller should then treat the
|
||||
prefetch as finished).
|
||||
"""
|
||||
# Sync completed tokens and per-pool hit pages across ATTN groups, taking
|
||||
# the minimum so every rank agrees on the same usable prefix length.
|
||||
@@ -2375,57 +2453,118 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
for transfer, count in zip(pool_transfers, pool_hit_pages)
|
||||
)
|
||||
if pool_transfers and not all_succeeded:
|
||||
# Drop the KV beliefs from the first page any pool failed to serve;
|
||||
# the next insert then re-writes that span through one FULL check,
|
||||
# restoring the missing aux pages.
|
||||
# Identify which non-KV components fell short. KV-derived
|
||||
# sidecars (indices_from_pool == KV) are excluded — they share
|
||||
# the KV hit length and are always covered when KV succeeds.
|
||||
failed_components = [
|
||||
ct
|
||||
for ct, xfers in comp_xfers.items()
|
||||
if any(x.indices_from_pool != PoolName.KV for x in xfers)
|
||||
]
|
||||
# Only SWA has a cheap rebuild path (_translate_full_to_swa);
|
||||
# the missing window is rebuilt at load-back via
|
||||
# SWARebuild/RecoverSWAWithLockedFull. Mamba SSM states require
|
||||
# O(seq_len) recompute — not degradable.
|
||||
rebuildable = all(ct != ComponentType.MAMBA for ct in failed_components)
|
||||
|
||||
if completed_tokens == 0 or not rebuildable:
|
||||
# Full discard — nothing usable from any pool, or a
|
||||
# non-rebuildable pool (Mamba) fell short so the whole
|
||||
# result is unusable.
|
||||
# Drop the KV beliefs from the first page any pool failed to serve;
|
||||
# the next insert then re-writes that span through one FULL check,
|
||||
# restoring the missing aux pages.
|
||||
invalidation_hashes = (
|
||||
operation.sidecar_hash_values
|
||||
if operation.sidecar_hash_values is not None
|
||||
else hash_value
|
||||
)
|
||||
trimmed_pages = len(invalidation_hashes) - len(hash_value)
|
||||
keep_pages = trimmed_pages + completed_tokens // self.page_size
|
||||
for transfer, count in zip(pool_transfers, pool_hit_pages):
|
||||
if transfer.keys is None:
|
||||
keep_pages = 0
|
||||
elif count < len(transfer.keys):
|
||||
# Aux transfers key the chain's trailing pages.
|
||||
keep_pages = min(
|
||||
keep_pages,
|
||||
max(0, len(invalidation_hashes) - len(transfer.keys)),
|
||||
)
|
||||
self.storage_existence_cache.invalidate_beyond(
|
||||
PoolName.KV, invalidation_hashes, keep_pages=keep_pages
|
||||
)
|
||||
# The controller's prefetch IO thread already releases the untransferred
|
||||
# tail (host_indices[completed_tokens:])
|
||||
self.cache_controller.append_host_mem_release(
|
||||
host_indices=host_indices[:completed_tokens],
|
||||
extra_pools=pool_transfers if operation.pool_transfers_done else None,
|
||||
)
|
||||
self._finish_storage_prefetch(
|
||||
request, fulfilled_tokens=0, reason="storage_transfer"
|
||||
)
|
||||
if anchor_lock_params is not None:
|
||||
self.dec_host_lock_ref(last_host_node_id, anchor_lock_params)
|
||||
if self.buffer_pipeline is not None:
|
||||
self.buffer_pipeline.pop_prefix_ctx(request)
|
||||
self.buffer_pipeline.release_anchor_lock(request)
|
||||
del self.ongoing_prefetch[request]
|
||||
self.cache_controller.prefetch_tokens_occupied -= (
|
||||
self._prefetch_occupied_span(prefetch_key, host_indices)
|
||||
)
|
||||
self.prefetch_loaded_tokens_by_reqid[request] = 0
|
||||
self.prefetch_loaded_storage_start_by_reqid.pop(request, None)
|
||||
logger.warning(
|
||||
"HiCache hybrid prefetch discarded req=%s completed=%d requested=%d "
|
||||
"kv_beliefs_kept_pages=%d",
|
||||
request.rid,
|
||||
completed_tokens,
|
||||
expected_tokens,
|
||||
keep_pages,
|
||||
)
|
||||
return False
|
||||
|
||||
# Fail-soft degrade: retain the Full KV (+ KV-derived sidecar)
|
||||
# result and discard **only** the failed non-KV independent
|
||||
# pools (SWA) whose storage fetch fell short. Their host
|
||||
# buffers are released here and their component entries are
|
||||
# removed from comp_xfers so the caller's normal commit /
|
||||
# staging path never touches the freed memory. At load-back
|
||||
# time the missing SWA window is rebuilt via the existing
|
||||
# RecoverSWAWithLockedFull / SWARebuild path in swa.py.
|
||||
# Belief self-heal is scoped to the retained KV prefix.
|
||||
invalidation_hashes = (
|
||||
operation.sidecar_hash_values
|
||||
if operation.sidecar_hash_values is not None
|
||||
else hash_value
|
||||
)
|
||||
trimmed_pages = len(invalidation_hashes) - len(hash_value)
|
||||
keep_pages = trimmed_pages + completed_tokens // self.page_size
|
||||
for transfer, count in zip(pool_transfers, pool_hit_pages):
|
||||
if transfer.keys is None:
|
||||
keep_pages = 0
|
||||
elif count < len(transfer.keys):
|
||||
# Aux transfers key the chain's trailing pages.
|
||||
keep_pages = min(
|
||||
keep_pages,
|
||||
max(0, len(invalidation_hashes) - len(transfer.keys)),
|
||||
)
|
||||
self.storage_existence_cache.invalidate_beyond(
|
||||
PoolName.KV, invalidation_hashes, keep_pages=keep_pages
|
||||
PoolName.KV,
|
||||
invalidation_hashes,
|
||||
keep_pages=trimmed_pages + completed_tokens // self.page_size,
|
||||
)
|
||||
# The controller's prefetch IO thread already releases the untransferred
|
||||
# tail (host_indices[completed_tokens:])
|
||||
self.cache_controller.append_host_mem_release(
|
||||
host_indices=host_indices[:completed_tokens],
|
||||
extra_pools=pool_transfers if operation.pool_transfers_done else None,
|
||||
)
|
||||
self._finish_storage_prefetch(
|
||||
request, fulfilled_tokens=0, reason="storage_transfer"
|
||||
)
|
||||
if anchor_lock_params is not None:
|
||||
self.dec_host_lock_ref(last_host_node_id, anchor_lock_params)
|
||||
if self.buffer_pipeline is not None:
|
||||
self.buffer_pipeline.pop_prefix_ctx(request)
|
||||
self.buffer_pipeline.release_anchor_lock(request)
|
||||
del self.ongoing_prefetch[request]
|
||||
self.cache_controller.prefetch_tokens_occupied -= (
|
||||
self._prefetch_occupied_span(prefetch_key, host_indices)
|
||||
)
|
||||
self.prefetch_loaded_tokens_by_reqid[request] = 0
|
||||
self.prefetch_loaded_storage_start_by_reqid.pop(request, None)
|
||||
# Remove failed non-KV components from comp_xfers (same dict as
|
||||
# the caller's) so the commit / staging path won't touch freed
|
||||
# host buffers.
|
||||
for ct in failed_components:
|
||||
comp_xfers.pop(ct, None)
|
||||
# Retain only the KV-fetched prefix; the caller re-reads these
|
||||
# off the operation after the check passes.
|
||||
operation.completed_tokens = completed_tokens
|
||||
operation.hash_value = hash_value[: completed_tokens // self.page_size]
|
||||
logger.warning(
|
||||
"HiCache hybrid prefetch discarded req=%s completed=%d requested=%d "
|
||||
"kv_beliefs_kept_pages=%d",
|
||||
"HiCache hybrid prefetch degraded req=%s completed=%d "
|
||||
"requested=%d retained_kv=%d discarded_components=%s",
|
||||
request.rid,
|
||||
completed_tokens,
|
||||
expected_tokens,
|
||||
keep_pages,
|
||||
completed_tokens,
|
||||
[ct.value for ct in failed_components],
|
||||
)
|
||||
return False
|
||||
return True
|
||||
return True
|
||||
|
||||
def _record_storage_prefetch_hit(
|
||||
@@ -2897,7 +3036,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
self.ongoing_prefetch[request] = info._replace(host_indices=host_indices)
|
||||
if buffer_mode:
|
||||
cc.prefetch_tokens_occupied += alloc_len
|
||||
cc.prefetch_buffer.put(operation)
|
||||
cc.dispatch_prefetch(operation)
|
||||
return True
|
||||
|
||||
def _drain_and_alloc_storage_hit():
|
||||
@@ -2978,6 +3117,14 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
operation.pool_transfers_done = True
|
||||
if ack.completed_req:
|
||||
if is_current:
|
||||
if _hc_timing_enabled():
|
||||
_hc_enq = getattr(operation, "final_ack_enqueue_ts", None)
|
||||
if _hc_enq is not None:
|
||||
logger.info(
|
||||
"HC_TIMING stage=ack_drain req=%s wait_ms=%.1f",
|
||||
operation.request_id,
|
||||
(time.monotonic() - _hc_enq) * 1e3,
|
||||
)
|
||||
# check_prefetch_progress() is not called for this rid yet.
|
||||
# Let us insert the prefetch result into the radix tree.
|
||||
self._handle_prefetch_result(operation)
|
||||
|
||||
@@ -4488,9 +4488,12 @@ class DeepseekV4Model(nn.Module):
|
||||
forward_batch,
|
||||
cp_all_tokens=cp_extend,
|
||||
)
|
||||
# Only multimodal placeholders become image_token_id, so a text-only
|
||||
# batch skips this full-residual where.
|
||||
if (
|
||||
self.config.model_type == "deepseek_v41"
|
||||
and self.config.vision_n_layers > 0
|
||||
and forward_batch.contains_mm_inputs()
|
||||
):
|
||||
hidden_states = torch.where(
|
||||
(input_ids == self.config.image_token_id)[:, None, None],
|
||||
@@ -4715,7 +4718,9 @@ class DeepseekV4Model(nn.Module):
|
||||
hidden_states = self.embed_tokens(input_ids)
|
||||
else:
|
||||
hidden_states = input_embeds
|
||||
hidden_states = hidden_states.unsqueeze(1).repeat(1, self.hc_mult, 1)
|
||||
from sglang.kernels.ops.layernorm.mhc import hc_broadcast
|
||||
|
||||
hidden_states = hc_broadcast(hidden_states, self.hc_mult)
|
||||
else:
|
||||
assert pp_proxy_tensors is not None
|
||||
hidden_states = pp_proxy_tensors["hidden_states"]
|
||||
|
||||
@@ -157,7 +157,7 @@ class Session:
|
||||
or carry_fill is input_ids_unpadded
|
||||
):
|
||||
# Unexpected type or aliased with an origin array (extending it
|
||||
# below would double-append): let _refresh_fill_ids rebuild.
|
||||
# below would double-append): let refresh_fill_ids rebuild.
|
||||
carry_fill = None
|
||||
else:
|
||||
del carry_fill[self.committed_fill_len :]
|
||||
|
||||
@@ -4052,7 +4052,8 @@ def get_cuda_graph_batch_size_alignment() -> int:
|
||||
alignment *= 2
|
||||
if require_gathered_buffer():
|
||||
alignment *= get_parallel().attn_tp_size
|
||||
if alignment % get_parallel().attn_cp_size != 0:
|
||||
# TODO: unverified on NVIDIA; drop the gate once validated on CUDA.
|
||||
if not is_hip() and alignment % get_parallel().attn_cp_size != 0:
|
||||
alignment *= get_parallel().attn_cp_size
|
||||
return alignment
|
||||
|
||||
|
||||
@@ -2592,6 +2592,107 @@ fn backup_kv_action_chains_unbacked_ancestors_first() {
|
||||
assert_eq!(action.node_ids, vec![c]);
|
||||
}
|
||||
|
||||
// Mirrors UnifiedRadixCache._execute_and_commit_kv_backup: back up each chain
|
||||
// node and stamp every node its transfers cover with that node's ack.
|
||||
// It skips the inc_lock_ref the Python side takes; the asserts do not depend on it.
|
||||
fn execute_backup_kv_for_test(tc: &mut UnifiedTreeCore<Vec<i64>>, backup: &BackupKV) {
|
||||
for &node_id in &backup.node_ids {
|
||||
let (device_value, mut comp_xfers) = tc.build_backup_spec(node_id).expect("backup spec");
|
||||
if device_value.numel() == 0 && comp_xfers.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let mut publish_node_ids = Vec::new();
|
||||
for transfer in comp_xfers.values_mut().flatten() {
|
||||
transfer.host_indices = transfer.device_indices.as_ref().map(Tensor::copy);
|
||||
publish_node_ids.extend(transfer.nodes_to_load.iter().flatten().copied());
|
||||
}
|
||||
if !publish_node_ids.contains(&node_id) {
|
||||
publish_node_ids.push(node_id);
|
||||
}
|
||||
tc.commit_backup(node_id, device_value.copy(), comp_xfers)
|
||||
.expect("live test node");
|
||||
tc.mark_write_through_pending(publish_node_ids, /* ack_id = */ node_id)
|
||||
.expect("live test nodes");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_back_swa_publish_leaves_an_ancestor_pending_under_another_ack_alone() {
|
||||
// c(Full unbacked) -> b(Full backed) -> t1(Full backed)
|
||||
// -> d(Full unbacked) -> t2(Full backed)
|
||||
// Every node holds a device-only SWA value inside the window.
|
||||
let mut tc = UnifiedTreeCore::new(
|
||||
CacheInitParams {
|
||||
is_write_back: true,
|
||||
enable_hicache: true,
|
||||
has_swa_host_pool: true,
|
||||
swa_sliding_window_size: Some(8),
|
||||
..Default::default()
|
||||
},
|
||||
vec![FULL, SWA],
|
||||
);
|
||||
let (key_t1, key_t2) = (vec![1, 2, 3, 4], vec![1, 2, 5, 6]);
|
||||
tc.insert(&insert_params(&key_t1, &[10, 11, 12, 13]));
|
||||
tc.insert(&insert_params(&key_t2, &[10, 11, 14, 15]));
|
||||
tc.insert(&insert_params(&vec![1, 2, 3], &[10, 11, 12]));
|
||||
tc.insert(&insert_params(&vec![1, 2, 5], &[10, 11, 14]));
|
||||
let child = |parent, page: &[i64]| {
|
||||
tc.arena
|
||||
.child_on_page(parent, /* extra_key = */ None, page)
|
||||
.expect("child on page")
|
||||
};
|
||||
let c_idx = child(tc.arena.root(), &[1]);
|
||||
let b_idx = child(c_idx, &[3]);
|
||||
let t1_idx = child(b_idx, &[4]);
|
||||
let d_idx = child(c_idx, &[5]);
|
||||
let t2_idx = child(d_idx, &[6]);
|
||||
let [c, b, t1, d, t2] = [c_idx, b_idx, t1_idx, d_idx, t2_idx].map(|idx| tc.arena.node(idx).id);
|
||||
for idx in [c_idx, b_idx, t1_idx, d_idx, t2_idx] {
|
||||
let full = tc.arena.node(idx).device_value(FULL).copy();
|
||||
tc.set_component_device_value(tc.arena.node(idx).id, SWA, full.copy())
|
||||
.expect("live test node");
|
||||
if [b_idx, t1_idx, t2_idx].contains(&idx) {
|
||||
tc.arena.set_host_value(idx, FULL, full);
|
||||
}
|
||||
}
|
||||
|
||||
// Two requests in one batch finish at t1 and t2; neither ack drains.
|
||||
for (key, target) in [(&key_t1, t1), (&key_t2, t2)] {
|
||||
let result = tc.insert(&insert_params(key, &[20, 21, 22, 23]));
|
||||
let backups: Vec<&BackupKV> = result
|
||||
.cache_actions
|
||||
.iter()
|
||||
.filter_map(|action| match action {
|
||||
CacheAction::BackupKV(backup) => Some(backup),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(backups.len(), 1);
|
||||
assert_eq!(backups[0].node_ids, vec![target]);
|
||||
execute_backup_kv_for_test(&mut tc, backups[0]);
|
||||
}
|
||||
|
||||
let pending = |node_id| {
|
||||
tc.arena
|
||||
.node(tc.arena.resolve(node_id).expect("live test node"))
|
||||
.write_through_pending_id
|
||||
};
|
||||
for (node_id, ack_id) in [(c, t1), (b, t1), (t1, t1), (d, t2), (t2, t2)] {
|
||||
assert_eq!(pending(node_id), Some(ack_id));
|
||||
}
|
||||
// Write-back defers the unbacked Full prefix to eviction.
|
||||
assert!(
|
||||
!tc.arena
|
||||
.node(tc.arena.resolve(c).expect("live test node"))
|
||||
.backuped()
|
||||
);
|
||||
assert!(
|
||||
!tc.arena
|
||||
.node(tc.arena.resolve(d).expect("live test node"))
|
||||
.backuped()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backup_kv_action_stops_at_an_externally_stored_or_pending_ancestor() {
|
||||
let mut tc = core();
|
||||
|
||||
@@ -1833,7 +1833,8 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
&& self.needs_incremental_component_backup_(target_node_id)
|
||||
}
|
||||
|
||||
/// Refresh the LRUs and append terminal backup actions.
|
||||
/// Refresh the LRUs and append the insert backup: a new-leaf write-through,
|
||||
/// or an SWA window publish on an existing backed node.
|
||||
fn insert_tail_step_(&mut self, state: &mut InsertWalkState<K>) {
|
||||
let target_node_id = state
|
||||
.target_node_id
|
||||
@@ -1850,10 +1851,8 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
}
|
||||
|
||||
if self.should_backup_after_insert_(state, target_node_id) {
|
||||
let backup = self.build_backup_kv_action_(
|
||||
self.arena.node(target_node_id),
|
||||
/* write_back = */ false,
|
||||
);
|
||||
let backup =
|
||||
self.build_backup_kv_action_(self.arena.node(target_node_id), self.is_write_back);
|
||||
state.pending_actions.push(CacheAction::BackupKV(backup));
|
||||
}
|
||||
}
|
||||
@@ -3611,7 +3610,8 @@ impl<K: ChildKeyType> UnifiedTreeCore<K> {
|
||||
Ok(order)
|
||||
}
|
||||
|
||||
/// Build the backup action for a node and its not-yet-persisted ancestors.
|
||||
/// Build the backup action for a node; write-through also chains its
|
||||
/// unbacked ancestors.
|
||||
pub fn build_backup_kv_action_(&self, node: &Node<K>, write_back: bool) -> BackupKV {
|
||||
let mut chain = vec![node.id];
|
||||
if !write_back {
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
"""Benchmark the fused ratio-2 decode compressor (``c2_decode_norm_rope_store``).
|
||||
|
||||
One CTA per token: pair-pool against the state ring, RMSNorm, RoPE, quantize and
|
||||
store one compressed KV row. The line axis is the cache page format.
|
||||
|
||||
Shape axes:
|
||||
- ``bs``: requests per batch.
|
||||
- ``draft_len``: 1 is plain decode; > 1 is a target-verify block of that many
|
||||
consecutive positions per request.
|
||||
- ``parity``: the first position's parity. On decode, ``odd`` rows complete a
|
||||
pair and run the full norm/RoPE/store path, ``even`` rows only park in the ring.
|
||||
|
||||
Constants follow the DeepSeek-V4 deployment: ``head_dim = 512``, ``rope_dim = 64``,
|
||||
a 256-token FULL page, hence 128 compressed slots per page.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.jit.benchmark import marker
|
||||
from sglang.kernels.jit.benchmark.utils import (
|
||||
DEFAULT_DEVICE,
|
||||
create_empty,
|
||||
create_random,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout
|
||||
from sglang.kernels.ops.attention.dsv4.low_ratio_compress import (
|
||||
c2_decode_norm_rope_store,
|
||||
)
|
||||
from sglang.srt.mem_cache.deepseek_v4_memory_pool import get_compress_state_ring_size
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(
|
||||
est_time=30, stage="base-b-kernel-benchmark", runner_config="1-gpu-large"
|
||||
)
|
||||
|
||||
HEAD_DIM = 512
|
||||
ROPE_DIM = 64
|
||||
PAGE_SIZE = 256 // 2
|
||||
MAX_POS = 65536
|
||||
EPS = 1e-6
|
||||
|
||||
|
||||
@marker.parametrize("parity", ["odd", "even", "half"], ["half"])
|
||||
@marker.parametrize("draft_len", [1, 2, 4, 8], [1, 8])
|
||||
@marker.parametrize("bs", marker.range(14, pattern="pow2"), [1, 1024])
|
||||
@marker.benchmark("layout", ["V4", "V41", "V41_FP4"])
|
||||
def benchmark(bs: int, draft_len: int, parity: str, layout: str):
|
||||
if parity == "half" and bs % 2 != 0:
|
||||
marker.skip("half parity requires even batch size")
|
||||
layout = KVLayout[layout]
|
||||
num_tokens = bs * draft_len
|
||||
ring_size = get_compress_state_ring_size(2, draft_len if draft_len > 1 else 0)
|
||||
torch.manual_seed(0)
|
||||
|
||||
kv_input = create_random(num_tokens, 2 * HEAD_DIM, dtype=torch.float32)
|
||||
kv_state = create_random(bs * ring_size, 2 * HEAD_DIM, dtype=torch.float32)
|
||||
norm_weight = create_random(HEAD_DIM)
|
||||
freqs_cis = create_random(MAX_POS, ROPE_DIM, dtype=torch.float32)
|
||||
req_ids = torch.randperm(bs, device=DEFAULT_DEVICE)
|
||||
start = torch.randint(1, MAX_POS // 2 - draft_len, (bs,), device=DEFAULT_DEVICE)
|
||||
if parity == "odd":
|
||||
start = start * 2 + 1
|
||||
elif parity == "even":
|
||||
start = start * 2
|
||||
else:
|
||||
start = start * 2
|
||||
start[::2] += 1 # half odd, half even
|
||||
|
||||
offsets = torch.arange(draft_len, device=DEFAULT_DEVICE)
|
||||
positions = (start[:, None] + offsets).flatten()
|
||||
req = req_ids.repeat_interleave(draft_len)
|
||||
|
||||
# Distinct scattered slots; slot 0 is left alone, it marks a padded row.
|
||||
num_slots = num_tokens + 1
|
||||
num_pages = -(-num_slots // PAGE_SIZE)
|
||||
slots = torch.randperm(num_pages * PAGE_SIZE - 1, device=DEFAULT_DEVICE)
|
||||
slots = slots[:num_tokens]
|
||||
raw_out_loc = (slots + 1) * 2 + 1
|
||||
k_cache = create_empty(
|
||||
num_pages,
|
||||
layout.page_bytes(PAGE_SIZE),
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
out = create_empty(num_tokens, HEAD_DIM)
|
||||
|
||||
# Traffic of the rows that complete a pair; parked rows write the state instead.
|
||||
num_complete = int((positions % 2 == 1).sum())
|
||||
num_parked = num_tokens - num_complete
|
||||
row_bytes = 2 * HEAD_DIM * 4
|
||||
complete_bytes = (
|
||||
2 * row_bytes # kv_input + partner
|
||||
+ ROPE_DIM * 4 # freqs_cis row
|
||||
+ HEAD_DIM * 2 # out
|
||||
+ layout.bytes_per_token
|
||||
)
|
||||
footprint = (
|
||||
num_complete * complete_bytes
|
||||
+ num_parked * 3 * row_bytes # kv_input + partner read, state write
|
||||
+ num_tokens * 3 * 8 # positions, req, raw_out_loc
|
||||
+ norm_weight.nbytes
|
||||
)
|
||||
return marker.do_bench(
|
||||
c2_decode_norm_rope_store,
|
||||
input_args=(kv_input, kv_state, norm_weight, positions, req, raw_out_loc),
|
||||
input_kwargs=dict(
|
||||
eps=EPS,
|
||||
freqs_cis=freqs_cis,
|
||||
k_cache=k_cache,
|
||||
page_size=PAGE_SIZE,
|
||||
ring_size=ring_size,
|
||||
draft_len=draft_len,
|
||||
layout=layout,
|
||||
out=out,
|
||||
),
|
||||
memory_args=None,
|
||||
memory_output=None,
|
||||
extra_memory_footprint=footprint,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
benchmark.run()
|
||||
@@ -0,0 +1,191 @@
|
||||
"""The ratio-2 compressor's eager-extend path against its decode path.
|
||||
|
||||
``c2_prefill_norm_rope_store`` pairs rows inside a chunk and parks each request's
|
||||
last ``ring_size`` rows with a separate write-back kernel, while
|
||||
``c2_decode_norm_rope_store`` takes every partner from the ring and parks every
|
||||
pending row there. Driving one token stream both ways must leave the same main-KV
|
||||
bytes and the same latents, so a chunk boundary that parks the wrong rows shows up
|
||||
in the next chunk's output instead of staying invisible in the first chunk's.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from typing import List, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout
|
||||
from sglang.kernels.ops.attention.dsv4.low_ratio_compress import (
|
||||
c2_decode_norm_rope_store,
|
||||
c2_prefill_norm_rope_store,
|
||||
)
|
||||
from sglang.srt.utils import is_cuda
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
# The c2 JIT kernels are the SM100 build; the only Blackwell runner config is the
|
||||
# four-GPU one.
|
||||
register_cuda_ci(est_time=60, stage="base-b-kernel-unit", runner_config="4-gpu-b200")
|
||||
|
||||
HEAD_DIM = 512
|
||||
ROPE_DIM = 64
|
||||
PAGE_SIZE = 64
|
||||
NUM_PAGES = 16
|
||||
EPS = 1e-6
|
||||
# Request r's full slots start here, even so that a (2k, 2k+1) pair shares a
|
||||
# compressed slot; never 0, which marks a padded row.
|
||||
SLOT_BASE = 64
|
||||
|
||||
|
||||
def _sm100() -> bool:
|
||||
return is_cuda() and torch.cuda.get_device_capability()[0] == 10
|
||||
|
||||
|
||||
class Stream:
|
||||
"""One token stream: per-request prefix and chunk sizes, laid out as the rows
|
||||
the backend would hand each call."""
|
||||
|
||||
def __init__(self, prefixes: List[int], chunks: List[List[int]]):
|
||||
self.prefixes = prefixes
|
||||
self.chunks = chunks # chunks[c][r] = rows of request r in chunk c
|
||||
|
||||
def chunk_rows(self, c: int) -> List[Tuple[int, int]]:
|
||||
"""(request, position) of every row of chunk ``c``, requests in order."""
|
||||
rows = []
|
||||
for r, n in enumerate(self.chunks[c]):
|
||||
start = self.prefixes[r] + sum(self.chunks[j][r] for j in range(c))
|
||||
rows.extend((r, start + i) for i in range(n))
|
||||
return rows
|
||||
|
||||
|
||||
def _make_inputs(gen, rows: int) -> torch.Tensor:
|
||||
"""``[rows, 2 * HEAD_DIM]`` fp32: the kv half then the score half."""
|
||||
return torch.randn(
|
||||
rows, 2 * HEAD_DIM, generator=gen, device="cuda", dtype=torch.float32
|
||||
)
|
||||
|
||||
|
||||
def _row_tensors(rows: List[Tuple[int, int]], pad: int):
|
||||
"""``req``, ``positions`` and ``raw_out_loc`` for ``rows``, plus ``pad`` padded
|
||||
rows. A padded row still has to address a live ring slot, so it takes request 0
|
||||
and position 0; only ``raw_out_loc == 0`` marks it."""
|
||||
req = [r for r, _ in rows] + [0] * pad
|
||||
pos = [p for _, p in rows] + [0] * pad
|
||||
loc = [SLOT_BASE * (1 + r) + p for r, p in rows] + [0] * pad
|
||||
to = lambda v, d: torch.tensor(v, device="cuda", dtype=d)
|
||||
return to(req, torch.int64), to(pos, torch.int32), to(loc, torch.int32)
|
||||
|
||||
|
||||
class TestC2PrefillRing(CustomTestCase):
|
||||
def _run(self, stream: Stream, layout: KVLayout, ring_size: int, seed: int):
|
||||
"""The fused chunked path and the row-at-a-time decode path over the same
|
||||
stream; returns (cache, latents) for each."""
|
||||
gen = torch.Generator(device="cuda").manual_seed(seed)
|
||||
num_reqs = len(stream.prefixes)
|
||||
weight = torch.randn(
|
||||
HEAD_DIM, generator=gen, device="cuda", dtype=torch.float32
|
||||
).to(torch.bfloat16)
|
||||
angles = torch.randn(4096, ROPE_DIM // 2, generator=gen, device="cuda")
|
||||
freqs = torch.view_as_real(
|
||||
torch.polar(torch.ones_like(angles), angles)
|
||||
).flatten(-2)
|
||||
|
||||
# One fp32 input row per (request, position) in the stream, shared by both
|
||||
# paths so any difference is the pairing, not the data.
|
||||
all_rows = [
|
||||
row for c in range(len(stream.chunks)) for row in stream.chunk_rows(c)
|
||||
]
|
||||
inputs = {row: _make_inputs(gen, 1)[0] for row in all_rows}
|
||||
|
||||
state_rows = num_reqs * ring_size + ring_size
|
||||
caches, latents = [], []
|
||||
for fused in (True, False):
|
||||
cache = torch.zeros(
|
||||
NUM_PAGES,
|
||||
layout.page_bytes(PAGE_SIZE),
|
||||
dtype=torch.uint8,
|
||||
device="cuda",
|
||||
)
|
||||
state = torch.zeros(
|
||||
state_rows, 2 * HEAD_DIM, dtype=torch.float32, device="cuda"
|
||||
)
|
||||
out = {}
|
||||
if fused:
|
||||
for c in range(len(stream.chunks)):
|
||||
rows = stream.chunk_rows(c)
|
||||
pad = 2
|
||||
req, pos, loc = _row_tensors(rows, pad)
|
||||
kv = torch.stack(
|
||||
[inputs[row] for row in rows]
|
||||
+ [torch.zeros(2 * HEAD_DIM, device="cuda")] * pad
|
||||
)
|
||||
starts, lens = [], []
|
||||
at = 0
|
||||
for r in range(num_reqs):
|
||||
starts.append(at)
|
||||
lens.append(stream.chunks[c][r])
|
||||
at += stream.chunks[c][r]
|
||||
latent = c2_prefill_norm_rope_store(
|
||||
kv,
|
||||
state,
|
||||
weight,
|
||||
pos,
|
||||
req,
|
||||
loc,
|
||||
EPS,
|
||||
freqs,
|
||||
cache,
|
||||
torch.tensor(starts, device="cuda", dtype=torch.int32),
|
||||
torch.tensor(lens, device="cuda", dtype=torch.int32),
|
||||
page_size=PAGE_SIZE,
|
||||
ring_size=ring_size,
|
||||
layout=layout,
|
||||
)
|
||||
for i, row in enumerate(rows):
|
||||
out[row] = latent[i].clone()
|
||||
else:
|
||||
for row in all_rows:
|
||||
req, pos, loc = _row_tensors([row], 0)
|
||||
latent = c2_decode_norm_rope_store(
|
||||
inputs[row][None, :],
|
||||
state,
|
||||
weight,
|
||||
pos,
|
||||
req,
|
||||
loc,
|
||||
EPS,
|
||||
freqs,
|
||||
cache,
|
||||
page_size=PAGE_SIZE,
|
||||
ring_size=ring_size,
|
||||
layout=layout,
|
||||
)
|
||||
out[row] = latent[0].clone()
|
||||
caches.append(cache)
|
||||
# Only a completing (odd position) row writes a latent.
|
||||
latents.append(
|
||||
torch.stack([out[row] for row in all_rows if row[1] % 2 == 1])
|
||||
)
|
||||
return caches, latents
|
||||
|
||||
@unittest.skipUnless(_sm100(), "the c2 JIT kernels are the SM100 build")
|
||||
def test_chunked_extend_matches_decode(self):
|
||||
# Request 0 starts even, request 1 odd and spans more rows than the ring,
|
||||
# request 2 odd with a single row in its first chunk.
|
||||
stream = Stream(
|
||||
prefixes=[0, 7, 1],
|
||||
chunks=[[5, 4, 1], [3, 6, 2], [1, 1, 1]],
|
||||
)
|
||||
for layout in (KVLayout.V41, KVLayout.V41_FP4):
|
||||
for ring_size in (2, 8):
|
||||
with self.subTest(layout=layout.name, ring_size=ring_size):
|
||||
caches, latents = self._run(stream, layout, ring_size, seed=7)
|
||||
self.assertTrue(
|
||||
torch.equal(caches[0], caches[1]),
|
||||
f"main-KV bytes differ: {(caches[0] != caches[1]).sum()} of "
|
||||
f"{caches[0].numel()}",
|
||||
)
|
||||
self.assertTrue(torch.equal(latents[0], latents[1]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -10,6 +10,9 @@ Checks the Triton kernel against a per-row torch reference:
|
||||
3. ``test_non_trailing_query_positions``: absolute ``query_pos`` that are not
|
||||
the trailing extend tokens, with a cross-chunk ``query_start_loc`` offset.
|
||||
4. ``test_swa_only_layer``: ``topk == 0`` writes only the window.
|
||||
5. ``test_reused_buffers_are_overwritten_whole``: the kernel is handed
|
||||
uninitialized and already-written buffers, so it must overwrite every row it
|
||||
is given rather than rely on a ``-1`` fill pass.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
@@ -80,12 +83,15 @@ def _check(**kwargs):
|
||||
assert torch.equal(got_idx, ref_idx)
|
||||
|
||||
|
||||
def _trailing_case(seq_lens, extend_lens, topk, compress_ratio, seed=0):
|
||||
"""Rows are the trailing ``extend_lens[r]`` tokens of request ``r``."""
|
||||
def _trailing_case(seq_lens, extend_lens, topk, compress_ratio, seed=0, pad_rows=0):
|
||||
"""Rows are the trailing ``extend_lens[r]`` tokens of request ``r``, followed by
|
||||
``pad_rows`` rows past the last request."""
|
||||
gen = torch.Generator(device="cpu").manual_seed(seed)
|
||||
query_pos = []
|
||||
for seq_len, extend_len in zip(seq_lens, extend_lens):
|
||||
query_pos.extend(range(seq_len - extend_len, seq_len))
|
||||
# Past the last request, so the kernel never reads these positions.
|
||||
query_pos.extend([0] * pad_rows)
|
||||
num_tokens = len(query_pos)
|
||||
topk_indices = torch.randint(
|
||||
0, 1 << 20, (num_tokens, topk), generator=gen, dtype=torch.int32
|
||||
@@ -163,6 +169,48 @@ def test_non_trailing_query_positions():
|
||||
)
|
||||
|
||||
|
||||
def test_reused_buffers_are_overwritten_whole():
|
||||
"""Callers hand the kernel uninitialized or already-written buffers, so every
|
||||
row it is given must be overwritten in full. A row that keeps part of an
|
||||
earlier call's longer selection, or a row past the last request that keeps
|
||||
anything at all, is a stale index attention would read as live."""
|
||||
width = -(-(64 + WINDOW) // TOPK_ALIGNMENT) * TOPK_ALIGNMENT
|
||||
# Long selections first: every row fills 64 + WINDOW entries.
|
||||
wide = _trailing_case([512, 640], [8, 8], topk=64, compress_ratio=4)
|
||||
num_tokens = wide["topk_indices"].shape[0]
|
||||
out_indices = torch.empty((num_tokens, width), dtype=torch.int32, device=DEVICE)
|
||||
out_lens = torch.empty(num_tokens, dtype=torch.int32, device=DEVICE)
|
||||
|
||||
# An uninitialized buffer is poison, not -1.
|
||||
out_indices.fill_(1 << 30)
|
||||
out_lens.fill_(1 << 30)
|
||||
got_idx, got_lens = combine_topk_swa_indices(
|
||||
**wide, out_indices=out_indices, out_lens=out_lens
|
||||
)
|
||||
assert got_idx.data_ptr() == out_indices.data_ptr()
|
||||
ref_idx, ref_lens = _reference(**wide)
|
||||
assert torch.equal(got_lens, ref_lens)
|
||||
assert torch.equal(got_idx, ref_idx)
|
||||
assert int(got_lens[0]) == 64 + WINDOW
|
||||
|
||||
# Same buffers, now holding the first call's output: shorter selections, a
|
||||
# zero-length request, and rows past the last request.
|
||||
narrow = _trailing_case(
|
||||
[20, 36, 48], [2, 0, 3], topk=64, compress_ratio=4, pad_rows=num_tokens - 5
|
||||
)
|
||||
assert narrow["topk_indices"].shape[0] == num_tokens
|
||||
got_idx, got_lens = combine_topk_swa_indices(
|
||||
**narrow, out_indices=out_indices, out_lens=out_lens
|
||||
)
|
||||
ref_idx, ref_lens = _reference(**narrow)
|
||||
assert torch.equal(got_lens, ref_lens), (got_lens.tolist(), ref_lens.tolist())
|
||||
assert torch.equal(got_idx, ref_idx)
|
||||
# The shorter rows and the padding rows must not keep the first call's tails.
|
||||
assert int(got_lens[0]) < 64 + WINDOW
|
||||
assert (got_lens[5:] == 0).all()
|
||||
assert (got_idx[5:] == -1).all()
|
||||
|
||||
|
||||
def test_swa_only_layer():
|
||||
case = _trailing_case([200, 50], [2, 2], topk=0, compress_ratio=4)
|
||||
case["topk_indices"] = torch.zeros((4, 1), dtype=torch.int32, device=DEVICE)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import struct
|
||||
import threading
|
||||
import unittest
|
||||
from collections import defaultdict
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import ANY, Mock, call, patch
|
||||
|
||||
@@ -18,6 +19,8 @@ from sglang.srt.disaggregation.common.staging_handler import (
|
||||
handle_staging_req,
|
||||
)
|
||||
from sglang.srt.disaggregation.common.utils import (
|
||||
FastQueue,
|
||||
TransferKVChunk,
|
||||
group_concurrent_contiguous,
|
||||
pack_int_lists,
|
||||
pack_list_of_buffers,
|
||||
@@ -68,6 +71,227 @@ register_cpu_ci(est_time=11, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class TestDisaggregationWire(unittest.TestCase):
|
||||
def test_mooncake_stride_mismatch_only_fails_affected_request(self):
|
||||
for layer_ids in ([], [7]):
|
||||
for outcome in ("mismatch", "cleared", "transport_failure"):
|
||||
with self.subTest(layer_ids=layer_ids, outcome=outcome):
|
||||
manager = object.__new__(MooncakeKVManager)
|
||||
manager.kv_args = SimpleNamespace(
|
||||
kv_data_ptrs=[],
|
||||
state_types=[StateType.SWA, StateType.DSA],
|
||||
state_data_ptrs=[[0x1000], [0x3000]],
|
||||
state_item_lens=[[32], [16]],
|
||||
state_dim_per_tensor=[[], []],
|
||||
state_layer_ids=[layer_ids, layer_ids],
|
||||
aux_data_ptrs=[0x5000],
|
||||
aux_item_lens=[8],
|
||||
)
|
||||
manager.is_mla_backend = True
|
||||
manager.is_hybrid_mla_backend = False
|
||||
manager.attn_tp_size = manager.attn_cp_size = manager.pp_size = 1
|
||||
manager.attn_tp_rank = manager.attn_cp_rank = manager.pp_rank = 0
|
||||
manager.enable_trace = manager.enable_staging = False
|
||||
manager.enable_custom_mem_pool = False
|
||||
manager.max_transfer_batch_indices = 0
|
||||
manager.bootstrap_port = 1234
|
||||
manager.session_lock = threading.Lock()
|
||||
manager.failure_lock = threading.Lock()
|
||||
manager.failed_sessions = set()
|
||||
manager.session_failures = defaultdict(int)
|
||||
manager.failure_records = {}
|
||||
manager._staging_outstanding = defaultdict(int)
|
||||
manager.request_status = {
|
||||
42: KVPoll.Transferring,
|
||||
43: KVPoll.Transferring,
|
||||
}
|
||||
manager.req_to_decode_prefix_len = {42: 0, 43: 0}
|
||||
manager.transfer_infos = {}
|
||||
manager.decode_kv_args_table = {}
|
||||
queue = FastQueue()
|
||||
for room in (42, 43):
|
||||
req = TransferInfo(
|
||||
room=room,
|
||||
endpoint="127.0.0.1",
|
||||
dst_port=1234,
|
||||
mooncake_session_id=f"decode:{room}",
|
||||
dst_kv_indices=np.array([], dtype=np.int32),
|
||||
dst_aux_index=5,
|
||||
dst_state_indices=[[5], [5]],
|
||||
required_dst_info_num=1,
|
||||
is_dummy=False,
|
||||
)
|
||||
manager.transfer_infos[room] = {req.mooncake_session_id: req}
|
||||
manager.decode_kv_args_table[req.mooncake_session_id] = (
|
||||
SimpleNamespace(
|
||||
requires_dcp_relayout=False,
|
||||
dst_state_data_ptrs=[[0x2000], [0x4000]],
|
||||
dst_state_item_lens=[
|
||||
[
|
||||
64
|
||||
if room == 42 and outcome != "transport_failure"
|
||||
else 32
|
||||
],
|
||||
[16],
|
||||
],
|
||||
dst_state_dim_per_tensor=[[], []],
|
||||
dst_state_layer_ids=[layer_ids, layer_ids],
|
||||
dst_attn_tp_size=1,
|
||||
dst_aux_ptrs=[0x6000],
|
||||
)
|
||||
)
|
||||
queue.put(
|
||||
TransferKVChunk(
|
||||
room=room,
|
||||
prefill_kv_indices=np.array([], dtype=np.int32),
|
||||
index_slice=slice(0, 0),
|
||||
is_last_chunk=True,
|
||||
prefill_aux_index=2,
|
||||
state_indices=[[2], [2]],
|
||||
)
|
||||
)
|
||||
queue.put(None)
|
||||
|
||||
def send_status(endpoint, parts, **kwargs):
|
||||
if outcome == "cleared" and parts[0] == b"42":
|
||||
manager.request_status.pop(42)
|
||||
|
||||
manager._send_multipart_locked = Mock(side_effect=send_status)
|
||||
manager._transfer_data = Mock(
|
||||
side_effect=lambda session, blocks: (
|
||||
-1
|
||||
if session == "decode:42" and outcome == "transport_failure"
|
||||
else 0
|
||||
)
|
||||
)
|
||||
with (
|
||||
get_context().override_server_args(enable_unified_memory=False),
|
||||
patch.object(
|
||||
envs.SGLANG_MOONCAKE_SEND_AUX_TCP, "get", return_value=False
|
||||
),
|
||||
):
|
||||
manager.transfer_worker(queue, executor=None)
|
||||
|
||||
self.assertEqual(
|
||||
manager.request_status.get(42),
|
||||
None if outcome == "cleared" else KVPoll.Failed,
|
||||
)
|
||||
self.assertEqual(manager.request_status[43], KVPoll.Success)
|
||||
expected_reason = (
|
||||
"Failed to send state components of 42 to 127.0.0.1:1234"
|
||||
if outcome == "transport_failure"
|
||||
else f"{StateType.SWA} item length mismatch for "
|
||||
+ (
|
||||
"paired entries src[0]=32 dst[0]=64"
|
||||
if layer_ids
|
||||
else "positional entry 0: prefill=32 decode=64"
|
||||
)
|
||||
)
|
||||
self.assertEqual(manager.failure_records[42], expected_reason)
|
||||
self.assertEqual(
|
||||
[
|
||||
c.args[1]
|
||||
for c in manager._send_multipart_locked.call_args_list
|
||||
],
|
||||
[[b"42", b"0", b"0"], [b"43", b"4", b"0"]],
|
||||
)
|
||||
good_writes = [
|
||||
call("decode:43", [(0x1040, 0x20A0, 32)]),
|
||||
call("decode:43", [(0x3020, 0x4050, 16)]),
|
||||
call("decode:43", [(0x5010, 0x6028, 8)]),
|
||||
]
|
||||
expected_writes = (
|
||||
[
|
||||
call("decode:42", [(0x1040, 0x20A0, 32)]),
|
||||
call("decode:42", [(0x3020, 0x4050, 16)]),
|
||||
]
|
||||
if outcome == "transport_failure"
|
||||
else []
|
||||
) + good_writes
|
||||
self.assertEqual(
|
||||
manager._transfer_data.call_args_list, expected_writes
|
||||
)
|
||||
self.assertEqual(
|
||||
manager.failed_sessions,
|
||||
{"decode:42"} if outcome == "transport_failure" else set(),
|
||||
)
|
||||
self.assertEqual(
|
||||
dict(manager.session_failures),
|
||||
{"decode:42": 1} if outcome == "transport_failure" else {},
|
||||
)
|
||||
self.assertFalse(manager._staging_outstanding)
|
||||
self.assertFalse(manager.transfer_infos)
|
||||
self.assertFalse(manager.req_to_decode_prefix_len)
|
||||
|
||||
def test_mooncake_state_stride_matches_pp_slice(self):
|
||||
cases = [
|
||||
([], [64, 128], [32, 64, 128, 256], [1, 2]),
|
||||
(
|
||||
[4, 128, 4, 128],
|
||||
[32, 64, 512, 2048],
|
||||
[16, 32, 64, 128, 256, 512, 1024, 2048],
|
||||
[1, 2, 5, 7],
|
||||
),
|
||||
]
|
||||
for ratios, src_lens, dst_lens, dst_entries in cases:
|
||||
with self.subTest(ratios=ratios):
|
||||
manager = object.__new__(MooncakeKVManager)
|
||||
manager.kv_args = SimpleNamespace(
|
||||
kv_data_ptrs=[],
|
||||
prefill_start_layer=1,
|
||||
prefill_end_layer=3,
|
||||
mla_compression_ratios=ratios,
|
||||
)
|
||||
manager.is_mla_backend = True
|
||||
manager.pp_size = 2
|
||||
manager.enable_custom_mem_pool = False
|
||||
manager.max_transfer_batch_indices = 0
|
||||
manager._transfer_data = Mock(return_value=0)
|
||||
src_ptrs = [0x1000 * (i + 1) for i in range(len(src_lens))]
|
||||
dst_ptrs = [0x1000 * (i + 9) for i in range(len(dst_lens))]
|
||||
manager._send_kvcache_generic(
|
||||
mooncake_session_id="decode",
|
||||
executor=None,
|
||||
src_data_ptrs=src_ptrs,
|
||||
dst_data_ptrs=dst_ptrs,
|
||||
item_lens=src_lens,
|
||||
dst_item_lens=dst_lens,
|
||||
prefill_data_indices=np.array([2], dtype=np.int32),
|
||||
dst_data_indices=np.array([5], dtype=np.int32),
|
||||
state_type=StateType.SWA,
|
||||
)
|
||||
manager._transfer_data.assert_called_once_with(
|
||||
"decode",
|
||||
[
|
||||
(
|
||||
src_ptrs[i] + 2 * length,
|
||||
dst_ptrs[j] + 5 * length,
|
||||
length,
|
||||
)
|
||||
for i, (j, length) in enumerate(zip(dst_entries, src_lens))
|
||||
],
|
||||
)
|
||||
|
||||
def test_sender_clear_keeps_abort_ack_until_writes_drain(self):
|
||||
manager = object.__new__(MooncakeKVManager)
|
||||
sender = object.__new__(MooncakeKVSender)
|
||||
sender.kv_mgr, sender.bootstrap_room = manager, 42
|
||||
for outstanding in (0, 1):
|
||||
with self.subTest(outstanding=outstanding):
|
||||
manager.request_status = {42: KVPoll.Failed}
|
||||
manager._staging_outstanding = {42: outstanding}
|
||||
manager._deferred_ack_targets = {42: ("127.0.0.1", 1234)}
|
||||
manager._deferred_ack_fanout_snapshots = {}
|
||||
manager.transfer_infos = {}
|
||||
with patch.object(manager, "_send_abort_ack") as ack:
|
||||
sender.clear()
|
||||
if outstanding:
|
||||
ack.assert_not_called()
|
||||
manager._staging_outstanding[42] = 0
|
||||
manager._maybe_ack_drained_abort(42)
|
||||
ack.assert_called_once_with("127.0.0.1", 1234, 42)
|
||||
manager._maybe_ack_drained_abort(42)
|
||||
ack.assert_called_once()
|
||||
|
||||
def test_mooncake_registration_staging_fields(self):
|
||||
msg = [
|
||||
b"room",
|
||||
@@ -401,6 +625,54 @@ class TestQwen4StateWire(unittest.TestCase):
|
||||
[(0, 2), (1, 3)],
|
||||
)
|
||||
|
||||
def _qsa_compressed_manager(self, *, src_item_len):
|
||||
manager = object.__new__(MooncakeKVManager)
|
||||
manager.request_status = {42: KVPoll.Transferring}
|
||||
manager.attn_tp_size = 4
|
||||
manager.pp_size = 1
|
||||
manager.is_mla_backend = False
|
||||
manager.is_hybrid_mla_backend = False
|
||||
manager.kv_args = SimpleNamespace(
|
||||
engine_rank=0,
|
||||
state_types=[StateType.QSA_COMPRESSED],
|
||||
state_data_ptrs=[[1000]],
|
||||
state_item_lens=[[src_item_len]],
|
||||
state_dim_per_tensor=[[]],
|
||||
state_layer_ids=[[24]],
|
||||
)
|
||||
manager.sent = []
|
||||
manager._send_kvcache_generic = lambda **kw: manager.sent.append(kw) or 0
|
||||
return manager
|
||||
|
||||
def test_qsa_compressed_layout_mismatch_rejected_before_transfer(self):
|
||||
"""Equal attention TP with fp8 on one peer only halves that peer's
|
||||
compressed item length. The transfer uses the source length as the
|
||||
destination stride, so the mismatch must be rejected before any write."""
|
||||
req = SimpleNamespace(
|
||||
room=42, mooncake_session_id="session", dst_state_indices=[[3]]
|
||||
)
|
||||
decode_info = SimpleNamespace(
|
||||
dst_attn_tp_size=4,
|
||||
dst_state_data_ptrs=[[2000]],
|
||||
dst_state_item_lens=[[16]],
|
||||
dst_state_dim_per_tensor=[[]],
|
||||
dst_state_layer_ids=[[24]],
|
||||
)
|
||||
manager = self._qsa_compressed_manager(src_item_len=32)
|
||||
with self.assertRaisesRegex(RuntimeError, "QSA_COMPRESSED layout differs"):
|
||||
MooncakeKVManager.maybe_send_extra(
|
||||
manager, req, [[3]], None, target_rank_registration_info=decode_info
|
||||
)
|
||||
self.assertEqual(manager.sent, [])
|
||||
|
||||
manager = self._qsa_compressed_manager(src_item_len=16)
|
||||
rc = MooncakeKVManager.maybe_send_extra(
|
||||
manager, req, [[3]], None, target_rank_registration_info=decode_info
|
||||
)
|
||||
self.assertEqual(rc, 0)
|
||||
self.assertEqual(len(manager.sent), 1)
|
||||
self.assertEqual(manager.sent[0]["item_lens"], [16])
|
||||
|
||||
def test_replicated_state_tp_policy(self):
|
||||
for src_tp, dst_tp, rank, expected in (
|
||||
(4, 1, 0, True),
|
||||
|
||||
@@ -1,14 +1,18 @@
|
||||
import concurrent.futures
|
||||
import ctypes
|
||||
import unittest
|
||||
from threading import Event
|
||||
from collections import defaultdict
|
||||
from queue import SimpleQueue
|
||||
from threading import Event, Lock
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, call, patch
|
||||
|
||||
import numpy as np
|
||||
|
||||
from sglang.srt.disaggregation.base.conn import StateType
|
||||
from sglang.srt.disaggregation.mooncake.conn import MooncakeKVManager
|
||||
from sglang.srt.disaggregation.base.conn import KVPoll, StateType
|
||||
from sglang.srt.disaggregation.common.utils import TransferKVChunk
|
||||
from sglang.srt.disaggregation.mooncake.conn import MooncakeKVManager, MooncakeKVSender
|
||||
from sglang.srt.disaggregation.utils import DisaggregationMode
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
@@ -407,5 +411,136 @@ class TestDcpPackLifetime(CustomTestCase):
|
||||
self.assertEqual(observed, [11])
|
||||
|
||||
|
||||
class TestMooncakeEarlySend(unittest.TestCase):
|
||||
def test_sender_queues_event_once_without_waiting_on_scheduler(self):
|
||||
for last in (False, True):
|
||||
with self.subTest(last=last):
|
||||
ready = MagicMock()
|
||||
manager = MagicMock()
|
||||
sender = SimpleNamespace(
|
||||
kv_mgr=manager,
|
||||
bootstrap_room=1,
|
||||
aux_index=0,
|
||||
trace_ctx=MagicMock(),
|
||||
_early_send_wait_event=ready,
|
||||
_prepare_send_indices=lambda ids, state: (
|
||||
ids,
|
||||
slice(0, 1),
|
||||
last,
|
||||
False,
|
||||
),
|
||||
_record_transfer_indices=MagicMock(),
|
||||
)
|
||||
MooncakeKVSender.send(sender, [0])
|
||||
self.assertIs(
|
||||
manager.add_transfer_request.call_args.kwargs["wait_event"], ready
|
||||
)
|
||||
ready.synchronize.assert_not_called()
|
||||
MooncakeKVSender.send(sender, [1])
|
||||
self.assertIsNone(
|
||||
manager.add_transfer_request.call_args.kwargs["wait_event"]
|
||||
)
|
||||
|
||||
def test_manager_preserves_event_in_queued_chunk(self):
|
||||
queue = SimpleQueue()
|
||||
ready = MagicMock()
|
||||
manager = SimpleNamespace(
|
||||
disaggregation_mode=DisaggregationMode.PREFILL,
|
||||
request_status={1: KVPoll.WaitingForInput},
|
||||
check_status=lambda room: KVPoll.WaitingForInput,
|
||||
transfer_infos={1: {"127.0.0.1:80": object()}},
|
||||
transfer_queues=[queue],
|
||||
)
|
||||
MooncakeKVManager.add_transfer_request(
|
||||
manager, 1, [0], slice(0, 1), False, wait_event=ready
|
||||
)
|
||||
self.assertIs(queue.get_nowait().wait_event, ready)
|
||||
ready.synchronize.assert_not_called()
|
||||
|
||||
def _worker(self, event):
|
||||
chunk = TransferKVChunk(
|
||||
1, [0], slice(0, 1), False, None, None, wait_event=event
|
||||
)
|
||||
queue = SimpleQueue()
|
||||
queue.put(chunk)
|
||||
queue.put(None)
|
||||
peer = SimpleNamespace(
|
||||
is_dummy=False,
|
||||
mooncake_session_id="peer",
|
||||
dst_device_kv_indices=None,
|
||||
dst_kv_indices=[10],
|
||||
)
|
||||
registration = SimpleNamespace(
|
||||
requires_dcp_relayout=False,
|
||||
dst_kv_ptrs=[1000],
|
||||
dst_kv_layer_ids=[0],
|
||||
dst_kv_item_len=16,
|
||||
dst_attn_tp_size=1,
|
||||
)
|
||||
manager = SimpleNamespace(
|
||||
enable_trace=False,
|
||||
enable_staging=False,
|
||||
enable_deferred_decode_kv_release=True,
|
||||
_staging_outstanding=defaultdict(int),
|
||||
request_status={1: KVPoll.WaitingForInput},
|
||||
transfer_infos={1: {"peer": peer}},
|
||||
_prefill_unique_rank=lambda: 0,
|
||||
session_lock=Lock(),
|
||||
failed_sessions=set(),
|
||||
# Read under session_lock alongside failed_sessions on every
|
||||
# non-dummy req, and invoked by the worker's except path.
|
||||
state_layout_rejections={},
|
||||
poison_deferred_ack_room=MagicMock(),
|
||||
decode_kv_args_table={"peer": registration},
|
||||
_get_dsa_cache_transfer_skip_flags=lambda reg: (False, False),
|
||||
kv_args=SimpleNamespace(kv_data_ptrs=[1]),
|
||||
is_mla_backend=True,
|
||||
send_kvcache=MagicMock(return_value=0),
|
||||
_maybe_ack_drained_abort=MagicMock(),
|
||||
bootstrap_port=8998,
|
||||
)
|
||||
manager.check_status = lambda room: manager.request_status[room]
|
||||
return manager, queue
|
||||
|
||||
def test_worker_waits_before_reading_kv(self):
|
||||
ready = MagicMock()
|
||||
manager, queue = self._worker(ready)
|
||||
|
||||
def transfer(*args, **kwargs):
|
||||
ready.synchronize.assert_called_once_with()
|
||||
return 0
|
||||
|
||||
manager.send_kvcache.side_effect = transfer
|
||||
MooncakeKVManager.transfer_worker(manager, queue, MagicMock())
|
||||
manager.send_kvcache.assert_called_once()
|
||||
|
||||
def test_abort_during_wait_prevents_transfer_and_releases_ack(self):
|
||||
ready = MagicMock()
|
||||
manager, queue = self._worker(ready)
|
||||
ready.synchronize.side_effect = lambda: manager.request_status.update(
|
||||
{1: KVPoll.Failed}
|
||||
)
|
||||
MooncakeKVManager.transfer_worker(manager, queue, MagicMock())
|
||||
ready.synchronize.assert_called_once_with()
|
||||
manager.send_kvcache.assert_not_called()
|
||||
manager._maybe_ack_drained_abort.assert_called_once_with(1)
|
||||
self.assertNotIn(1, manager._staging_outstanding)
|
||||
|
||||
def test_failed_room_skips_wait(self):
|
||||
ready = MagicMock()
|
||||
manager, queue = self._worker(ready)
|
||||
manager.request_status[1] = KVPoll.Failed
|
||||
MooncakeKVManager.transfer_worker(manager, queue, MagicMock())
|
||||
ready.synchronize.assert_not_called()
|
||||
manager.send_kvcache.assert_not_called()
|
||||
manager._maybe_ack_drained_abort.assert_called_once_with(1)
|
||||
self.assertNotIn(1, manager._staging_outstanding)
|
||||
|
||||
def test_worker_preserves_no_event_path(self):
|
||||
manager, queue = self._worker(None)
|
||||
MooncakeKVManager.transfer_worker(manager, queue, MagicMock())
|
||||
manager.send_kvcache.assert_called_once()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -219,6 +219,7 @@ class TestKPoolStreamScheduling(unittest.TestCase):
|
||||
)
|
||||
batch = SimpleNamespace(
|
||||
forward_mode=ForwardMode.EXTEND,
|
||||
seq_lens=torch.tensor([8192 + num_tokens]),
|
||||
seq_lens_cpu=torch.tensor([8192 + num_tokens]),
|
||||
)
|
||||
prepare_qk = Mock(return_value=(x, x, None, None))
|
||||
|
||||
@@ -28,7 +28,7 @@ class _FakeReq:
|
||||
"""Carries the fill-id state convert_decode_to_extend touches, with the
|
||||
real Req methods so the array bookkeeping is not re-implemented here."""
|
||||
|
||||
_refresh_fill_ids = Req._refresh_fill_ids
|
||||
refresh_fill_ids = Req.refresh_fill_ids
|
||||
set_extend_range = Req.set_extend_range
|
||||
|
||||
def __init__(self, *, num_prompt_tokens: int, num_output_tokens: int):
|
||||
|
||||
@@ -132,7 +132,7 @@ class _FakeReq:
|
||||
self.full_untruncated_fill_ids = list(range(origin_len + output_len))
|
||||
self.extend_range = None
|
||||
|
||||
def _refresh_fill_ids(self):
|
||||
def refresh_fill_ids(self):
|
||||
self.full_untruncated_fill_ids = self.origin_input_ids + self.output_ids
|
||||
|
||||
@property
|
||||
|
||||
@@ -103,6 +103,19 @@ class MockReq:
|
||||
def get_fill_ids(self):
|
||||
return self.full_untruncated_fill_ids[: self.extend_range.end]
|
||||
|
||||
rid = "mock-req"
|
||||
skip_radix_cache_insert = False
|
||||
finished_reason = None
|
||||
|
||||
def finished(self):
|
||||
return self.finished_reason is not None
|
||||
|
||||
def refresh_fill_ids(self):
|
||||
pass
|
||||
|
||||
def owned_kv_len(self):
|
||||
return self.kv.kv_committed_len
|
||||
|
||||
|
||||
def _make_req(fill_ids, req_pool_idx=0, cache_protected_len=0, last_node=None):
|
||||
return MockReq(fill_ids, req_pool_idx, cache_protected_len, last_node)
|
||||
|
||||
@@ -24,6 +24,21 @@ register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class TestDSV4CompressedPools(CustomTestCase):
|
||||
def test_swa_key_page_size_uses_physical_paged_size(self):
|
||||
pool = DeepSeekV4TokenToKVPool.__new__(DeepSeekV4TokenToKVPool)
|
||||
pool.request_window = None
|
||||
pool.swa_page_size = 256
|
||||
for physical_page_size in (256, 64):
|
||||
with self.subTest(physical_page_size=physical_page_size):
|
||||
pool.swa_kv_pool = SimpleNamespace(page_size=physical_page_size)
|
||||
self.assertEqual(pool.get_swa_key_page_size(), physical_page_size)
|
||||
|
||||
def test_swa_key_page_size_without_paged_pool(self):
|
||||
pool = DeepSeekV4TokenToKVPool.__new__(DeepSeekV4TokenToKVPool)
|
||||
pool.swa_kv_pool = None
|
||||
pool.request_window = SimpleNamespace(page_size=256)
|
||||
self.assertEqual(pool.get_swa_key_page_size(), 256)
|
||||
|
||||
def test_physical_kv_pages_cover_reserved_logical_page(self):
|
||||
size = 8192
|
||||
self.assertEqual(_num_dsv4_physical_kv_pages(size, 256, 256), 33)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Exercise container and ancestor budgets using synthetic procfs/cgroup files."""
|
||||
|
||||
import functools
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
@@ -8,6 +9,8 @@ from unittest.mock import Mock, patch
|
||||
from sglang.srt.mem_cache import host_memory
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
_cgroup_memory_headroom = host_memory._cgroup_memory_headroom
|
||||
|
||||
register_cpu_ci(est_time=1, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
@@ -31,7 +34,7 @@ class TestHostMemory(unittest.TestCase):
|
||||
f"1 0 0:1 {mount_root} {escaped} rw - {filesystem} cgroup {options}\n"
|
||||
)
|
||||
|
||||
def memory(self, path, usage, maximum="max", high="max", v1=False):
|
||||
def memory(self, path, usage, maximum="max", high="max", v1=False, stat=None):
|
||||
directory = self.mount / path
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
files = (
|
||||
@@ -39,6 +42,11 @@ class TestHostMemory(unittest.TestCase):
|
||||
if v1
|
||||
else {"memory.max": maximum, "memory.high": high, "memory.current": usage}
|
||||
)
|
||||
prefix = "total_" if v1 else ""
|
||||
stat = {f"{prefix}active_file": 0, f"{prefix}inactive_file": 0} | (stat or {})
|
||||
files["memory.stat"] = "".join(
|
||||
f"{key} {value}\n" for key, value in stat.items()
|
||||
)
|
||||
for name, value in files.items():
|
||||
(directory / name).write_text(str(value))
|
||||
|
||||
@@ -74,6 +82,32 @@ class TestHostMemory(unittest.TestCase):
|
||||
self.memory("task", 400, 1000, v1=True)
|
||||
self.assertEqual(host_memory._cgroup_memory_headroom(self.proc), 600)
|
||||
|
||||
def test_reclaimable_page_cache_is_not_used(self):
|
||||
"""Page cache charged to the cgroup, such as a checkpoint just read, is
|
||||
reclaimed under the limit and must not shrink the headroom. Shared
|
||||
memory is counted in file/cache but cannot be reclaimed."""
|
||||
v2 = {"file": 700, "shmem": 200, "active_file": 200, "inactive_file": 300}
|
||||
v1 = {
|
||||
"cache": 700,
|
||||
"shmem": 200,
|
||||
"active_file": 0,
|
||||
"inactive_file": 0,
|
||||
"total_active_file": 200,
|
||||
"total_inactive_file": 300,
|
||||
}
|
||||
for is_v1, stat, usage, expected in [
|
||||
(False, v2, 900, 600),
|
||||
(True, v1, 900, 600),
|
||||
# v1 usage is approximate and may trail the cache counters.
|
||||
(True, v1, 400, 1000),
|
||||
]:
|
||||
with self.subTest(v1=is_v1, usage=usage):
|
||||
self.configure(v1=is_v1)
|
||||
self.memory("task/engine", usage, 1000, v1=is_v1, stat=stat)
|
||||
self.assertEqual(
|
||||
host_memory._cgroup_memory_headroom(self.proc), expected
|
||||
)
|
||||
|
||||
def test_independent_engines_have_separate_allowances(self):
|
||||
# Both engines see the same host RAM but have different charged usage.
|
||||
for task, usage, expected in [("a", 300, 700), ("b", 600, 400)]:
|
||||
@@ -98,9 +132,14 @@ class TestHostMemory(unittest.TestCase):
|
||||
)
|
||||
|
||||
def test_host_availability_is_also_a_bound(self):
|
||||
for cgroup, expected in [(None, 100), (200, 100), (50, 50)]:
|
||||
for cgroup, allow_fallback, expected in [
|
||||
(None, False, 100),
|
||||
(200, False, 100),
|
||||
(50, False, 50),
|
||||
(50, True, 50),
|
||||
]:
|
||||
with (
|
||||
self.subTest(cgroup=cgroup),
|
||||
self.subTest(cgroup=cgroup, allow_fallback=allow_fallback),
|
||||
patch.object(
|
||||
host_memory.psutil,
|
||||
"virtual_memory",
|
||||
@@ -110,20 +149,57 @@ class TestHostMemory(unittest.TestCase):
|
||||
host_memory, "_cgroup_memory_headroom", return_value=cgroup
|
||||
),
|
||||
):
|
||||
self.assertEqual(host_memory.available_host_memory_bytes(), expected)
|
||||
self.assertEqual(
|
||||
host_memory.available_host_memory_bytes(
|
||||
allow_cgroup_fallback=allow_fallback
|
||||
),
|
||||
expected,
|
||||
)
|
||||
|
||||
def test_unmounted_memory_cgroup_fails(self):
|
||||
self.configure()
|
||||
(self.proc / "self/mountinfo").write_text("")
|
||||
with self.assertRaisesRegex(RuntimeError, "Cannot locate"):
|
||||
host_memory._cgroup_memory_headroom(self.proc)
|
||||
def available(self, host_available=5000, *, allow_cgroup_fallback=False):
|
||||
"""Public sizing entry point over the synthetic procfs."""
|
||||
with (
|
||||
patch.object(
|
||||
host_memory.psutil,
|
||||
"virtual_memory",
|
||||
return_value=Mock(available=host_available),
|
||||
),
|
||||
patch.object(
|
||||
host_memory,
|
||||
"_cgroup_memory_headroom",
|
||||
functools.partial(_cgroup_memory_headroom, self.proc),
|
||||
),
|
||||
):
|
||||
return host_memory.available_host_memory_bytes(
|
||||
allow_cgroup_fallback=allow_cgroup_fallback
|
||||
)
|
||||
|
||||
def test_missing_usage_for_known_limit_fails(self):
|
||||
self.configure()
|
||||
self.memory("task/engine", 100, 1000)
|
||||
(self.mount / "task/engine/memory.current").unlink()
|
||||
with self.assertRaises(FileNotFoundError):
|
||||
host_memory._cgroup_memory_headroom(self.proc)
|
||||
def without_cgroupfs(self, v1=False):
|
||||
# A container sharing the host cgroup namespace, no cgroupfs mounted.
|
||||
self.configure("/system.slice/engine.scope", v1=v1)
|
||||
(self.proc / "self/mountinfo").write_text("1 0 0:1 / /proc rw - proc proc rw\n")
|
||||
|
||||
def test_unavailable_cgroup_requires_explicit_sizing(self):
|
||||
for v1 in [False, True]:
|
||||
for mounted in [False, True]:
|
||||
with self.subTest(v1=v1, mounted=mounted):
|
||||
if mounted:
|
||||
self.configure("/system.slice/engine.scope", v1=v1)
|
||||
else:
|
||||
self.without_cgroupfs(v1=v1)
|
||||
with self.assertRaisesRegex(RuntimeError, "set --hicache-size"):
|
||||
self.available()
|
||||
self.assertEqual(self.available(allow_cgroup_fallback=True), 5000)
|
||||
|
||||
def test_missing_counters_for_known_limit_requires_explicit_sizing(self):
|
||||
for name in ("memory.current", "memory.stat"):
|
||||
with self.subTest(name=name):
|
||||
self.configure()
|
||||
self.memory("task/engine", 100, 1000)
|
||||
(self.mount / "task/engine" / name).unlink()
|
||||
with self.assertRaisesRegex(RuntimeError, "set --hicache-size"):
|
||||
self.available()
|
||||
self.assertEqual(self.available(allow_cgroup_fallback=True), 5000)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -641,6 +641,11 @@ class _GraftReq:
|
||||
def get_fill_ids(self):
|
||||
return array("q", self.fill_ids)
|
||||
|
||||
def refresh_fill_ids(self):
|
||||
pass # fill ids are fixed for the stand-in
|
||||
|
||||
def finished(self):
|
||||
return self.finished_reason is not None
|
||||
|
||||
class TestRotationGraftDecline(CustomTestCase):
|
||||
"""The overlap disagg-prefill loop plans batch t+1 before batch t's radix
|
||||
|
||||
@@ -37,15 +37,17 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
||||
c.pp_size, c.tp_size = 4, 2
|
||||
c.pp_group, c.pp_prefetch_command_group = "pp", "command"
|
||||
c.pp_prefetch_command_thread = None
|
||||
c.prefetch_hits_sync_groups = c.prefetch_completion_sync_groups = ["pp", "tp"]
|
||||
c.prefetch_hits_sync_groups = ["pp", "tp"]
|
||||
c.num_prefetch_io_lanes = 1
|
||||
c.prefetch_completion_sync_groups_by_lane = [["pp", "tp"]]
|
||||
c.pp_prefetch_states, c.pp_prefetch_decisions = {}, {}
|
||||
c.pp_prefetch_state_lock = threading.Lock()
|
||||
c.pp_prefetch_command_queue = Queue()
|
||||
c.prefetch_queue = Queue()
|
||||
c.prefetch_sync_queue = Queue()
|
||||
c.prefetch_sync_queues = [Queue()]
|
||||
c.ack_prefetch_queue = Queue()
|
||||
c.host_mem_release_queue = Queue()
|
||||
c.prefetch_buffer = Queue()
|
||||
c.prefetch_io_queues = [Queue()]
|
||||
c.storage_stop_event = threading.Event()
|
||||
c.prefetch_tokens_occupied = 0
|
||||
c.mem_pool_host = Mock(page_size=4)
|
||||
@@ -103,7 +105,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
||||
|
||||
def sync_acks(self, *acks):
|
||||
for ack in acks:
|
||||
self.c.prefetch_sync_queue.put(ack)
|
||||
self.c.prefetch_sync_queues[0].put(ack)
|
||||
with patch.object(
|
||||
self.c.storage_stop_event,
|
||||
"is_set",
|
||||
@@ -183,7 +185,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
||||
self.assertFalse(cache.check_prefetch_progress(handle))
|
||||
c._storage_hit_query.reset_mock()
|
||||
self.run_commands(ticket)
|
||||
operation = c.prefetch_buffer.get_nowait()
|
||||
operation = c.prefetch_io_queues[0].get_nowait()
|
||||
self.assertEqual(operation.handle, handle)
|
||||
self.assertEqual(
|
||||
operation.hash_value, get_storage_hash_str(ticket.prefetch_key, page_size=4)
|
||||
@@ -237,7 +239,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
||||
),
|
||||
):
|
||||
c.prefetch_io_aux_func()
|
||||
acks = [c.prefetch_sync_queue.get_nowait() for _ in range(8)]
|
||||
acks = [c.prefetch_sync_queues[0].get_nowait() for _ in range(8)]
|
||||
self.assertEqual([a.rid for a in acks], ["hit"] * 4 + ["next"] * 4)
|
||||
self.assertEqual(
|
||||
[a.completed_tokens for a in acks], [0, 0, None, None, 4, 8, None, None]
|
||||
@@ -261,7 +263,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
||||
cache.finish(ticket.handle, CacheRequestOutcome.ABORT)
|
||||
self.assertIs(c.pp_prefetch_decisions["hit"], PPPrefetchDecision.CANCELLED)
|
||||
self.run_commands(ticket)
|
||||
operation = c.prefetch_buffer.get_nowait()
|
||||
operation = c.prefetch_io_queues[0].get_nowait()
|
||||
self.sync_acks(PrefetchAck("hit", operation, completed_tokens=4))
|
||||
c.mem_pool_host.free.assert_not_called()
|
||||
self.sync_acks(PrefetchAck("hit", operation, completed_req=True))
|
||||
@@ -293,7 +295,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
||||
c.pp_prefetch_states.clear()
|
||||
c.mem_pool_host.alloc.reset_mock()
|
||||
self.run_commands(ticket)
|
||||
operation = c.prefetch_buffer.get_nowait()
|
||||
operation = c.prefetch_io_queues[0].get_nowait()
|
||||
self.assertEqual(
|
||||
c.mem_pool_host.alloc.call_args_list,
|
||||
[
|
||||
@@ -311,7 +313,7 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
||||
c.mem_pool_host.get_pool.side_effect = KeyError(PoolName.SWA)
|
||||
with self.assertLogs(level="ERROR"):
|
||||
self.run_commands(ticket)
|
||||
self.assertTrue(c.prefetch_buffer.get_nowait().is_terminated())
|
||||
self.assertTrue(c.prefetch_io_queues[0].get_nowait().is_terminated())
|
||||
self.assertEqual(c.mem_pool_host.free.call_count, 1)
|
||||
self.assertEqual(c.mem_pool_host.free.call_args.kwargs["pool"], PoolName.KV)
|
||||
|
||||
@@ -387,8 +389,8 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
||||
self.assertEqual(invocation.kwargs, {"src": 1})
|
||||
self.assertEqual(c.pp_prefetch_command_queue.unfinished_tasks, 0)
|
||||
self.assertTrue(c.pp_prefetch_command_queue.empty())
|
||||
self.assertIs(c.prefetch_buffer.get_nowait(), operation)
|
||||
self.assertTrue(c.prefetch_buffer.empty())
|
||||
self.assertIs(c.prefetch_io_queues[0].get_nowait(), operation)
|
||||
self.assertTrue(c.prefetch_io_queues[0].empty())
|
||||
c.mem_pool_host.alloc.assert_called_once()
|
||||
|
||||
def test_idle_downstream_skips_empty_broadcasts_before_ticket_and_stop(self):
|
||||
@@ -404,8 +406,8 @@ class TestPPPrefetchTicket(unittest.TestCase):
|
||||
patch.object(c.pp_prefetch_command_queue, "task_done") as task_done,
|
||||
):
|
||||
c.pp_prefetch_command_thread_func()
|
||||
self.assertEqual(c.prefetch_buffer.get_nowait().request_id, "hit")
|
||||
self.assertTrue(c.prefetch_buffer.empty())
|
||||
self.assertEqual(c.prefetch_io_queues[0].get_nowait().request_id, "hit")
|
||||
self.assertTrue(c.prefetch_io_queues[0].empty())
|
||||
c.mem_pool_host.alloc.assert_called_once()
|
||||
task_done.assert_not_called()
|
||||
|
||||
|
||||
@@ -167,7 +167,7 @@ class TestSharedPrefillAdmission(unittest.TestCase):
|
||||
with get_parallel().override(attn_dcp_size=1):
|
||||
scheduler.init_req_max_new_tokens(req)
|
||||
self.assertEqual(req.sampling_params.max_new_tokens, 1)
|
||||
req._refresh_fill_ids()
|
||||
req.refresh_fill_ids()
|
||||
|
||||
cache = SimpleNamespace(
|
||||
sliding_window_size=page_size,
|
||||
|
||||
@@ -78,7 +78,7 @@ class TestSessionTokenShare(CustomTestCase):
|
||||
if baked is None:
|
||||
baked = len(output)
|
||||
req.output_ids.extend(output[:baked])
|
||||
req._refresh_fill_ids()
|
||||
req.refresh_fill_ids()
|
||||
req.output_ids.extend(output[baked:])
|
||||
self.session.finish_req(req)
|
||||
|
||||
@@ -114,7 +114,7 @@ class TestSessionTokenShare(CustomTestCase):
|
||||
r2 = self._create("r2", [50, 51])
|
||||
self.assertEqual(list(r2.origin_input_ids), in1 + out1 + [50, 51])
|
||||
r2.output_ids.extend([6, 7])
|
||||
r2._refresh_fill_ids()
|
||||
r2.refresh_fill_ids()
|
||||
self.session.abort_req()
|
||||
self.assertEqual(self.session.committed_origin_len, len(in1))
|
||||
|
||||
@@ -149,7 +149,7 @@ class TestSessionTokenShare(CustomTestCase):
|
||||
# array, then output trimmed to finished_len (like _trim_overshoot)
|
||||
# before finish.
|
||||
r1.output_ids.extend([1, 2, 3, 4, 5, 6])
|
||||
r1._refresh_fill_ids()
|
||||
r1.refresh_fill_ids()
|
||||
del r1.output_ids[4:]
|
||||
self.session.finish_req(r1)
|
||||
self.assertEqual(
|
||||
@@ -161,7 +161,7 @@ class TestSessionTokenShare(CustomTestCase):
|
||||
r2 = self._create("r2", [50])
|
||||
self.assertEqual(list(r2.origin_input_ids), in1 + [1, 2, 3, 4] + [50])
|
||||
self.assertEqual(len(r2.full_untruncated_fill_ids), 0) # carry skipped
|
||||
r2._refresh_fill_ids()
|
||||
r2.refresh_fill_ids()
|
||||
self.assertEqual(list(r2.full_untruncated_fill_ids), list(r2.origin_input_ids))
|
||||
|
||||
|
||||
|
||||
@@ -3635,14 +3635,14 @@ class UnifiedRadixCacheSuite:
|
||||
gate = threading.Event()
|
||||
swa_release_q = cc.extra_host_mem_release_queues.get(PoolName.SWA)
|
||||
|
||||
def _page_transfer_hook(_self, op):
|
||||
def _page_transfer_hook(_self, op, sync_queue):
|
||||
# Signal that the worker has reached the sidecar step, then block
|
||||
# until the main thread releases the gate.
|
||||
entered.set()
|
||||
gate.wait(timeout=10.0)
|
||||
# Forward to the real implementation so the worker runs the
|
||||
# terminate-aware release path itself.
|
||||
original_sidecar(_self, op)
|
||||
original_sidecar(_self, op, sync_queue)
|
||||
|
||||
# Simulate a slow prefetch IO. Hook on _page_transfer.
|
||||
with mock.patch.object(
|
||||
@@ -5290,7 +5290,7 @@ class UnifiedRadixCacheSuite:
|
||||
cc = cache.cache_controller
|
||||
|
||||
# Fake _reduce_prefetch_ack to drop SWA pool_hits when drop_swa is True.
|
||||
def fake_reduce(ack):
|
||||
def fake_reduce(ack, groups=None):
|
||||
if drop_swa and ack.pool_hits is not None:
|
||||
if PoolName.SWA.value in ack.pool_hits:
|
||||
ack.pool_hits[PoolName.SWA.value] = 0
|
||||
@@ -5368,9 +5368,11 @@ class UnifiedRadixCacheSuite:
|
||||
self.skipTest("fixture does not exercise SWA L3 prefetch")
|
||||
return storage_dir, seq
|
||||
|
||||
def test_tp_swa_prefetch_dropped_when_peer_misses(self):
|
||||
"""A peer rank missing the SWA window drops the whole prefetch result
|
||||
on every rank (TP-consistent all-or-nothing)."""
|
||||
def test_tp_swa_prefetch_degrades_to_kv_only_when_peer_misses(self):
|
||||
"""A peer rank missing the SWA window degrades the prefetch to KV-only
|
||||
on every rank (TP-consistent fail-soft: the fetched KV prefix is
|
||||
retained, only the failed SWA pool is discarded and rebuilt at
|
||||
load-back)."""
|
||||
setup = self._setup_swa_tp_prefetch()
|
||||
if setup is None:
|
||||
return
|
||||
@@ -5382,7 +5384,10 @@ class UnifiedRadixCacheSuite:
|
||||
self._consume_prefetch(cons, seq, CacheRequestHandle("drop", 0))
|
||||
|
||||
m = cons.match_prefix(MatchPrefixParams(key=RadixKey(array("q", seq))))
|
||||
self.assertEqual(m.host_hit_length, 0)
|
||||
# Fail-soft: the KV pool fetched a usable prefix and it is retained.
|
||||
self.assertEqual(m.host_hit_length, len(seq))
|
||||
# TP consistency: every rank agrees SWA itself is missing — the
|
||||
# degraded span carries no SWA host mirror (rebuilt from Full KV).
|
||||
self.assertFalse(
|
||||
self._swa_host_on_path(cons, seq), "SWA must be dropped when a peer misses"
|
||||
)
|
||||
@@ -5409,9 +5414,11 @@ class UnifiedRadixCacheSuite:
|
||||
)
|
||||
cons.sanity_check()
|
||||
|
||||
def test_tp_swa_prefetch_drop_frees_host_pool(self):
|
||||
"""A dropped SWA prefetch must return its whole host buffer to the pool
|
||||
(no leak, no over-free)."""
|
||||
def test_tp_swa_prefetch_degrade_frees_failed_swa_host_pool(self):
|
||||
"""A degraded prefetch (peer missing SWA) must return the failed SWA
|
||||
pool's host buffer to the pool (no leak, no over-free) while the
|
||||
retained KV span stays occupied — the fail-soft degrade never
|
||||
double-frees the kept side (TP-consistent on every rank)."""
|
||||
setup = self._setup_swa_tp_prefetch()
|
||||
if setup is None:
|
||||
return
|
||||
@@ -5420,18 +5427,25 @@ class UnifiedRadixCacheSuite:
|
||||
cons = self._l3_consumer(storage_dir)
|
||||
cons.tp_world_size = 2
|
||||
self._patch_tp_prefetch_sync(cons, drop_swa=True)
|
||||
avail_before = cons.swa_kv_pool_host.available_size()
|
||||
avail_before = self._host_avail_sizes(cons)
|
||||
self._consume_prefetch(cons, seq, CacheRequestHandle("drop", 0))
|
||||
|
||||
self.assertEqual(
|
||||
cons.match_prefix(
|
||||
MatchPrefixParams(key=RadixKey(array("q", seq)))
|
||||
).host_hit_length,
|
||||
0,
|
||||
len(seq),
|
||||
)
|
||||
# Whole window dropped -> its host buffer is fully released back.
|
||||
cons.drain_storage_control_queues() # Drain the release queue.
|
||||
self.assertEqual(cons.swa_kv_pool_host.available_size(), avail_before)
|
||||
avail_after = self._host_avail_sizes(cons)
|
||||
# Failed pool: its whole SWA window is released back (no leak).
|
||||
self.assertEqual(
|
||||
cons.swa_kv_pool_host.available_size(), avail_before[PoolName.SWA]
|
||||
)
|
||||
# Retained side: the KV graft keeps its host buffer occupied — the
|
||||
# degrade must not release the KV it just decided to keep.
|
||||
self.assertLess(avail_after[PoolName.KV], avail_before[PoolName.KV])
|
||||
cons.sanity_check()
|
||||
|
||||
def test_hicache_write_back_evict_drops_unbacked_leaf_when_host_full(self):
|
||||
"""Write-back eviction will keep freeing device KV when the host pool
|
||||
@@ -6636,35 +6650,32 @@ class UnifiedRadixCacheSuite:
|
||||
leaf, ComponentType.SWA, EvictLayer.HOST
|
||||
)
|
||||
cache._free_values(evicted.device_frees, evicted.host_frees)
|
||||
full_host_pool = cache.cache_controller.mem_pool_host
|
||||
swa_host_pool = cache.components[ComponentType.SWA]._swa_kv_pool_host
|
||||
full_available_before = full_host_pool.available_size()
|
||||
swa_available_before = swa_host_pool.available_size()
|
||||
|
||||
result = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", tokens))))
|
||||
|
||||
# Fail-soft semantics (#38348): the both-layers-absent SWA hole on a
|
||||
# backuped node is a valid match boundary — the FULL span hits on
|
||||
# host (no storage round-trip), and the missing SWA window is rebuilt
|
||||
# from Full KV when the continuing request's insert overlaps the hole.
|
||||
self.assertEqual(result.full_kv_hit_length, len(tokens))
|
||||
self.assertEqual(result.swa_branching_seqlen, len(tokens))
|
||||
self.assertEqual(result.host_hit_length, leaf_len)
|
||||
self.assertIsNone(result.swa_branching_seqlen)
|
||||
self.assertEqual(result.swa_host_hit_length, 0)
|
||||
|
||||
# Load-back materializes the FULL span on device; the continuing
|
||||
# request's insert overlaps the hole node and rebuilds its in-window
|
||||
# SWA from Full KV.
|
||||
self._load_back_node(cache, leaf)
|
||||
self._insert(
|
||||
cache,
|
||||
allocator,
|
||||
req_to_token_pool,
|
||||
tokens[: result.swa_branching_seqlen],
|
||||
)
|
||||
cache.writing_check(write_back=True)
|
||||
# Full was already backed up, so only the SWA window is allocated.
|
||||
self.assertEqual(full_host_pool.available_size(), full_available_before)
|
||||
self.assertEqual(
|
||||
swa_host_pool.available_size(),
|
||||
swa_available_before - leaf_len,
|
||||
cache, allocator, req_to_token_pool, tokens + self._make_seq(2000, 1)
|
||||
)
|
||||
|
||||
rematch = cache.match_prefix(
|
||||
MatchPrefixParams(key=RadixKey(array("q", tokens)))
|
||||
)
|
||||
self.assertEqual(len(rematch.device_indices), result.swa_branching_seqlen)
|
||||
rematch = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", tokens))))
|
||||
self.assertEqual(len(rematch.device_indices), len(tokens))
|
||||
self.assertIsNone(rematch.swa_branching_seqlen)
|
||||
# The hole node's in-window tail now carries device SWA again.
|
||||
tail = _device_value(cache, rematch.last_device_node, ComponentType.SWA)
|
||||
self.assertIsNotNone(tail)
|
||||
|
||||
def test_swa_branching_seqlen_caps_insert_after_forward(self):
|
||||
if (
|
||||
@@ -7833,10 +7844,17 @@ class UnifiedRadixCacheSuite:
|
||||
self.assertEqual(xfer.nodes_to_load, [y])
|
||||
self.assertEqual(int(xfer.host_indices.numel()), ps)
|
||||
|
||||
with self.assertRaises(AssertionError):
|
||||
cache.tree_core.build_hicache_transfers(
|
||||
ComponentType.SWA, y, CacheTransferPhase.LOAD_BACK
|
||||
)
|
||||
# Anchored on Y, the walk reaches N's both-layers-absent SWA hole:
|
||||
# the fail-soft LOAD_BACK builder stops collecting there instead of
|
||||
# asserting — the missing SWA window is rebuilt from Full KV at
|
||||
# load-back via SWARebuild/RecoverSWAWithLockedFull.
|
||||
transfers = cache.tree_core.build_hicache_transfers(
|
||||
ComponentType.SWA, y, CacheTransferPhase.LOAD_BACK
|
||||
)
|
||||
self.assertEqual(len(transfers), 1)
|
||||
xfer = transfers[0]
|
||||
self.assertEqual(xfer.nodes_to_load, [y])
|
||||
self.assertEqual(int(xfer.host_indices.numel()), ps)
|
||||
|
||||
def test_hicache_swa_finalize_anchored_on_best_match_node(self):
|
||||
cache, _, _, y, x, _ = self._swa_anchor_setup()
|
||||
@@ -8236,6 +8254,68 @@ class UnifiedRadixCacheSuite:
|
||||
|
||||
cache.sanity_check()
|
||||
|
||||
def test_write_back_swa_publish_leaves_an_ancestor_pending_under_another_ack_alone(
|
||||
self,
|
||||
):
|
||||
"""Write-back builds the insert backup as [target] only, so a second SWA
|
||||
window publish never re-marks an ancestor another ack still holds."""
|
||||
if not self.cfg.has_swa:
|
||||
self.skipTest("requires SWA")
|
||||
if self._skip_unsupported_hicache_test():
|
||||
return
|
||||
ps = self.cfg.page_size
|
||||
if self.cfg.sliding_window_size <= 2 * ps:
|
||||
self.skipTest("window must span past the leaf and its parent to reach c")
|
||||
|
||||
cache, allocator, req_to_token_pool = build_fixture(self.cfg)
|
||||
self._init_hicache(cache, write_policy="write_back")
|
||||
|
||||
# c(Full unbacked) -> b(Full backed) -> t1(Full backed)
|
||||
# -> d(Full unbacked) -> t2(Full backed)
|
||||
# Every node holds a device-only SWA value inside the window.
|
||||
c_seq = self._make_seq(1, 2)
|
||||
t1_seq = c_seq + self._make_seq(1000, 1) + self._make_seq(2000, 1)
|
||||
t2_seq = c_seq + self._make_seq(3000, 1) + self._make_seq(4000, 1)
|
||||
for seq in (t1_seq, t2_seq, t1_seq[:-ps], t2_seq[:-ps]):
|
||||
self._insert(cache, allocator, req_to_token_pool, seq)
|
||||
|
||||
def leaf(seq):
|
||||
return cache.match_prefix(
|
||||
MatchPrefixParams(key=RadixKey(array("q", seq)))
|
||||
).last_device_node
|
||||
|
||||
c, b, t1 = self._path_chain(cache, leaf(t1_seq))
|
||||
_, d, t2 = self._path_chain(cache, leaf(t2_seq))
|
||||
for node in (b, t1, t2):
|
||||
cache.tree_core.set_component_host_value_raw(
|
||||
node,
|
||||
ComponentType.FULL,
|
||||
_device_value(cache, node, ComponentType.FULL).clone(),
|
||||
)
|
||||
for node in (c, b, t1, d, t2):
|
||||
self.assertIsNotNone(_device_value(cache, node, ComponentType.SWA))
|
||||
self.assertIsNone(_host_value(cache, node, ComponentType.SWA))
|
||||
|
||||
# Two requests in one batch finish at t1 and t2; neither ack drains.
|
||||
with mock.patch.object(
|
||||
cache,
|
||||
"_execute_and_commit_kv_backup",
|
||||
wraps=cache._execute_and_commit_kv_backup,
|
||||
) as backups:
|
||||
for seq, target in ((t1_seq, t1), (t2_seq, t2)):
|
||||
backups.reset_mock()
|
||||
self._insert(cache, allocator, req_to_token_pool, seq)
|
||||
self.assertEqual(
|
||||
[call.args[0].node_ids for call in backups.call_args_list],
|
||||
[[target]],
|
||||
)
|
||||
|
||||
for node, ack in ((c, t1), (b, t1), (t1, t1), (d, t2), (t2, t2)):
|
||||
self.assertEqual(cache.tree_core.get_write_through_pending_id(node), ack)
|
||||
# Write-back defers the unbacked Full prefix to eviction.
|
||||
self.assertFalse(cache.tree_core.is_backuped(c))
|
||||
self.assertFalse(cache.tree_core.is_backuped(d))
|
||||
|
||||
|
||||
class UnifiedLRUListBoundedRefreshTest(CustomTestCase):
|
||||
components = (ComponentType.FULL, ComponentType.SWA)
|
||||
|
||||
@@ -37,10 +37,15 @@ class C_HiCacheController(BaseHook):
|
||||
def wrapped_init(self, *args, **kwargs):
|
||||
self.sim_prefetch_buffer = Queue()
|
||||
result = original_init(self, *args, **kwargs)
|
||||
# The real IO thread normally creates this queue. The simulator
|
||||
# replaces that thread, so initialize the handoff queue here.
|
||||
if hasattr(self, "prefetch_hit_queue"):
|
||||
self.prefetch_buffer = Queue()
|
||||
# The real IO threads normally create the handoff queues. The
|
||||
# simulator replaces those threads, so initialize the per-lane
|
||||
# handoff queues here.
|
||||
if hasattr(self, "prefetch_hit_queue") and not getattr(
|
||||
self, "prefetch_io_queues", None
|
||||
):
|
||||
self.prefetch_io_queues = [
|
||||
Queue() for _ in range(getattr(self, "num_prefetch_io_lanes", 1))
|
||||
]
|
||||
return result
|
||||
|
||||
def wrapped_append_host_mem_release(self, host_indices):
|
||||
@@ -182,11 +187,10 @@ class C_HiCacheController(BaseHook):
|
||||
|
||||
# Feed operations whose host pages were allocated by the scheduler
|
||||
# into the virtual-time transfer loop.
|
||||
prefetch_buffer = getattr(self, "prefetch_buffer", None)
|
||||
if prefetch_buffer is not None:
|
||||
while not prefetch_buffer.empty():
|
||||
for io_queue in getattr(self, "prefetch_io_queues", None) or []:
|
||||
while True:
|
||||
try:
|
||||
operation = prefetch_buffer.get(block=False)
|
||||
operation = io_queue.get(block=False)
|
||||
if operation is not None:
|
||||
self.sim_prefetch_buffer.put(operation)
|
||||
except Empty:
|
||||
@@ -295,10 +299,12 @@ class C_HybridCacheController(BaseHook):
|
||||
|
||||
def wrapped_init(self, *args, **kwargs):
|
||||
result = original_init(self, *args, **kwargs)
|
||||
if hasattr(self, "prefetch_hit_queue") and not hasattr(
|
||||
self, "prefetch_buffer"
|
||||
if hasattr(self, "prefetch_hit_queue") and not getattr(
|
||||
self, "prefetch_io_queues", None
|
||||
):
|
||||
self.prefetch_buffer = Queue()
|
||||
self.prefetch_io_queues = [
|
||||
Queue() for _ in range(getattr(self, "num_prefetch_io_lanes", 1))
|
||||
]
|
||||
return result
|
||||
|
||||
def wrapped_storage_hit_query(self, operator):
|
||||
|
||||
Reference in New Issue
Block a user