Compare commits
165
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a493fad77e | ||
|
|
38aff6c842 | ||
|
|
f60160bce6 | ||
|
|
b419518774 | ||
|
|
148936bcb9 | ||
|
|
6ceccac658 | ||
|
|
1d2610bbc2 | ||
|
|
e766b3a0e0 | ||
|
|
e4391748e9 | ||
|
|
c8d06424fa | ||
|
|
97c7eabb33 | ||
|
|
e9b436dc2d | ||
|
|
fc63ae5392 | ||
|
|
9df2d40324 | ||
|
|
6a8a6e399f | ||
|
|
e6c0737541 | ||
|
|
9374e03f13 | ||
|
|
86883ae89f | ||
|
|
43c2844c18 | ||
|
|
9e288079e4 | ||
|
|
738d5a50d0 | ||
|
|
e80c8608dc | ||
|
|
5351d5491b | ||
|
|
8a2e1403a3 | ||
|
|
1910eccb8e | ||
|
|
6217b9de46 | ||
|
|
9dd446b5c1 | ||
|
|
f76eaf0f92 | ||
|
|
45fdc97082 | ||
|
|
f690861265 | ||
|
|
e4bfdc949b | ||
|
|
59e3aee6d9 | ||
|
|
406081d3e3 | ||
|
|
1975265315 | ||
|
|
534417b82e | ||
|
|
ed0a454730 | ||
|
|
04ee2d03d7 | ||
|
|
9d5db13e53 | ||
|
|
7e20f0af79 | ||
|
|
0b74ab2ac4 | ||
|
|
3c7ec33d26 | ||
|
|
09ca4957ee | ||
|
|
7921442a5c | ||
|
|
05ef104208 | ||
|
|
d44620f285 | ||
|
|
5df4f2c036 | ||
|
|
db1e8cdbe3 | ||
|
|
f3ac513d92 | ||
|
|
0054c5b8fe | ||
|
|
6da0a40b02 | ||
|
|
caf3403086 | ||
|
|
617f19ba61 | ||
|
|
ba3318ffbd | ||
|
|
8279d3d6df | ||
|
|
ef4cccb6a0 | ||
|
|
98b950366d | ||
|
|
87ab6c0155 | ||
|
|
0308920bdb | ||
|
|
57db2a4ae9 | ||
|
|
7fd7fd58a5 | ||
|
|
71a72b8fea | ||
|
|
6c9518fcd8 | ||
|
|
3a4ab451ac | ||
|
|
5d38fc96a9 | ||
|
|
261ebbc55b | ||
|
|
53fb9be7ee | ||
|
|
e5be84f795 | ||
|
|
39ceba8a3d | ||
|
|
b9d259441c | ||
|
|
e7dc175e94 | ||
|
|
6612af9e0e | ||
|
|
71c6d8bc55 | ||
|
|
dbc661602e | ||
|
|
054dcee896 | ||
|
|
73a25267ac | ||
|
|
bcad628d30 | ||
|
|
baba18bc7d | ||
|
|
0afcb334a4 | ||
|
|
33e6381b7d | ||
|
|
9c4ff32562 | ||
|
|
7d5759b2d9 | ||
|
|
ac11e9162d | ||
|
|
84fc632088 | ||
|
|
8d9bce30a7 | ||
|
|
5f2ffa70b8 | ||
|
|
f4ef648b16 | ||
|
|
6469cf4197 | ||
|
|
3f8751c6ed | ||
|
|
e4f1310d6f | ||
|
|
a42ab6fce0 | ||
|
|
fa8ad18b5c | ||
|
|
7f1edbba80 | ||
|
|
99c317b131 | ||
|
|
2b2ede8211 | ||
|
|
b4dbcebbec | ||
|
|
30ba5818e7 | ||
|
|
c9bfb24bcc | ||
|
|
3d109d50be | ||
|
|
4eabf5e228 | ||
|
|
7d67d57804 | ||
|
|
491989284f | ||
|
|
53ea8fb2a7 | ||
|
|
efdbfd2cc3 | ||
|
|
8342b89e85 | ||
|
|
c92938d718 | ||
|
|
7578cdd218 | ||
|
|
21f7ce37dd | ||
|
|
236686454c | ||
|
|
1f6ca2cf68 | ||
|
|
bde1fcbf51 | ||
|
|
c4f3e3be33 | ||
|
|
9ef65b51c2 | ||
|
|
6ba34507f1 | ||
|
|
93d610c1fd | ||
|
|
4d0efcc921 | ||
|
|
f71fafa684 | ||
|
|
1068926cd6 | ||
|
|
48fb49baef | ||
|
|
2372cdc06e | ||
|
|
7b815cc98b | ||
|
|
d06c5dde42 | ||
|
|
8d0ee5e51e | ||
|
|
75758cb0a0 | ||
|
|
60e80ac7d1 | ||
|
|
4ef799aaa6 | ||
|
|
1be762368b | ||
|
|
f577b61b52 | ||
|
|
4792b61332 | ||
|
|
62f1ed099e | ||
|
|
bfd9bb2e03 | ||
|
|
6deac260bf | ||
|
|
842ad7c4d4 | ||
|
|
b605d2d370 | ||
|
|
e403cede0a | ||
|
|
735434fcde | ||
|
|
e17fbc9a2d | ||
|
|
5507fff133 | ||
|
|
4f0ca95477 | ||
|
|
b91137ab98 | ||
|
|
67d8368a84 | ||
|
|
db7d2cb7db | ||
|
|
f23179ce99 | ||
|
|
8266769b2d | ||
|
|
3b671d6086 | ||
|
|
7925735a3e | ||
|
|
6833498646 | ||
|
|
b48e2cb1eb | ||
|
|
bfeb7cd9b2 | ||
|
|
ddf5207630 | ||
|
|
104218d9ed | ||
|
|
92632a60ba | ||
|
|
4b3b367b63 | ||
|
|
a78da9b524 | ||
|
|
b081dd3d23 | ||
|
|
c74a4037fb | ||
|
|
b963295489 | ||
|
|
fa826e08b1 | ||
|
|
3810f531a8 | ||
|
|
2580c24d1b | ||
|
|
4f22146e51 | ||
|
|
12e3b82e52 | ||
|
|
8305f66fc8 | ||
|
|
21a4a16b4b | ||
|
|
fc954b7e08 | ||
|
|
c2059c4fb2 |
@@ -0,0 +1,57 @@
|
||||
# Gitea Actions 自动构建 sglang 镜像(海外节点,原版源)
|
||||
# 基底 lmsysorg/sglang:dev-dsv41(docker.io),依赖走 pypi.org 默认源。
|
||||
# 触发:push 到 dsv41-pd 分支。
|
||||
#
|
||||
# 前置条件(一次性,在 Gitea 实例上配置):
|
||||
# 1. 实例已注册 act_runner(Gitea Actions runner,标签含 ubuntu-latest)
|
||||
# 2. 仓库 Settings → Secrets 添加:
|
||||
# REGISTRY_USERNAME / REGISTRY_PASSWORD(推送镜像的账号,如 Gitea 访问令牌)
|
||||
# 3. 可选:Settings → Actions → Variables 添加 REGISTRY(默认 git.agentwithu.com,
|
||||
# 即 Gitea 自带容器 registry;也可填 docker.io 等)
|
||||
name: build-sglang-image
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [dsv41-pd, dsv41-pd-visioncp]
|
||||
# compose/部署配置 与 workflow 自身的改动不触发镜像构建(省 runner 与推送带宽)。
|
||||
# 改了 workflow 想重建镜像时,需伴随任意代码改动或手动触发。
|
||||
paths-ignore:
|
||||
- 'deploy/**'
|
||||
- '.gitea/**'
|
||||
|
||||
env:
|
||||
# 直接写死 Gitea 自带 registry(vars context 在该实例上求值异常会导致回退 docker.io)
|
||||
REGISTRY: git.agentwithu.com
|
||||
IMAGE_NAME: minke.yu/sglang
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Resolve tag
|
||||
id: meta
|
||||
run: |
|
||||
SHA9=$(git rev-parse --short=9 HEAD)
|
||||
echo "tag=${{ github.ref_name }}-${SHA9}-ci-$(date -u +%Y%m%d-%H%M)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
|
||||
- uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ secrets.REGISTRY_USERNAME }}
|
||||
password: ${{ secrets.REGISTRY_PASSWORD }}
|
||||
|
||||
- uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: Dockerfile.gitea
|
||||
push: true
|
||||
# cache-from: type=registry
|
||||
# cache-to: type=registry,mode=max
|
||||
tags: |
|
||||
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.tag }}
|
||||
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }}-latest
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
# Gitea Actions 用(海外节点):原版 docker.io 基底 + 原版 pypi
|
||||
# 与 b300 离线版(/data/ymk/build/Dockerfile)的区别:
|
||||
# - 基底直用 docker.io 的 lmsysorg/sglang:dev-dsv41(不走 umirror/DaoCloud)
|
||||
# - 不设 PIP_INDEX_URL,用默认 pypi.org
|
||||
# - 源码由 CI checkout 后经 COPY 进镜像(不用 in-image clone)
|
||||
FROM lmsysorg/sglang:dev-dsv41
|
||||
|
||||
# checkout 含 .git,setuptools-scm 可打戳(main 系分支 version 显示 dev 属正常)
|
||||
COPY . /sgl-workspace/sglang
|
||||
|
||||
# Rust gateway(sgl-model-gateway)编译,与 b300 离线版 deploy/b300-build/Dockerfile 同构。
|
||||
# 基底自带 rustup stable,RUSTUP_TOOLCHAIN=stable 绕过 rust-toolchain.toml 的 1.90 钉版;
|
||||
# python 扩展必须显式 -p,产物重命名为 sglang_router_rs.abi3.so 覆盖 python 包内同文件。
|
||||
# 海外 runner 直连 crates.io 无网络问题;cache mount 海外环境同样无害。
|
||||
RUN --mount=type=cache,target=/cargo-home \
|
||||
--mount=type=cache,target=/cargo-target \
|
||||
cd /sgl-workspace/sglang/sgl-model-gateway \
|
||||
&& CARGO_HOME=/cargo-home CARGO_TARGET_DIR=/cargo-target RUSTUP_TOOLCHAIN=stable \
|
||||
cargo build --release -p sgl-model-gateway -p sgl-model-gateway-python \
|
||||
&& cp /cargo-target/release/sgl-model-gateway /opt/sglang/bin/sgl-model-gateway \
|
||||
&& cp /cargo-target/release/libsglang_router_rs.so \
|
||||
/opt/sglang/lib/python3.12/site-packages/sglang_router/sglang_router_rs.abi3.so
|
||||
|
||||
RUN pip install --no-cache-dir -e /sgl-workspace/sglang/python
|
||||
@@ -0,0 +1,77 @@
|
||||
# 优化版(2026-10-08 v3):torch/nvidia 全家桶永远复用基底,全量路径零重下。
|
||||
# 要点:
|
||||
# 1. pyproject build-system.requires 含 torch==2.13.0,pip 隔离构建环境每次都重下 ~6GB。
|
||||
# 基底镜像已装 torch 2.13 → 用 --no-build-isolation 复用,构建环境零下载。
|
||||
# 2. 老全量路径让 pip 解析全部依赖:bare `torchvision` 等会让解析器弃用基底已装的
|
||||
# 本地版本(torch 2.13.0+cu130),改用 PyPI 的 torch==2.13.0,连带重下整套 nvidia
|
||||
# 依赖(9/23 实测 nvidia-nccl-cu13 等 6GB+,限流下 2h40m);且基底 NCCL 是
|
||||
# 刻意装高的 2.30.7(torch 元数据钉 ==2.29.7),任何全量解析都会把它降级回去。
|
||||
# → v3 不再让 pip 碰重型依赖:check_deps.py 在构建期对照 pyproject 与基底
|
||||
# 已装版本,把增量分三类——重型(torch*/nvidia-*/cuda-*)永不进解析、
|
||||
# 不满足即让构建失败并提示决策(显式失败好过静默重装);
|
||||
# kernel wheel(sglang-kernel/flashinfer*/sgl-deep-gemm)要跟随源码版本,
|
||||
# 钉版变动时以 pip --no-deps 单独装(防止顺藤摸瓜解析 torch);
|
||||
# 其余轻量依赖只装未满足的增量。
|
||||
# 3. 基底缺 setuptools-scm(版本打戳要用),单层预装 + pip cache mount,只下载一次。
|
||||
# 4. RUN 的 pip cache mount 持久化在宿主机,轻量依赖增量不再重复下载。
|
||||
FROM uhub.service.ucloud.cn/umirror/sglang:dev-dsv41
|
||||
|
||||
ARG PIP_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
ENV PIP_INDEX_URL=${PIP_INDEX}
|
||||
|
||||
# 构建后端(pyproject build-system.requires,torch 除外——基底已有)
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install "setuptools>=61" "setuptools-scm>=8" "setuptools-rust>=1.11" wheel
|
||||
|
||||
# 依赖增量计算脚本(对照 pyproject 与基底已装版本,产出轻量/kernel 增量清单)
|
||||
# 注意 build context 是 /data/ymk 且 .dockerignore 排除了 build/,只能从 sglang/ 树内 COPY
|
||||
COPY sglang/deploy/b300-build/check_deps.py /tmp/check_deps.py
|
||||
|
||||
# 整个源码树(含 .git,用于 setuptools-scm 版本打戳)
|
||||
COPY sglang/ /sgl-workspace/sglang/
|
||||
|
||||
# Rust gateway(sgl-model-gateway)编译。基底自带 rustup stable(实测 cargo 1.98.1),
|
||||
# RUSTUP_TOOLCHAIN=stable 绕过仓库 rust-toolchain.toml 钉的 1.90(避免重复下载工具链);
|
||||
# b300 直连 crates.io(稀疏索引 + static.crates.io 均通),无需任何镜像源。
|
||||
# cache mount 持久化 registry/target:冷编译 ~15min(二进制 ~11min + python 扩展 ~4min),
|
||||
# 增量重建秒~分钟级。python 扩展必须显式 -p(workspace 根包默认不编译扩展),
|
||||
# 产物 libsglang_router_rs.so 需重命名为 sglang_router_rs.abi3.so 覆盖 python 包内同文件
|
||||
# (与 H20 线上部署方式一致)。2026-10-09 b300 实测验证。
|
||||
RUN --mount=type=cache,target=/cargo-home \
|
||||
--mount=type=cache,target=/cargo-target \
|
||||
cd /sgl-workspace/sglang/sgl-model-gateway \
|
||||
&& CARGO_HOME=/cargo-home CARGO_TARGET_DIR=/cargo-target RUSTUP_TOOLCHAIN=stable \
|
||||
cargo build --release -p sgl-model-gateway -p sgl-model-gateway-python \
|
||||
&& cp /cargo-target/release/sgl-model-gateway /opt/sglang/bin/sgl-model-gateway \
|
||||
&& cp /cargo-target/release/libsglang_router_rs.so \
|
||||
/opt/sglang/lib/python3.12/site-packages/sglang_router/sglang_router_rs.abi3.so
|
||||
|
||||
# 计算依赖增量(重型 torch/nvidia/cuda 永不进入清单;不满足则 check_deps 非零、构建失败)
|
||||
RUN python /tmp/check_deps.py /sgl-workspace/sglang/python/pyproject.toml \
|
||||
/tmp/missing_light.txt /tmp/missing_kernel.txt \
|
||||
&& echo "== kernel wheel delta (pip --no-deps) ==" && cat /tmp/missing_kernel.txt \
|
||||
&& echo "== lightweight dep delta ==" && cat /tmp/missing_light.txt
|
||||
|
||||
# kernel wheel 增量(如 sglang-kernel 升版):--no-deps 防止 pip 顺藤摸瓜解析 torch
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
if grep -q '[^[:space:]]' /tmp/missing_kernel.txt; then \
|
||||
pip install --no-deps -r /tmp/missing_kernel.txt; \
|
||||
else \
|
||||
echo "== no kernel wheel delta"; \
|
||||
fi
|
||||
|
||||
# 轻量依赖增量(通常为空或几个小包),重型全家桶原样复用基底。
|
||||
# ⚠️ 必须 --no-deps:即使目标只是 xgrammar 这种小包,只要依赖图里出现 torch,
|
||||
# pip 就会校验 torch+cu130 元数据钉死的 nvidia-nccl-cu13==2.29.7,发现基底装的是
|
||||
# 2.30.7 便"好心"下载 206MB 降级(10-08 v3 首跑实测踩中)。基底的传递依赖本来就
|
||||
# 齐全,增量小包的二级依赖也一样;若有漏网,b300 冒烟会在 import 阶段显式暴露。
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
if grep -q '[^[:space:]]' /tmp/missing_light.txt; then \
|
||||
pip install --no-deps -r /tmp/missing_light.txt; \
|
||||
else \
|
||||
echo "== no lightweight delta, skipping dependency install"; \
|
||||
fi
|
||||
|
||||
# editable 安装 sglang 本体,--no-deps 保证 pip 完全不碰依赖图
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install --no-deps --no-build-isolation -e /sgl-workspace/sglang/python
|
||||
@@ -0,0 +1,65 @@
|
||||
# fast 路径(2026-10-08 v3.1):python/pyproject.toml 未变化时使用(build.sh 自动判断)。
|
||||
# 与 Dockerfile 全量路径同构:同样跑 check_deps.py 增量安装。
|
||||
# 原因:全量构建若装过 kernel wheel 增量(如 sglang-kernel 0.4.7),随后 pyproject 未变的
|
||||
# fast 构建若只 --no-deps,会静默退回基底的旧 kernel(0.4.6.post1)——镜像内容漂移。
|
||||
# check_deps 本地比对 ~0.5s,增量命中 pip cache 秒级,成本可忽略。
|
||||
FROM uhub.service.ucloud.cn/umirror/sglang:dev-dsv41
|
||||
|
||||
ARG PIP_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
ENV PIP_INDEX_URL=${PIP_INDEX}
|
||||
|
||||
# 构建后端(pyproject build-system.requires,torch 除外——基底已有)
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install "setuptools>=61" "setuptools-scm>=8" "setuptools-rust>=1.11" wheel
|
||||
|
||||
# 依赖增量计算脚本(对照 pyproject 与基底已装版本,产出轻量/kernel 增量清单)
|
||||
# 注意 build context 是 /data/ymk 且 .dockerignore 排除了 build/,只能从 sglang/ 树内 COPY
|
||||
COPY sglang/deploy/b300-build/check_deps.py /tmp/check_deps.py
|
||||
|
||||
# 整个源码树(含 .git,用于 setuptools-scm 版本打戳)
|
||||
COPY sglang/ /sgl-workspace/sglang/
|
||||
|
||||
# Rust gateway(sgl-model-gateway)编译。基底自带 rustup stable(实测 cargo 1.98.1),
|
||||
# RUSTUP_TOOLCHAIN=stable 绕过仓库 rust-toolchain.toml 钉的 1.90(避免重复下载工具链);
|
||||
# b300 直连 crates.io(稀疏索引 + static.crates.io 均通),无需任何镜像源。
|
||||
# cache mount 持久化 registry/target:冷编译 ~15min,增量重建秒~分钟级。
|
||||
# python 扩展必须显式 -p(workspace 根包默认不编译扩展),产物 libsglang_router_rs.so
|
||||
# 需重命名为 sglang_router_rs.abi3.so 覆盖 python 包内同文件(与 H20 线上部署方式一致)。
|
||||
RUN --mount=type=cache,target=/cargo-home \
|
||||
--mount=type=cache,target=/cargo-target \
|
||||
cd /sgl-workspace/sglang/sgl-model-gateway \
|
||||
&& CARGO_HOME=/cargo-home CARGO_TARGET_DIR=/cargo-target RUSTUP_TOOLCHAIN=stable \
|
||||
cargo build --release -p sgl-model-gateway -p sgl-model-gateway-python \
|
||||
&& cp /cargo-target/release/sgl-model-gateway /opt/sglang/bin/sgl-model-gateway \
|
||||
&& cp /cargo-target/release/libsglang_router_rs.so \
|
||||
/opt/sglang/lib/python3.12/site-packages/sglang_router/sglang_router_rs.abi3.so
|
||||
|
||||
# 计算依赖增量(重型 torch/nvidia/cuda 永不进入清单;不满足则 check_deps 非零、构建失败)
|
||||
RUN python /tmp/check_deps.py /sgl-workspace/sglang/python/pyproject.toml \
|
||||
/tmp/missing_light.txt /tmp/missing_kernel.txt \
|
||||
&& echo "== kernel wheel delta (pip --no-deps) ==" && cat /tmp/missing_kernel.txt \
|
||||
&& echo "== lightweight dep delta ==" && cat /tmp/missing_light.txt
|
||||
|
||||
# kernel wheel 增量(如 sglang-kernel 升版):--no-deps 防止 pip 顺藤摸瓜解析 torch
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
if grep -q '[^[:space:]]' /tmp/missing_kernel.txt; then \
|
||||
pip install --no-deps -r /tmp/missing_kernel.txt; \
|
||||
else \
|
||||
echo "== no kernel wheel delta"; \
|
||||
fi
|
||||
|
||||
# 轻量依赖增量(通常为空或几个小包),重型全家桶原样复用基底。
|
||||
# ⚠️ 必须 --no-deps:即使目标只是 xgrammar 这种小包,只要依赖图里出现 torch,
|
||||
# pip 就会校验 torch+cu130 元数据钉死的 nvidia-nccl-cu13==2.29.7,发现基底装的是
|
||||
# 2.30.7 便"好心"下载 206MB 降级(10-08 v3 首跑实测踩中)。基底的传递依赖本来就
|
||||
# 齐全,增量小包的二级依赖也一样;若有漏网,b300 冒烟会在 import 阶段显式暴露。
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
if grep -q '[^[:space:]]' /tmp/missing_light.txt; then \
|
||||
pip install --no-deps -r /tmp/missing_light.txt; \
|
||||
else \
|
||||
echo "== no lightweight delta, skipping dependency install"; \
|
||||
fi
|
||||
|
||||
# editable 安装 sglang 本体,--no-deps 保证 pip 完全不碰依赖图
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install --no-deps --no-build-isolation -e /sgl-workspace/sglang/python
|
||||
@@ -0,0 +1,33 @@
|
||||
#!/bin/bash
|
||||
# 用法: /data/ymk/build/build.sh [tag]
|
||||
# 默认 tag: ymkymx/sglang:<分支>-<sha9>-local-<UTC构建日期时间>(命名规则见 AGENTS.md)
|
||||
# 另打 <分支>-latest-local 别名。
|
||||
#
|
||||
# 依赖路径自动选择:
|
||||
# python/pyproject.toml 的 sha256 与上次成功构建一致 → Dockerfile.fast(--no-deps,秒级)
|
||||
# 不一致(或 FULL=1)→ Dockerfile 全量依赖解析,成功后记录新 hash
|
||||
set -euo pipefail
|
||||
cd /data/ymk
|
||||
SHA=$(git -C sglang rev-parse --short=9 HEAD)
|
||||
BR=$(git -C sglang rev-parse --abbrev-ref HEAD)
|
||||
DT=$(date -u +%Y%m%d-%H%M)
|
||||
TAG=${1:-ymkymx/sglang:$BR-$SHA-local-$DT}
|
||||
|
||||
HASH_FILE=build/.last_pyproject_sha256
|
||||
CUR=$(sha256sum sglang/python/pyproject.toml | cut -d' ' -f1)
|
||||
PREV=$(cat "$HASH_FILE" 2>/dev/null || echo none)
|
||||
|
||||
DOCKERFILE=build/Dockerfile
|
||||
if [ "$CUR" = "$PREV" ] && [ "${FULL:-0}" != "1" ]; then
|
||||
DOCKERFILE=build/Dockerfile.fast
|
||||
echo "== pyproject 未变,走 fast 路径(--no-deps)"
|
||||
else
|
||||
echo "== pyproject 有变化或 FULL=1,走全量依赖解析"
|
||||
fi
|
||||
|
||||
echo "== building $TAG from sglang@$(git -C sglang log --oneline -1) [$DOCKERFILE]"
|
||||
docker build -f "$DOCKERFILE" -t "$TAG" .
|
||||
# 构建成功才记录 hash / 打别名
|
||||
echo "$CUR" > "$HASH_FILE"
|
||||
docker tag "$TAG" "ymkymx/sglang:$BR-latest-local"
|
||||
echo "== done: $TAG (别名 $BR-latest-local)"
|
||||
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compute dependency deltas for a b300 sglang image build, in three classes.
|
||||
|
||||
Reads pyproject.toml, evaluates every requirement against the *installed*
|
||||
distributions in the base image, and writes two requirements files:
|
||||
|
||||
argv[2] (light): unsatisfied lightweight requirements → `pip install -r`
|
||||
argv[3] (kernel): unsatisfied kernel wheels (sglang-kernel, flashinfer*,
|
||||
sgl-deep-gemm) → `pip install --no-deps -r`
|
||||
These track the sglang source being built and are meant
|
||||
to be upgraded from the index when pyproject bumps them;
|
||||
--no-deps keeps pip from walking torch's metadata.
|
||||
|
||||
The PyTorch stack (torch/torchvision/torchaudio/torchcodec/
|
||||
torch_memory_saver), the nvidia-* CUDA family and cuda-* packages are NEVER
|
||||
emitted. If one of them is missing or its installed version no longer
|
||||
satisfies the pinned specifier (e.g. upstream bumps nvidia-cutlass-dsl), the
|
||||
script fails the build loudly instead of letting pip download ~6GB or
|
||||
"fix" the base image's deliberately newer nvidia-nccl-cu13 2.30.7 back to
|
||||
torch's metadata pin ==2.29.7 (any pip full-resolution would do both —
|
||||
2026-09-23/24 builds did, throttled to 2h40m).
|
||||
"""
|
||||
|
||||
import sys
|
||||
import tomllib
|
||||
from importlib import metadata as md
|
||||
|
||||
from packaging.requirements import Requirement
|
||||
|
||||
# Never resolved by pip: reused from the base image or the build fails.
|
||||
HEAVY_PREFIXES = ("torch", "nvidia-", "cuda-")
|
||||
# Kernel wheels: upgraded via pip --no-deps when pyproject pins move.
|
||||
KERNEL_PREFIXES = ("sglang-kernel", "flashinfer", "sgl-deep-gemm")
|
||||
|
||||
|
||||
def norm(name: str) -> str:
|
||||
return name.lower().replace("_", "-")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
pyproject, light_out, kernel_out = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
with open(pyproject, "rb") as f:
|
||||
deps = tomllib.load(f)["project"]["dependencies"]
|
||||
|
||||
light: list[str] = []
|
||||
kernel: list[str] = []
|
||||
heavy_unsat: list[str] = []
|
||||
for dep in deps:
|
||||
req = Requirement(dep)
|
||||
if req.marker is not None and not req.marker.evaluate():
|
||||
continue
|
||||
try:
|
||||
installed = md.version(req.name)
|
||||
except md.PackageNotFoundError:
|
||||
installed = None
|
||||
satisfied = installed is not None and req.specifier.contains(
|
||||
installed, prereleases=True
|
||||
)
|
||||
if satisfied:
|
||||
continue
|
||||
name = norm(req.name)
|
||||
if name.startswith(HEAVY_PREFIXES):
|
||||
heavy_unsat.append(f"{dep} (installed: {installed})")
|
||||
elif name.startswith(KERNEL_PREFIXES):
|
||||
kernel.append(dep)
|
||||
else:
|
||||
light.append(dep)
|
||||
|
||||
if heavy_unsat:
|
||||
print("=" * 70, file=sys.stderr)
|
||||
print("FATAL: heavy base-image deps no longer satisfy pyproject:", file=sys.stderr)
|
||||
for item in heavy_unsat:
|
||||
print(f" {item}", file=sys.stderr)
|
||||
print(
|
||||
"Refusing to let pip resolve these (would re-download ~6GB and/or\n"
|
||||
"downgrade NCCL). Decide explicitly: bump the base image, or adjust\n"
|
||||
"the pin in pyproject.toml.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
with open(light_out, "w") as f:
|
||||
f.write("\n".join(light) + ("\n" if light else ""))
|
||||
with open(kernel_out, "w") as f:
|
||||
f.write("\n".join(kernel) + ("\n" if kernel else ""))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,33 @@
|
||||
# B300 ds41 部署 compose 档案
|
||||
|
||||
b300-01 上 DeepSeek-V4.1-Flash 各部署方案的 docker compose 存档。
|
||||
**本目录是 source of truth**;b300 上的运行目录 `/data/ymk/ds41/` 是工作副本。
|
||||
|
||||
方案说明、参数矩阵、特殊 env、已知坑见飞书部署手册(组内共享):
|
||||
《B300 sglang 部署手册》 https://u04wb5irxz.feishu.cn/docx/FOi8dbnybodHr0xIfkPcQ6gtnRx
|
||||
|
||||
## 同步流程
|
||||
|
||||
```bash
|
||||
# 本机(D:\B300\sglang)
|
||||
git add deploy/ && git commit -m "deploy: ..."
|
||||
git push origin dsv41-pd-visioncp # Gitea(paths-ignore,不触发镜像构建)
|
||||
git push b300 dsv41-pd-visioncp # b300 裸仓库
|
||||
|
||||
# b300-01
|
||||
git -C /data/ymk/sglang pull # deploy/ 出现在 /data/ymk/sglang/deploy/b300-ds41/
|
||||
```
|
||||
|
||||
## CI 说明
|
||||
|
||||
`.gitea/workflows/build-image.yaml` 的 push 触发器带 `paths-ignore: ['deploy/**', '.gitea/**']`,
|
||||
改本目录不会触发镜像构建。注意:workflow 自身的改动也不再自动触发,需要重建镜像时
|
||||
伴随代码改动 push 或手动触发。
|
||||
|
||||
## 目录规则
|
||||
|
||||
- 命名:`dockerserve-<拓扑>-<特性>-<卡位>.yml`(如 `-b4` = 后 4 卡)
|
||||
- 退役文件不要留在主目录:移入 `archive-YYYYMMDD/`(b300 侧同样执行)
|
||||
- 禁止提交 `.bak` 备份文件
|
||||
- `mooncake-store.json` 是 hicache L3 mooncake store 配置,被
|
||||
`dockerserve-pd2-p-r1/r2.yml` 通过 `SGLANG_HICACHE_MOONCAKE_CONFIG_PATH` 引用
|
||||
@@ -0,0 +1,27 @@
|
||||
cd /data/ymk/ds41
|
||||
HDR="Authorization=Bearer deepseek-v4-flash-0731-key"
|
||||
PORT=30040
|
||||
MODEL=/data/models/DeepSeek-V4.1-Flash
|
||||
|
||||
# 一律 --warmup-requests 0:默认 warmup 会把 prompt#0 种进 L1/L2 污染 L3 测量(已踩过)
|
||||
run() {
|
||||
echo "===== [$(date +%H:%M:%S)] RUN: $1 (np=$2) ====="
|
||||
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--warmup-requests 0 --num-prompts "$2" --max-concurrency "$2" --seed "$3" --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR" 2>&1 | grep -E 'Input token throughput|Mean TTFT|P99 TTFT|Successful'
|
||||
sleep 2
|
||||
}
|
||||
flush() { curl -sf -m 10 -X POST http://127.0.0.1:$PORT/flush_cache > /dev/null && echo "[$(date +%H:%M:%S)] flushed"; sleep 3; }
|
||||
|
||||
echo "===== PRIME COLD mc8(冷基线 + 写入 L3) ====="
|
||||
run PRIME_COLD 8 1541391
|
||||
flush
|
||||
run MC1_L3 1 1541391
|
||||
flush
|
||||
run MC2_L3 2 1541391
|
||||
flush
|
||||
run MC4_L3 4 1541391
|
||||
flush
|
||||
run MC8_L3 8 1541391
|
||||
echo "===== [$(date +%H:%M:%S)] ALL DONE ====="
|
||||
@@ -0,0 +1,204 @@
|
||||
#!/bin/bash
|
||||
# bench_suite_v2.sh — DeepSeek-V4.1-Flash on b300-01,2026-10-09 新版基线方法
|
||||
#
|
||||
# 相对 v1 的方法变化:
|
||||
# 1. 服务刚起数据不准 → 每个测试模式自带 warmup 轮(结果丢弃,标记 WARM)
|
||||
# 2. prefill 冷/热双轮:冷轮写入 hicache(L1/L3),同 seed 热轮测纯 cache-hit TPS
|
||||
# 3. decode 阶梯并发:步长 32,上限 = cuda-graph-max-bs-decode 的 80%;
|
||||
# 每档先 warm(灌 L1)再测,prefill 全 cache hit;
|
||||
# 报告 TPOT + 整实例输出 TPS + 推算实际并发(= 输出TPS × TPOT / 1000)
|
||||
# 4. l3 模式(3p1d 经 router):大语料溢出 L1 后重放,验证 L3 命中
|
||||
#
|
||||
# 在宿主机执行,bench 客户端通过 docker exec 进容器跑(需要 /data 挂载)。
|
||||
#
|
||||
# 用法:
|
||||
# bench_suite_v2.sh warmup <container> <port>
|
||||
# bench_suite_v2.sh prefill <tag> <container> <port>
|
||||
# bench_suite_v2.sh decode <tag> <container> <port> [cg_max_bs=256]
|
||||
# bench_suite_v2.sh l3 <tag> <container> <port> [num_prompts=24]
|
||||
# bench_suite_v2.sh report <log>
|
||||
# 产物: /data/ymk/ds41/bench_v2_<tag>.log
|
||||
set -u
|
||||
|
||||
BASE=/data/ymk/ds41
|
||||
MODEL=${MODEL:-/data/models/DeepSeek-V4.1-Flash}
|
||||
HDR="Authorization=Bearer deepseek-v4-flash-0731-key"
|
||||
|
||||
usage() { sed -n 2,20p "$0" >&2; exit 1; }
|
||||
|
||||
# $1=container $2=port $3=tag -> 生成可执行脚本路径并 echo
|
||||
emit_common() {
|
||||
cat <<EOF
|
||||
cd $BASE
|
||||
HDR="$HDR"
|
||||
PORT=$2
|
||||
run() {
|
||||
local name="\$1"; shift
|
||||
echo "===== [\$(date +%H:%M:%S)] START: \$name ====="
|
||||
python3 -m sglang.bench_serving "\$@"
|
||||
echo "===== [\$(date +%H:%M:%S)] END(exit=\$?): \$name ====="
|
||||
sleep 5
|
||||
}
|
||||
EOF
|
||||
}
|
||||
|
||||
# ---------------- warmup ----------------
|
||||
cmd_warmup() {
|
||||
local c=${1:?container} port=${2:?port}
|
||||
local gen=$BASE/.bench_v2_warmup.sh
|
||||
{ emit_common "$c" "$port"; cat <<'EOF'
|
||||
run WARMUP_a --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
|
||||
--dataset-name random-ids --random-input-len 4096 --random-output-len 16 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 777001 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
run WARMUP_b --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
|
||||
--dataset-name random-ids --random-input-len 16000 --random-output-len 1 \
|
||||
--num-prompts 2 --max-concurrency 1 --seed 777002 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
echo "===== [$(date +%H:%M:%S)] ALL DONE ====="
|
||||
EOF
|
||||
} | sed "s|MODELPH|$MODEL|g" > "$gen"
|
||||
chmod +x "$gen"
|
||||
docker exec "$c" bash "$gen"
|
||||
}
|
||||
|
||||
# ---------------- prefill(冷/热双轮) ----------------
|
||||
cmd_prefill() {
|
||||
local tag=${1:?tag} c=${2:?container} port=${3:?port}
|
||||
local p=$(date +%H%M%S) gen=$BASE/.bench_v2_gen_${tag}.sh
|
||||
{ emit_common "$c" "$port"
|
||||
# warmup(丢弃)
|
||||
cat <<'EOF'
|
||||
run WARMUP --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
|
||||
--dataset-name random-ids --random-input-len 4096 --random-output-len 16 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 777000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
EOF
|
||||
# 冷轮
|
||||
for stage in "bs1_16000 16000 1 4 1" "bs1_160000 160000 1 4 2" "bs16_4096 4096 16 32 3"; do
|
||||
set -- $stage
|
||||
cat <<EOF
|
||||
run COLD_$1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len $2 --random-output-len 1 \\
|
||||
--num-prompts $4 --max-concurrency $3 --seed ${p}$5 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
done
|
||||
# 热轮(同 seed → cache hit)
|
||||
for stage in "bs1_16000 16000 1 4 1" "bs1_160000 160000 1 4 2" "bs16_4096 4096 16 32 3"; do
|
||||
set -- $stage
|
||||
cat <<EOF
|
||||
run HIT_$1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len $2 --random-output-len 1 \\
|
||||
--num-prompts $4 --max-concurrency $3 --seed ${p}$5 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
done
|
||||
echo 'echo "===== [$(date +%H:%M:%S)] ALL DONE ====="'
|
||||
} | sed "s|MODELPH|$MODEL|g" > "$gen"
|
||||
chmod +x "$gen"
|
||||
nohup docker exec "$c" bash "$gen" > "$BASE/bench_v2_${tag}.log" 2>&1 &
|
||||
echo "launched pid=$! seeds=${p}1..3 log=$BASE/bench_v2_${tag}.log"
|
||||
}
|
||||
|
||||
# ---------------- decode 阶梯 ----------------
|
||||
cmd_decode() {
|
||||
local tag=${1:?tag} c=${2:?container} port=${3:?port} cg=${4:-256}
|
||||
local maxc=$(( cg * 80 / 100 / 32 * 32 )) # 80% 向下取整到 32 的倍数
|
||||
local p=$(date +%H%M%S) gen=$BASE/.bench_v2_gen_${tag}.sh
|
||||
echo "ladder: 32..$maxc step 32 (cg_max_bs=$cg)"
|
||||
{ emit_common "$c" "$port"
|
||||
cat <<'EOF'
|
||||
run WARMUP --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 128 \
|
||||
--num-prompts 16 --max-concurrency 16 --seed 777000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
EOF
|
||||
# bs1 延迟地板(warm + measure)
|
||||
cat <<EOF
|
||||
run WARM_bs1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
|
||||
--num-prompts 4 --max-concurrency 1 --seed ${p}01 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
run DEC_bs1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
|
||||
--num-prompts 4 --max-concurrency 1 --seed ${p}01 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
local cc
|
||||
for (( cc=32; cc<=maxc; cc+=32 )); do
|
||||
local np=$(( cc * 2 ))
|
||||
cat <<EOF
|
||||
run WARM_c${cc} --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
|
||||
--num-prompts $np --max-concurrency $cc --seed ${p}1${cc} --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
run DEC_c${cc} --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
|
||||
--num-prompts $np --max-concurrency $cc --seed ${p}1${cc} --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
done
|
||||
echo 'echo "===== [$(date +%H:%M:%S)] ALL DONE ====="'
|
||||
} | sed "s|MODELPH|$MODEL|g" > "$gen"
|
||||
chmod +x "$gen"
|
||||
nohup docker exec "$c" bash "$gen" > "$BASE/bench_v2_${tag}.log" 2>&1 &
|
||||
echo "launched pid=$! log=$BASE/bench_v2_${tag}.log"
|
||||
}
|
||||
|
||||
# ---------------- L3 语料溢出(3p1d 经 router) ----------------
|
||||
cmd_l3() {
|
||||
local tag=${1:?tag} c=${2:?container} port=${3:?port} np=${4:-48}
|
||||
local p=$(date +%H%M%S) gen=$BASE/.bench_v2_gen_${tag}.sh
|
||||
{ emit_common "$c" "$port"
|
||||
cat <<EOF
|
||||
run WARMUP --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 4096 --random-output-len 16 \\
|
||||
--num-prompts 8 --max-concurrency 8 --seed 777000 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
run L3_COLD_160k --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \\
|
||||
--num-prompts $np --max-concurrency 8 --seed ${p}1 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
echo "===== [\$(date +%H:%M:%S)] DRAIN: sleep 120 等 L2→L3 备份排空 ====="
|
||||
sleep 120
|
||||
run L3_HIT_160k --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \\
|
||||
--num-prompts $np --max-concurrency 8 --seed ${p}1 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
echo 'echo "===== [$(date +%H:%M:%S)] ALL DONE ====="'
|
||||
} > "$gen"
|
||||
chmod +x "$gen"
|
||||
nohup docker exec "$c" bash "$gen" > "$BASE/bench_v2_${tag}.log" 2>&1 &
|
||||
echo "launched pid=$! corpus=${np}x160k log=$BASE/bench_v2_${tag}.log"
|
||||
}
|
||||
|
||||
# ---------------- report ----------------
|
||||
cmd_report() {
|
||||
local log=${1:?log required}
|
||||
awk '
|
||||
/START:/ { name=$4; sub(/:$/, "", name); tps=""; itps=""; tpot=""; ttft=""; acc="" }
|
||||
/Input token throughput/ { itps=$NF }
|
||||
/Output token throughput/ { tps=$NF }
|
||||
/Mean TPOT/ { tpot=$NF }
|
||||
/Mean TTFT/ { ttft=$NF }
|
||||
/[Aa]ccept/ { acc=$0; gsub(/^ +/, "", acc) }
|
||||
/END\(exit/ {
|
||||
line=sprintf("%-22s in_tps=%-10s out_tps=%-10s tpot_ms=%-8s ttft_ms=%-8s", name, itps, tps, tpot, ttft)
|
||||
if (tps != "" && tpot != "" && tpot+0 > 0)
|
||||
line=line sprintf(" eff_concurrency=%.1f", tps * tpot / 1000)
|
||||
print line
|
||||
if (acc != "") print " " acc
|
||||
}
|
||||
' "$log"
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
warmup) shift; cmd_warmup "$@" ;;
|
||||
prefill) shift; cmd_prefill "$@" ;;
|
||||
decode) shift; cmd_decode "$@" ;;
|
||||
l3) shift; cmd_l3 "$@" ;;
|
||||
report) shift; cmd_report "$@" ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
@@ -0,0 +1,61 @@
|
||||
cd /data/ymk/ds41
|
||||
HDR="Authorization=Bearer deepseek-v4-flash-0731-key"
|
||||
PORT=30040
|
||||
MODEL=/data/models/DeepSeek-V4.1-Flash
|
||||
|
||||
run() {
|
||||
local name="$1"; shift
|
||||
echo "===== [$(date +%H:%M:%S)] START: $name ====="
|
||||
python3 -m sglang.bench_serving "$@" 2>&1 | grep -vE '^Namespace|warn'
|
||||
echo "===== [$(date +%H:%M:%S)] END(exit=$?): $name ====="
|
||||
sleep 3
|
||||
}
|
||||
|
||||
r160() { # $1=name $2=seed $3=num_prompts $4=mc
|
||||
run "$1" --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--num-prompts "$3" --max-concurrency "$4" --seed "$2" --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
}
|
||||
|
||||
# WARMUP
|
||||
run WARMUP --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 4096 --random-output-len 16 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 777000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
|
||||
# R0 纯计算极限(集合 A 首次出现,L1/L2/L3 全空)
|
||||
r160 R0_COMPUTE_COLD 1541391 8 8
|
||||
# R1 L1 极限(A 全量在 L1 设备池:1.28M < 1.6M)
|
||||
r160 R1_L1_HIT 1541391 8 8
|
||||
# R2 填充集 B(12×160k=1.92M > L1 1.6M),把 A 挤出 L1 到 L2
|
||||
r160 R2_FILLER_EVICT 999000 12 8
|
||||
# R3 L2 极限(A 已被挤出 L1,从 L2 host 池 load-back)
|
||||
r160 R3_L2_HIT 1541391 8 8
|
||||
# R4 flush L1+L2(L3 保留),走 L3 prefetch
|
||||
echo "===== [$(date +%H:%M:%S)] FLUSH L1+L2 ====="
|
||||
curl -sf -m 10 -X POST http://127.0.0.1:$PORT/flush_cache && echo "flush ok"
|
||||
sleep 5
|
||||
r160 R4_L3_HIT 1541391 8 8
|
||||
# R5 嫁接后 L2 复核
|
||||
r160 R5_L2_AFTER_GRAFT 1541391 8 8
|
||||
|
||||
# R6 混合负载:新集合 C(纯计算,4×160k mc4)+ 集合 A(命中,8×160k mc8)并发打
|
||||
echo "===== [$(date +%H:%M:%S)] START: R6_MIX_PARALLEL ====="
|
||||
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--num-prompts 4 --max-concurrency 4 --seed 555000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR" > /data/ymk/ds41/.tier_r6_coldclient.log 2>&1 &
|
||||
COLD_PID=$!
|
||||
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 1541391 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR" > /data/ymk/ds41/.tier_r6_hitclient.log 2>&1 &
|
||||
HIT_PID=$!
|
||||
wait $COLD_PID $HIT_PID
|
||||
echo "===== [$(date +%H:%M:%S)] END: R6_MIX_PARALLEL ====="
|
||||
echo "--- R6 cold client (compute) ---"
|
||||
grep -E 'Input token throughput|Mean TTFT|Request throughput' /data/ymk/ds41/.tier_r6_coldclient.log
|
||||
echo "--- R6 hit client (cache) ---"
|
||||
grep -E 'Input token throughput|Mean TTFT|Request throughput' /data/ymk/ds41/.tier_r6_hitclient.log
|
||||
echo "===== [$(date +%H:%M:%S)] ALL DONE ====="
|
||||
@@ -0,0 +1,27 @@
|
||||
cd /data/ymk/ds41
|
||||
HDR="Authorization=Bearer deepseek-v4-flash-0731-key"
|
||||
PORT=30040
|
||||
MODEL=/data/models/DeepSeek-V4.1-Flash
|
||||
|
||||
echo "===== [$(date +%H:%M:%S)] FLUSH L1+L2(A 只剩 L3) ====="
|
||||
curl -sf -m 10 -X POST http://127.0.0.1:$PORT/flush_cache && echo "flush ok"
|
||||
sleep 5
|
||||
|
||||
echo "===== [$(date +%H:%M:%S)] START: R7_MIX_L3_PLUS_COMPUTE ====="
|
||||
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--num-prompts 4 --max-concurrency 4 --seed 666000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR" > /data/ymk/ds41/.tier_r7_coldclient.log 2>&1 &
|
||||
COLD_PID=$!
|
||||
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 1541391 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR" > /data/ymk/ds41/.tier_r7_hitclient.log 2>&1 &
|
||||
HIT_PID=$!
|
||||
wait $COLD_PID $HIT_PID
|
||||
echo "===== [$(date +%H:%M:%S)] END: R7 ====="
|
||||
echo "--- R7 cold client (compute, 4x160k mc4) ---"
|
||||
grep -E 'Input token throughput|Mean TTFT|Request throughput' /data/ymk/ds41/.tier_r7_coldclient.log
|
||||
echo "--- R7 hit client (L3, 8x160k mc8) ---"
|
||||
grep -E 'Input token throughput|Mean TTFT|Request throughput' /data/ymk/ds41/.tier_r7_hitclient.log
|
||||
echo "===== [$(date +%H:%M:%S)] ALL DONE ====="
|
||||
@@ -0,0 +1,172 @@
|
||||
# 3x cp2-P + 1x dp2-D PD 分离 + L3 mooncake 互联缓存 + engram host table
|
||||
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-cp2-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-cp2-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-cp2-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-cp2-d
|
||||
environment:
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
@@ -0,0 +1,225 @@
|
||||
# 【L3 dummy client 变体 2026-10-09 深夜,定型=shm+best_effort】sglang→standalone_storage,数据面经本机 mooncake_client(50052 IPC+进程映射),不碰 TCP 传输引擎
|
||||
# 实测 24x160k 热/冷:shm+best_effort +26.4%(唯一正收益);wait_complete 因 all-or-nothing discard 等待白等反而 -17.7%。TCP 传输引擎两组皆负。
|
||||
# 【L3 cache-hit 测试变体 2026-10-09】基于 dockerserve-3p1d-otlp.yml,仅改 P:
|
||||
# --max-total-tokens 800000(L1 缩到 ~0.8M token)+ --hicache-ratio 1.0(L2=L1=0.8M token。⚠️ 教训:ratio 0.05 时 L2 太小,build_storage_backup_spec 拿不到 host 页会静默跳过 L3 写入,热轮无数据可中;V4 hicache 不支持 --hicache-size)
|
||||
# 使 ~2.6M token/P 的语料(48×160k)溢出 L1+L2(1.6M),热轮强制走 L3 mooncake 命中。
|
||||
# 【2026-10-10 L2 放大】hicache-ratio 1.0→4.0(L2=3.2M token/P,宿主可用内存 ~1.9TB 充裕):
|
||||
# #38348 fail-soft 移植后 bench 显示收益被容量掩盖——24x160k 下 host_capacity 拒绝
|
||||
# 0.75M~2.97M token/P(L2 装不下,prefetch 未发起即被拒退回重算)。放大后验证:
|
||||
# ①热轮请求 L3/L2 命中不被容量拒绝;②L3 hit 嫁接进 L2 树后在 L2 长期留存。
|
||||
# 与主模板同容器名/端口,用前必须先 down 主栈。D 与 router 不变。
|
||||
# 原注释: 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram + OTLP trace(2026-09-27)
|
||||
# 基于 dockerserve-3p1d-soak.yml 增加: --enable-trace --trace-modules request,mooncake
|
||||
# --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name ds41-3p1d-<实例名>
|
||||
# 依赖: b300-01 k8s cw jaeger-trace(jaegertracing/all-in-one:1.76.0,badger 6h TTL,hostNetwork 4317)
|
||||
# 安全性: BatchSpanProcessor 导出失败只丢 span;Jaeger 宕机不影响推理服务
|
||||
# 调优旋钮(默认未开): SGLANG_TRACE_ASYNC=1(异步 span 创建), SGLANG_TRACE_LEVEL(默认 3)
|
||||
# router(ds41-router-3p1d,rust)无 OTLP 支持,不产生 span,追踪从各 sglang HTTP 层开始
|
||||
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-baba18bc7d-local-20261010-2310
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python:/sgl-workspace/sglang/python
|
||||
- /data/ymk/ds41/fake-cargo:/root/.cargo/bin/cargo:ro
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
RUSTUP_TOOLCHAIN: "1.92-x86_64-unknown-linux-gnu"
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store-dummy.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 4.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p1
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 4.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p2
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 4.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p3
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
# G9 P0 验证(2026-10-11):D 单独升镜像拿 /query_cache_hit 修复(71c6d8bc5),
|
||||
# 并开 decode radix + hicache L3 客户端(复用 dummy store,只读查询)。
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-6612af9e0-local-20261011-0740
|
||||
container_name: ds41-3p1d-d1
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
RUSTUP_TOOLCHAIN: "1.92-x86_64-unknown-linux-gnu"
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store-dummy.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
--disaggregation-decode-enable-radix-cache
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-d1
|
||||
@@ -0,0 +1,239 @@
|
||||
# 【G9 P0 验证栈:真实 store 3p1d 2026-10-11】基于 dockerserve-3p1d-otlp.yml。
|
||||
# 用途:验证 P0 改动(/v1/loads 排队 token + drain rate 字段、P 侧 /query_cache_hit)。
|
||||
# 与主模板同容器名/端口,用前必须先 down 主栈;router(ds41-router-3p1d)独立不在本文件。
|
||||
# 2026-10-11 改造要点(vs 原 l3test):
|
||||
# - 镜像升 6612af9e0(含 G9 P0 + /query_cache_hit KeyError 修复)+ scheduler.py
|
||||
# 热补丁(L3 查询门限从 enable_decode_hicache 放宽到 enable_hicache_storage,
|
||||
# 否则 P 节点永远查不到 L3)。
|
||||
# - 每实例独立 MOONCAKE_LOCAL_HOSTNAME(a/b/c/d),extra_hosts 互配全部 rank 别名
|
||||
# (真实 store 跨实例副本写要求对端 rank 主机名可解析,见 realstore-x2 注释;
|
||||
# rank 命名规则 {hostname}-d{dp}t{tp}p{pp}c{attn_cp},commit 84fc632088;
|
||||
# D 为 dp2 dp-attention(attn_tp=1),后缀推断为 -d0t0p0c0/-d1t1p0c0,未实证)。
|
||||
# - ⚠️ D 侧 cache 方案实证否决(2026-10-11):SWA 模型下
|
||||
# --disaggregation-decode-enable-radix-cache 先要 SGLANG_ENABLE_UNIFIED_RADIX_TREE=1,
|
||||
# 开了又报 "supports only device-resident cache and is incompatible with
|
||||
# --enable-hierarchical-cache"(kv_cache_builder.py 硬 ValueError)——D 挂不了 L3。
|
||||
# 结论:/query_cache_hit 放 P(L3 全局统一,P 有完整 hicache),D 保持生产配置。
|
||||
# - D mem-fraction 0.80:曾因崩溃 d1 泄漏 ~61GiB 临时降 0.75,全栈 down 掉旧容器后
|
||||
# 驱动即回收(GPU 全零),已恢复 0.80。D max-running 保持 256(512 时 SWA 池挤爆 KV 池)。
|
||||
# P 侧保持 l3test 原状:--max-total-tokens 800000 + ratio 1.0(L1/L2 各 ~0.8M token),
|
||||
# write_through 保证 prefill 后 L3 有数据,供 /query_cache_hit 对拍。
|
||||
# 原注释: 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram + OTLP trace(2026-09-27)
|
||||
# 依赖: mooncake-master 容器(50051,起栈前 restart 清旧 segment 注册);
|
||||
# b300-01 k8s cw jaeger-trace(4317,导出失败只丢 span 不影响服务)
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-e7dc175e9-local-20261011-0940
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-rank-hosts: &rank-hosts
|
||||
# mooncake rank 主机名别名:P(tp2/cp2 interleave)-d0t0p0c0/-d0t1p0c1;
|
||||
# D(dp2 dp-attention)-d0t0p0c0/-d1t1p0c0(推断,起栈后核对 master 注册名)
|
||||
- "localhost-a-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-a-d0t1p0c1:127.0.0.1"
|
||||
- "localhost-b-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-b-d0t1p0c1:127.0.0.1"
|
||||
- "localhost-c-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-c-d0t1p0c1:127.0.0.1"
|
||||
- "localhost-d-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-d-d1t1p0c0:127.0.0.1"
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p1
|
||||
extra_hosts: *rank-hosts
|
||||
environment:
|
||||
<<: *p-environment
|
||||
MOONCAKE_LOCAL_HOSTNAME: localhost-a
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p1
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p2
|
||||
extra_hosts: *rank-hosts
|
||||
environment:
|
||||
<<: *p-environment
|
||||
MOONCAKE_LOCAL_HOSTNAME: localhost-b
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p2
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p3
|
||||
extra_hosts: *rank-hosts
|
||||
environment:
|
||||
<<: *p-environment
|
||||
MOONCAKE_LOCAL_HOSTNAME: localhost-c
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p3
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-d1
|
||||
extra_hosts: *rank-hosts
|
||||
# ⚠️ D 保持生产配置(2026-10-11 实证):SWA 模型下
|
||||
# --disaggregation-decode-enable-radix-cache 与 --enable-hierarchical-cache 互斥
|
||||
# (kv_cache_builder.py 硬 ValueError,"supports only device-resident cache"),
|
||||
# D 无法挂 L3 —— /query_cache_hit 改由 P 节点回答(L3 全局统一,P 有完整 hicache)。
|
||||
# MOONCAKE_LOCAL_HOSTNAME 保留备用(无害)。
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
MOONCAKE_LOCAL_HOSTNAME: localhost-d
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-d1
|
||||
@@ -0,0 +1,196 @@
|
||||
# 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram + OTLP trace(2026-09-27)
|
||||
# 基于 dockerserve-3p1d-soak.yml 增加: --enable-trace --trace-modules request,mooncake
|
||||
# --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name ds41-3p1d-<实例名>
|
||||
# 依赖: b300-01 k8s cw jaeger-trace(jaegertracing/all-in-one:1.76.0,badger 6h TTL,hostNetwork 4317)
|
||||
# 安全性: BatchSpanProcessor 导出失败只丢 span;Jaeger 宕机不影响推理服务
|
||||
# 调优旋钮(默认未开): SGLANG_TRACE_ASYNC=1(异步 span 创建), SGLANG_TRACE_LEVEL(默认 3)
|
||||
# router(ds41-router-3p1d,rust)无 OTLP 支持,不产生 span,追踪从各 sglang HTTP 层开始
|
||||
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave --chunked-prefill-size 32768 --max-prefill-tokens 32768
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p1
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave --chunked-prefill-size 32768 --max-prefill-tokens 32768
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p2
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave --chunked-prefill-size 32768 --max-prefill-tokens 32768
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p3
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-d1
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-d1
|
||||
@@ -0,0 +1,196 @@
|
||||
# 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram + OTLP trace(2026-09-27)
|
||||
# 基于 dockerserve-3p1d-soak.yml 增加: --enable-trace --trace-modules request,mooncake
|
||||
# --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name ds41-3p1d-<实例名>
|
||||
# 依赖: b300-01 k8s cw jaeger-trace(jaegertracing/all-in-one:1.76.0,badger 6h TTL,hostNetwork 4317)
|
||||
# 安全性: BatchSpanProcessor 导出失败只丢 span;Jaeger 宕机不影响推理服务
|
||||
# 调优旋钮(默认未开): SGLANG_TRACE_ASYNC=1(异步 span 创建), SGLANG_TRACE_LEVEL(默认 3)
|
||||
# router(ds41-router-3p1d,rust)无 OTLP 支持,不产生 span,追踪从各 sglang HTTP 层开始
|
||||
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p1
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p2
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p3
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-d1
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-d1
|
||||
@@ -0,0 +1,174 @@
|
||||
# 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram(2026-09-25,mr256,nsys on p1/d1)
|
||||
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-prof-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
nsys launch --session-new=p1prof --cuda-graph-trace=node sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-prof-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-prof-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-prof-d1
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
nsys launch --session-new=d1prof --cuda-graph-trace=node sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
@@ -0,0 +1,174 @@
|
||||
# 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram(2026-09-25,mr256,nsys on p1/d1)
|
||||
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-d1
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
@@ -0,0 +1,43 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
container_name: ds41-sglang-b4
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30001:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
- PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.60
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--dp-size 4
|
||||
--enable-dp-attention
|
||||
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,43 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
|
||||
container_name: ds41-sglang-b4
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30001:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
- PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.60
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--enable-decoder-swa-bounded-replay
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
|
||||
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,42 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
|
||||
container_name: ds41-sglang-b4
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30001:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--enable-decoder-swa-bounded-replay
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,148 @@
|
||||
# 【阶段A 新旧基线 2026-10-18】4×2卡 standalone 并行微基准(aiperf 口径)
|
||||
# 目的:同一台 b300-01 上旧版(tag 26.10.18 线上镜像)vs 新版(g9i-est-td @ e4391748e,
|
||||
# 对齐上游 main 6c3db0f520,torch 2.14.1 自制基底)同配置对比。
|
||||
# 配置来源:线上 k8s ds41-3p1d-g9i 的 P/D pod spec,去掉 --disaggregation-* 与
|
||||
# mooncake L3 storage backend(微基准只测 L1/L2 冷/热;L3 在阶段 B PD 合测再测)。
|
||||
# 布局:old-cp2=GPU0,1:30500 old-dp2=GPU2,3:30600 new-cp2=GPU4,5:30700 new-dp2=GPU6,7:30800
|
||||
# 注意:
|
||||
# - 新镜像 flashinfer 0.7.2 首次 autotune 暖机 ~15-20min,用独立 cache 目录
|
||||
# (/data/ymk/cache/sglang-e43917)防与旧版缓存串扰;旧版复用 /data/ymk/cache/sglang。
|
||||
# - standalone 无 PD 传输,不带 pid:host / NVLink mooncake 四件套。
|
||||
# - 启动后核对三件套(max_total_num_tokens / max_running_requests / available_gpu_mem)。
|
||||
# 用法:docker compose -p bencha -f dockerserve-bencha-4inst.yml up -d
|
||||
# bench: docker exec aiperf-cmp bash /data/ymk/ds41/bench_aiperf.sh <mode> <tag> <port> [args]
|
||||
|
||||
x-common: &common
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
environment: &common-env
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
old-cp2:
|
||||
<<: *common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-baba18bc7d-local-20261010-2310
|
||||
container_name: bencha-old-cp2
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
environment:
|
||||
<<: *common-env
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.85
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30500
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
|
||||
old-dp2:
|
||||
<<: *common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-6612af9e0-local-20261011-0740
|
||||
container_name: bencha-old-dp2
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
environment:
|
||||
<<: *common-env
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--dp-size 2 --enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.85
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 160
|
||||
--max-running-requests 300
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30600
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
|
||||
new-cp2:
|
||||
<<: *common
|
||||
image: ymkymx/sglang:g9i-est-td-1d2610bbc-local-20261018-1506
|
||||
container_name: bencha-new-cp2
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang-e43917:/root/.cache/sglang
|
||||
environment:
|
||||
<<: *common-env
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.85
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30700
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
|
||||
new-dp2:
|
||||
<<: *common
|
||||
image: ymkymx/sglang:g9i-est-td-1d2610bbc-local-20261018-1506
|
||||
container_name: bencha-new-dp2
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang-e43917:/root/.cache/sglang
|
||||
environment:
|
||||
<<: *common-env
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--dp-size 2 --enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.85
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 160
|
||||
--max-running-requests 300
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30800
|
||||
--enable-cache-report --enable-metrics
|
||||
--skip-server-warmup
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
@@ -0,0 +1,229 @@
|
||||
# 【阶段B 新旧基线 2026-10-18】两套 1P1D PD 并行:旧版 vs 新版同部署对比
|
||||
# 旧栈:P=GPU0,1:32100 D=GPU2,3:32200 router=32300 master=50051(镜像 ∈ tag 26.10.18)
|
||||
# 新栈:P=GPU4,5:32400 D=GPU6,7:32500 router=32600 master=50061(g9i-est-td-e4391748e)
|
||||
# 配置 = 线上 k8s P/D spec 原样(mem 0.85、ratio 1.0、write_through、wait_complete、
|
||||
# dspark b5、cp2-P / dp2-D),仅改端口/bootstrap/master;OTLP trace 不陪跑。
|
||||
# NVLink-intra:pid:host + MC_INTRA 双拼写 + AUX_TCP(与线上 k8s spec 逐字对齐;CUSTOM_MEM_POOL 实测触发段错误,不设)。
|
||||
# 两套各挂各的 mooncake master(rank 名在同 master 下必须唯一,混挂会撞 localhost-d0tXpYcZ)。
|
||||
# 新栈复用阶段A已暖的 cache /data/ymk/cache/sglang-e43917(flashinfer 0.7.2 autotune 已跑过)。
|
||||
# ⚠️ 用前必须 down 阶段A(docker compose -p bencha down)。
|
||||
# 用法:docker compose -p benchb -f dockerserve-benchb-1p1d-x2.yml up -d
|
||||
# 入口:旧栈 http://127.0.0.1:32300 新栈 http://127.0.0.1:32600(api-key deepseek-v4-flash-0731-key)
|
||||
|
||||
x-common: &common
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-pd-env: &pd-env
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
x-p-extra-hosts: &p-extra-hosts
|
||||
# mooncake rank 后缀主机名(84fc632088 要求可解析)
|
||||
- "localhost-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-d0t1p0c1:127.0.0.1"
|
||||
|
||||
services:
|
||||
# ---------- 旧栈(tag 26.10.18) ----------
|
||||
master-old:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-baba18bc7d-local-20261010-2310
|
||||
container_name: benchb-master-old
|
||||
network_mode: host
|
||||
command: >
|
||||
/opt/sglang/bin/mooncake_master --port 50051
|
||||
--enable_http_metadata_server=true --http_metadata_server_port=8080
|
||||
--eviction_high_watermark_ratio=0.90
|
||||
|
||||
p-old:
|
||||
<<: *common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-baba18bc7d-local-20261010-2310
|
||||
container_name: benchb-p-old
|
||||
extra_hosts: *p-extra-hosts
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
environment:
|
||||
<<: *pd-env
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store-tcp.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.85
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 32100
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9601
|
||||
|
||||
d-old:
|
||||
<<: *common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-6612af9e0-local-20261011-0740
|
||||
container_name: benchb-d-old
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
environment:
|
||||
<<: *pd-env
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--dp-size 2 --enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.85
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 160
|
||||
--max-running-requests 300
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 32200
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9602
|
||||
|
||||
router-old:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-baba18bc7d-local-20261010-2310
|
||||
container_name: benchb-router-old
|
||||
network_mode: host
|
||||
entrypoint: sglang-router
|
||||
command: >
|
||||
launch --mini-lb --pd-disaggregation
|
||||
--prefill http://127.0.0.1:32100 9601
|
||||
--decode http://127.0.0.1:32200
|
||||
--host 0.0.0.0 --port 32300
|
||||
--api-key deepseek-v4-flash-0731-key
|
||||
|
||||
# ---------- 新栈(g9i-est-td @ e4391748e) ----------
|
||||
master-new:
|
||||
image: ymkymx/sglang:g9i-est-td-148936bcb-local-20261018-1803
|
||||
container_name: benchb-master-new
|
||||
network_mode: host
|
||||
command: >
|
||||
/opt/sglang/bin/mooncake_master --port 50061
|
||||
--enable_http_metadata_server=true --http_metadata_server_port=8081
|
||||
--metrics_port 9004
|
||||
--eviction_high_watermark_ratio=0.90
|
||||
|
||||
p-new:
|
||||
<<: *common
|
||||
image: ymkymx/sglang:g9i-est-td-148936bcb-local-20261018-1803
|
||||
container_name: benchb-p-new
|
||||
extra_hosts: *p-extra-hosts
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang-e43917:/root/.cache/sglang
|
||||
environment:
|
||||
<<: *pd-env
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store-tcp-50061.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.85
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 32400
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9611
|
||||
|
||||
d-new:
|
||||
<<: *common
|
||||
image: ymkymx/sglang:g9i-est-td-148936bcb-local-20261018-1803
|
||||
container_name: benchb-d-new
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang-e43917:/root/.cache/sglang
|
||||
environment:
|
||||
<<: *pd-env
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--dp-size 2 --enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.85
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 160
|
||||
--max-running-requests 300
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 32500
|
||||
--enable-cache-report --enable-metrics
|
||||
--skip-server-warmup
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9612
|
||||
|
||||
router-new:
|
||||
image: ymkymx/sglang:g9i-est-td-148936bcb-local-20261018-1803
|
||||
container_name: benchb-router-new
|
||||
network_mode: host
|
||||
entrypoint: sglang-router
|
||||
command: >
|
||||
launch --mini-lb --pd-disaggregation
|
||||
--prefill http://127.0.0.1:32400 9611
|
||||
--decode http://127.0.0.1:32500
|
||||
--host 0.0.0.0 --port 32600
|
||||
--api-key deepseek-v4-flash-0731-key
|
||||
@@ -0,0 +1,160 @@
|
||||
# 4×cp2 独立实例(非 PD)+ hicache L3(mooncake) + dspark + engram host table 卸载(2026-09-23 晚)
|
||||
# 用户指定组合:4 cp2 hicache l3 dspark roundrobin loadbalance engram offload
|
||||
# 每实例:tp2 ep2 + interleave CP2 + dspark b5 + hicache L3 write_through(size 0) + engram host table
|
||||
# (tp2 权重必须靠 engram 卸载才放得下,见 goals-track G1.9 反转)
|
||||
# GPU: a=0,1 b=2,3 c=4,5 d=6,7;端口: 30000/30010/30020/30030;rr router=30002
|
||||
# 参考:dockerserve-cpdspark-l3.yml(hicache 参数)、dockerserve-3cp2-pd.yml(cp2/engram 参数)
|
||||
x-common: &common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
# 热补丁(2026-09-23):CP 对称 idle 检查,修 health-check × hicache drain 死锁
|
||||
# 源文件:/data/ymk/sglang(dsv41-pd-visioncp 分支工作区已改);补丁脚本见
|
||||
# D:\B300\experiments\cp2x4\patch_healthcheck_cp_idle.py
|
||||
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
|
||||
environment: &env
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_MS_AUTO_DISC: "0"
|
||||
MOONCAKE_MASTER: 127.0.0.1:50051
|
||||
MOONCAKE_TE_META_DATA_SERVER: P2PHANDSHAKE
|
||||
MOONCAKE_PROTOCOL: tcp
|
||||
MOONCAKE_GLOBAL_SEGMENT_SIZE: 300gb
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
a:
|
||||
<<: *common
|
||||
container_name: ds41-cp2-a
|
||||
environment:
|
||||
<<: *env
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
b:
|
||||
<<: *common
|
||||
container_name: ds41-cp2-b
|
||||
environment:
|
||||
<<: *env
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
c:
|
||||
<<: *common
|
||||
container_name: ds41-cp2-c
|
||||
environment:
|
||||
<<: *env
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
d:
|
||||
<<: *common
|
||||
container_name: ds41-cp2-d2
|
||||
environment:
|
||||
<<: *env
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30030
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
@@ -0,0 +1,44 @@
|
||||
# cp8 单机非 PD 测试(2026-09-23 晚)
|
||||
# 背景:dp8 单机 dspark 健康但长上下文 prefill 结构性慢(单请求只落 1 rank),
|
||||
# 换 cp8(tp8 + interleave prefill CP8,单请求 prefill 切到 8 卡)对照。
|
||||
# 关 engram、无 hicache L3、无 PD。cp+dspark 无 replay 是已知健康组合
|
||||
# (cp4+dspark+replay 三元组必崩,本配置不开 replay)。
|
||||
# GPU 0-7,端口 30000,容器 ds41-cp8,project cp8
|
||||
services:
|
||||
cp8:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-cp8
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
environment:
|
||||
SGLANG_RAGGED_VERIFY_MODE: "static"
|
||||
CUDA_VISIBLE_DEVICES: "0,1,2,3,4,5,6,7"
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 8 --ep-size 8
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
@@ -0,0 +1,50 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536
|
||||
container_name: ds41-cpdspark-l3-b
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
- MC_MS_AUTO_DISC=0
|
||||
- MOONCAKE_MASTER=127.0.0.1:50051
|
||||
- MOONCAKE_TE_META_DATA_SERVER=P2PHANDSHAKE
|
||||
- MOONCAKE_PROTOCOL=tcp
|
||||
- MOONCAKE_GLOBAL_SEGMENT_SIZE=400gb
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4 --ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,51 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536
|
||||
container_name: ds41-cpdspark-l3-b
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
- MC_MS_AUTO_DISC=0
|
||||
- MOONCAKE_MASTER=127.0.0.1:50051
|
||||
- MOONCAKE_TE_META_DATA_SERVER=P2PHANDSHAKE
|
||||
- MOONCAKE_PROTOCOL=tcp
|
||||
- MOONCAKE_GLOBAL_SEGMENT_SIZE=400gb
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4 --ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,50 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536
|
||||
container_name: ds41-cpdspark-l3-a
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
- MC_MS_AUTO_DISC=0
|
||||
- MOONCAKE_MASTER=127.0.0.1:50051
|
||||
- MOONCAKE_TE_META_DATA_SERVER=P2PHANDSHAKE
|
||||
- MOONCAKE_PROTOCOL=tcp
|
||||
- MOONCAKE_GLOBAL_SEGMENT_SIZE=400gb
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4 --ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,51 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536
|
||||
container_name: ds41-cpdspark-l3-a
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
- MC_MS_AUTO_DISC=0
|
||||
- MOONCAKE_MASTER=127.0.0.1:50051
|
||||
- MOONCAKE_TE_META_DATA_SERVER=P2PHANDSHAKE
|
||||
- MOONCAKE_PROTOCOL=tcp
|
||||
- MOONCAKE_GLOBAL_SEGMENT_SIZE=400gb
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4 --ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,44 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-92632a60b-20260922-2100
|
||||
container_name: ds41-cpdspark-nr-b4
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30001:30001"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,44 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
container_name: ds41-cpdspark-nr
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30000:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,44 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
|
||||
container_name: ds41-cpdspark
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30000:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--enable-decoder-swa-bounded-replay
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,49 @@
|
||||
# 单独 D 节点基线(非 PD role):3p1d 的 d1 配置去掉 disaggregation,直接对外服务
|
||||
# 用途:测单 D 节点 decode 基线(阶梯并发 32..80%*cuda-graph-max-bs-decode,
|
||||
# 报 TPOT / 实际并发 / 整实例输出 TPS;prefill 走 L1 radix cache-hit)
|
||||
# GPU 6,7;端口 30001;无 hicache 依赖(L1 即可支撑重复 prompt 命中)
|
||||
# 2026-10-09 建:配合 bench_suite_v2.sh decode
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
d1s:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-d1-standalone
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
@@ -0,0 +1,62 @@
|
||||
# dp2 对照(2026-09-24 上午):验证 dp2(dp-attention)下 dspark 是否正常。
|
||||
# 与 dockerserve-tp2.yml 逐参数对齐,唯一差异 = 加 --dp-size 2 --enable-dp-attention --enable-dp-lm-head。
|
||||
# GPU 6-7,端口 30030 直连。镜像/补丁挂载/L3/engram/tok8 全部相同。
|
||||
# 注:dp+dspark 不能开 --enable-decoder-swa-bounded-replay(本配置本来就没开)。
|
||||
x-common: &common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
|
||||
environment: &env
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_MS_AUTO_DISC: "0"
|
||||
MOONCAKE_MASTER: 127.0.0.1:50051
|
||||
MOONCAKE_TE_META_DATA_SERVER: P2PHANDSHAKE
|
||||
MOONCAKE_PROTOCOL: tcp
|
||||
MOONCAKE_GLOBAL_SEGMENT_SIZE: 300gb
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
dp2:
|
||||
<<: *common
|
||||
container_name: ds41-dp2
|
||||
environment:
|
||||
<<: *env
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--dp-size 2 --enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30030
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
@@ -0,0 +1,42 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
|
||||
container_name: ds41-dp4test
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30000:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--enable-decoder-swa-bounded-replay
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,44 @@
|
||||
# dp8+tp8 单机非 PD 对照测试(2026-09-23 晚)
|
||||
# 背景:3cp2+dp2 PD 部署 dspark accept 回归(32k ctx accept len 3.14→1.31,
|
||||
# 见 experiments/3cp2-pd/bench-20260923.md「未决问题」)。本配置退回单机验证
|
||||
# dspark 健康度:engram host table 关闭、无 hicache L3、无 PD、无 mooncake。
|
||||
# 参数基准:dockerserve-3cp2-pd.yml 的 d 服务(dp 路径),去掉 PD/L3/engram 相关。
|
||||
# GPU 0-7,端口 30000,容器 ds41-dp8,project dp8
|
||||
services:
|
||||
dp8:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-dp8
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
environment:
|
||||
SGLANG_RAGGED_VERIFY_MODE: "static"
|
||||
CUDA_VISIBLE_DEVICES: "0,1,2,3,4,5,6,7"
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 8 --ep-size 8 --dp-size 8
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
@@ -0,0 +1,61 @@
|
||||
# 单独 P 节点基线(非 PD role):3p1d 的 p1 配置去掉 disaggregation,直接对外服务
|
||||
# 用途:测单 P 节点 prefill 基线吞吐(冷)+ hicache cache-hit TPS(热)
|
||||
# GPU 0,1;端口 30000;依赖 mooncake-master(hicache L3,127.0.0.1:50051)
|
||||
# 2026-10-09 建:配合 bench_suite_v2.sh prefill
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
p1s:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-p1-standalone
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
@@ -0,0 +1,47 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
|
||||
container_name: ds41-prefill
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30000:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-decoder-swa-bounded-replay
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2.5
|
||||
--hicache-write-policy write_back
|
||||
--disaggregation-mode prefill
|
||||
--optimistic-prefill-attempts 4
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,51 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-pd-decode
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--dp-size 4
|
||||
--enable-dp-attention
|
||||
--enable-dp-lm-head
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30001
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
@@ -0,0 +1,50 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536
|
||||
container_name: ds41-pd-decode
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--dp-size 4
|
||||
--enable-dp-attention
|
||||
--enable-dp-lm-head
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30001
|
||||
--enable-cache-report
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,55 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-c74a4037f-20260921-1300
|
||||
container_name: ds41-pd-prefill-dp
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--dp-size 4
|
||||
--enable-dp-attention
|
||||
--enable-dp-lm-head
|
||||
--load-balance-method total_tokens
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30010
|
||||
--enable-cache-report
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2.5
|
||||
--hicache-write-policy write_back
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-bootstrap-port 8918
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,53 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-pd-prefill
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2.5
|
||||
--hicache-write-policy write_back
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
@@ -0,0 +1,52 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536
|
||||
container_name: ds41-pd-prefill
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2.5
|
||||
--hicache-write-policy write_back
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,57 @@
|
||||
# PD2 二分 Round 2 D 侧(2026-09-24):干净 D + 嫌疑项②
|
||||
# SGLANG_DISAGGREGATION_QUEUE_SIZE=16 + SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=32
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-pd2-decode
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=6,7
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
- SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1
|
||||
- SGLANG_DISAGGREGATION_QUEUE_SIZE=16
|
||||
- SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=32
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2
|
||||
--ep-size 2
|
||||
--dp-size 2
|
||||
--enable-dp-attention
|
||||
--enable-dp-lm-head
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30030
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
@@ -0,0 +1,54 @@
|
||||
# PD 干净对照实验 D 节点(2026-09-24 上午):dp2 decode,配 dockerserve-pd2-p.yml 使用。
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-pd2-decode
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=6,7
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
- SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2
|
||||
--ep-size 2
|
||||
--dp-size 2
|
||||
--enable-dp-attention
|
||||
--enable-dp-lm-head
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30030
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
@@ -0,0 +1,68 @@
|
||||
# PD2 二分 Round 1(2026-09-24):干净基线 + 嫌疑项①「P 侧 L3 hicache 块 + mooncake-store.json」
|
||||
# 相对 dockerserve-pd2-p.yml 的差异:
|
||||
# env 加 SGLANG_HICACHE_MOONCAKE_CONFIG_PATH=/data/ymk/ds41/mooncake-store.json
|
||||
# hicache 参数块换成坏部署同款:page_first_direct + direct + write_through + mooncake +
|
||||
# wait_complete + size 0(替代 write_back L2)
|
||||
# D 侧不变(坏部署 D 本无 L3)。探针:bs16 low_entropy decode 看 accept len。
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-pd2-prefill
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
- SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1
|
||||
- SGLANG_HICACHE_MOONCAKE_CONFIG_PATH=/data/ymk/ds41/mooncake-store.json
|
||||
- SGLANG_HICACHE_PREFETCH_IO_THREADS=4
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2
|
||||
--ep-size 2
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30020
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
@@ -0,0 +1,66 @@
|
||||
# PD2 二分 Round 2 P 侧(2026-09-24):R1(L3 块)之上再加嫌疑项②
|
||||
# SGLANG_DISAGGREGATION_QUEUE_SIZE=16 + SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=32
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-pd2-prefill
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
- SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1
|
||||
- SGLANG_HICACHE_MOONCAKE_CONFIG_PATH=/data/ymk/ds41/mooncake-store.json
|
||||
- SGLANG_HICACHE_PREFETCH_IO_THREADS=4
|
||||
- SGLANG_DISAGGREGATION_QUEUE_SIZE=16
|
||||
- SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=32
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2
|
||||
--ep-size 2
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30020
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
@@ -0,0 +1,59 @@
|
||||
# PD 干净对照实验(2026-09-24 上午):验证「cp2-P → dp2-D」传输路径下 dspark 是否正常。
|
||||
# 用户疑问:单机六形态健康不等于 PD 链路健康,可能两边传输没对齐。
|
||||
# 以已验证可用的 dockerserve-pd-{p,d}-vision.yml 为底,缩到 2+2 卡;P 加 engram host table
|
||||
# (2 卡权重放不下,见 G1.9)。P=GPU4-5 端口 30020,D=GPU6-7 端口 30030,mini_lb=30004。
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-pd2-prefill
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
- SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2
|
||||
--ep-size 2
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30020
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2.5
|
||||
--hicache-write-policy write_back
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
@@ -0,0 +1,42 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
|
||||
container_name: ds41-replay-off
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30000:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,42 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
|
||||
container_name: ds41-replay-on
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30000:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-decoder-swa-bounded-replay
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,76 @@
|
||||
# 【跨 P 实例 L3 读写验证 2026-10-10】dockerserve-tier-cp2-realstore.yml 的第二实例:
|
||||
# 与 ds41-tier-t2(A:GPU 0,1,端口 30040)同时跑,共用 mooncake master 50051。
|
||||
# 关键差异:
|
||||
# - GPU 2,3,端口 30041,容器名 ds41-tier-t3
|
||||
# - MOONCAKE_LOCAL_HOSTNAME=localhost-b:mooncake 客户端 hostname 前缀按实例区分
|
||||
# (rank 后缀 -d0t0p0c0/-d0t1p0c1 由 84fc632088 拼在后面的),否则两实例 rank 同名
|
||||
# → master preferred-segment 分不出段归属 → 副本落错进程段 → 跨进程回退 TCP(实例内)
|
||||
# 且跨实例段地址冲突。extra_hosts 别名同步加 -b 前缀(P2P handshake 要求可解析,
|
||||
# ac11e9162d)。
|
||||
# 验证路径:A prime 写 L3 → flush B 的 L1/L2 → B 读同 seed prompts = 跨实例 TCP P2P 命中。
|
||||
x-common: &common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
shm_size: 32gb
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python:/sgl-workspace/sglang/python
|
||||
- /data/ymk/ds41/fake-cargo:/root/.cargo/bin/cargo:ro
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
t3:
|
||||
<<: *common
|
||||
container_name: ds41-tier-t3
|
||||
extra_hosts:
|
||||
- "localhost-b-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-b-d0t1p0c1:127.0.0.1"
|
||||
# 跨实例 P2P:对端实例(realstore compose 的 t2)rank 主机名也要能解析
|
||||
- "localhost-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-d0t1p0c1:127.0.0.1"
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: 1
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_TIMING: 1
|
||||
MOONCAKE_LOCAL_HOSTNAME: localhost-b
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: 4
|
||||
CUDA_VISIBLE_DEVICES: 2,3
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--chunked-prefill-size 32768
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30041
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 4.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--max-total-tokens 1600000
|
||||
@@ -0,0 +1,73 @@
|
||||
# 【L3 真实 store A/B 2026-10-10】同 dockerserve-tier-cp2.yml,唯一差异:
|
||||
# L3 从 dummy standalone(mooncake-store-dummy.json,50052 守护进程)
|
||||
# 切到真实 mooncake store(mooncake-store.json:master 50051 + P2PHANDSHAKE + tcp,
|
||||
# 客户端注册 300GB 自身内存段)——与 k8s 线上 ds41-3p1d 的 L3 模式一致。
|
||||
# 目的:区分 "sglang L3 路径本身慢" 还是 "dummy 后端 per-key 路径慢"(mc 扫描 A/B)。
|
||||
x-common: &common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
shm_size: 32gb
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python:/sgl-workspace/sglang/python
|
||||
- /data/ymk/ds41/fake-cargo:/root/.cargo/bin/cargo:ro
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
t2:
|
||||
<<: *common
|
||||
container_name: ds41-tier-t2
|
||||
extra_hosts:
|
||||
- "localhost-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-d0t1p0c1:127.0.0.1"
|
||||
# 跨实例 P2P:对端实例(x2 compose 的 t3)rank 主机名也要能解析,
|
||||
# 否则 master 把副本分配到对端段后本端 open segment 失败(TRANSFER_FAIL)
|
||||
- "localhost-b-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-b-d0t1p0c1:127.0.0.1"
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: 1
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_TIMING: 1
|
||||
# L3 预取传输并行 lane 数(crc32(rid) 分 lane,每 lane 独立 gloo 组;
|
||||
# 实例内所有 rank 必须一致)。2026-10-10 A/B 见 l3-hicache-deep-dive 文档。
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: 4
|
||||
CUDA_VISIBLE_DEVICES: 0,1
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--chunked-prefill-size 32768
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30040
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 4.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--max-total-tokens 1600000
|
||||
@@ -0,0 +1,65 @@
|
||||
# 【prefill 分层极限测试 2026-10-10】单实例 cp2(tp2+ep2+prefill-cp interleave)+ dspark b5
|
||||
# + hicache L2(ratio 4.0=6.4M token) + L3(shm dummy) + best_effort
|
||||
# 目的:测 纯计算 / L1 / L2 / L3 四档 prefill 极限吞吐 + 混合负载异步聚合验证
|
||||
# L1=1.6M token(装得下 8×160k=1.28M 工作集);L2 6.4M;L3 shm 64GB
|
||||
# GPU 0,1(用前停 p3d1 栈);端口 30040;共用 mooncake-store(50052)/master(50051)
|
||||
x-common: &common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
shm_size: 32gb
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python:/sgl-workspace/sglang/python
|
||||
- /data/ymk/ds41/fake-cargo:/root/.cargo/bin/cargo:ro
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
t2:
|
||||
<<: *common
|
||||
container_name: ds41-tier-t2
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: 1
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store-dummy.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: 4
|
||||
SGLANG_HICACHE_TIMING: 1
|
||||
SGLANG_HICACHE_STORAGE_BATCH_SIZE: 512
|
||||
CUDA_VISIBLE_DEVICES: 0,1
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--chunked-prefill-size 32768
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30040
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 4.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--max-total-tokens 1600000
|
||||
@@ -0,0 +1,92 @@
|
||||
# 纯 tp2 单实例对照(2026-09-24 早):与 dockerserve-cp2x4.yml 逐参数对齐,唯一差异 = 去掉
|
||||
# --enable-prefill-cp --cp-strategy interleave(即无 CP),用于测「tp2 vs cp2」的 prefill 效率差。
|
||||
# GPU 4-5(a)/6-7(b),端口 30020/30030;2×tp2 聚合经 rr router 30003。
|
||||
# 镜像/补丁挂载/L3/engram/tok8 全部与 cp2x4 相同。
|
||||
x-common: &common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
# 与 cp2x4 保持同一份 scheduler.py(含未 commit 的 CP idle 补丁),保证唯一变量是 CP 开关
|
||||
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
|
||||
environment: &env
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_MS_AUTO_DISC: "0"
|
||||
MOONCAKE_MASTER: 127.0.0.1:50051
|
||||
MOONCAKE_TE_META_DATA_SERVER: P2PHANDSHAKE
|
||||
MOONCAKE_PROTOCOL: tcp
|
||||
MOONCAKE_GLOBAL_SEGMENT_SIZE: 300gb
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
tp2:
|
||||
<<: *common
|
||||
container_name: ds41-tp2
|
||||
environment:
|
||||
<<: *env
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
b:
|
||||
<<: *common
|
||||
container_name: ds41-tp2-b
|
||||
environment:
|
||||
<<: *env
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30030
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
@@ -0,0 +1,43 @@
|
||||
services:
|
||||
sglang:
|
||||
image: uhub.service.ucloud.cn/umirror/sglang:dev-dsv41
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
|
||||
ports:
|
||||
- "30000:30000"
|
||||
|
||||
volumes:
|
||||
- /data:/data
|
||||
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--enable-decoder-swa-bounded-replay
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
|
||||
@@ -0,0 +1,454 @@
|
||||
# ds41-3p1d: DeepSeek-V4.1-Flash 3P1D PD 分离(cs32k + OTLP trace),RBG 版
|
||||
# 拓扑: 3×prefill(tp2+ep2+cp2, cs32k, dspark, hicache L3) + 1×decode(tp2+ep2+dp2, cg256 mr320, dspark,
|
||||
# load-balance-method=total_requests 按请求数分流) + mooncake-master(hicache L3 store) + router(mini-lb PD)
|
||||
# 全部钉 b300-01(单机 8 卡,hostNetwork + hostPID——mooncake NVLink-intra 跨 Pod CUDA IPC 必需,
|
||||
# 等同 docker compose 的 pid: host;参考 AGENTS.md「部署四要素」)
|
||||
# 镜像已侧载 containerd: docker.io/ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831(IfNotPresent)
|
||||
# 参数来源: deploy/b300-ds41/dockerserve-3p1d-otlp-cs32k.yml(docker compose 版)
|
||||
# 对外: 仅 ClusterIP Service ds41-3p1d:50002(router 用普通 Pod 网络,不占宿主机端口,
|
||||
# 天然不对宿主机/外网暴露);new-api-gw 用集群 DNS http://ds41-3p1d:50002/v1 访问
|
||||
# trace: OTLP → 127.0.0.1:4317(同节点 jaeger-trace cw,badger 6h 轮窗)
|
||||
apiVersion: workloads.x-k8s.io/v1alpha2
|
||||
kind: RoleBasedGroup
|
||||
metadata:
|
||||
name: ds41-3p1d
|
||||
labels:
|
||||
app: ds41-3p1d
|
||||
sglang-model: ds-v41-flash
|
||||
spec:
|
||||
roleTemplates:
|
||||
- name: prefill-base
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
sglang-model: ds-v41-flash
|
||||
sglang-role: prefill
|
||||
spec:
|
||||
hostNetwork: true
|
||||
hostPID: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: b300-01
|
||||
containers:
|
||||
- name: sglang
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec sglang serve \
|
||||
--trust-remote-code \
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash \
|
||||
--tp 2 --ep-size 2 \
|
||||
--enable-prefill-cp --cp-strategy interleave \
|
||||
--chunked-prefill-size 32768 --max-prefill-tokens 32768 \
|
||||
--mem-fraction-static 0.75 \
|
||||
--attention-backend dsv4 \
|
||||
--moe-runner-backend flashinfer_mxfp4 \
|
||||
--cuda-graph-max-bs-decode 32 \
|
||||
--reasoning-parser auto --tool-call-parser auto \
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' \
|
||||
--max-running-requests 256 \
|
||||
--prefill-max-requests 8 \
|
||||
--tokenizer-worker-num 8 \
|
||||
--host 0.0.0.0 --port "$SGLANG_PORT" \
|
||||
--enable-cache-report --enable-metrics \
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5 \
|
||||
--enable-hierarchical-cache \
|
||||
--hicache-ratio 2 \
|
||||
--hicache-mem-layout page_first_direct \
|
||||
--hicache-io-backend direct \
|
||||
--hicache-write-policy write_through \
|
||||
--hicache-storage-backend mooncake \
|
||||
--hicache-storage-prefetch-policy wait_complete \
|
||||
--hicache-size 0 \
|
||||
--disaggregation-mode prefill \
|
||||
--disaggregation-transfer-backend mooncake \
|
||||
--disaggregation-bootstrap-port "$SGLANG_BOOTSTRAP_PORT" \
|
||||
--enable-trace \
|
||||
--trace-modules request,mooncake \
|
||||
--otlp-traces-endpoint 127.0.0.1:4317 \
|
||||
--otlp-service-name "$OTLP_NAME"
|
||||
env:
|
||||
- name: SGLANG_TORCH_PROFILER_DIR
|
||||
value: /data/ymk/prof/torch
|
||||
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||
value: "1"
|
||||
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||
value: static
|
||||
- name: MC_INTRANODE_NVLINK
|
||||
value: "true"
|
||||
- name: MC_INTRA_NVLINK
|
||||
value: "true"
|
||||
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||
value: "1"
|
||||
- name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH
|
||||
value: /data/ymk/ds41/mooncake-store.json
|
||||
- name: SGLANG_HICACHE_PREFETCH_IO_THREADS
|
||||
value: "4"
|
||||
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||
value: "16"
|
||||
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||
value: "32"
|
||||
- name: TZ
|
||||
value: Asia/Shanghai
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: metrics
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 90
|
||||
resources:
|
||||
requests:
|
||||
cpu: "32"
|
||||
memory: 220Gi
|
||||
nvidia.com/gpu: "2"
|
||||
limits:
|
||||
cpu: "96"
|
||||
memory: 950Gi
|
||||
nvidia.com/gpu: "2"
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
- name: sglang-cache
|
||||
mountPath: /root/.cache/sglang
|
||||
- name: dshm
|
||||
mountPath: /dev/shm
|
||||
volumes:
|
||||
- name: data
|
||||
hostPath:
|
||||
path: /data
|
||||
type: Directory
|
||||
- name: sglang-cache
|
||||
hostPath:
|
||||
path: /data/ymk/cache/sglang
|
||||
type: Directory
|
||||
- name: dshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 32Gi
|
||||
- name: decode-base
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
sglang-model: ds-v41-flash
|
||||
sglang-role: decode
|
||||
spec:
|
||||
hostNetwork: true
|
||||
hostPID: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: b300-01
|
||||
containers:
|
||||
- name: sglang
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec sglang serve \
|
||||
--trust-remote-code \
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash \
|
||||
--tp 2 --ep-size 2 --dp-size 2 \
|
||||
--enable-dp-attention --enable-dp-lm-head \
|
||||
--mem-fraction-static 0.83 \
|
||||
--attention-backend dsv4 \
|
||||
--moe-runner-backend flashinfer_mxfp4 \
|
||||
--cuda-graph-max-bs-decode 256 \
|
||||
--load-balance-method total_requests \
|
||||
--reasoning-parser auto --tool-call-parser auto \
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' \
|
||||
--max-running-requests 320 \
|
||||
--tokenizer-worker-num 8 \
|
||||
--host 0.0.0.0 --port 30030 \
|
||||
--enable-cache-report --enable-metrics \
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5 \
|
||||
--disaggregation-mode decode \
|
||||
--disaggregation-transfer-backend mooncake \
|
||||
--disaggregation-bootstrap-port 9001 \
|
||||
--enable-trace \
|
||||
--trace-modules request,mooncake \
|
||||
--otlp-traces-endpoint 127.0.0.1:4317 \
|
||||
--otlp-service-name ds41-3p1d-d1
|
||||
env:
|
||||
- name: SGLANG_TORCH_PROFILER_DIR
|
||||
value: /data/ymk/prof/torch
|
||||
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||
value: "1"
|
||||
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||
value: static
|
||||
- name: MC_INTRANODE_NVLINK
|
||||
value: "true"
|
||||
- name: MC_INTRA_NVLINK
|
||||
value: "true"
|
||||
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||
value: "1"
|
||||
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||
value: "16"
|
||||
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||
value: "32"
|
||||
- name: TZ
|
||||
value: Asia/Shanghai
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 30030
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: metrics
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 90
|
||||
resources:
|
||||
requests:
|
||||
cpu: "32"
|
||||
memory: 100Gi
|
||||
nvidia.com/gpu: "2"
|
||||
limits:
|
||||
cpu: "96"
|
||||
memory: 600Gi
|
||||
nvidia.com/gpu: "2"
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
- name: sglang-cache
|
||||
mountPath: /root/.cache/sglang
|
||||
- name: dshm
|
||||
mountPath: /dev/shm
|
||||
volumes:
|
||||
- name: data
|
||||
hostPath:
|
||||
path: /data
|
||||
type: Directory
|
||||
- name: sglang-cache
|
||||
hostPath:
|
||||
path: /data/ymk/cache/sglang
|
||||
type: Directory
|
||||
- name: dshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 32Gi
|
||||
roles:
|
||||
- name: mooncake-master
|
||||
replicas: 1
|
||||
standalonePattern:
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-mooncake-master
|
||||
spec:
|
||||
hostNetwork: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: b300-01
|
||||
containers:
|
||||
- name: mooncake-master
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec /opt/sglang/bin/mooncake_master \
|
||||
--port 50051 \
|
||||
--enable_http_metadata_server=true \
|
||||
--http_metadata_server_port=18080 \
|
||||
--eviction_high_watermark_ratio=0.90
|
||||
readinessProbe:
|
||||
tcpSocket:
|
||||
port: 50051
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
failureThreshold: 60
|
||||
resources:
|
||||
requests:
|
||||
cpu: "1"
|
||||
memory: 8Gi
|
||||
limits:
|
||||
cpu: "4"
|
||||
memory: 64Gi
|
||||
- name: p1
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- mooncake-master
|
||||
standalonePattern:
|
||||
templateRef:
|
||||
name: prefill-base
|
||||
patch:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-p1
|
||||
spec:
|
||||
containers:
|
||||
- name: sglang
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: "0,1"
|
||||
- name: SGLANG_PORT
|
||||
value: "30000"
|
||||
- name: SGLANG_BOOTSTRAP_PORT
|
||||
value: "8998"
|
||||
- name: OTLP_NAME
|
||||
value: ds41-3p1d-p1
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 30000
|
||||
- name: p2
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- mooncake-master
|
||||
standalonePattern:
|
||||
templateRef:
|
||||
name: prefill-base
|
||||
patch:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-p2
|
||||
spec:
|
||||
containers:
|
||||
- name: sglang
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: "2,3"
|
||||
- name: SGLANG_PORT
|
||||
value: "30010"
|
||||
- name: SGLANG_BOOTSTRAP_PORT
|
||||
value: "8999"
|
||||
- name: OTLP_NAME
|
||||
value: ds41-3p1d-p2
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 30010
|
||||
- name: p3
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- mooncake-master
|
||||
standalonePattern:
|
||||
templateRef:
|
||||
name: prefill-base
|
||||
patch:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-p3
|
||||
spec:
|
||||
containers:
|
||||
- name: sglang
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: "4,5"
|
||||
- name: SGLANG_PORT
|
||||
value: "30020"
|
||||
- name: SGLANG_BOOTSTRAP_PORT
|
||||
value: "9000"
|
||||
- name: OTLP_NAME
|
||||
value: ds41-3p1d-p3
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 30020
|
||||
- name: d1
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- mooncake-master
|
||||
standalonePattern:
|
||||
templateRef:
|
||||
name: decode-base
|
||||
patch:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-d1
|
||||
spec:
|
||||
containers:
|
||||
- name: sglang
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: "6,7"
|
||||
- name: router
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- p1
|
||||
- p2
|
||||
- p3
|
||||
- d1
|
||||
standalonePattern:
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-router
|
||||
spec:
|
||||
# 入口不走 hostNetwork:普通 Pod 网络 + ClusterIP,天然不对宿主机/外网暴露。
|
||||
# P/D 地址用节点内网 IP(hostNetwork 监听在 192.168.1.243 上,Calico 可达)。
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: b300-01
|
||||
containers:
|
||||
- name: router
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec sglang-router launch \
|
||||
--mini-lb --pd-disaggregation \
|
||||
--host 0.0.0.0 --port 50002 \
|
||||
--prefill http://192.168.1.243:30000 8998 \
|
||||
--prefill http://192.168.1.243:30010 8999 \
|
||||
--prefill http://192.168.1.243:30020 9000 \
|
||||
--decode http://192.168.1.243:30030
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 50002
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 30
|
||||
resources:
|
||||
requests:
|
||||
cpu: "1"
|
||||
memory: 4Gi
|
||||
limits:
|
||||
cpu: "8"
|
||||
memory: 16Gi
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: ds41-3p1d
|
||||
labels:
|
||||
app: ds41-3p1d
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
rbg.workloads.x-k8s.io/group-name: ds41-3p1d
|
||||
rbg.workloads.x-k8s.io/role-name: router
|
||||
ports:
|
||||
- name: http
|
||||
port: 50002
|
||||
targetPort: 50002
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PodMonitor
|
||||
metadata:
|
||||
name: ds41-3p1d
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
sglang-model: ds-v41-flash
|
||||
podMetricsEndpoints:
|
||||
- interval: 15s
|
||||
path: /metrics
|
||||
port: metrics
|
||||
@@ -0,0 +1,2 @@
|
||||
#!/bin/sh
|
||||
exit 1
|
||||
@@ -0,0 +1,451 @@
|
||||
# ds41-3p1d: DeepSeek-V4.1-Flash 3P1D PD 分离(cs32k + OTLP trace),RBG 版
|
||||
# 拓扑: 3×prefill(tp2+ep2+cp2, cs32k, dspark, hicache L3) + 1×decode(tp2+ep2+dp2, cg256 mr320, dspark,
|
||||
# load-balance-method=total_requests 按请求数分流) + mooncake-master(hicache L3 store) + router(mini-lb PD)
|
||||
# 全部钉 b300-01(单机 8 卡,hostNetwork + hostPID——mooncake NVLink-intra 跨 Pod CUDA IPC 必需,
|
||||
# 等同 docker compose 的 pid: host;参考 AGENTS.md「部署四要素」)
|
||||
# 镜像已侧载 containerd: docker.io/ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix(IfNotPresent)
|
||||
# 参数来源: deploy/b300-ds41/dockerserve-3p1d-otlp-cs32k.yml(docker compose 版)
|
||||
# 对外: 仅 ClusterIP Service ds41-3p1d:50002(router 用普通 Pod 网络,不占宿主机端口,
|
||||
# 天然不对宿主机/外网暴露);new-api-gw 用集群 DNS http://ds41-3p1d:50002/v1 访问
|
||||
# trace: OTLP → 127.0.0.1:4317(同节点 jaeger-trace cw,badger 6h 轮窗)
|
||||
apiVersion: workloads.x-k8s.io/v1alpha2
|
||||
kind: RoleBasedGroup
|
||||
metadata:
|
||||
name: ds41-3p1d
|
||||
labels:
|
||||
app: ds41-3p1d
|
||||
sglang-model: ds-v41-flash
|
||||
spec:
|
||||
roleTemplates:
|
||||
- name: prefill-base
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
sglang-model: ds-v41-flash
|
||||
sglang-role: prefill
|
||||
spec:
|
||||
hostNetwork: true
|
||||
hostPID: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: b300-01
|
||||
containers:
|
||||
- name: sglang
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec sglang serve \
|
||||
--trust-remote-code \
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash \
|
||||
--tp 2 --ep-size 2 \
|
||||
--enable-prefill-cp --cp-strategy interleave \
|
||||
--chunked-prefill-size 32768 --max-prefill-tokens 32768 \
|
||||
--mem-fraction-static 0.75 \
|
||||
--attention-backend dsv4 \
|
||||
--moe-runner-backend flashinfer_mxfp4 \
|
||||
--cuda-graph-max-bs-decode 32 \
|
||||
--reasoning-parser auto --tool-call-parser auto \
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' \
|
||||
--max-running-requests 256 \
|
||||
--tokenizer-worker-num 8 \
|
||||
--host 0.0.0.0 --port "$SGLANG_PORT" \
|
||||
--enable-cache-report --enable-metrics \
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5 \
|
||||
--enable-hierarchical-cache \
|
||||
--hicache-ratio 2 \
|
||||
--hicache-mem-layout page_first_direct \
|
||||
--hicache-io-backend direct \
|
||||
--hicache-write-policy write_through \
|
||||
--hicache-storage-backend mooncake \
|
||||
--hicache-storage-prefetch-policy wait_complete \
|
||||
--hicache-size 0 \
|
||||
--disaggregation-mode prefill \
|
||||
--disaggregation-transfer-backend mooncake \
|
||||
--disaggregation-bootstrap-port "$SGLANG_BOOTSTRAP_PORT" \
|
||||
--enable-trace \
|
||||
--trace-modules request,mooncake \
|
||||
--otlp-traces-endpoint 127.0.0.1:4317 \
|
||||
--otlp-service-name "$OTLP_NAME"
|
||||
env:
|
||||
- name: SGLANG_TORCH_PROFILER_DIR
|
||||
value: /data/ymk/prof/torch
|
||||
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||
value: "1"
|
||||
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||
value: static
|
||||
- name: MC_INTRANODE_NVLINK
|
||||
value: "true"
|
||||
- name: MC_INTRA_NVLINK
|
||||
value: "true"
|
||||
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||
value: "1"
|
||||
- name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH
|
||||
value: /data/ymk/ds41/mooncake-store.json
|
||||
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||
value: "16"
|
||||
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||
value: "32"
|
||||
- name: TZ
|
||||
value: Asia/Shanghai
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: metrics
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 90
|
||||
resources:
|
||||
requests:
|
||||
cpu: "32"
|
||||
memory: 220Gi
|
||||
nvidia.com/gpu: "2"
|
||||
limits:
|
||||
cpu: "96"
|
||||
memory: 950Gi
|
||||
nvidia.com/gpu: "2"
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
- name: sglang-cache
|
||||
mountPath: /root/.cache/sglang
|
||||
- name: dshm
|
||||
mountPath: /dev/shm
|
||||
volumes:
|
||||
- name: data
|
||||
hostPath:
|
||||
path: /data
|
||||
type: Directory
|
||||
- name: sglang-cache
|
||||
hostPath:
|
||||
path: /data/ymk/cache/sglang
|
||||
type: Directory
|
||||
- name: dshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 32Gi
|
||||
- name: decode-base
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
sglang-model: ds-v41-flash
|
||||
sglang-role: decode
|
||||
spec:
|
||||
hostNetwork: true
|
||||
hostPID: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: b300-01
|
||||
containers:
|
||||
- name: sglang
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec sglang serve \
|
||||
--trust-remote-code \
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash \
|
||||
--tp 2 --ep-size 2 --dp-size 2 \
|
||||
--enable-dp-attention --enable-dp-lm-head \
|
||||
--mem-fraction-static 0.83 \
|
||||
--attention-backend dsv4 \
|
||||
--moe-runner-backend flashinfer_mxfp4 \
|
||||
--cuda-graph-max-bs-decode 256 \
|
||||
--load-balance-method total_requests \
|
||||
--reasoning-parser auto --tool-call-parser auto \
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' \
|
||||
--max-running-requests 320 \
|
||||
--tokenizer-worker-num 8 \
|
||||
--host 0.0.0.0 --port 30030 \
|
||||
--enable-cache-report --enable-metrics \
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5 \
|
||||
--disaggregation-mode decode \
|
||||
--disaggregation-transfer-backend mooncake \
|
||||
--disaggregation-bootstrap-port 9001 \
|
||||
--enable-trace \
|
||||
--trace-modules request,mooncake \
|
||||
--otlp-traces-endpoint 127.0.0.1:4317 \
|
||||
--otlp-service-name ds41-3p1d-d1
|
||||
env:
|
||||
- name: SGLANG_TORCH_PROFILER_DIR
|
||||
value: /data/ymk/prof/torch
|
||||
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||
value: "1"
|
||||
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||
value: static
|
||||
- name: MC_INTRANODE_NVLINK
|
||||
value: "true"
|
||||
- name: MC_INTRA_NVLINK
|
||||
value: "true"
|
||||
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||
value: "1"
|
||||
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||
value: "16"
|
||||
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||
value: "32"
|
||||
- name: TZ
|
||||
value: Asia/Shanghai
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 30030
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: metrics
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 90
|
||||
resources:
|
||||
requests:
|
||||
cpu: "32"
|
||||
memory: 100Gi
|
||||
nvidia.com/gpu: "2"
|
||||
limits:
|
||||
cpu: "96"
|
||||
memory: 600Gi
|
||||
nvidia.com/gpu: "2"
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
- name: sglang-cache
|
||||
mountPath: /root/.cache/sglang
|
||||
- name: dshm
|
||||
mountPath: /dev/shm
|
||||
volumes:
|
||||
- name: data
|
||||
hostPath:
|
||||
path: /data
|
||||
type: Directory
|
||||
- name: sglang-cache
|
||||
hostPath:
|
||||
path: /data/ymk/cache/sglang
|
||||
type: Directory
|
||||
- name: dshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 32Gi
|
||||
roles:
|
||||
- name: mooncake-master
|
||||
replicas: 1
|
||||
standalonePattern:
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-mooncake-master
|
||||
spec:
|
||||
hostNetwork: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: b300-01
|
||||
containers:
|
||||
- name: mooncake-master
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec /opt/sglang/bin/mooncake_master \
|
||||
--port 50051 \
|
||||
--enable_http_metadata_server=true \
|
||||
--http_metadata_server_port=18080 \
|
||||
--eviction_high_watermark_ratio=0.90
|
||||
readinessProbe:
|
||||
tcpSocket:
|
||||
port: 50051
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
failureThreshold: 60
|
||||
resources:
|
||||
requests:
|
||||
cpu: "1"
|
||||
memory: 8Gi
|
||||
limits:
|
||||
cpu: "4"
|
||||
memory: 64Gi
|
||||
- name: p1
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- mooncake-master
|
||||
standalonePattern:
|
||||
templateRef:
|
||||
name: prefill-base
|
||||
patch:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-p1
|
||||
spec:
|
||||
containers:
|
||||
- name: sglang
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: "0,1"
|
||||
- name: SGLANG_PORT
|
||||
value: "30000"
|
||||
- name: SGLANG_BOOTSTRAP_PORT
|
||||
value: "8998"
|
||||
- name: OTLP_NAME
|
||||
value: ds41-3p1d-p1
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 30000
|
||||
- name: p2
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- mooncake-master
|
||||
standalonePattern:
|
||||
templateRef:
|
||||
name: prefill-base
|
||||
patch:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-p2
|
||||
spec:
|
||||
containers:
|
||||
- name: sglang
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: "2,3"
|
||||
- name: SGLANG_PORT
|
||||
value: "30010"
|
||||
- name: SGLANG_BOOTSTRAP_PORT
|
||||
value: "8999"
|
||||
- name: OTLP_NAME
|
||||
value: ds41-3p1d-p2
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 30010
|
||||
- name: p3
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- mooncake-master
|
||||
standalonePattern:
|
||||
templateRef:
|
||||
name: prefill-base
|
||||
patch:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-p3
|
||||
spec:
|
||||
containers:
|
||||
- name: sglang
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: "4,5"
|
||||
- name: SGLANG_PORT
|
||||
value: "30020"
|
||||
- name: SGLANG_BOOTSTRAP_PORT
|
||||
value: "9000"
|
||||
- name: OTLP_NAME
|
||||
value: ds41-3p1d-p3
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 30020
|
||||
- name: d1
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- mooncake-master
|
||||
standalonePattern:
|
||||
templateRef:
|
||||
name: decode-base
|
||||
patch:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-d1
|
||||
spec:
|
||||
containers:
|
||||
- name: sglang
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: "6,7"
|
||||
- name: router
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- p1
|
||||
- p2
|
||||
- p3
|
||||
- d1
|
||||
standalonePattern:
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-router
|
||||
spec:
|
||||
# 入口不走 hostNetwork:普通 Pod 网络 + ClusterIP,天然不对宿主机/外网暴露。
|
||||
# P/D 地址用节点内网 IP(hostNetwork 监听在 192.168.1.243 上,Calico 可达)。
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: b300-01
|
||||
containers:
|
||||
- name: router
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec sglang-router launch \
|
||||
--mini-lb --pd-disaggregation \
|
||||
--host 0.0.0.0 --port 50002 \
|
||||
--prefill http://192.168.1.243:30000 8998 \
|
||||
--prefill http://192.168.1.243:30010 8999 \
|
||||
--prefill http://192.168.1.243:30020 9000 \
|
||||
--decode http://192.168.1.243:30030
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 50002
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 30
|
||||
resources:
|
||||
requests:
|
||||
cpu: "1"
|
||||
memory: 4Gi
|
||||
limits:
|
||||
cpu: "8"
|
||||
memory: 16Gi
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: ds41-3p1d
|
||||
labels:
|
||||
app: ds41-3p1d
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
rbg.workloads.x-k8s.io/group-name: ds41-3p1d
|
||||
rbg.workloads.x-k8s.io/role-name: router
|
||||
ports:
|
||||
- name: http
|
||||
port: 50002
|
||||
targetPort: 50002
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PodMonitor
|
||||
metadata:
|
||||
name: ds41-3p1d
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
sglang-model: ds-v41-flash
|
||||
podMetricsEndpoints:
|
||||
- interval: 15s
|
||||
path: /metrics
|
||||
port: metrics
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"standalone_storage": true,
|
||||
"client_server_address": "127.0.0.1:50052"
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"master_server_address": "127.0.0.1:50051",
|
||||
"metadata_server": "P2PHANDSHAKE",
|
||||
"global_segment_size": "300gb",
|
||||
"protocol": "tcp"
|
||||
}
|
||||
@@ -256,12 +256,15 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
)
|
||||
|
||||
cfg = resolving_view(server_args)
|
||||
if model_config_of(server_args).hf_config.model_type != "deepseek_v41":
|
||||
hf_config = model_config_of(server_args).hf_config
|
||||
if hf_config.model_type != "deepseek_v41":
|
||||
if cfg.enable_encoder_swa_bounded_replay:
|
||||
raise ValueError(
|
||||
"--enable-encoder-swa-bounded-replay requires DeepSeek-V4.1"
|
||||
)
|
||||
return
|
||||
if hf_config.vision_n_layers > 0 and cfg.enable_prefill_cp:
|
||||
_validate_deepseek_v41_vision_prefill_cp(server_args)
|
||||
if (
|
||||
cfg.dsv4_attn_backend == "trtllm"
|
||||
and cfg.cuda_graph_config.prefill.backend
|
||||
@@ -291,7 +294,8 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
cfg.cuda_graph_config.prefill.backend != Backend.DISABLED
|
||||
and get_platform().is_hip,
|
||||
),
|
||||
("context parallelism", cfg.attn_cp_size > 1),
|
||||
# Prefill CP declares attn_cp_size and DP attention only later.
|
||||
("context parallelism", cfg.attn_cp_size > 1 or cfg.enable_prefill_cp),
|
||||
# ROCm replays hits in a forward of their own, which only the hitting
|
||||
# attention-DP ranks would enter; CUDA folds them into the extend.
|
||||
("DP attention on ROCm", get_platform().is_hip and attn_dp_enabled_of(cfg)),
|
||||
@@ -377,6 +381,10 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
and prefill_graph.max_seq_len is None
|
||||
and not cp_breakable_prefill
|
||||
):
|
||||
# The non-CP captured low-ratio indexer scores a static context width.
|
||||
# CP BCG runs these sources eagerly with live prefix metadata, so this
|
||||
# default would only force long-prefix CP batches back to eager.
|
||||
# Explicit max_seq_len values still constrain both paths.
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"validate_deepseek_v41_features",
|
||||
@@ -437,3 +445,42 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
enable_decoder_swa_bounded_replay=enabled,
|
||||
)
|
||||
logger.info("DeepSeek-V4.1 decoder SWA bounded replay resolved to %s", enabled)
|
||||
|
||||
|
||||
|
||||
|
||||
def _validate_deepseek_v41_vision_prefill_cp(server_args: ServerArgs) -> None:
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
|
||||
|
||||
cfg = resolving_view(server_args)
|
||||
if cfg.cp_strategy != "interleave":
|
||||
raise ValueError(
|
||||
"DeepSeek-V4.1 vision with prefill CP requires --cp-strategy "
|
||||
f"interleave; got {cfg.cp_strategy!r}."
|
||||
)
|
||||
if cfg.cuda_graph_config.prefill.backend != Backend.DISABLED:
|
||||
# The CP runner merges image features eagerly; no capture path replays it.
|
||||
locked = getattr(server_args, "_cuda_graph_config_locked", set())
|
||||
if (Phase.PREFILL, "backend") in locked:
|
||||
raise ValueError(
|
||||
"DeepSeek-V4.1 vision with prefill CP runs eager prefill; remove "
|
||||
"the explicit prefill CUDA graph backend."
|
||||
)
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"validate_deepseek_v41_features",
|
||||
cuda_graph_config=with_phase(
|
||||
cfg.cuda_graph_config, Phase.PREFILL, backend=Backend.DISABLED
|
||||
),
|
||||
)
|
||||
logger.warning(
|
||||
"Disabling the prefill CUDA graph for DeepSeek-V4.1 vision with prefill CP."
|
||||
)
|
||||
if (
|
||||
str(cfg.speculative_algorithm).upper() == "DSPARK"
|
||||
and cfg.enable_decoder_swa_bounded_replay
|
||||
):
|
||||
raise ValueError(
|
||||
"DeepSeek-V4.1 vision with prefill CP does not support DSpark together "
|
||||
"with --enable-decoder-swa-bounded-replay yet."
|
||||
)
|
||||
|
||||
@@ -1226,20 +1226,33 @@ class CommonKVManager(BaseKVManager):
|
||||
"enable DSpark with the same block size and target/draft KV "
|
||||
"layout. Upgrade both servers together."
|
||||
)
|
||||
|
||||
decode_tp_size_aligns_prefill_world_size = (
|
||||
info.attn_cp_size > 1
|
||||
and (self.is_mla_backend or self.is_hybrid_mla_backend)
|
||||
and self.attn_cp_size == 1
|
||||
and info.attn_tp_size * info.attn_cp_size == self.attn_tp_size
|
||||
)
|
||||
if (
|
||||
info.attn_tp_size != self.attn_tp_size
|
||||
and not decode_tp_size_aligns_prefill_world_size
|
||||
# Combined branch (40323-series + 40177): prefill CP can also pair
|
||||
# with a DP-attention decode server. MLA KV is replicated across
|
||||
# prefill CP ranks, so per-rank layouts match when attn_tp matches.
|
||||
dp_decode_with_prefill_cp = (
|
||||
info.attn_cp_size > 1
|
||||
and self.attn_cp_size == 1
|
||||
and (self.is_mla_backend or self.is_hybrid_mla_backend)
|
||||
and info.attn_tp_size == self.attn_tp_size
|
||||
)
|
||||
non_cp_mla_layout = info.attn_cp_size == self.attn_cp_size == 1 and (
|
||||
self.is_mla_backend or self.is_hybrid_mla_backend
|
||||
)
|
||||
if info.attn_tp_size != self.attn_tp_size and not (
|
||||
decode_tp_size_aligns_prefill_world_size
|
||||
or dp_decode_with_prefill_cp
|
||||
or non_cp_mla_layout
|
||||
):
|
||||
raise RuntimeError(
|
||||
"DeepSeek-V4.1 DSpark PD requires the same TP size on both "
|
||||
"servers (including prefill CP ranks for MLA)"
|
||||
"DeepSeek-V4.1 DSpark PD requires matching attention TP "
|
||||
"unless both servers use CP=1 with an MLA KV layout, "
|
||||
"or prefill runs CP with an MLA KV layout"
|
||||
)
|
||||
|
||||
# Every send path rejects head-sharded MXFP8 block scales across
|
||||
|
||||
@@ -87,6 +87,115 @@ FAILED_SESSION_RECOVERIES = Counter(
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Intra-node NVLink transport helpers.
|
||||
#
|
||||
# Mooncake's IntraNodeNvlinkTransport can only register and reach *device*
|
||||
# memory (it IPC-opens the remote cudaMalloc segments). Host-resident regions
|
||||
# (aux buffers, some state components) cannot be registered: one host region
|
||||
# makes the whole registerLocalMemoryBatch fail, and the engine then rolls
|
||||
# back *every* region, leaving the segment descriptor empty and all KV
|
||||
# transfers failing with "Requested address ... not found". When the
|
||||
# intra-node NVLink transport is active we therefore
|
||||
# 1. register only device-memory regions, and
|
||||
# 2. route blocks whose source is host memory over the ordered zmq channel
|
||||
# (same ordering guarantee the aux TCP path relies on) instead of the
|
||||
# transfer engine.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
import ctypes as _ctypes
|
||||
|
||||
_CUDA_MEMORY_TYPE_DEVICE = 2
|
||||
|
||||
try:
|
||||
from cuda.bindings import runtime as _cudart
|
||||
except ImportError: # pragma: no cover - cuda-python is always present in images
|
||||
_cudart = None
|
||||
|
||||
|
||||
def _is_device_pointer(ptr: int) -> bool:
|
||||
"""Probe a *local* pointer with cudaPointerGetAttributes.
|
||||
|
||||
Only valid for pointers owned by this process (never probe remote
|
||||
segment addresses). Returns False on any error so the caller falls back
|
||||
to the safe host path.
|
||||
"""
|
||||
if _cudart is None:
|
||||
# Cannot tell; assume device so behavior stays unchanged.
|
||||
return True
|
||||
err, attr = _cudart.cudaPointerGetAttributes(int(ptr))
|
||||
if int(err) != 0:
|
||||
# Clear the error so subsequent CUDA calls are not poisoned.
|
||||
_cudart.cudaGetLastError()
|
||||
return False
|
||||
return int(attr.type) == _CUDA_MEMORY_TYPE_DEVICE
|
||||
|
||||
|
||||
def _read_bytes_from_address(addr: int, length: int) -> Optional[bytes]:
|
||||
if length <= 0:
|
||||
return b""
|
||||
if _is_device_pointer(addr):
|
||||
buf = bytearray(length)
|
||||
# cudaMemcpyDeviceToHost = 2; synchronous default-stream copy.
|
||||
err, = _cudart.cudaMemcpy(
|
||||
_ctypes.addressof((_ctypes.c_char * length).from_buffer(buf)),
|
||||
int(addr),
|
||||
length,
|
||||
2,
|
||||
)
|
||||
if int(err) != 0:
|
||||
logger.error(
|
||||
f"cudaMemcpy D2H failed (err={err}) for addr {hex(addr)} len {length}"
|
||||
)
|
||||
return None
|
||||
return bytes(buf)
|
||||
return _ctypes.string_at(int(addr), length)
|
||||
|
||||
|
||||
def _write_bytes_to_address(addr: int, data: bytes) -> bool:
|
||||
if not data:
|
||||
return True
|
||||
if _is_device_pointer(addr):
|
||||
buf = _ctypes.create_string_buffer(data, len(data))
|
||||
# cudaMemcpyHostToDevice = 1; synchronous default-stream copy.
|
||||
err, = _cudart.cudaMemcpy(
|
||||
int(addr), _ctypes.addressof(buf), len(data), 1
|
||||
)
|
||||
if int(err) != 0:
|
||||
logger.error(
|
||||
f"cudaMemcpy H2D failed (err={err}) for addr {hex(addr)} "
|
||||
f"len {len(data)}"
|
||||
)
|
||||
return False
|
||||
return True
|
||||
_ctypes.memmove(int(addr), data, len(data))
|
||||
return True
|
||||
|
||||
|
||||
_NVLINK_INTRA_ACTIVE = None
|
||||
|
||||
|
||||
def _nvlink_intra_transport_active() -> bool:
|
||||
"""Whether mooncake installed the intra-node NVLink transport.
|
||||
|
||||
Mirrors the env probing in mooncake's transfer_engine_impl.cpp: the
|
||||
transport is installed iff MC_INTRANODE_NVLINK is set (any value), or an
|
||||
equivalent protocol selection was made.
|
||||
"""
|
||||
global _NVLINK_INTRA_ACTIVE
|
||||
if _NVLINK_INTRA_ACTIVE is None:
|
||||
active = bool(
|
||||
os.environ.get("MC_INTRANODE_NVLINK")
|
||||
or os.environ.get("MC_INTRA_NVLINK")
|
||||
)
|
||||
if not active:
|
||||
proto = (os.environ.get("MOONCAKE_PROTOCOL") or "").strip().lower()
|
||||
active = proto in ("nvlink_intra", "nvlink-intra", "intra_nvlink")
|
||||
_NVLINK_INTRA_ACTIVE = active
|
||||
return _NVLINK_INTRA_ACTIVE
|
||||
|
||||
|
||||
# decode
|
||||
@dataclasses.dataclass
|
||||
class TransferInfo:
|
||||
@@ -240,6 +349,7 @@ def _uses_flat_entry_layout(
|
||||
|
||||
class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
AUX_DATA_HEADER = b"AUX_DATA"
|
||||
STATE_DATA_HEADER = b"STATE_DATA"
|
||||
# Implements teardown() below, so runtime PD role switching is supported.
|
||||
supports_role_switch = True
|
||||
# Bootstrap thread defers the ABORT ack until the transfer worker drains.
|
||||
@@ -261,6 +371,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
super().__init__(args, disaggregation_mode, server_args, is_mla_backend)
|
||||
self.init_engine()
|
||||
self.register_buffer_to_engine()
|
||||
# session_id -> (endpoint, dst_port, room), used to route host-memory
|
||||
# transfer blocks over zmq when the intra-node NVLink transport is
|
||||
# active (it cannot reach host memory). Populated on bootstrap.
|
||||
self._session_endpoint_map = {}
|
||||
self.enable_staging = envs.SGLANG_DISAGG_STAGING_BUFFER.get()
|
||||
self.max_transfer_batch_indices = (
|
||||
envs.SGLANG_MOONCAKE_MAX_TRANSFER_BATCH_INDICES.get()
|
||||
@@ -359,6 +473,13 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
Deduped because the unified memory pool reports one raw buffer as both
|
||||
its KV and its mamba state component, and double registration fails in
|
||||
the engine.
|
||||
|
||||
When the intra-node NVLink transport is active, host-memory regions
|
||||
(aux buffers, some state components) are skipped: the transport only
|
||||
accepts device memory, and a single host region fails the whole batch
|
||||
and triggers a full engine-side rollback that would unregister the KV
|
||||
pools too. Host-resident payloads are instead exchanged over the
|
||||
ordered zmq channel (see _transfer_data / send_aux).
|
||||
"""
|
||||
regions: List[Tuple[int, int]] = []
|
||||
seen: Set[Tuple[int, int]] = set()
|
||||
@@ -383,6 +504,24 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
self.kv_args.state_data_ptrs, self.kv_args.state_data_lens
|
||||
):
|
||||
add(ptrs, lens)
|
||||
|
||||
if _nvlink_intra_transport_active():
|
||||
device_regions = []
|
||||
skipped = []
|
||||
for ptr, length in regions:
|
||||
if _is_device_pointer(ptr):
|
||||
device_regions.append((ptr, length))
|
||||
else:
|
||||
skipped.append((ptr, length))
|
||||
if skipped:
|
||||
logger.info(
|
||||
"Intra-node NVLink transport: skipping %d host-memory "
|
||||
"regions from engine registration (they will be exchanged "
|
||||
"over the zmq channel instead): %s",
|
||||
len(skipped),
|
||||
[(hex(p), l) for p, l in skipped[:8]],
|
||||
)
|
||||
regions = device_regions
|
||||
return regions
|
||||
|
||||
def register_buffer_to_engine(self):
|
||||
@@ -793,10 +932,63 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
if not transfer_blocks:
|
||||
return 0
|
||||
|
||||
src_addrs, dst_addrs, lengths = zip(*transfer_blocks)
|
||||
return self.engine.batch_transfer_sync(
|
||||
mooncake_session_id, list(src_addrs), list(dst_addrs), list(lengths)
|
||||
)
|
||||
if not _nvlink_intra_transport_active():
|
||||
src_addrs, dst_addrs, lengths = zip(*transfer_blocks)
|
||||
return self.engine.batch_transfer_sync(
|
||||
mooncake_session_id, list(src_addrs), list(dst_addrs), list(lengths)
|
||||
)
|
||||
|
||||
# Intra-node NVLink transport can only move device memory. Partition
|
||||
# blocks by the *local source* pointer (probing a local pointer is
|
||||
# safe; the remote dst is never probed): device-sourced blocks go
|
||||
# through the engine as usual, host-sourced blocks are shipped over
|
||||
# the ordered zmq channel and written into the peer's buffer by the
|
||||
# receiver (see _handle_state_data). This mirrors the aux TCP path.
|
||||
device_blocks = []
|
||||
host_blocks = []
|
||||
for src, dst, length in transfer_blocks:
|
||||
if _is_device_pointer(src):
|
||||
device_blocks.append((src, dst, length))
|
||||
else:
|
||||
host_blocks.append((src, dst, length))
|
||||
|
||||
rc = 0
|
||||
if device_blocks:
|
||||
src_addrs, dst_addrs, lengths = zip(*device_blocks)
|
||||
rc = self.engine.batch_transfer_sync(
|
||||
mooncake_session_id, list(src_addrs), list(dst_addrs), list(lengths)
|
||||
)
|
||||
if rc == 0 and host_blocks:
|
||||
rc = self._send_host_blocks_tcp(mooncake_session_id, host_blocks)
|
||||
return rc
|
||||
|
||||
def _send_host_blocks_tcp(self, mooncake_session_id, host_blocks):
|
||||
target = self._session_endpoint_map.get(mooncake_session_id)
|
||||
if target is None:
|
||||
logger.error(
|
||||
f"No zmq endpoint known for mooncake session "
|
||||
f"{mooncake_session_id}; cannot deliver {len(host_blocks)} "
|
||||
"host-memory transfer blocks"
|
||||
)
|
||||
return -1
|
||||
endpoint, dst_port, room = target
|
||||
na = NetworkAddress(endpoint, dst_port)
|
||||
for src, dst, length in host_blocks:
|
||||
data = _read_bytes_from_address(src, length)
|
||||
if data is None:
|
||||
return -1
|
||||
self._send_multipart_locked(
|
||||
na.to_tcp(),
|
||||
[
|
||||
MooncakeKVManager.STATE_DATA_HEADER,
|
||||
str(room).encode("ascii"),
|
||||
str(int(dst)).encode("ascii"),
|
||||
struct.pack(">I", len(data)),
|
||||
data,
|
||||
],
|
||||
is_ipv6=na.is_ipv6,
|
||||
)
|
||||
return 0
|
||||
|
||||
def _state_stride_mismatch(
|
||||
self,
|
||||
@@ -1744,8 +1936,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
):
|
||||
# TODO(shangming): Fix me when nvlink_transport of Mooncake is bug-free
|
||||
if (
|
||||
self.enable_custom_mem_pool and self.custom_mem_pool_type == "NVLINK"
|
||||
) or envs.SGLANG_MOONCAKE_SEND_AUX_TCP.get():
|
||||
(self.enable_custom_mem_pool and self.custom_mem_pool_type == "NVLINK")
|
||||
or envs.SGLANG_MOONCAKE_SEND_AUX_TCP.get()
|
||||
or _nvlink_intra_transport_active()
|
||||
):
|
||||
return self.send_aux_tcp(req, prefill_aux_index, dst_aux_ptrs)
|
||||
|
||||
transfer_blocks = []
|
||||
@@ -1828,6 +2022,71 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
f"Received AUX_DATA for bootstrap_room {room} with length:{len(data)}"
|
||||
)
|
||||
|
||||
def _host_transfer_regions(self):
|
||||
"""Address ranges this process published as transfer targets.
|
||||
|
||||
Used to validate STATE_DATA writes. Built lazily because kv_args is
|
||||
fully populated only after registration.
|
||||
"""
|
||||
regions = getattr(self, "_host_transfer_regions_cache", None)
|
||||
if regions is None:
|
||||
regions = []
|
||||
for ptr, length in zip(
|
||||
self.kv_args.kv_data_ptrs or [], self.kv_args.kv_data_lens or []
|
||||
):
|
||||
regions.append((int(ptr), int(ptr) + int(length)))
|
||||
for ptr, length in zip(
|
||||
self.kv_args.aux_data_ptrs or [], self.kv_args.aux_data_lens or []
|
||||
):
|
||||
regions.append((int(ptr), int(ptr) + int(length)))
|
||||
for ptrs, lens in zip(
|
||||
self.kv_args.state_data_ptrs or [], self.kv_args.state_data_lens or []
|
||||
):
|
||||
for ptr, length in zip(ptrs or [], lens or []):
|
||||
regions.append((int(ptr), int(ptr) + int(length)))
|
||||
self._host_transfer_regions_cache = regions
|
||||
return regions
|
||||
|
||||
def _handle_state_data(self, msg: List[bytes]):
|
||||
"""Handle STATE_DATA messages received by the decode thread.
|
||||
|
||||
Carries one host-memory transfer block that could not go through the
|
||||
intra-node NVLink transport. Written directly into the local buffer at
|
||||
the destination address; ordering against the final status message is
|
||||
guaranteed by the shared per-endpoint zmq socket.
|
||||
"""
|
||||
room = int(msg[1].decode("ascii"))
|
||||
dst_addr = int(msg[2].decode("ascii"))
|
||||
data_length = struct.unpack(">I", msg[3])[0]
|
||||
data = msg[4]
|
||||
|
||||
if len(data) != data_length:
|
||||
logger.error(f"STATE_DATA length mismatch for bootstrap_room {room}")
|
||||
return
|
||||
|
||||
in_region = any(
|
||||
start <= dst_addr and dst_addr + len(data) <= end
|
||||
for start, end in self._host_transfer_regions()
|
||||
)
|
||||
if not in_region:
|
||||
logger.error(
|
||||
f"STATE_DATA for bootstrap_room {room} targets unknown region "
|
||||
f"{hex(dst_addr)}..{hex(dst_addr + len(data))}; dropping"
|
||||
)
|
||||
return
|
||||
|
||||
if not _write_bytes_to_address(dst_addr, data):
|
||||
logger.error(
|
||||
f"STATE_DATA write failed for bootstrap_room {room} at "
|
||||
f"{hex(dst_addr)} len {len(data)}"
|
||||
)
|
||||
return
|
||||
|
||||
logger.debug(
|
||||
f"Received STATE_DATA for bootstrap_room {room} at {hex(dst_addr)} "
|
||||
f"with length:{len(data)}"
|
||||
)
|
||||
|
||||
def _get_dsa_cache_transfer_skip_flags(
|
||||
self, info: Optional[KVArgsRegisterInfo]
|
||||
) -> Tuple[bool, bool]:
|
||||
@@ -2789,6 +3048,8 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
)
|
||||
):
|
||||
if kv_chunk.room in self.transfer_infos:
|
||||
for sid in self.transfer_infos[kv_chunk.room]:
|
||||
self._session_endpoint_map.pop(sid, None)
|
||||
self.transfer_infos.pop(kv_chunk.room)
|
||||
self.req_to_decode_prefix_len.pop(kv_chunk.room, None)
|
||||
if self.enable_staging:
|
||||
@@ -2948,6 +3209,11 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
self.transfer_infos[room][mooncake_session_id] = (
|
||||
TransferInfo.from_zmq(waiting_req_bytes)
|
||||
)
|
||||
self._session_endpoint_map[mooncake_session_id] = (
|
||||
self.transfer_infos[room][mooncake_session_id].endpoint,
|
||||
self.transfer_infos[room][mooncake_session_id].dst_port,
|
||||
room,
|
||||
)
|
||||
# NOTE: after bootstrapping we can mark the req as waiting for input
|
||||
if len(self.transfer_infos[room]) == required_dst_info_num:
|
||||
self.resolve_kv_replica_factor(self.transfer_infos[room])
|
||||
@@ -2976,6 +3242,9 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
if msg[0] == MooncakeKVManager.AUX_DATA_HEADER:
|
||||
self._handle_aux_data(msg)
|
||||
continue
|
||||
if msg[0] == MooncakeKVManager.STATE_DATA_HEADER:
|
||||
self._handle_state_data(msg)
|
||||
continue
|
||||
|
||||
# Staging: prefill notifies a chunk written to staging buffer
|
||||
if msg[0] == b"CHUNK_READY":
|
||||
@@ -3052,12 +3321,22 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
# add further chunks into the transfer queue.
|
||||
return
|
||||
|
||||
# NOTE(shangming): sharding according to the dst_infos to make sure
|
||||
# requests with the same dst_sessions will be added into the same
|
||||
# queue, which enables early abort with failed sessions.
|
||||
# Shard by dst session ports AND bootstrap_room. The port term alone
|
||||
# degenerates to a constant when all requests target a single decode
|
||||
# instance (its session ports are fixed), collapsing every transfer
|
||||
# onto 1/16 of the lanes and throttling P->D intake (measured at
|
||||
# client concurrency 300: #transfer-req piled at 100+ while decode
|
||||
# itself stayed healthy). Mixing in bootstrap_room keeps all chunks
|
||||
# of one request on the same lane (per-request FIFO order preserved)
|
||||
# while spreading distinct rooms across all lanes. Abort/cleanup is
|
||||
# unaffected: chunks are dropped per-room at dequeue via
|
||||
# request_status (see the worker loop) and per-request via
|
||||
# failed_sessions before send — no lane-level purge exists, and the
|
||||
# deferred abort ack is room-scoped (_staging_outstanding). Same
|
||||
# trade-off as mori/conn.py which shards by bootstrap_room alone.
|
||||
dst_infos = self.transfer_infos[bootstrap_room].keys()
|
||||
session_port_sum = sum(int(session.rsplit(":", 1)[1]) for session in dst_infos)
|
||||
shard_idx = session_port_sum % len(self.transfer_queues)
|
||||
shard_idx = (session_port_sum + bootstrap_room) % len(self.transfer_queues)
|
||||
|
||||
if trace_ctx is None:
|
||||
trace_ctx = TraceNullContext()
|
||||
|
||||
@@ -21,6 +21,8 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import zlib
|
||||
import time
|
||||
from array import array
|
||||
from collections import deque
|
||||
@@ -112,6 +114,64 @@ PREFILL_INPUT_POLL_INTERVAL_S = 0.0001
|
||||
|
||||
_is_npu = is_npu()
|
||||
|
||||
# DEBUG-ONLY (p2 wedge trap 2026-10-13): per-step batch-decision fingerprint
|
||||
# and event-loop phase logging. Same env as custom_all_reduce_v2 ARDBG.
|
||||
# 1 = sampled/one-line, 2 = + stacks on ONE_SHOT_PUSH. Off by default.
|
||||
_AR_SEQ_DEBUG = int(os.environ.get("SGLANG_AR_SEQ_DEBUG", "0"))
|
||||
_plan_seq = 0
|
||||
_plan_none_seq = 0
|
||||
|
||||
|
||||
def _dbg_plan(self, batch) -> None:
|
||||
global _plan_seq, _plan_none_seq
|
||||
if not _AR_SEQ_DEBUG:
|
||||
return
|
||||
_plan_seq += 1
|
||||
rank = torch.distributed.get_rank()
|
||||
if batch is None:
|
||||
# 空转步采样记录,避免 idle 自旋刷爆日志
|
||||
_plan_none_seq += 1
|
||||
if _plan_none_seq & 511:
|
||||
return
|
||||
logger.info("[ARDBG] plan rank=%s step=%d batch=None", rank, _plan_seq)
|
||||
return
|
||||
reqs = batch.reqs
|
||||
fp = 0
|
||||
ext_total = 0
|
||||
for r in reqs:
|
||||
# crc32 而非 hash():后者按进程随机化种子,跨 rank 不可比
|
||||
fp = (fp * 131 + zlib.crc32(r.rid.encode())) & 0xFFFFFFFFFFFFFFFF
|
||||
er = getattr(r, "extend_range", None)
|
||||
el = int(er.length) if er is not None else -1
|
||||
fp = (fp * 131 + el) & 0xFFFFFFFFFFFFFFFF
|
||||
fp = (fp * 131 + int(getattr(r, "prefix_len", -1))) & 0xFFFFFFFFFFFFFFFF
|
||||
if el > 0:
|
||||
ext_total += el
|
||||
logger.info(
|
||||
"[ARDBG] plan rank=%s step=%d nreq=%d ext=%d fp=%016x",
|
||||
rank,
|
||||
_plan_seq,
|
||||
len(reqs),
|
||||
ext_total,
|
||||
fp,
|
||||
)
|
||||
|
||||
|
||||
def _dbg_phase(label: str, batch=None) -> None:
|
||||
if not _AR_SEQ_DEBUG:
|
||||
return
|
||||
if batch is None and label in ("inflight_enter", "inflight_exit"):
|
||||
# 空转迭代的 inflight 相位价值低,采样记录
|
||||
if _plan_seq & 255:
|
||||
return
|
||||
logger.info(
|
||||
"[ARDBG] phase rank=%s step=%s %s nreq=%s",
|
||||
torch.distributed.get_rank(),
|
||||
_plan_seq,
|
||||
label,
|
||||
len(batch.reqs) if batch is not None else "-",
|
||||
)
|
||||
|
||||
|
||||
def should_force_retry(req: Req) -> bool:
|
||||
"""Test hook to force a request into optimistic prefill retry."""
|
||||
@@ -656,6 +716,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
if batch:
|
||||
set_schedule_time_batch(batch)
|
||||
|
||||
_dbg_plan(self, batch)
|
||||
return NextBatchPlan(batch_to_run=batch, running_batch=running_batch)
|
||||
|
||||
@torch.no_grad()
|
||||
@@ -689,13 +750,17 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
if batch:
|
||||
if self.enable_staging:
|
||||
self.maybe_prefetch_staging_for_batch(batch)
|
||||
_dbg_phase("run_batch_enter", batch)
|
||||
result = self.run_batch(batch)
|
||||
_dbg_phase("run_batch_exit", batch)
|
||||
self.process_batch_result(batch, result)
|
||||
else:
|
||||
self._sched_idled = True
|
||||
self.on_idle()
|
||||
|
||||
_dbg_phase("inflight_enter")
|
||||
self.process_disagg_prefill_inflight_queue()
|
||||
_dbg_phase("inflight_exit")
|
||||
|
||||
# Update last_batch
|
||||
self.last_batch = batch
|
||||
@@ -788,15 +853,18 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
# Skip batch creation next round to avoid repeated preparation failures.
|
||||
skip_batch_creation = batch is None
|
||||
|
||||
# Launch the current batch
|
||||
if batch:
|
||||
if self.enable_staging:
|
||||
self.maybe_prefetch_staging_for_batch(batch)
|
||||
batch_result = self.run_batch(batch)
|
||||
self._apply_war_barrier()
|
||||
self.result_queue.append((batch.copy(), batch_result))
|
||||
else:
|
||||
batch_result = None
|
||||
# Launch the current batch
|
||||
if batch:
|
||||
if self.enable_staging:
|
||||
self.maybe_prefetch_staging_for_batch(batch)
|
||||
_dbg_phase("run_batch_enter", batch)
|
||||
batch_result = self.run_batch(batch)
|
||||
_dbg_phase("run_batch_exit", batch)
|
||||
self._apply_war_barrier()
|
||||
self.result_queue.append((batch.copy(), batch_result))
|
||||
else:
|
||||
batch_result = None
|
||||
self._sched_idled = True
|
||||
|
||||
if self.last_batch:
|
||||
if self.enable_continuous_input_polling:
|
||||
@@ -818,7 +886,9 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
# When the server is idle, do self-check and re-init some states
|
||||
self.on_idle()
|
||||
|
||||
_dbg_phase("inflight_enter")
|
||||
self.process_disagg_prefill_inflight_queue()
|
||||
_dbg_phase("inflight_exit")
|
||||
|
||||
# Run sample of the current batch
|
||||
# It depends on the result of the last batch (e.g., grammar), so we run it after the last batch is processed.
|
||||
@@ -988,6 +1058,10 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
continue
|
||||
|
||||
self.checkpoint_disagg_prefill(req)
|
||||
# G9: the disagg-prefill path bypasses batch_result_processor's
|
||||
# prefill loop, so record the drain event here — final chunk
|
||||
# done, request heading to the inflight (KV transfer) queue.
|
||||
self.record_prefill_drain(req.seqlen)
|
||||
self.disagg_prefill_inflight_queue.append(req)
|
||||
if self.spec_algorithm.is_eagle() and draft_input is not None:
|
||||
req.output_topk_p = draft_input.topk_p[i]
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
from collections import deque
|
||||
from contextlib import nullcontext
|
||||
@@ -46,6 +48,32 @@ if is_npu():
|
||||
DSV4NPUTokenToKVPool,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# DEBUG-ONLY (p2 wedge repro 2026-10-13): env-gated seq logging for host-side
|
||||
# gloo all-reduce sync points. Mirrors SGLANG_AR_SEQ_DEBUG in
|
||||
# custom_all_reduce_v2.py. 1 = one-line, 2 = + python stack. Not for prod.
|
||||
_AR_SEQ_DEBUG = int(os.environ.get("SGLANG_AR_SEQ_DEBUG", "0"))
|
||||
_host_ar_seq = 0
|
||||
|
||||
|
||||
def _dbg_log_host_ar(phase: str, polls) -> None:
|
||||
global _host_ar_seq
|
||||
if not _AR_SEQ_DEBUG:
|
||||
return
|
||||
seq = _host_ar_seq
|
||||
_host_ar_seq += 1
|
||||
# 空转迭代(无 poller)采样 1/256,避免 idle 自旋刷爆日志
|
||||
if (not polls) and seq & 255:
|
||||
return
|
||||
logger.info(
|
||||
"[ARDBG] host rank=%s seq=%d phase=%s polls=%s",
|
||||
dist.get_rank(),
|
||||
seq,
|
||||
phase,
|
||||
polls,
|
||||
)
|
||||
|
||||
#########################
|
||||
# Constants & Enums
|
||||
#########################
|
||||
@@ -289,13 +317,20 @@ def poll_and_all_reduce_attn_cp_tp_group(
|
||||
attn_cp_cpu_group: dist.ProcessGroup,
|
||||
attn_tp_cpu_group: dist.ProcessGroup,
|
||||
):
|
||||
# DEBUG-ONLY
|
||||
_dbg_log_host_ar("enter", len(pollers))
|
||||
# First sync across attn-tp ranks so all TP participants for a given (dp, cp)
|
||||
# shard observe the same status transitions.
|
||||
polls = poll_and_all_reduce(pollers, attn_tp_cpu_group)
|
||||
# DEBUG-ONLY
|
||||
_dbg_log_host_ar("tp-done", polls)
|
||||
|
||||
# Then sync across attn-cp ranks, so all TPxCP participants in one DP shard
|
||||
# converge to the same global status.
|
||||
return _all_reduce_polls(polls, attn_cp_cpu_group)
|
||||
result = _all_reduce_polls(polls, attn_cp_cpu_group)
|
||||
# DEBUG-ONLY
|
||||
_dbg_log_host_ar("cp-done", result)
|
||||
return result
|
||||
|
||||
|
||||
def poll_and_all_reduce_with_staging(
|
||||
|
||||
@@ -25,6 +25,8 @@ the kernel captured in the graph dereferences its row at replay time.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import traceback
|
||||
from contextlib import contextmanager
|
||||
from typing import List, NamedTuple, Optional, Tuple
|
||||
|
||||
@@ -70,6 +72,9 @@ MB = 1024 * 1024
|
||||
_ALIGN_BYTES = 1024
|
||||
_SEMAPHORE_BYTES = 128
|
||||
_MAX_GRAPH_INPUTS = 131072
|
||||
# DEBUG-ONLY (p2 wedge repro 2026-10-13): env-gated seq logging for TP device
|
||||
# all-reduce calls. 1 = one-line per call, 2 = + python stack. Not for prod.
|
||||
_AR_SEQ_DEBUG = int(os.environ.get("SGLANG_AR_SEQ_DEBUG", "0"))
|
||||
# resolved once at import time; explicit constructor sizes take precedence
|
||||
_DEFAULT_MAX_SIZE = envs.SGLANG_CUSTOM_ALL_REDUCE_V2_MAX_SIZE_KB.get() * 1024
|
||||
# forced per-direction workspace sizes; highest priority, override both the
|
||||
@@ -335,6 +340,9 @@ class CustomAllReduceV2:
|
||||
self.pull_enabled,
|
||||
)
|
||||
dist.barrier(group=self.group)
|
||||
# DEBUG-ONLY: per-instance seq counter for SGLANG_AR_SEQ_DEBUG logging
|
||||
self._dbg_seq = 0
|
||||
self._dbg_rank = self.rank
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Algo selection
|
||||
@@ -463,6 +471,27 @@ class CustomAllReduceV2:
|
||||
assert config is not None, f"No config for {nbytes = }"
|
||||
algo, use_graph, use_multicast = config
|
||||
graph_params = self._allocate_graph_row(input, nbytes) if use_graph else None
|
||||
if _AR_SEQ_DEBUG:
|
||||
seq = self._dbg_seq
|
||||
self._dbg_seq += 1
|
||||
# 采样 1/1024:生产高频路径下全量日志会拖垮 P 节点
|
||||
if seq & 1023 == 0:
|
||||
logger.info(
|
||||
"[ARDBG] dev rank=%s seq=%d nbytes=%d algo=%s graph=%s capturing=%s",
|
||||
self._dbg_rank,
|
||||
seq,
|
||||
nbytes,
|
||||
getattr(algo, "name", algo),
|
||||
use_graph,
|
||||
torch.cuda.is_current_stream_capturing(),
|
||||
)
|
||||
if _AR_SEQ_DEBUG >= 2 and getattr(algo, "name", algo) == "ONE_SHOT_PUSH":
|
||||
logger.info(
|
||||
"[ARDBG] stack rank=%s seq=%d\n%s",
|
||||
self._dbg_rank,
|
||||
seq,
|
||||
"".join(traceback.format_stack(limit=15)),
|
||||
)
|
||||
return custom_all_reduce(
|
||||
self.obj,
|
||||
input,
|
||||
|
||||
@@ -1815,6 +1815,18 @@ async def openai_v1_chat_completions(
|
||||
)
|
||||
|
||||
|
||||
@app.post("/query_cache_hit", dependencies=[Depends(validate_json_request)])
|
||||
async def query_cache_hit(request: ChatCompletionRequest, raw_request: Request):
|
||||
"""G9: probe prefix-cache hit (L1/L2 + L3 storage) for a chat request.
|
||||
|
||||
Deployed on decode engines; the PD router calls it before dispatch to
|
||||
decide conditional bypass. Accepts a normal chat body or `input_ids`.
|
||||
"""
|
||||
return await raw_request.app.state.openai_serving_chat.query_cache_hit(
|
||||
request, raw_request
|
||||
)
|
||||
|
||||
|
||||
@app.post(
|
||||
"/v1/embeddings",
|
||||
response_class=ORJSONResponse,
|
||||
|
||||
@@ -1174,6 +1174,73 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
return "text", processed_messages.prompt_ids
|
||||
return "input_ids", processed_messages.prompt_ids
|
||||
|
||||
async def query_cache_hit(
|
||||
self, request: ChatCompletionRequest, raw_request: Request = None
|
||||
) -> ORJSONResponse:
|
||||
"""G9 probe: report how much of this chat request's prompt is already
|
||||
cached (per-rank-best L1/L2 radix + shared L3 storage) without running
|
||||
any prefill. The router uses it for conditional PD-bypass decisions.
|
||||
|
||||
Multimodal requests are rejected (v1): their sentinel-token expansion
|
||||
costs ~10-100ms per image for only ~2% of traffic; the router skips
|
||||
them for bypass anyway.
|
||||
"""
|
||||
try:
|
||||
internal_req, _ = self._convert_to_internal_request(request, raw_request)
|
||||
except Exception as e:
|
||||
return ORJSONResponse(
|
||||
status_code=400,
|
||||
content={"error": f"failed to convert chat request: {e}"},
|
||||
)
|
||||
if internal_req.contains_mm_input():
|
||||
return ORJSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "multimodal requests are not supported by /query_cache_hit"},
|
||||
)
|
||||
try:
|
||||
# 只取 input_ids,不走 _tokenize_one_request——它要求 rid_to_state
|
||||
# 里已有 ReqState(正常生成路径在 tokenize 前注册),纯查询没有。
|
||||
# dsv41 系 chat encoding 直接渲染 prompt_ids;否则对渲染后的
|
||||
# text 走 tokenizer(_tokenize_texts 与真实路径同一入口)。
|
||||
if internal_req.input_ids is not None:
|
||||
input_ids = list(internal_req.input_ids)
|
||||
elif internal_req.text:
|
||||
input_ids, _ = await self.tokenizer_manager._tokenize_texts(
|
||||
internal_req.text, False
|
||||
)
|
||||
else:
|
||||
return ORJSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "request produced neither input_ids nor text"},
|
||||
)
|
||||
from sglang.srt.managers.io_struct import QueryCacheHitReqInput
|
||||
|
||||
result = await self.tokenizer_manager.query_cache_hit(
|
||||
QueryCacheHitReqInput(
|
||||
input_ids=list(input_ids),
|
||||
extra_key=getattr(internal_req, "extra_key", None),
|
||||
cache_salt=getattr(internal_req, "cache_salt", None),
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
return ORJSONResponse(
|
||||
status_code=500, content={"error": f"cache-hit query failed: {e}"}
|
||||
)
|
||||
if result.error is not None:
|
||||
return ORJSONResponse(status_code=500, content={"error": result.error})
|
||||
hit = result.l1_hit_tokens + result.l2_hit_tokens + result.l3_hit_tokens
|
||||
return ORJSONResponse(
|
||||
content={
|
||||
"total_tokens": result.total_tokens,
|
||||
"l1_hit_tokens": result.l1_hit_tokens,
|
||||
"l2_hit_tokens": result.l2_hit_tokens,
|
||||
"l3_hit_tokens": result.l3_hit_tokens,
|
||||
"hit_tokens": hit,
|
||||
"new_tokens": max(0, result.total_tokens - hit),
|
||||
"dp_rank": result.dp_rank,
|
||||
}
|
||||
)
|
||||
|
||||
def _can_reuse_text_only_prompt_ids(
|
||||
self, processed_messages: MessageProcessingResult, is_multimodal: bool
|
||||
) -> bool:
|
||||
|
||||
@@ -788,6 +788,16 @@ class Envs:
|
||||
SGLANG_HICACHE_HF3FS_CONFIG_PATH = EnvStr(None)
|
||||
SGLANG_HICACHE_DECODE_OFFLOAD_STRIDE = EnvInt(None)
|
||||
SGLANG_HICACHE_SKIP_HOST_DUPLICATE_RECLAIM = EnvBool(False)
|
||||
# Per-stage timing logs for the L3 storage prefetch path (query / transfer /
|
||||
# ack reduce / drain / graft / load-back). Emits "HC_TIMING ..." lines.
|
||||
SGLANG_HICACHE_TIMING = EnvBool(False)
|
||||
# Pages per batched storage IO call (query / get / put chunking).
|
||||
SGLANG_HICACHE_STORAGE_BATCH_SIZE = EnvInt(128)
|
||||
# Number of parallel prefetch IO lanes (worker threads) per controller.
|
||||
# Operations are sharded by crc32(request_id) so every rank maps a request
|
||||
# to the same lane; each lane owns dedicated completion sync groups.
|
||||
# MUST be identical across all TP/CP/PP ranks of an instance.
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS = EnvInt(4)
|
||||
SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR = EnvStr(None)
|
||||
# File-backend LRU eviction (opt-in; sizes accept SI/IEC suffixes, "0" disables).
|
||||
SGLANG_HICACHE_FILE_BACKEND_MAX_SIZE = EnvStr(None)
|
||||
|
||||
@@ -205,6 +205,7 @@ def _create_flashmla_metadata():
|
||||
# only for that shape and go stale silently if FlashMLA retunes it.
|
||||
_FLASHMLA_SCHED_BLOCK_SIZE_N = 64
|
||||
_FLASHMLA_SCHED_FIXED_OVERHEAD = 5
|
||||
_FLASHMLA_SCHED_MAX_SHARED_BYTES = 48 << 10
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=None)
|
||||
@@ -241,7 +242,9 @@ def _maybe_precompute_flashmla_sched_meta(
|
||||
|
||||
b, s_q = q.shape[0], q.shape[1]
|
||||
num_sm_parts = max(_num_sms(q.device.index) // s_q, 1)
|
||||
if 4 * (5 * b + 1 + num_sm_parts * META_INTS) > 48 * 1024:
|
||||
# Skip when shared memory exceeds the limit.
|
||||
smem_bytes = 4 * (b * 5 + 1 + num_sm_parts * 8)
|
||||
if smem_bytes > _FLASHMLA_SCHED_MAX_SHARED_BYTES:
|
||||
return
|
||||
meta = torch.empty((num_sm_parts, META_INTS), dtype=torch.int32, device=q.device)
|
||||
num_splits = torch.empty((b + 1,), dtype=torch.int32, device=q.device)
|
||||
@@ -710,7 +713,14 @@ class DSV4AttnMetadata:
|
||||
if src_val is None and dst_val is None:
|
||||
continue
|
||||
assert dst_val is not None, f"{field_name=} {src_val=} {dst_val=}"
|
||||
dst_val.copy_(src_val)
|
||||
shape_mismatch = dst_val.shape != src_val.shape
|
||||
assert not shape_mismatch or field_name in self._CP_GLOBAL_FIELDS, (
|
||||
f"Only CP-global replay metadata may use a shorter live prefix, "
|
||||
f"got {field_name=} {src_val.shape=} {dst_val.shape=}"
|
||||
)
|
||||
_copy_tensor_allowing_storage_alias(
|
||||
dst_val, src_val, pad_value=0 if shape_mismatch else None
|
||||
)
|
||||
|
||||
# Safe to replace: captured kernels read only the per-replay objects, or
|
||||
# the field is produced in-graph before the attention graph break reads it.
|
||||
@@ -1020,6 +1030,27 @@ def _prefill_graph_max_seq_len() -> Optional[int]:
|
||||
return get_exec().graph.cuda_graph_config.prefill.max_seq_len
|
||||
|
||||
|
||||
def _copy_tensor_allowing_storage_alias(
|
||||
dst: torch.Tensor, src: torch.Tensor, *, pad_value: Optional[int] = None
|
||||
) -> None:
|
||||
"""Copy replay metadata while preserving capture-stable destination addresses."""
|
||||
if dst is src:
|
||||
return
|
||||
if dst.untyped_storage().data_ptr() == src.untyped_storage().data_ptr():
|
||||
src = src.clone()
|
||||
if dst.shape == src.shape:
|
||||
dst.copy_(src)
|
||||
return
|
||||
assert (
|
||||
pad_value is not None
|
||||
and dst.ndim == src.ndim
|
||||
and dst.shape[0] >= src.shape[0]
|
||||
and dst.shape[1:] == src.shape[1:]
|
||||
), f"Cannot copy replay metadata from {src.shape=} to {dst.shape=}"
|
||||
dst.fill_(pad_value)
|
||||
dst[: src.shape[0]].copy_(src)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DSV4Metadata:
|
||||
core_attn_metadata: DSV4AttnMetadata
|
||||
@@ -1303,6 +1334,11 @@ class DeepseekV4AttnBackend(
|
||||
] = None
|
||||
self.online_c128_mtp = OnlineC128MTPController(self)
|
||||
self.sparse_prefill_workspace = SparsePrefillWorkspace(self.device)
|
||||
# CP V4.1 consumers share compressed KV across layers. Separate ratio
|
||||
# workspaces keep those prefixes intact while each layer refreshes SWA.
|
||||
self.shared_compressed_prefill_workspaces = {
|
||||
ratio: SparsePrefillWorkspace(self.device) for ratio in (1, 2)
|
||||
}
|
||||
spec_alg = model_runner.spec_algorithm
|
||||
self.needs_cpu_seq_lens = not spec_alg.is_dspark() and (
|
||||
not _is_cuda or self.online_c128_mtp.enabled()
|
||||
@@ -4109,20 +4145,38 @@ class DeepseekV4AttnBackend(
|
||||
compress_ratio, core_attn_metadata, extra_page_size
|
||||
)
|
||||
n_compressed = flat_token_ids.shape[0]
|
||||
workspace = self.sparse_prefill_workspace.get(
|
||||
n_compressed + cache.swa_token_ids.shape[0]
|
||||
reuse_compressed = compress_ratio in (1, 2) and is_cp_active(forward_batch)
|
||||
workspace_pool = (
|
||||
self.shared_compressed_prefill_workspaces[compress_ratio]
|
||||
if reuse_compressed
|
||||
else self.sparse_prefill_workspace
|
||||
)
|
||||
workspace = workspace_pool.get(n_compressed + cache.swa_token_ids.shape[0])
|
||||
compressed_slice = workspace[:n_compressed]
|
||||
swa_slice = workspace[n_compressed:]
|
||||
|
||||
if compressed_slice is not None:
|
||||
dequantize_k_cache_paged(
|
||||
extra_k_cache,
|
||||
flat_token_ids,
|
||||
page_size=extra_page_size,
|
||||
out=compressed_slice,
|
||||
layout=token_to_kv_pool.get_extra_key_layout(layer_id),
|
||||
)
|
||||
source_key = None
|
||||
if reuse_compressed:
|
||||
source_layer = token_to_kv_pool.source_layer_of(layer_id)
|
||||
source_key = (source_layer, workspace.data_ptr())
|
||||
gather = cache.compressed[compress_ratio]
|
||||
# A source layer may have just updated its cache in place. Consumer
|
||||
# layers only reuse the compressed prefix; their top-k and SWA stay live.
|
||||
if (
|
||||
source_key is None
|
||||
or layer_id == source_key[0]
|
||||
or gather.dequantized_source != source_key
|
||||
):
|
||||
dequantize_k_cache_paged(
|
||||
extra_k_cache,
|
||||
flat_token_ids,
|
||||
page_size=extra_page_size,
|
||||
out=compressed_slice,
|
||||
layout=token_to_kv_pool.get_extra_key_layout(layer_id),
|
||||
)
|
||||
if source_key is not None:
|
||||
gather.dequantized_source = source_key
|
||||
dequantize_k_cache_paged(
|
||||
token_to_kv_pool.get_swa_key_buffer_radix(layer_id),
|
||||
cache.swa_token_ids,
|
||||
|
||||
@@ -79,10 +79,11 @@ def use_dsv4_q8kv8_sparse_prefill(dsv4_prefill_backend: str = "auto") -> bool:
|
||||
class SparsePrefillWorkspace:
|
||||
"""Backend-owned scratch storage for sparse prefill KV dequantization.
|
||||
|
||||
The workspace contents are fully overwritten before every attention call,
|
||||
so token buckets and compression ratios can safely share one buffer. Sparse
|
||||
prefill executes eagerly and serially on the supported paths, which makes it
|
||||
safe to replace the scratch allocation when a larger extent is needed.
|
||||
Callers normally overwrite the entire workspace. Shared compressed-KV
|
||||
callers keep separate workspaces per ratio and track prefix validity in the
|
||||
per-forward gather cache, including the allocation address. Sparse prefill
|
||||
executes eagerly and serially on the supported paths, so the allocation can
|
||||
be replaced when a larger extent is needed.
|
||||
"""
|
||||
|
||||
def __init__(self, device: torch.device):
|
||||
@@ -295,14 +296,18 @@ class CompressedGather:
|
||||
# chunk-invariant per request; subsequent layers only overwrite that prefix.
|
||||
combined_indices: Optional[torch.Tensor] = None
|
||||
combined_lens: Optional[torch.Tensor] = None
|
||||
# Valid only for this forward's gather layout. Each ratio has its own
|
||||
# workspace; its compressed prefix survives consumer layers' SWA writes.
|
||||
dequantized_source: Optional[tuple[int, int]] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class SparsePrefillChunkCache:
|
||||
"""Cache prefill-chunk metadata shared across layers.
|
||||
|
||||
Fields depend on request/token mappings and compressed page tables, not
|
||||
per-layer k_cache; per-layer top-k combinations are recomputed into reused
|
||||
Gather layouts depend on request/token mappings and compressed page tables.
|
||||
Shared-source dequantization keys live only for this forward; per-layer
|
||||
top-k combinations are recomputed into reused
|
||||
buffers.
|
||||
"""
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ import torch
|
||||
|
||||
from sglang.srt.arg_groups.overrides import (
|
||||
attention_backends_of,
|
||||
model_config_of,
|
||||
resolved_view,
|
||||
resolving_view,
|
||||
)
|
||||
@@ -35,10 +36,12 @@ from sglang.srt.layers.cp.interleave import InterleaveCPStrategy
|
||||
from sglang.srt.layers.cp.padding import get_cp_padding_align_size
|
||||
from sglang.srt.layers.cp.utils import (
|
||||
cp_gather_after_forward,
|
||||
cp_shard_hidden_states,
|
||||
cp_split_before_forward,
|
||||
prepare_cp_forward,
|
||||
)
|
||||
from sglang.srt.layers.cp.zigzag import ZigzagCPStrategy
|
||||
from sglang.srt.layers.logits_processor import LogitsMetadata
|
||||
from sglang.srt.model_executor.forward_batch_info import PPProxyTensors
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -118,6 +121,8 @@ class PrefillCPBCGInput:
|
||||
|
||||
input_embeds: torch.Tensor
|
||||
positions: torch.Tensor
|
||||
input_ids: Optional[torch.Tensor] = None
|
||||
num_token_non_padded: Optional[torch.Tensor] = None
|
||||
bucket_local_tokens: Dict[int, int] = field(default_factory=dict)
|
||||
live_local_tokens: int = 0
|
||||
|
||||
@@ -136,6 +141,8 @@ class PrefillCPBCGInput:
|
||||
(runner.max_num_tokens,),
|
||||
dtype=torch.int64,
|
||||
),
|
||||
input_ids=torch.zeros((runner.max_num_tokens,), dtype=torch.int64),
|
||||
num_token_non_padded=torch.zeros((), dtype=torch.int32),
|
||||
)
|
||||
|
||||
def required_local_tokens(self, extend_seq_lens: Any) -> Optional[int]:
|
||||
@@ -248,6 +255,7 @@ class PrefillCPBCGInput:
|
||||
raw_tokens = int(forward_batch.extend_num_tokens)
|
||||
global_input_ids = forward_batch.input_ids[:raw_tokens]
|
||||
global_positions = forward_batch.positions[:raw_tokens]
|
||||
local_input_ids = cp_shard_hidden_states(global_input_ids, forward_batch)
|
||||
global_input_embeds = runner.model_runner.model.get_input_embeddings()(
|
||||
global_input_ids
|
||||
)
|
||||
@@ -278,12 +286,31 @@ class PrefillCPBCGInput:
|
||||
|
||||
input_embeds = self.input_embeds[:captured_local_tokens]
|
||||
positions = self.positions[:captured_local_tokens]
|
||||
assert self.input_ids is not None
|
||||
input_ids = self.input_ids[:captured_local_tokens]
|
||||
input_embeds.zero_()
|
||||
positions.zero_()
|
||||
input_ids.zero_()
|
||||
input_embeds[:live_local_tokens].copy_(local_input_embeds)
|
||||
positions[:live_local_tokens].copy_(local_positions)
|
||||
input_ids[:live_local_tokens].copy_(local_input_ids)
|
||||
forward_batch.input_embeds = input_embeds
|
||||
forward_batch.positions = positions
|
||||
forward_batch._cp_positions = positions
|
||||
# Keep the global input_ids field intact: the runner uses its length to
|
||||
# select the global capture bucket. The DSV4 body consumes this fixed,
|
||||
# rank-local view for hash routing and MegaMoE.
|
||||
forward_batch._cp_input_ids = input_ids
|
||||
forward_batch.input_ids_global = input_ids
|
||||
if forward_batch.num_token_non_padded is not None:
|
||||
assert self.num_token_non_padded is not None
|
||||
metadata = forward_batch.attn_cp_metadata
|
||||
logical_tokens = (
|
||||
metadata.per_rank_logical_token or metadata.per_rank_actual_token
|
||||
)
|
||||
strategy = get_cp_strategy()
|
||||
assert strategy is not None
|
||||
self.num_token_non_padded.fill_(logical_tokens[strategy.cp_rank])
|
||||
forward_batch.num_token_non_padded = self.num_token_non_padded
|
||||
self.live_local_tokens = live_local_tokens
|
||||
|
||||
if isinstance(get_cp_strategy(), InterleaveCPStrategy):
|
||||
@@ -373,7 +400,10 @@ def execute_prefill_cp_bcg(
|
||||
forward_batch.input_ids,
|
||||
hidden_states,
|
||||
model.lm_head,
|
||||
forward_batch,
|
||||
logits_metadata,
|
||||
aux_hidden_states,
|
||||
**logits_kwargs,
|
||||
)
|
||||
if tail is not None:
|
||||
output.hidden_states_token_indices = tail.token_indices
|
||||
return output
|
||||
|
||||
@@ -17,6 +17,7 @@ limitations under the License.
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import zlib
|
||||
from dataclasses import dataclass
|
||||
from queue import Empty, Queue
|
||||
from typing import TYPE_CHECKING, Any, Callable, List, NamedTuple, Optional
|
||||
@@ -32,6 +33,17 @@ from sglang.srt.mem_cache.hicache_storage import (
|
||||
count_pool_hits,
|
||||
)
|
||||
|
||||
from sglang.srt.environ import envs as _envs
|
||||
|
||||
_HC_TIMING = None
|
||||
|
||||
|
||||
def _hc_timing_enabled() -> bool:
|
||||
global _HC_TIMING
|
||||
if _HC_TIMING is None:
|
||||
_HC_TIMING = _envs.SGLANG_HICACHE_TIMING.get()
|
||||
return _HC_TIMING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
||||
from sglang.srt.mem_cache.memory_pool_host import LogicalHostPool
|
||||
@@ -318,7 +330,18 @@ class HiCacheController:
|
||||
self.attn_tp_group = attn_tp_group
|
||||
self.pp_group = pp_group
|
||||
self.prefetch_hits_sync_groups: List[torch.distributed.ProcessGroup] = []
|
||||
self.prefetch_completion_sync_groups: List[torch.distributed.ProcessGroup] = []
|
||||
# One set of completion sync groups per prefetch IO lane. Each lane's
|
||||
# worker/sync threads form a serial pipeline, so the collective
|
||||
# sequence on a lane's groups is identical across ranks as long as
|
||||
# lane assignment is deterministic (see dispatch_prefetch).
|
||||
self.prefetch_completion_sync_groups_by_lane: List[
|
||||
List[torch.distributed.ProcessGroup]
|
||||
] = []
|
||||
# Read once at construction; attach/reset reuse the stored value so
|
||||
# lane count never changes underneath live sync groups.
|
||||
self.num_prefetch_io_lanes = max(
|
||||
1, _envs.SGLANG_HICACHE_PREFETCH_IO_THREADS.get()
|
||||
)
|
||||
self.mem_pool_device_allocator = token_to_kv_pool_allocator
|
||||
mem_pool_device = token_to_kv_pool_allocator.get_kvcache()
|
||||
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool
|
||||
@@ -347,8 +370,8 @@ class HiCacheController:
|
||||
self.storage_stop_event = threading.Event()
|
||||
|
||||
# Storage control queues, (re)created whenever the storage threads start.
|
||||
self.prefetch_buffer: Optional[Queue[PrefetchOperation]] = None
|
||||
self.prefetch_sync_queue: Optional[Queue[PrefetchAck]] = None
|
||||
self.prefetch_io_queues: List[Queue[PrefetchOperation]] = []
|
||||
self.prefetch_sync_queues: List[Queue[PrefetchAck]] = []
|
||||
self.prefetch_hit_queue: Optional[Queue[StorageOperation]] = None
|
||||
self.ack_prefetch_queue = Queue[PrefetchAck]()
|
||||
self.ack_backup_queue: Optional[Queue[StorageOperation]] = None
|
||||
@@ -457,27 +480,39 @@ class HiCacheController:
|
||||
self.prefetch_thread = threading.Thread(
|
||||
target=self.prefetch_thread_func, daemon=True
|
||||
)
|
||||
self.prefetch_io_aux_thread = threading.Thread(
|
||||
target=self.prefetch_io_aux_func, daemon=True
|
||||
)
|
||||
self.prefetch_sync_thread = threading.Thread(
|
||||
target=self.prefetch_sync_thread_func, daemon=True
|
||||
)
|
||||
self.prefetch_io_aux_threads = [
|
||||
threading.Thread(
|
||||
target=self.prefetch_io_aux_func, args=(lane,), daemon=True
|
||||
)
|
||||
for lane in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
self.prefetch_sync_threads = [
|
||||
threading.Thread(
|
||||
target=self.prefetch_sync_thread_func, args=(lane,), daemon=True
|
||||
)
|
||||
for lane in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
self.backup_thread = threading.Thread(
|
||||
target=self.backup_thread_func, daemon=True
|
||||
)
|
||||
self.prefetch_queue = Queue()
|
||||
self.backup_queue = Queue()
|
||||
self.prefetch_buffer = Queue()
|
||||
self.prefetch_sync_queue = Queue()
|
||||
self.prefetch_io_queues = [
|
||||
Queue() for _ in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
self.prefetch_sync_queues = [
|
||||
Queue() for _ in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
self.prefetch_hit_queue = Queue()
|
||||
self.ack_prefetch_queue = Queue()
|
||||
self.ack_backup_queue = Queue()
|
||||
self.host_mem_release_queue = Queue()
|
||||
|
||||
self.prefetch_thread.start()
|
||||
self.prefetch_io_aux_thread.start()
|
||||
self.prefetch_sync_thread.start()
|
||||
for thread in self.prefetch_io_aux_threads:
|
||||
thread.start()
|
||||
for thread in self.prefetch_sync_threads:
|
||||
thread.start()
|
||||
self.backup_thread.start()
|
||||
|
||||
def _stop_storage_threads(self):
|
||||
@@ -498,10 +533,10 @@ class HiCacheController:
|
||||
self.prefetch_queue.put_nowait(None)
|
||||
if hasattr(self, "backup_queue"):
|
||||
self.backup_queue.put_nowait(None)
|
||||
if hasattr(self, "prefetch_buffer"):
|
||||
self.prefetch_buffer.put_nowait(None)
|
||||
if hasattr(self, "prefetch_sync_queue"):
|
||||
self.prefetch_sync_queue.put_nowait(None)
|
||||
for queue in getattr(self, "prefetch_io_queues", []) or []:
|
||||
queue.put_nowait(None)
|
||||
for queue in getattr(self, "prefetch_sync_queues", []) or []:
|
||||
queue.put_nowait(None)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -511,10 +546,8 @@ class HiCacheController:
|
||||
threads.append(self.prefetch_thread)
|
||||
if hasattr(self, "backup_thread"):
|
||||
threads.append(self.backup_thread)
|
||||
if hasattr(self, "prefetch_io_aux_thread"):
|
||||
threads.append(self.prefetch_io_aux_thread)
|
||||
if hasattr(self, "prefetch_sync_thread"):
|
||||
threads.append(self.prefetch_sync_thread)
|
||||
threads.extend(getattr(self, "prefetch_io_aux_threads", []) or [])
|
||||
threads.extend(getattr(self, "prefetch_sync_threads", []) or [])
|
||||
|
||||
for t in threads:
|
||||
try:
|
||||
@@ -598,7 +631,13 @@ class HiCacheController:
|
||||
# Use dedicated gloo groups so storage prefetch sync is isolated
|
||||
# from other collectives and consistent across CPxTP participants.
|
||||
self.prefetch_hits_sync_groups = self._create_sync_groups()
|
||||
self.prefetch_completion_sync_groups = self._create_sync_groups()
|
||||
# Each IO lane gets its own completion groups: per-lane collectives
|
||||
# stay ordered because a lane is a serial pipeline and dispatch is
|
||||
# deterministic (crc32(request_id) % num_prefetch_io_lanes).
|
||||
self.prefetch_completion_sync_groups_by_lane = [
|
||||
self._create_sync_groups()
|
||||
for _ in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
|
||||
# Select the get and set functions
|
||||
self.page_get_func = self._generic_page_get
|
||||
@@ -634,9 +673,10 @@ class HiCacheController:
|
||||
except Exception:
|
||||
pass
|
||||
self._destroy_sync_groups(self.prefetch_hits_sync_groups)
|
||||
self._destroy_sync_groups(self.prefetch_completion_sync_groups)
|
||||
for lane_groups in self.prefetch_completion_sync_groups_by_lane:
|
||||
self._destroy_sync_groups(lane_groups)
|
||||
self.prefetch_hits_sync_groups = []
|
||||
self.prefetch_completion_sync_groups = []
|
||||
self.prefetch_completion_sync_groups_by_lane = []
|
||||
try:
|
||||
if (
|
||||
hasattr(self, "storage_backend")
|
||||
@@ -674,11 +714,11 @@ class HiCacheController:
|
||||
raise RuntimeError("Stop storage threads failed; detach aborted.") from e
|
||||
|
||||
# Best-effort destroy process groups created for storage ops.
|
||||
self._destroy_sync_groups(
|
||||
self.prefetch_hits_sync_groups + self.prefetch_completion_sync_groups
|
||||
)
|
||||
self._destroy_sync_groups(self.prefetch_hits_sync_groups)
|
||||
for lane_groups in self.prefetch_completion_sync_groups_by_lane:
|
||||
self._destroy_sync_groups(lane_groups)
|
||||
self.prefetch_hits_sync_groups = []
|
||||
self.prefetch_completion_sync_groups = []
|
||||
self.prefetch_completion_sync_groups_by_lane = []
|
||||
|
||||
# Best-effort close (some backends rely on GC/destructor).
|
||||
try:
|
||||
@@ -774,8 +814,10 @@ class HiCacheController:
|
||||
if self.enable_storage:
|
||||
self.prefetch_queue.queue.clear()
|
||||
self.backup_queue.queue.clear()
|
||||
self.prefetch_buffer.queue.clear()
|
||||
self.prefetch_sync_queue.queue.clear()
|
||||
for queue in self.prefetch_io_queues:
|
||||
queue.queue.clear()
|
||||
for queue in self.prefetch_sync_queues:
|
||||
queue.queue.clear()
|
||||
self.prefetch_hit_queue.queue.clear()
|
||||
self.ack_prefetch_queue.queue.clear()
|
||||
self.ack_backup_queue.queue.clear()
|
||||
@@ -788,18 +830,26 @@ class HiCacheController:
|
||||
self.prefetch_thread = threading.Thread(
|
||||
target=self.prefetch_thread_func, daemon=True
|
||||
)
|
||||
self.prefetch_io_aux_thread = threading.Thread(
|
||||
target=self.prefetch_io_aux_func, daemon=True
|
||||
)
|
||||
self.prefetch_sync_thread = threading.Thread(
|
||||
target=self.prefetch_sync_thread_func, daemon=True
|
||||
)
|
||||
self.prefetch_io_aux_threads = [
|
||||
threading.Thread(
|
||||
target=self.prefetch_io_aux_func, args=(lane,), daemon=True
|
||||
)
|
||||
for lane in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
self.prefetch_sync_threads = [
|
||||
threading.Thread(
|
||||
target=self.prefetch_sync_thread_func, args=(lane,), daemon=True
|
||||
)
|
||||
for lane in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
self.backup_thread = threading.Thread(
|
||||
target=self.backup_thread_func, daemon=True
|
||||
)
|
||||
self.prefetch_thread.start()
|
||||
self.prefetch_io_aux_thread.start()
|
||||
self.prefetch_sync_thread.start()
|
||||
for thread in self.prefetch_io_aux_threads:
|
||||
thread.start()
|
||||
for thread in self.prefetch_sync_threads:
|
||||
thread.start()
|
||||
self.backup_thread.start()
|
||||
|
||||
def has_inflight_device_transfers(self) -> bool:
|
||||
@@ -1099,7 +1149,23 @@ class HiCacheController:
|
||||
count += 1
|
||||
return count
|
||||
|
||||
def _page_transfer(self, operation: PrefetchOperation) -> int:
|
||||
def dispatch_prefetch(self, operation: PrefetchOperation) -> None:
|
||||
"""Route a prefetch operation to its IO lane.
|
||||
|
||||
Lane assignment must be deterministic and identical on every rank:
|
||||
each lane owns dedicated completion sync groups, and the collective
|
||||
sequence on a lane is the filtered dispatch order. crc32 (not the
|
||||
per-process salted builtin hash) keeps the mapping stable across
|
||||
processes. All chunks of one operation stay in one lane, preserving
|
||||
the per-operation ack order the sync thread reduces over.
|
||||
"""
|
||||
lane = (
|
||||
zlib.crc32(str(operation.request_id).encode())
|
||||
% self.num_prefetch_io_lanes
|
||||
)
|
||||
self.prefetch_io_queues[lane].put(operation)
|
||||
|
||||
def _page_transfer(self, operation: PrefetchOperation, sync_queue: Queue) -> int:
|
||||
# Transfer batch by batch
|
||||
prefix_keys = operation.prefix_keys
|
||||
kv_derived_transfers = [
|
||||
@@ -1111,8 +1177,8 @@ class HiCacheController:
|
||||
completed_pages = 0
|
||||
for i in range(0, len(operation.hash_value), STORAGE_BATCH_SIZE):
|
||||
# When an error is occurred, we should keep looping and produce the same number of
|
||||
# PrefetchAck as other ranks do, because prefetch_sync_thread (i.e. consumer of
|
||||
# prefetch_sync_queue) perform reduce on the results. This is so tricky.
|
||||
# PrefetchAck as other ranks do, because the prefetch sync thread (i.e. consumer of
|
||||
# the lane sync queue) perform reduce on the results. This is so tricky.
|
||||
if all_success and operation.is_terminated():
|
||||
all_success = False
|
||||
if all_success:
|
||||
@@ -1124,6 +1190,7 @@ class HiCacheController:
|
||||
# Get one batch token, and update the completed_tokens if succeed
|
||||
extra_info = HiCacheStorageExtraInfo(prefix_keys=prefix_keys)
|
||||
|
||||
_hc_tb = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
try:
|
||||
hit_pages = self._page_transfer_kv_batch(
|
||||
operation,
|
||||
@@ -1138,6 +1205,14 @@ class HiCacheController:
|
||||
operation.request_id,
|
||||
)
|
||||
hit_pages = 0
|
||||
if _hc_timing_enabled():
|
||||
logger.info(
|
||||
"HC_TIMING stage=xfer_batch req=%s keys=%d ms=%.1f hit=%d",
|
||||
operation.request_id,
|
||||
len(batch_hashes),
|
||||
(time.monotonic() - _hc_tb) * 1e3,
|
||||
hit_pages,
|
||||
)
|
||||
# Check termination
|
||||
if hit_pages != len(batch_hashes):
|
||||
all_success = False
|
||||
@@ -1149,7 +1224,7 @@ class HiCacheController:
|
||||
completed_tokens=completed_pages * self.page_size,
|
||||
operation=operation,
|
||||
)
|
||||
self.prefetch_sync_queue.put(ack)
|
||||
sync_queue.put(ack)
|
||||
return completed_pages
|
||||
|
||||
def _page_transfer_kv_batch(
|
||||
@@ -1191,21 +1266,36 @@ class HiCacheController:
|
||||
# Clamp to minimal number of hits.
|
||||
return min([kv_hits, *sidecar_hits.values()])
|
||||
|
||||
def prefetch_io_aux_func(self):
|
||||
def prefetch_io_aux_func(self, lane: int = 0):
|
||||
"""
|
||||
Auxiliary function conducting IO operations for prefetching.
|
||||
|
||||
Each lane has its own worker thread and queues; an operation's whole
|
||||
chunk sequence stays on one lane so the sync thread sees it in order.
|
||||
"""
|
||||
io_queue = self.prefetch_io_queues[lane]
|
||||
sync_queue = self.prefetch_sync_queues[lane]
|
||||
while not self.storage_stop_event.is_set():
|
||||
try:
|
||||
operation = self.prefetch_buffer.get(block=True, timeout=1)
|
||||
operation = io_queue.get(block=True, timeout=1)
|
||||
except Empty:
|
||||
continue
|
||||
if operation is None:
|
||||
continue
|
||||
_hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
try:
|
||||
self._page_transfer(operation)
|
||||
self._page_transfer(operation, sync_queue)
|
||||
finally:
|
||||
self.prefetch_sync_queue.put(
|
||||
if _hc_timing_enabled():
|
||||
logger.info(
|
||||
"HC_TIMING stage=xfer req=%s lane=%d wait_ms=%.1f xfer_ms=%.1f",
|
||||
operation.request_id,
|
||||
lane,
|
||||
(_hc_t0 - getattr(operation, "query_done_ts", operation.start_time))
|
||||
* 1e3,
|
||||
(time.monotonic() - _hc_t0) * 1e3,
|
||||
)
|
||||
sync_queue.put(
|
||||
PrefetchAck(
|
||||
rid=operation.request_id,
|
||||
completed_req=True,
|
||||
@@ -1266,7 +1356,16 @@ class HiCacheController:
|
||||
for start in range(0, len(page_hashes), STORAGE_BATCH_SIZE):
|
||||
batch_hashes = page_hashes[start : start + STORAGE_BATCH_SIZE]
|
||||
extra_info = HiCacheStorageExtraInfo(prefix_keys=prefix_keys)
|
||||
_hc_tb = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
hit_page_num = self.storage_backend.batch_exists(batch_hashes, extra_info)
|
||||
if _hc_timing_enabled():
|
||||
logger.info(
|
||||
"HC_TIMING stage=query_batch req=%s keys=%d ms=%.1f hit=%d",
|
||||
operation.request_id,
|
||||
len(batch_hashes),
|
||||
(time.monotonic() - _hc_tb) * 1e3,
|
||||
hit_page_num,
|
||||
)
|
||||
hash_value.extend(batch_hashes[:hit_page_num])
|
||||
storage_query_count += hit_page_num * self.page_size
|
||||
if hit_page_num < len(batch_hashes):
|
||||
@@ -1285,6 +1384,7 @@ class HiCacheController:
|
||||
operation = self.prefetch_queue.get(block=True, timeout=1)
|
||||
if operation is None:
|
||||
continue
|
||||
_hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
try:
|
||||
if operation.is_terminated():
|
||||
hash_value, storage_hit_count = [], 0
|
||||
@@ -1298,10 +1398,22 @@ class HiCacheController:
|
||||
operation.request_id,
|
||||
)
|
||||
hash_value, storage_hit_count = [], 0
|
||||
|
||||
_hc_t1 = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
storage_hit_count = self._sync_prefetch_hit_query(
|
||||
operation, storage_hit_count
|
||||
)
|
||||
if _hc_timing_enabled():
|
||||
_hc_t2 = time.monotonic()
|
||||
operation.query_done_ts = _hc_t2
|
||||
logger.info(
|
||||
"HC_TIMING stage=query req=%s wait_ms=%.1f query_ms=%.1f "
|
||||
"hit_reduce_ms=%.1f hit_tokens=%d",
|
||||
operation.request_id,
|
||||
(_hc_t0 - operation.start_time) * 1e3,
|
||||
(_hc_t1 - _hc_t0) * 1e3,
|
||||
(_hc_t2 - _hc_t1) * 1e3,
|
||||
storage_hit_count,
|
||||
)
|
||||
|
||||
# Record the TP-synced hit count; the scheduler thread decides
|
||||
# at drain time whether to revoke (below threshold) or allocate.
|
||||
@@ -1406,28 +1518,41 @@ class HiCacheController:
|
||||
except Empty:
|
||||
continue
|
||||
|
||||
def prefetch_sync_thread_func(self):
|
||||
def prefetch_sync_thread_func(self, lane: int = 0):
|
||||
"""Synchronize prefetch results across all PP and TP ranks."""
|
||||
sync_queue = self.prefetch_sync_queues[lane]
|
||||
groups = self.prefetch_completion_sync_groups_by_lane[lane]
|
||||
while not self.storage_stop_event.is_set():
|
||||
try:
|
||||
ack = self.prefetch_sync_queue.get(block=True, timeout=1)
|
||||
ack = sync_queue.get(block=True, timeout=1)
|
||||
if ack is None:
|
||||
continue
|
||||
self._reduce_prefetch_ack(ack)
|
||||
self._reduce_prefetch_ack(ack, groups)
|
||||
if _hc_timing_enabled() and ack.completed_req:
|
||||
ack.operation.final_ack_enqueue_ts = time.monotonic()
|
||||
self.ack_prefetch_queue.put(ack)
|
||||
except Empty:
|
||||
continue
|
||||
|
||||
def _reduce_prefetch_ack(self, ack: PrefetchAck) -> None:
|
||||
def _reduce_prefetch_ack(
|
||||
self, ack: PrefetchAck, groups: List[torch.distributed.ProcessGroup]
|
||||
) -> None:
|
||||
"""Synchronize all ranks to agree on a PrefetchAck."""
|
||||
if ack.completed_tokens is not None:
|
||||
# Determine the minimal successful prefix of tokens.
|
||||
_hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
completed_tokens_tensor = torch.tensor(
|
||||
ack.completed_tokens, dtype=torch.int
|
||||
)
|
||||
self._all_reduce(
|
||||
completed_tokens_tensor,
|
||||
torch.distributed.ReduceOp.MIN,
|
||||
self.prefetch_completion_sync_groups,
|
||||
groups,
|
||||
)
|
||||
ack.completed_tokens = completed_tokens_tensor.item()
|
||||
if _hc_timing_enabled():
|
||||
logger.info(
|
||||
"HC_TIMING stage=ack_reduce req=%s ms=%.1f",
|
||||
ack.operation.request_id,
|
||||
(time.monotonic() - _hc_t0) * 1e3,
|
||||
)
|
||||
|
||||
@@ -2352,6 +2352,27 @@ class SetInternalStateReqOutput(BaseReq, kw_only=True):
|
||||
updated: bool
|
||||
|
||||
|
||||
class QueryCacheHitReqInput(BaseReq, kw_only=True):
|
||||
"""G9: read-only probe of prefix-cache hit for a token sequence.
|
||||
|
||||
Sent to every DP rank of a (decode) engine; each rank answers with its own
|
||||
L1/L2 radix match plus the shared L3 storage hit continuing from it.
|
||||
"""
|
||||
|
||||
input_ids: List[int]
|
||||
extra_key: Optional[str] = None
|
||||
cache_salt: Optional[str] = None
|
||||
|
||||
|
||||
class QueryCacheHitReqOutput(BaseReq, kw_only=True):
|
||||
total_tokens: int = 0
|
||||
l1_hit_tokens: int = 0
|
||||
l2_hit_tokens: int = 0
|
||||
l3_hit_tokens: int = 0
|
||||
dp_rank: int = 0
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
class ProfileReqType(Enum):
|
||||
START_PROFILE = 1
|
||||
STOP_PROFILE = 2
|
||||
|
||||
@@ -197,6 +197,12 @@ class LoadSnapshot(msgspec.Struct, omit_defaults=True):
|
||||
num_running_reqs: int = 0
|
||||
num_waiting_reqs: int = 0
|
||||
num_waiting_uncached_tokens: int = 0
|
||||
# G9 queue-time estimation (总输入长度口径, self-consistent numerator/denominator):
|
||||
# sum of total input lengths (req.seqlen) over all pending queues, and the
|
||||
# self-learned drain rate in the same unit — completed prefills' total
|
||||
# input tokens / wall second over a sliding window.
|
||||
queued_total_input_tokens: int = 0
|
||||
drain_rate_tokens_per_s: float = 0.0
|
||||
num_used_tokens: int = 0
|
||||
num_total_tokens: int = 0
|
||||
# num_total_tokens minus tokens still awaiting a KV transfer (equal to it
|
||||
|
||||
@@ -0,0 +1,486 @@
|
||||
"""One owner rank encodes each image span and broadcasts it to the ranks that
|
||||
run the same prefill chunk; every agreement precedes the payload it guards."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from contextlib import contextmanager
|
||||
from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence, Tuple
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
disable_symmetric_memory_context,
|
||||
restore_symmetric_memory_context,
|
||||
)
|
||||
from sglang.srt.mem_cache.multimodal_cache import EmbeddingResult, MultiModalStaticCache
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SpanKey = Tuple[Optional[int], int]
|
||||
SpanEncoder = Callable[[List[Any]], torch.Tensor | List[torch.Tensor]]
|
||||
SpanSignature = Callable[[Any, int], Tuple[Any, ...]]
|
||||
|
||||
LOCAL_HIT = 0
|
||||
OWNER_CACHE_BROADCAST = 1
|
||||
OWNER_ENCODE_BROADCAST = 2
|
||||
|
||||
PHASE_PREPARE = "prepare"
|
||||
PHASE_FEATURES = "features"
|
||||
PHASE_FINALIZE = "finalize"
|
||||
|
||||
|
||||
class MmOwnerProtocolError(RuntimeError):
|
||||
"""Raised with identical text on every group member after a group-agreed failure."""
|
||||
|
||||
|
||||
class ImageSpanRequest(msgspec.Struct, frozen=True):
|
||||
hash: Optional[int]
|
||||
span_len: int
|
||||
item: Any
|
||||
inside_chunk: bool
|
||||
duplicates: List[Any] = []
|
||||
|
||||
|
||||
class ImageSpanKey(msgspec.Struct, frozen=True):
|
||||
hash: Optional[int]
|
||||
span_len: int
|
||||
geometry: Optional[Tuple[Any, ...]]
|
||||
|
||||
|
||||
class RankManifest(msgspec.Struct, frozen=True):
|
||||
rank: int
|
||||
keys: List[ImageSpanKey]
|
||||
cached: List[bool]
|
||||
dtype: str
|
||||
width: int
|
||||
rids: List[str]
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
class OwnerPlan(msgspec.Struct, frozen=True):
|
||||
actions: List[int]
|
||||
owners: List[int]
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
class RankStatus(msgspec.Struct, frozen=True):
|
||||
rank: int
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
def select_owner_group(parallel) -> Optional[Any]:
|
||||
"""The group whose members all execute the same requests, or None when a
|
||||
single rank already encodes every image it sees."""
|
||||
replication = parallel.tp_size // parallel.attn_dp_size
|
||||
if replication <= 1:
|
||||
return None
|
||||
if parallel.attn_cp_size == 1:
|
||||
group = parallel.attn_tp_group
|
||||
elif parallel.attn_dp_size == 1 and parallel.attn_cp_size == parallel.tp_size:
|
||||
group = parallel.attn_cp_group
|
||||
else:
|
||||
return None
|
||||
return group if group.world_size == replication else None
|
||||
|
||||
|
||||
def has_owner_span_work(
|
||||
mm_inputs: Sequence[Any],
|
||||
extend_prefix_lens: Sequence[int],
|
||||
extend_seq_lens: Sequence[int],
|
||||
) -> bool:
|
||||
"""Host-side mirror of the per-image scheduling path: does any raw
|
||||
single-span image overlap the chunk on every rank of the group."""
|
||||
for mm_input, prefix_len, extend_len in zip(
|
||||
mm_inputs, extend_prefix_lens, extend_seq_lens
|
||||
):
|
||||
if mm_input is None or extend_len <= 0:
|
||||
continue
|
||||
items = [item for item in mm_input.mm_items if item is not None]
|
||||
if not items or any(
|
||||
item.precomputed_embeddings is not None or len(item.offsets) != 1
|
||||
for item in items
|
||||
):
|
||||
continue
|
||||
for item in items:
|
||||
start, end = item.offsets[0]
|
||||
if end >= prefix_len and start < prefix_len + extend_len:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class MmOwnerSession(msgspec.Struct):
|
||||
group: Any
|
||||
device: Any
|
||||
dtype: Any
|
||||
width: int
|
||||
rids: List[str]
|
||||
signature: Any
|
||||
engaged: bool
|
||||
phase: str = PHASE_PREPARE
|
||||
in_collective: bool = False
|
||||
|
||||
def resolve(
|
||||
self,
|
||||
requests: Sequence[ImageSpanRequest],
|
||||
cache: MultiModalStaticCache,
|
||||
encode: SpanEncoder,
|
||||
) -> Dict[SpanKey, torch.Tensor]:
|
||||
if not self.engaged:
|
||||
raise RuntimeError(
|
||||
"owner protocol reached for a chunk whose host metadata has no image span"
|
||||
)
|
||||
# Owners allocate different amounts than receivers, so none of these
|
||||
# buffers may come out of a symmetric pool.
|
||||
saved_context = disable_symmetric_memory_context()
|
||||
try:
|
||||
return _resolve_owner_features(self, requests, cache, encode)
|
||||
finally:
|
||||
restore_symmetric_memory_context(saved_context)
|
||||
|
||||
def features_ready(self) -> None:
|
||||
self._complete()
|
||||
self.phase = PHASE_FINALIZE
|
||||
|
||||
@contextmanager
|
||||
def uncaptured(self) -> Iterator[None]:
|
||||
# A failure inside a collective leaves the group in an unknown state;
|
||||
# no later exchange may try to agree on it.
|
||||
self.in_collective = True
|
||||
yield
|
||||
self.in_collective = False
|
||||
|
||||
@contextmanager
|
||||
def fence(self) -> Iterator[None]:
|
||||
try:
|
||||
yield
|
||||
except Exception as exc:
|
||||
self._fail(exc)
|
||||
raise
|
||||
self._complete()
|
||||
|
||||
def _fail(self, exc: BaseException) -> None:
|
||||
if (
|
||||
not self.engaged
|
||||
or self.in_collective
|
||||
or isinstance(exc, MmOwnerProtocolError)
|
||||
):
|
||||
raise exc
|
||||
text = _describe(self, self.phase, exc)
|
||||
if self.phase == PHASE_PREPARE:
|
||||
try:
|
||||
_exchange_manifest(self, _manifest(self, [], [], error=text))
|
||||
except MmOwnerProtocolError as agreed:
|
||||
raise agreed from exc
|
||||
_exchange_status(self, text, exc)
|
||||
|
||||
def _complete(self) -> None:
|
||||
if not self.engaged:
|
||||
return
|
||||
if self.phase == PHASE_PREPARE:
|
||||
raise RuntimeError(
|
||||
f"owner protocol {self.phase} completed without a manifest exchange"
|
||||
)
|
||||
error = None
|
||||
cause = None
|
||||
try:
|
||||
_synchronize(self.device)
|
||||
except Exception as exc:
|
||||
cause = exc
|
||||
error = _describe(self, self.phase, exc)
|
||||
_exchange_status(self, error, cause)
|
||||
|
||||
|
||||
def _manifest(
|
||||
session: MmOwnerSession,
|
||||
keys: List[ImageSpanKey],
|
||||
cached: List[bool],
|
||||
error: Optional[str] = None,
|
||||
) -> RankManifest:
|
||||
return RankManifest(
|
||||
rank=session.group.rank_in_group,
|
||||
keys=keys,
|
||||
cached=cached,
|
||||
dtype=str(session.dtype),
|
||||
width=session.width,
|
||||
rids=list(session.rids),
|
||||
error=error,
|
||||
)
|
||||
|
||||
|
||||
def _exchange_manifest(session: MmOwnerSession, manifest: RankManifest) -> OwnerPlan:
|
||||
group = session.group
|
||||
with session.uncaptured():
|
||||
manifests = group.all_gather_object(manifest)
|
||||
plan = _plan_or_error(session, manifests) if group.rank_in_group == 0 else None
|
||||
plan = group.broadcast_object(plan, src=0)
|
||||
session.phase = PHASE_FEATURES
|
||||
if plan.error is not None:
|
||||
raise MmOwnerProtocolError(plan.error)
|
||||
return plan
|
||||
|
||||
|
||||
def _plan_or_error(session: MmOwnerSession, manifests: List[RankManifest]) -> OwnerPlan:
|
||||
try:
|
||||
return _make_plan(manifests)
|
||||
except Exception as exc:
|
||||
return OwnerPlan(actions=[], owners=[], error=_describe(session, "plan", exc))
|
||||
|
||||
|
||||
def _exchange_status(
|
||||
session: MmOwnerSession, error: Optional[str], cause: Optional[BaseException]
|
||||
) -> None:
|
||||
with session.uncaptured():
|
||||
statuses = session.group.all_gather_object(
|
||||
RankStatus(rank=session.group.rank_in_group, error=error)
|
||||
)
|
||||
_raise_first_error(statuses, cause)
|
||||
|
||||
|
||||
def _resolve_owner_features(
|
||||
session: MmOwnerSession,
|
||||
requests: Sequence[ImageSpanRequest],
|
||||
cache: MultiModalStaticCache,
|
||||
encode: SpanEncoder,
|
||||
) -> Dict[SpanKey, torch.Tensor]:
|
||||
group = session.group
|
||||
features: Dict[SpanKey, torch.Tensor] = {}
|
||||
keys: List[ImageSpanKey] = []
|
||||
cached: List[bool] = []
|
||||
error = None
|
||||
try:
|
||||
keys, cached = _pin_local_cache(session, requests, cache, features)
|
||||
except Exception as exc:
|
||||
error = _describe(session, "manifest", exc)
|
||||
plan = _exchange_manifest(session, _manifest(session, keys, cached, error))
|
||||
|
||||
if all(action == LOCAL_HIT for action in plan.actions):
|
||||
return features
|
||||
|
||||
buffers: Dict[int, torch.Tensor] = {}
|
||||
error = None
|
||||
try:
|
||||
buffers = _prepare_transfers(session, requests, keys, plan, features, encode)
|
||||
_synchronize(session.device)
|
||||
except Exception as exc:
|
||||
error = _describe(session, "encode", exc)
|
||||
_exchange_status(session, error, None)
|
||||
|
||||
with session.uncaptured():
|
||||
for index, (action, owner) in enumerate(zip(plan.actions, plan.owners)):
|
||||
if action != LOCAL_HIT:
|
||||
group.broadcast(buffers[index], src=owner)
|
||||
|
||||
for index, key in enumerate(keys):
|
||||
if plan.actions[index] == LOCAL_HIT:
|
||||
continue
|
||||
span = buffers[index]
|
||||
features[(key.hash, key.span_len)] = span
|
||||
cache.set(key.hash, EmbeddingResult(embedding=span))
|
||||
return features
|
||||
|
||||
|
||||
def _pin_local_cache(
|
||||
session: MmOwnerSession,
|
||||
requests: Sequence[ImageSpanRequest],
|
||||
cache: MultiModalStaticCache,
|
||||
features: Dict[SpanKey, torch.Tensor],
|
||||
) -> Tuple[List[ImageSpanKey], List[bool]]:
|
||||
keys: List[ImageSpanKey] = []
|
||||
cached: List[bool] = []
|
||||
for request in requests:
|
||||
if request.hash is None:
|
||||
raise ValueError(
|
||||
f"image span of {request.span_len} tokens has no content hash"
|
||||
)
|
||||
geometry = session.signature(request.item, request.span_len)
|
||||
for duplicate in request.duplicates:
|
||||
other = session.signature(duplicate, request.span_len)
|
||||
if other != geometry:
|
||||
raise ValueError(
|
||||
f"image hash {request.hash} ({request.span_len} tokens) occurs "
|
||||
f"with different geometry: {geometry} vs {other}"
|
||||
)
|
||||
keys.append(
|
||||
ImageSpanKey(
|
||||
hash=request.hash, span_len=request.span_len, geometry=geometry
|
||||
)
|
||||
)
|
||||
span = _valid_cached_span(session, cache, request)
|
||||
if span is not None:
|
||||
features[(request.hash, request.span_len)] = span
|
||||
cached.append(span is not None)
|
||||
return keys, cached
|
||||
|
||||
|
||||
def _valid_cached_span(
|
||||
session: MmOwnerSession,
|
||||
cache: MultiModalStaticCache,
|
||||
request: ImageSpanRequest,
|
||||
) -> Optional[torch.Tensor]:
|
||||
entry = cache.get_single(request.hash)
|
||||
if entry is None:
|
||||
return None
|
||||
span = entry.embedding
|
||||
if (
|
||||
span.dim() == 2
|
||||
and span.shape[0] == request.span_len
|
||||
and span.shape[1] == session.width
|
||||
and span.dtype == session.dtype
|
||||
and span.device == session.device
|
||||
):
|
||||
return span
|
||||
logger.warning(
|
||||
"Discarding cached multimodal embedding that cannot serve the current "
|
||||
"image span: cache_key=%s expected=(%d, %d, %s) cached=(%s, %s).",
|
||||
request.hash,
|
||||
request.span_len,
|
||||
session.width,
|
||||
session.dtype,
|
||||
tuple(span.shape),
|
||||
span.dtype,
|
||||
)
|
||||
cache.free(request.hash, None)
|
||||
return None
|
||||
|
||||
|
||||
def _make_plan(manifests: List[RankManifest]) -> OwnerPlan:
|
||||
for manifest in manifests:
|
||||
if manifest.error is not None:
|
||||
return OwnerPlan(actions=[], owners=[], error=manifest.error)
|
||||
lead = manifests[0]
|
||||
for manifest in manifests[1:]:
|
||||
if (manifest.keys, manifest.dtype, manifest.width, manifest.rids) != (
|
||||
lead.keys,
|
||||
lead.dtype,
|
||||
lead.width,
|
||||
lead.rids,
|
||||
):
|
||||
return OwnerPlan(
|
||||
actions=[],
|
||||
owners=[],
|
||||
error=(
|
||||
"image manifest mismatch between group ranks 0 and "
|
||||
f"{manifest.rank}: rids={lead.rids} vs {manifest.rids}, "
|
||||
f"keys={lead.keys} vs {manifest.keys}, "
|
||||
f"dtype={lead.dtype} vs {manifest.dtype}, "
|
||||
f"width={lead.width} vs {manifest.width}"
|
||||
),
|
||||
)
|
||||
replication = len(manifests)
|
||||
actions: List[int] = []
|
||||
owners: List[int] = []
|
||||
for index, key in enumerate(lead.keys):
|
||||
owner = key.hash % replication
|
||||
if all(manifest.cached[index] for manifest in manifests):
|
||||
action = LOCAL_HIT
|
||||
elif manifests[owner].cached[index]:
|
||||
action = OWNER_CACHE_BROADCAST
|
||||
else:
|
||||
action = OWNER_ENCODE_BROADCAST
|
||||
actions.append(action)
|
||||
owners.append(owner)
|
||||
return OwnerPlan(actions=actions, owners=owners)
|
||||
|
||||
|
||||
def _prepare_transfers(
|
||||
session: MmOwnerSession,
|
||||
requests: Sequence[ImageSpanRequest],
|
||||
keys: List[ImageSpanKey],
|
||||
plan: OwnerPlan,
|
||||
features: Dict[SpanKey, torch.Tensor],
|
||||
encode: SpanEncoder,
|
||||
) -> Dict[int, torch.Tensor]:
|
||||
rank = session.group.rank_in_group
|
||||
buffers: Dict[int, torch.Tensor] = {}
|
||||
owned: List[int] = []
|
||||
for index, (action, owner) in enumerate(zip(plan.actions, plan.owners)):
|
||||
if action == LOCAL_HIT:
|
||||
continue
|
||||
if owner != rank:
|
||||
try:
|
||||
buffers[index] = _new_span_buffer(session, keys[index])
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
f"receive buffer for image hash {keys[index].hash} shape "
|
||||
f"{(keys[index].span_len, session.width)} {session.dtype} "
|
||||
f"failed: {type(exc).__name__}: {exc}"
|
||||
) from exc
|
||||
elif action == OWNER_CACHE_BROADCAST:
|
||||
key = (keys[index].hash, keys[index].span_len)
|
||||
buffers[index] = features[key].contiguous()
|
||||
else:
|
||||
owned.append(index)
|
||||
if owned:
|
||||
owned_hashes = [keys[index].hash for index in owned]
|
||||
try:
|
||||
encoded = encode([requests[index].item for index in owned])
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
f"owner encode of image hashes {owned_hashes} failed: "
|
||||
f"{type(exc).__name__}: {exc}"
|
||||
) from exc
|
||||
spans = _split_spans(encoded, [keys[index].span_len for index in owned])
|
||||
for index, span in zip(owned, spans):
|
||||
buffers[index] = _validated_span(session, keys[index], span)
|
||||
return buffers
|
||||
|
||||
|
||||
def _new_span_buffer(session: MmOwnerSession, key: ImageSpanKey) -> torch.Tensor:
|
||||
return torch.empty(
|
||||
(key.span_len, session.width), device=session.device, dtype=session.dtype
|
||||
)
|
||||
|
||||
|
||||
def _split_spans(
|
||||
encoded: torch.Tensor | List[torch.Tensor], span_lens: List[int]
|
||||
) -> List[torch.Tensor]:
|
||||
if isinstance(encoded, list):
|
||||
if len(encoded) != len(span_lens):
|
||||
raise ValueError(
|
||||
f"encoder returned {len(encoded)} spans for {len(span_lens)} images"
|
||||
)
|
||||
return [span.reshape(-1, span.shape[-1]) for span in encoded]
|
||||
encoded = encoded.reshape(-1, encoded.shape[-1])
|
||||
if encoded.shape[0] != sum(span_lens):
|
||||
raise ValueError(
|
||||
f"encoder returned {encoded.shape[0]} rows for spans of {span_lens}"
|
||||
)
|
||||
return list(torch.split(encoded, span_lens, dim=0))
|
||||
|
||||
|
||||
def _validated_span(
|
||||
session: MmOwnerSession, key: ImageSpanKey, span: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
expected = (key.span_len, session.width)
|
||||
if tuple(span.shape) != expected or span.dtype != session.dtype:
|
||||
raise ValueError(
|
||||
f"encoded span for hash={key.hash} has shape {tuple(span.shape)} "
|
||||
f"dtype {span.dtype}; expected {expected} {session.dtype}"
|
||||
)
|
||||
if span.device != session.device:
|
||||
span = span.to(session.device)
|
||||
return span.contiguous()
|
||||
|
||||
|
||||
def _synchronize(device) -> None:
|
||||
if device.type == "cuda":
|
||||
torch.cuda.current_stream(device).synchronize()
|
||||
|
||||
|
||||
def _describe(session: MmOwnerSession, stage: str, exc: BaseException) -> str:
|
||||
return (
|
||||
f"multimodal owner protocol failed during {stage} on group rank "
|
||||
f"{session.group.rank_in_group} (global rank "
|
||||
f"{session.group.ranks[session.group.rank_in_group]}, rids={list(session.rids)}): "
|
||||
f"{type(exc).__name__}: {exc}"
|
||||
)
|
||||
|
||||
|
||||
def _raise_first_error(
|
||||
statuses: List[RankStatus], cause: Optional[BaseException]
|
||||
) -> None:
|
||||
for status in statuses:
|
||||
if status.error is not None:
|
||||
raise MmOwnerProtocolError(status.error) from cause
|
||||
@@ -5,6 +5,7 @@ from typing import Callable, Dict, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.managers.mm_owner_embedding import ImageSpanRequest, MmOwnerSession
|
||||
from sglang.srt.managers.schedule_batch import MultimodalDataItem
|
||||
from sglang.srt.mem_cache.multimodal_cache import EmbeddingResult, MultiModalStaticCache
|
||||
from sglang.srt.multimodal.evs import EVSEmbeddingResult
|
||||
@@ -364,41 +365,24 @@ def _batch_encode_per_image_misses(
|
||||
unique_misses: Dict[Tuple[Optional[int], int], Tuple[MultimodalDataItem, int]] = {}
|
||||
hash_to_embedding: Dict[Tuple[Optional[int], int], torch.Tensor] = {}
|
||||
|
||||
# Phase 1a: find overlapping items per request and collect cache misses
|
||||
for req_info in per_image_requests:
|
||||
chunk_start = req_info.extend_prefix_len
|
||||
chunk_end = chunk_start + req_info.extend_seq_len # exclusive
|
||||
overlapping = []
|
||||
if req_info.extend_seq_len > 0:
|
||||
for idx, item in enumerate(req_info.items):
|
||||
token_count = _item_overlap(item, chunk_start, chunk_end)
|
||||
if token_count is not None:
|
||||
overlapping.append((idx, item, token_count))
|
||||
req_info.overlapping = overlapping
|
||||
|
||||
for _idx, item, expected_token_count in overlapping:
|
||||
cache_key = (item.hash, expected_token_count)
|
||||
if cache_key in hash_to_embedding:
|
||||
# Phase 1a: collect cache misses over the unique overlapping spans
|
||||
for span in _collect_image_span_requests(per_image_requests):
|
||||
cache_key = (span.hash, span.span_len)
|
||||
cached = embedding_cache.get_single(span.hash)
|
||||
if cached is not None:
|
||||
cached_embedding = cached.embedding
|
||||
cached_token_count = _embedding_token_count(cached_embedding)
|
||||
if cached_token_count == span.span_len:
|
||||
hash_to_embedding[cache_key] = cached_embedding
|
||||
continue
|
||||
cached = embedding_cache.get_single(item.hash)
|
||||
if cached is not None:
|
||||
cached_embedding = cached.embedding
|
||||
cached_token_count = _embedding_token_count(cached_embedding)
|
||||
if cached_token_count == expected_token_count:
|
||||
hash_to_embedding[cache_key] = cached_embedding
|
||||
else:
|
||||
_discard_mismatched_cached_embedding(
|
||||
item.hash, expected_token_count, cached_token_count
|
||||
)
|
||||
unique_misses[cache_key] = (item, expected_token_count)
|
||||
elif cache_key not in unique_misses:
|
||||
if (
|
||||
item.offsets[0][0] >= chunk_start
|
||||
and item.offsets[-1][1] < chunk_end
|
||||
and item.can_defer_cuda_ipc_feature_reconstruction()
|
||||
):
|
||||
item.model_specific_data[BORROW_CUDA_IPC_FEATURE_KEY] = True
|
||||
unique_misses[cache_key] = (item, expected_token_count)
|
||||
_discard_mismatched_cached_embedding(
|
||||
span.hash, span.span_len, cached_token_count
|
||||
)
|
||||
elif (
|
||||
span.inside_chunk and span.item.can_defer_cuda_ipc_feature_reconstruction()
|
||||
):
|
||||
span.item.model_specific_data[BORROW_CUDA_IPC_FEATURE_KEY] = True
|
||||
unique_misses[cache_key] = (span.item, span.span_len)
|
||||
|
||||
# Phase 1b: single ViT call for all unique cache misses
|
||||
if unique_misses:
|
||||
@@ -437,6 +421,54 @@ def _batch_encode_per_image_misses(
|
||||
return hash_to_embedding
|
||||
|
||||
|
||||
def _collect_image_span_requests(
|
||||
per_image_requests: List[PerImageRequestInfo],
|
||||
) -> List[ImageSpanRequest]:
|
||||
spans: Dict[
|
||||
Tuple[Optional[int], int],
|
||||
Tuple[MultimodalDataItem, bool, List[MultimodalDataItem]],
|
||||
] = {}
|
||||
for req_info in per_image_requests:
|
||||
chunk_start = req_info.extend_prefix_len
|
||||
chunk_end = chunk_start + req_info.extend_seq_len # exclusive
|
||||
overlapping = []
|
||||
if req_info.extend_seq_len > 0:
|
||||
for idx, (item, (start, end)) in enumerate(
|
||||
zip(req_info.items, req_info.items_offset)
|
||||
):
|
||||
if end >= chunk_start and start < chunk_end:
|
||||
overlapping.append((idx, item, start, end))
|
||||
|
||||
for _idx, item, start, end in overlapping:
|
||||
cache_key = (item.hash, end - start + 1)
|
||||
if cache_key in spans:
|
||||
spans[cache_key][2].append(item)
|
||||
continue
|
||||
spans[cache_key] = (item, start >= chunk_start and end < chunk_end, [])
|
||||
req_info.overlapping = [
|
||||
(idx, item, end - start + 1) for idx, item, start, end in overlapping
|
||||
]
|
||||
return [
|
||||
ImageSpanRequest(
|
||||
hash=item_hash,
|
||||
span_len=span_len,
|
||||
item=item,
|
||||
inside_chunk=inside_chunk,
|
||||
duplicates=duplicates,
|
||||
)
|
||||
for (item_hash, span_len), (item, inside_chunk, duplicates) in spans.items()
|
||||
]
|
||||
|
||||
|
||||
def _owner_span_encoder(data_embedding_func: DataEmbeddingFunc, device: torch.device):
|
||||
def encode(items: List[MultimodalDataItem]):
|
||||
if not _can_skip_pre_embed_feature_move(data_embedding_func):
|
||||
_move_items_to_device(items, device, data_embedding_func)
|
||||
return data_embedding_func(items)
|
||||
|
||||
return encode
|
||||
|
||||
|
||||
def _get_chunked_embedding_by_item(
|
||||
data_embedding_func: DataEmbeddingFunc,
|
||||
embedding_items_per_req: List[MultimodalDataItem],
|
||||
@@ -546,6 +578,7 @@ def _get_chunked_prefill_embedding(
|
||||
extend_length: List[int],
|
||||
items_offset_list: List[List[Tuple[int, int]]],
|
||||
input_ids: torch.Tensor,
|
||||
mm_owner: Optional[MmOwnerSession] = None,
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor]:
|
||||
"""
|
||||
Chunked prefill embedding: encode items across all requests and extract
|
||||
@@ -613,7 +646,22 @@ def _get_chunked_prefill_embedding(
|
||||
|
||||
# Phase 1: batch encode all per-image cache misses in ONE ViT call
|
||||
hash_to_embedding: Dict[Tuple[Optional[int], int], torch.Tensor] = {}
|
||||
if per_image_requests:
|
||||
if per_image_requests and mm_owner is not None:
|
||||
# The owner protocol must see every overlapping span before any local
|
||||
# cache filtering: a rank-local hit can never skip a group collective.
|
||||
span_requests = _collect_image_span_requests(per_image_requests)
|
||||
if mm_owner.engaged:
|
||||
hash_to_embedding = mm_owner.resolve(
|
||||
span_requests,
|
||||
cache=embedding_cache,
|
||||
encode=_owner_span_encoder(data_embedding_func, device),
|
||||
)
|
||||
elif span_requests:
|
||||
raise RuntimeError(
|
||||
"owner eligibility saw no image span in this chunk, but "
|
||||
f"scheduling found {len(span_requests)}"
|
||||
)
|
||||
elif per_image_requests:
|
||||
hash_to_embedding = _batch_encode_per_image_misses(
|
||||
data_embedding_func, per_image_requests, device
|
||||
)
|
||||
@@ -717,6 +765,7 @@ def get_embedding_and_mask(
|
||||
prefix_length: List[int],
|
||||
extend_length: List[int],
|
||||
items_offset_list: List[List[Tuple[int, int]]],
|
||||
mm_owner: Optional[MmOwnerSession] = None,
|
||||
) -> Tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor]:
|
||||
"""
|
||||
Generate multimodal embeddings and create a mask for identifying their positions in the input sequence.
|
||||
@@ -757,6 +806,7 @@ def get_embedding_and_mask(
|
||||
extend_length,
|
||||
items_offset_list,
|
||||
input_ids,
|
||||
mm_owner=mm_owner,
|
||||
)
|
||||
if embedding is None:
|
||||
return None, None, input_ids
|
||||
|
||||
@@ -11,6 +11,7 @@ import sys
|
||||
from abc import abstractmethod
|
||||
from array import array
|
||||
from collections import defaultdict
|
||||
from contextlib import nullcontext
|
||||
from multiprocessing import shared_memory
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
@@ -26,6 +27,7 @@ from sglang.srt.managers.io_struct import (
|
||||
TokenizedEmbeddingReqInput,
|
||||
TokenizedGenerateReqInput,
|
||||
)
|
||||
from sglang.srt.managers.mm_owner_embedding import MmOwnerSession
|
||||
|
||||
# Preserve the existing initialization import for downstream callers.
|
||||
from sglang.srt.managers.mm_schedule import (
|
||||
@@ -448,6 +450,7 @@ def embed_mm_inputs(
|
||||
data_embedding_func_mapping: Dict[Modality, DataEmbeddingFunc] = None,
|
||||
placeholder_tokens: dict[Modality, List[int]] = None,
|
||||
use_deepstack: Dict[Modality, bool] = {},
|
||||
mm_owner: Optional[MmOwnerSession] = None,
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""
|
||||
Embed multimodal inputs and integrate them with text token embeddings.
|
||||
@@ -529,6 +532,7 @@ def embed_mm_inputs(
|
||||
prefix_length=extend_prefix_lens,
|
||||
extend_length=extend_seq_lens,
|
||||
items_offset_list=items_offsets,
|
||||
mm_owner=mm_owner,
|
||||
)
|
||||
|
||||
if use_deepstack.get(modality, None) and embedding is not None:
|
||||
@@ -549,7 +553,12 @@ def embed_mm_inputs(
|
||||
# filled with the hash values of the multimodal for the prefix matching in the radix attention.
|
||||
# There values are useless because their embeddings will be replaced by vision embeddings anyway.
|
||||
input_ids.clamp_(min=0, max=vocab_size - 1)
|
||||
input_embeds = input_embedding(input_ids)
|
||||
if mm_owner is not None:
|
||||
# The text embedding may all-reduce across TP; a rank-local failure in
|
||||
# feature preparation has to be agreed on before any rank enters it.
|
||||
mm_owner.features_ready()
|
||||
with mm_owner.uncaptured() if mm_owner is not None else nullcontext():
|
||||
input_embeds = input_embedding(input_ids)
|
||||
|
||||
# deepstack embedding
|
||||
if use_deepstack:
|
||||
@@ -576,7 +585,9 @@ def embed_mm_inputs(
|
||||
_scatter_mm_embedding(dest=input_embeds, mask=mask, src=embedding)
|
||||
if use_deepstack.get(modality, None):
|
||||
_scatter_mm_embedding(
|
||||
dest=input_deepstack_embeds, mask=mask, src=deepstack_embeddings[i]
|
||||
dest=input_deepstack_embeds,
|
||||
mask=mask,
|
||||
src=deepstack_embeddings[i],
|
||||
)
|
||||
|
||||
return input_embeds, other_info
|
||||
|
||||
@@ -1093,6 +1093,15 @@ class PrefillAdder:
|
||||
else AddReqResult.CONTINUE
|
||||
)
|
||||
|
||||
def can_share_extend_batch(self, req: Req) -> bool:
|
||||
# Token embedding overrides embed the batch's raw input_ids before the
|
||||
# model runs, and that lookup cannot index multimodal placeholder hash IDs.
|
||||
if req.positional_embed_overrides is not None:
|
||||
return all(r.multimodal_inputs is None for r in self.can_run_list)
|
||||
if req.multimodal_inputs is not None:
|
||||
return all(r.positional_embed_overrides is None for r in self.can_run_list)
|
||||
return True
|
||||
|
||||
def add_chunked_req(self, req: Req):
|
||||
if self.dllm_config is not None:
|
||||
_rem_tokens = self._get_dllm_remain_tokens(req)
|
||||
|
||||
@@ -164,6 +164,8 @@ from sglang.srt.managers.io_struct import (
|
||||
PauseGenerationReqInput,
|
||||
PdRoleSwitchReqInput,
|
||||
ProfileReq,
|
||||
QueryCacheHitReqInput,
|
||||
QueryCacheHitReqOutput,
|
||||
ReleaseMemoryOccupationReqInput,
|
||||
RemoveExternalCorpusReqInput,
|
||||
RemoveExternalCorpusReqOutput,
|
||||
@@ -1853,6 +1855,7 @@ class Scheduler(
|
||||
(ShutdownReq, self.handle_shutdown),
|
||||
(GetInternalStateReq, self.get_internal_state),
|
||||
(SetInternalStateReq, self.set_internal_state),
|
||||
(QueryCacheHitReqInput, self.handle_query_cache_hit),
|
||||
(RpcReqInput, self.handle_rpc_request),
|
||||
(ExpertDistributionReq, self.expert_distribution_handle),
|
||||
(LoadLoRAAdapterReqInput, self.load_lora_adapter),
|
||||
@@ -2180,9 +2183,13 @@ class Scheduler(
|
||||
vmm_errors = self._materialize_cuda_vmm_inputs(recv_req)
|
||||
|
||||
# Skip health check when server is busy — ongoing requests already carry health info.
|
||||
if is_health_check_generate_req(recv_req) and not self.is_fully_idle(
|
||||
for_health_check=True
|
||||
):
|
||||
# NOTE: the admit/skip decision must be identical on every CP/TP rank.
|
||||
# is_fully_idle() includes rank-local hicache drain queues, which diverge
|
||||
# across ranks right after activity; a divergent decision lets one rank
|
||||
# dispatch the health-check generate while others piggyback-skip, breaking
|
||||
# collective ordering (deadlock: one rank blocks in the hicache drain
|
||||
# all_reduce while another waits in the CP request broadcast).
|
||||
if is_health_check_generate_req(recv_req) and not self.is_sched_idle_cp_symmetric():
|
||||
self.return_health_check_ipcs.append(
|
||||
getattr(recv_req, "http_worker_ipc", None)
|
||||
)
|
||||
@@ -2516,6 +2523,11 @@ class Scheduler(
|
||||
self._prev_step: Optional[Tuple[int, float, bool]] = None
|
||||
self._prev_prefill_end_ts: Optional[float] = None
|
||||
self._sched_idled = False
|
||||
# G9: sliding-window drain events (monotonic_ts, total_input_tokens) for
|
||||
# the queue-time estimator. Recorded once per request when its final
|
||||
# prefill chunk completes (batch_result_processor), in the same
|
||||
# total-input-length unit as LoadSnapshot.queued_total_input_tokens.
|
||||
self.prefill_drain_window: deque = deque()
|
||||
self.load_inquirer = SchedulerLoadInquirer(
|
||||
disaggregation_mode=self.disaggregation_mode,
|
||||
server_args=self.server_args,
|
||||
@@ -2555,8 +2567,35 @@ class Scheduler(
|
||||
),
|
||||
get_total_prefill_busy_us=lambda: self.total_prefill_busy_us,
|
||||
get_decode_moment_totals=lambda: self.decode_moment_totals,
|
||||
get_prefill_drain_rate=self.get_prefill_drain_rate,
|
||||
)
|
||||
|
||||
PREFILL_DRAIN_WINDOW_S = 60.0
|
||||
|
||||
def record_prefill_drain(self, total_input_tokens: int) -> None:
|
||||
"""G9: one drain event per request whose final prefill chunk completed.
|
||||
|
||||
Tokens are the request's total input length (seqlen), matching the
|
||||
queued_total_input_tokens numerator so the router's
|
||||
wait = queued_tokens / drain_rate is unit-consistent (决策②).
|
||||
"""
|
||||
now = time.monotonic()
|
||||
self.prefill_drain_window.append((now, total_input_tokens))
|
||||
cutoff = now - self.PREFILL_DRAIN_WINDOW_S
|
||||
while self.prefill_drain_window and self.prefill_drain_window[0][0] < cutoff:
|
||||
self.prefill_drain_window.popleft()
|
||||
|
||||
def get_prefill_drain_rate(self) -> float:
|
||||
"""Completed prefill input tokens per wall second over the window."""
|
||||
now = time.monotonic()
|
||||
cutoff = now - self.PREFILL_DRAIN_WINDOW_S
|
||||
while self.prefill_drain_window and self.prefill_drain_window[0][0] < cutoff:
|
||||
self.prefill_drain_window.popleft()
|
||||
if not self.prefill_drain_window:
|
||||
return 0.0
|
||||
span = max(now - self.prefill_drain_window[0][0], 1.0)
|
||||
return sum(tokens for _, tokens in self.prefill_drain_window) / span
|
||||
|
||||
def init_output_streamer(self) -> None:
|
||||
self.output_streamer = self.get_output_streamer_class()(
|
||||
send_to_detokenizer=self.ipc_channels.send_to_detokenizer,
|
||||
@@ -2609,6 +2648,7 @@ class Scheduler(
|
||||
output_streamer=self.output_streamer,
|
||||
beam_coordinator=self.beam_coordinator,
|
||||
abort_request=self.abort_request,
|
||||
record_prefill_drain=self.record_prefill_drain,
|
||||
)
|
||||
|
||||
def init_internal_state(self) -> None:
|
||||
@@ -4140,6 +4180,8 @@ class Scheduler(
|
||||
break
|
||||
if self.enable_lora and not self.can_schedule_lora_req(req, running_loras):
|
||||
continue
|
||||
if not adder.can_share_extend_batch(req):
|
||||
break
|
||||
|
||||
# A forward batch runs one pooling mode, so setwise readout requests
|
||||
# (token_indices_to_pool) cannot share a batch with last-token ones.
|
||||
@@ -5247,6 +5289,31 @@ class Scheduler(
|
||||
else:
|
||||
self.metrics_reporter.record_scheduler_active(time.monotonic_ns())
|
||||
|
||||
def is_sched_idle_cp_symmetric(self) -> bool:
|
||||
"""Idle check using only state that is identical across CP/TP ranks.
|
||||
|
||||
Request/batch/queue state is collectively maintained (requests arrive
|
||||
via broadcast, batches are collectively scheduled), so every rank
|
||||
computes the same result. Rank-local hicache drain and disagg transfer
|
||||
queues are deliberately excluded: those are exactly the terms that
|
||||
diverge across ranks and caused the CP health-check deadlock
|
||||
(hicache drain all_reduce vs CP request broadcast cross-collective
|
||||
wait, seen on cp2/cp4 + hicache L3 right after router health checks).
|
||||
|
||||
Used only for health-check admission; all other idle logic keeps using
|
||||
is_fully_idle().
|
||||
"""
|
||||
return (
|
||||
self.running_batch.is_empty()
|
||||
and self.chunked_req is None
|
||||
and not self.dllm_manager.any_staging_reqs()
|
||||
and (self.last_batch is None or self.last_batch.is_empty())
|
||||
and (not self.enable_overlap or len(self.result_queue) == 0)
|
||||
and self._pp_microbatches_drained()
|
||||
and len(self.waiting_queue) == 0
|
||||
and len(self.grammar_manager.grammar_queue) == 0
|
||||
)
|
||||
|
||||
def is_fully_idle(self, for_health_check=False, ignore_waiting=False) -> bool:
|
||||
# Health check piggybacks on running requests in process_output.
|
||||
# Only running_batch + waiting_queue guarantee active GPU processing;
|
||||
@@ -5459,6 +5526,86 @@ class Scheduler(
|
||||
success = False
|
||||
return success
|
||||
|
||||
def handle_query_cache_hit(self, recv_req: QueryCacheHitReqInput):
|
||||
"""G9: read-only prefix-hit probe (L1/L2 radix + L3 storage).
|
||||
|
||||
The router calls this (via /query_cache_hit on the decode engine) to
|
||||
decide conditional PD-bypass. Runs inside the scheduler event loop so
|
||||
radix/storage state is consistent, and reuses the exact admission-time
|
||||
match + L3 query logic (mirrors DecodeHiCachePreallocMixin
|
||||
._build_decode_prefix_match) without creating a Req or mutating
|
||||
request state. Node access metadata (LRU) may be touched; a hit found
|
||||
here can still be evicted before the real request arrives (inherent
|
||||
TOCTOU, acceptable for an estimate).
|
||||
"""
|
||||
from sglang.srt.mem_cache.base_prefix_cache import MatchPrefixParams
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
|
||||
total = len(recv_req.input_ids)
|
||||
# 树节点键是 array("q"),RadixKey.match_at 断言两侧类型一致——
|
||||
# 查询侧必须同样用 array,否则树非空时匹配直接炸(2026-10-11 实测)。
|
||||
from array import array
|
||||
|
||||
query_ids = array("q", recv_req.input_ids)
|
||||
try:
|
||||
match_result = self.tree_cache.match_prefix(
|
||||
MatchPrefixParams(
|
||||
key=RadixKey(
|
||||
token_ids=query_ids,
|
||||
extra_key=recv_req.extra_key,
|
||||
cache_salt=recv_req.cache_salt,
|
||||
)
|
||||
)
|
||||
)
|
||||
l1_hit = len(match_result.device_indices)
|
||||
l2_hit = getattr(match_result, "host_hit_length", 0) or 0
|
||||
l3_hit = 0
|
||||
last_host_node = getattr(match_result, "last_host_node", None)
|
||||
# L3 查询条件:本调度器挂了 hicache 存储后端即可(P 节点 HiCache
|
||||
# write_through/write_back 场景),不必是 decode-hicache 模式。
|
||||
# SWA 模型的 decode radix 与 hicache 互斥(kv_cache_builder 硬限制),
|
||||
# 若只认 enable_decode_hicache,P 节点永远查不到 L3。
|
||||
can_query_l3 = getattr(self, "enable_decode_hicache", False) or getattr(
|
||||
self, "enable_hicache_storage", False
|
||||
)
|
||||
if (
|
||||
can_query_l3
|
||||
and last_host_node is not None
|
||||
and hasattr(self.tree_cache, "query_storage_hit_length")
|
||||
and (
|
||||
self.tree_cache.is_backuped(last_host_node)
|
||||
or self.tree_cache.is_root(last_host_node)
|
||||
)
|
||||
):
|
||||
matched_len = l1_hit + l2_hit
|
||||
suffix_tokens = query_ids[matched_len:]
|
||||
last_hash = self.tree_cache.get_last_hash_value(last_host_node)
|
||||
prefix_keys = (
|
||||
self.tree_cache.get_prefix_hash_values(last_host_node)
|
||||
if getattr(
|
||||
self.tree_cache, "hicache_storage_pass_prefix_keys", False
|
||||
)
|
||||
else None
|
||||
)
|
||||
l3_hit = self.tree_cache.query_storage_hit_length(
|
||||
last_host_node,
|
||||
suffix_tokens,
|
||||
last_hash,
|
||||
prefix_keys,
|
||||
extra_key=recv_req.extra_key,
|
||||
cache_salt=recv_req.cache_salt,
|
||||
)
|
||||
return QueryCacheHitReqOutput(
|
||||
total_tokens=total,
|
||||
l1_hit_tokens=l1_hit,
|
||||
l2_hit_tokens=l2_hit,
|
||||
l3_hit_tokens=l3_hit,
|
||||
dp_rank=int(get_parallel().dp_rank or 0),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"handle_query_cache_hit failed: {e}")
|
||||
return QueryCacheHitReqOutput(total_tokens=total, error=str(e))
|
||||
|
||||
def get_internal_state(self, recv_req: GetInternalStateReq):
|
||||
return self.internal_state.get_internal_state(recv_req)
|
||||
|
||||
|
||||
@@ -113,6 +113,8 @@ class SchedulerBatchResultProcessor:
|
||||
output_streamer: SchedulerOutputStreamer
|
||||
beam_coordinator: BeamCoordinator
|
||||
abort_request: Callable
|
||||
# G9: scheduler-side drain-event recorder for the queue-time estimator.
|
||||
record_prefill_drain: Callable
|
||||
|
||||
def process_batch_result_prebuilt(self, batch: ScheduleBatch):
|
||||
assert self.disaggregation_mode == DisaggregationMode.DECODE
|
||||
@@ -350,6 +352,9 @@ class SchedulerBatchResultProcessor:
|
||||
|
||||
if req.inflight_middle_chunks <= 0:
|
||||
req.time_stats.set_prefill_finished_time()
|
||||
# G9: exactly once per request (final chunk only) — feed the
|
||||
# queue-time estimator with the drained total input length.
|
||||
self.record_prefill_drain(req.seqlen)
|
||||
|
||||
if discard_finish_reason is not None:
|
||||
req.to_finish = discard_finish_reason
|
||||
|
||||
@@ -54,6 +54,7 @@ class SchedulerLoadInquirer:
|
||||
get_total_prefill_uncached_tokens: Callable
|
||||
get_total_prefill_busy_us: Callable
|
||||
get_decode_moment_totals: Callable
|
||||
get_prefill_drain_rate: Callable
|
||||
|
||||
def _get_num_pending_tokens(self, chunk_deduct: int = 0) -> int:
|
||||
"""Get the total number of tokens pending prefill.
|
||||
@@ -128,9 +129,10 @@ class SchedulerLoadInquirer:
|
||||
num_used_tokens, kv_token_usage = (
|
||||
self.pool_stats_observer.get_pool_stats().get_kv_token_stats()
|
||||
)
|
||||
num_total_tokens = num_used_tokens + sum(
|
||||
queued_total_input_tokens = sum(
|
||||
req.seqlen for queue in pending_token_queues for req in queue
|
||||
)
|
||||
num_total_tokens = num_used_tokens + queued_total_input_tokens
|
||||
num_active_tokens = max(0, num_total_tokens - awaiting_kv_tokens)
|
||||
|
||||
memory = None
|
||||
@@ -220,6 +222,8 @@ class SchedulerLoadInquirer:
|
||||
num_running_reqs=num_running_reqs,
|
||||
num_waiting_reqs=num_waiting_reqs,
|
||||
num_waiting_uncached_tokens=self.get_num_waiting_uncached_tokens(),
|
||||
queued_total_input_tokens=queued_total_input_tokens,
|
||||
drain_rate_tokens_per_s=round(self.get_prefill_drain_rate(), 2),
|
||||
num_used_tokens=num_used_tokens,
|
||||
num_total_tokens=num_total_tokens,
|
||||
num_active_tokens=num_active_tokens,
|
||||
|
||||
@@ -58,6 +58,8 @@ from sglang.srt.managers.io_struct import (
|
||||
ProfileReq,
|
||||
ProfileReqOutput,
|
||||
ProfileReqType,
|
||||
QueryCacheHitReqInput,
|
||||
QueryCacheHitReqOutput,
|
||||
ReleaseMemoryOccupationReqInput,
|
||||
ReleaseMemoryOccupationReqOutput,
|
||||
RemoveExternalCorpusReqInput,
|
||||
@@ -133,6 +135,7 @@ _COMMUNICATOR_SPECS = [
|
||||
("detach_hicache_storage", DetachHiCacheStorageReqOutput),
|
||||
("profile", ProfileReqOutput),
|
||||
("get_internal_state", GetInternalStateReqOutput),
|
||||
("query_cache_hit", QueryCacheHitReqOutput),
|
||||
("set_internal_state", SetInternalStateReqOutput),
|
||||
("expert_distribution", ExpertDistributionReqOutput),
|
||||
("begin_weight_update", BeginWeightUpdateReqOutput),
|
||||
@@ -973,6 +976,37 @@ class TokenizerControlMixin:
|
||||
self.auto_create_handle_loop()
|
||||
return await self.dumper_control_communicator(obj)
|
||||
|
||||
async def query_cache_hit(
|
||||
self: TokenizerManager, obj: QueryCacheHitReqInput
|
||||
) -> QueryCacheHitReqOutput:
|
||||
"""G9: fan the probe out to all DP ranks and keep the best hit.
|
||||
|
||||
The decode node's DP controller routes new requests to any rank, so
|
||||
the bypass decision uses the best per-rank L1/L2 (L3 is shared); the
|
||||
real request still goes through normal dispatch afterwards.
|
||||
"""
|
||||
self.auto_create_handle_loop()
|
||||
results: List[QueryCacheHitReqOutput] = await self.query_cache_hit_communicator(
|
||||
obj
|
||||
)
|
||||
ok = [r for r in results if r.error is None]
|
||||
if not ok:
|
||||
msg = (
|
||||
"; ".join(r.error or "empty response" for r in results)
|
||||
or "no scheduler responded"
|
||||
)
|
||||
return QueryCacheHitReqOutput(total_tokens=len(obj.input_ids), error=msg)
|
||||
best = max(
|
||||
ok, key=lambda r: r.l1_hit_tokens + r.l2_hit_tokens + r.l3_hit_tokens
|
||||
)
|
||||
return QueryCacheHitReqOutput(
|
||||
total_tokens=best.total_tokens,
|
||||
l1_hit_tokens=best.l1_hit_tokens,
|
||||
l2_hit_tokens=best.l2_hit_tokens,
|
||||
l3_hit_tokens=best.l3_hit_tokens,
|
||||
dp_rank=best.dp_rank,
|
||||
)
|
||||
|
||||
async def get_loads(
|
||||
self: TokenizerManager,
|
||||
include: Optional[List[str]] = None,
|
||||
|
||||
@@ -1345,6 +1345,16 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
raise ValueError(
|
||||
"encoder SWA replay cannot return cached prompt logprobs"
|
||||
)
|
||||
requests_embed_overrides = obj.positional_embed_overrides is not None or (
|
||||
isinstance(obj, EmbeddingReqInput)
|
||||
and obj.embed_overrides is not None
|
||||
and obj.embed_override_token_id is not None
|
||||
)
|
||||
if requests_embed_overrides and obj.contains_mm_input():
|
||||
raise ValueError(
|
||||
"embedding overrides cannot be combined with image, video, or audio "
|
||||
"inputs"
|
||||
)
|
||||
# A decision layout indexes the whole prompt, so it is checked before any
|
||||
# truncation below could cut the prompt.
|
||||
self._validate_joint_schema_request(obj, input_ids)
|
||||
|
||||
@@ -870,3 +870,15 @@ class PureSWATokenToKVPoolAllocator(SWATokenToKVPoolAllocator):
|
||||
|
||||
def is_swa_req_ring(allocator) -> bool:
|
||||
return isinstance(allocator, SWATokenToKVPoolAllocator) and allocator.swa_req_ring
|
||||
|
||||
|
||||
def swa_kept_out_of_tree(allocator) -> bool:
|
||||
"""Whether radix-tree nodes do not own the SWA state: a per-request SWA ring,
|
||||
or the DeepSeek-V4.1 encoder-replay request window, which replay rebuilds on a
|
||||
prefix hit. SWA bookkeeping must not gate prefix matching for either."""
|
||||
if not isinstance(allocator, SWATokenToKVPoolAllocator):
|
||||
return False
|
||||
if allocator.swa_req_ring:
|
||||
return True
|
||||
kvcache = getattr(allocator, "_kvcache", None)
|
||||
return getattr(kvcache, "request_window", None) is not None
|
||||
|
||||
@@ -20,7 +20,7 @@ if TYPE_CHECKING:
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Max pages per batched storage IO call.
|
||||
STORAGE_BATCH_SIZE = 128
|
||||
STORAGE_BATCH_SIZE = envs.SGLANG_HICACHE_STORAGE_BATCH_SIZE.get()
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -15,6 +15,7 @@ import torch
|
||||
|
||||
from sglang.srt.managers.cache_controller import (
|
||||
CacheOperation,
|
||||
_hc_timing_enabled,
|
||||
)
|
||||
from sglang.srt.managers.cache_controller import (
|
||||
HiCacheController as BaseHiCacheController,
|
||||
@@ -1332,7 +1333,7 @@ class HybridCacheController(BaseHiCacheController):
|
||||
if not self._allocate_pp_prefetch_buffers(ticket, operation):
|
||||
operation.host_indices = torch.empty(0, dtype=torch.int64)
|
||||
operation.mark_terminate()
|
||||
self.prefetch_buffer.put(operation)
|
||||
self.dispatch_prefetch(operation)
|
||||
except Exception:
|
||||
logger.exception("PP HiCache ticket processing failed.")
|
||||
if operation is None or not operation.hash_value:
|
||||
@@ -1342,7 +1343,7 @@ class HybridCacheController(BaseHiCacheController):
|
||||
operation.host_indices = torch.empty(0, dtype=torch.int64)
|
||||
operation.mark_terminate()
|
||||
# Preserve every KV/sidecar ACK even when local allocation fails.
|
||||
self.prefetch_buffer.put(operation)
|
||||
self.dispatch_prefetch(operation)
|
||||
finally:
|
||||
if is_source and objects:
|
||||
self.pp_prefetch_command_queue.task_done()
|
||||
@@ -1479,19 +1480,21 @@ class HybridCacheController(BaseHiCacheController):
|
||||
)
|
||||
return host_indices, device_indices, resolved_pool_transfers
|
||||
|
||||
def _page_transfer(self, operation: PrefetchOperation) -> bool:
|
||||
def _page_transfer(self, operation: PrefetchOperation, sync_queue: Queue) -> bool:
|
||||
with self.mem_pool_host.layout_lease():
|
||||
return self._page_transfer_with_stable_layout(operation)
|
||||
return self._page_transfer_with_stable_layout(operation, sync_queue)
|
||||
|
||||
def _page_transfer_with_stable_layout(self, operation: PrefetchOperation) -> None:
|
||||
def _page_transfer_with_stable_layout(
|
||||
self, operation: PrefetchOperation, sync_queue: Queue
|
||||
) -> None:
|
||||
# KV pools and KV-derived pools first — determines actual completed page count
|
||||
kv_completed_pages = super()._page_transfer(operation)
|
||||
kv_completed_pages = super()._page_transfer(operation, sync_queue)
|
||||
|
||||
# Read non-KV derived sidecar pool, e.g. SWA, Mamba.
|
||||
self._page_transfer_sidecar(operation, kv_completed_pages)
|
||||
self._page_transfer_sidecar(operation, kv_completed_pages, sync_queue)
|
||||
|
||||
def _page_transfer_sidecar(
|
||||
self, operation: PrefetchOperation, kv_completed_pages: int
|
||||
self, operation: PrefetchOperation, kv_completed_pages: int, sync_queue: Queue
|
||||
) -> None:
|
||||
if operation.pool_transfers is None:
|
||||
return
|
||||
@@ -1536,10 +1539,10 @@ class HybridCacheController(BaseHiCacheController):
|
||||
"HiCache sidecar prefetch failed for request %s",
|
||||
operation.request_id,
|
||||
)
|
||||
# Emit PrefetchAck to prefetch_sync_queue, even the operation has been canceled by the
|
||||
# Emit PrefetchAck to the lane sync queue, even the operation has been canceled by the
|
||||
# scheduler thread. The prefetch sync thread expects the same number of PrefetchAck objects
|
||||
# to perform all_reduce.
|
||||
self.prefetch_sync_queue.put(
|
||||
sync_queue.put(
|
||||
PrefetchAck(
|
||||
rid=operation.request_id,
|
||||
operation=operation,
|
||||
@@ -1795,9 +1798,11 @@ class HybridCacheController(BaseHiCacheController):
|
||||
pool.device_indices = source.device_indices
|
||||
return extra_pools
|
||||
|
||||
def _reduce_prefetch_ack(self, ack: PrefetchAck) -> None:
|
||||
def _reduce_prefetch_ack(
|
||||
self, ack: PrefetchAck, groups: List[torch.distributed.ProcessGroup]
|
||||
) -> None:
|
||||
# Handle KV-derived pool.
|
||||
super()._reduce_prefetch_ack(ack)
|
||||
super()._reduce_prefetch_ack(ack, groups)
|
||||
|
||||
# Handle other sidecar pools, e.g. SWA, Mamba.
|
||||
if ack.pool_hits is not None:
|
||||
@@ -1810,20 +1815,24 @@ class HybridCacheController(BaseHiCacheController):
|
||||
self._all_reduce(
|
||||
packed,
|
||||
torch.distributed.ReduceOp.MIN,
|
||||
self.prefetch_completion_sync_groups,
|
||||
groups,
|
||||
)
|
||||
for i, pool in enumerate(PoolName):
|
||||
ack.pool_hits[pool.value] = packed[i].item()
|
||||
|
||||
def prefetch_sync_thread_func(self) -> None:
|
||||
def prefetch_sync_thread_func(self, lane: int = 0) -> None:
|
||||
"""Synchronize progressive ACKs and publish PP ticket completion."""
|
||||
sync_queue = self.prefetch_sync_queues[lane]
|
||||
groups = self.prefetch_completion_sync_groups_by_lane[lane]
|
||||
while not self.storage_stop_event.is_set():
|
||||
try:
|
||||
ack = self.prefetch_sync_queue.get(block=True, timeout=1)
|
||||
ack = sync_queue.get(block=True, timeout=1)
|
||||
if ack is None:
|
||||
continue
|
||||
self._reduce_prefetch_ack(ack)
|
||||
self._reduce_prefetch_ack(ack, groups)
|
||||
if not getattr(ack.operation, "is_pp_broadcast", False):
|
||||
if _hc_timing_enabled() and ack.completed_req:
|
||||
ack.operation.final_ack_enqueue_ts = time.monotonic()
|
||||
self.ack_prefetch_queue.put(ack)
|
||||
continue
|
||||
|
||||
|
||||
@@ -465,7 +465,18 @@ class MooncakeStore(HiCacheStorage, MooncakeBaseStore):
|
||||
)
|
||||
device_name = ""
|
||||
if self.config.standalone_storage:
|
||||
if not isinstance(mem_pool.allocator, MooncakeHostTensorAllocator):
|
||||
# V4 unified cache hands us a LogicalHostPool anchor that owns no
|
||||
# physical buffers (kv_buffer=None, allocator=None); the real
|
||||
# buffers live in side pools registered later via
|
||||
# register_mem_host_pool_v2() -> register_buffer(ptr, size).
|
||||
# Only enforce the MooncakeHostTensorAllocator requirement when
|
||||
# the anchor actually owns memory.
|
||||
_anchor_owns_buffers = (
|
||||
getattr(mem_pool, "kv_buffer", None) is not None
|
||||
)
|
||||
if _anchor_owns_buffers and not isinstance(
|
||||
getattr(mem_pool, "allocator", None), MooncakeHostTensorAllocator
|
||||
):
|
||||
raise RuntimeError(
|
||||
"MooncakeStore with standalone_storage=True requires MooncakeHostTensorAllocator. "
|
||||
"Please set standalone_storage=False "
|
||||
@@ -509,6 +520,22 @@ class MooncakeStore(HiCacheStorage, MooncakeBaseStore):
|
||||
else:
|
||||
client_hostname = self.config.local_hostname
|
||||
transfer_engine = None
|
||||
# Each rank embeds its own Mooncake client process. Giving
|
||||
# every rank the same local_hostname makes the master's
|
||||
# preferred-segment allocation unable to tell their
|
||||
# segments apart, so replicas can land on a peer rank's
|
||||
# segment and the local-memcpy fast path (which requires
|
||||
# same-process endpoints) silently falls back to TCP.
|
||||
# Make the hostname unique per rank so own-writes land on
|
||||
# the own segment and reads can use local memcpy.
|
||||
if storage_config is not None:
|
||||
client_hostname = (
|
||||
f"{client_hostname}"
|
||||
f"-d{storage_config.dp_rank}"
|
||||
f"t{storage_config.tp_rank}"
|
||||
f"p{storage_config.pp_rank}"
|
||||
f"c{storage_config.attn_cp_rank}"
|
||||
)
|
||||
|
||||
setup_kwargs = {}
|
||||
if self.config.enable_ssd_offload:
|
||||
|
||||
@@ -368,6 +368,14 @@ class SWAComponent(TreeComponent):
|
||||
state["len"] = 0
|
||||
if not gates_prefix_match and (node.backuped or not node.evicted):
|
||||
return True
|
||||
# Fail-soft: a backuped node whose SWA is absent on *both*
|
||||
# layers (device tombstone plus no L3 backup — the hole a
|
||||
# degraded prefetch can graft) still has Full KV on host.
|
||||
# Treat only that case as a valid match boundary — SWA will
|
||||
# be rebuilt from Full KV at load-back via
|
||||
# SWARebuild/RecoverSWAWithLockedFull.
|
||||
if cd.host_value is None and node.backuped:
|
||||
return True
|
||||
return False
|
||||
state["len"] += len(node.key)
|
||||
return state["len"] >= sliding_window_size
|
||||
@@ -1156,7 +1164,12 @@ class SWAComponent(TreeComponent):
|
||||
cur is not self.tree_core.root_node and n_swa < self.sliding_window_size
|
||||
):
|
||||
cd = cur.component_data[ct]
|
||||
assert cd.host_value is not None or cd.value is not None
|
||||
# Fail-soft: a degraded prefetch may insert a backuped node
|
||||
# whose SWA was never backed up (tombstoned before backup).
|
||||
# Stop collecting — SWA will be rebuilt from Full KV at
|
||||
# load-back via SWARebuild/RecoverSWAWithLockedFull.
|
||||
if cd.host_value is None and cd.value is None:
|
||||
break
|
||||
if cd.value is not None:
|
||||
# device exists, skip it
|
||||
n_swa += len(cd.value)
|
||||
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import atexit
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from array import array
|
||||
@@ -23,7 +24,7 @@ import torch
|
||||
|
||||
from sglang.srt.distributed.communication_tags import P2PTag
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.managers.cache_controller import CacheOperation
|
||||
from sglang.srt.managers.cache_controller import CacheOperation, _hc_timing_enabled
|
||||
from sglang.srt.mem_cache.allocator.page_interleave import (
|
||||
page_interleave_shard_size,
|
||||
)
|
||||
@@ -67,6 +68,7 @@ from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
from sglang.srt.mem_cache.storage_prefetch import StoragePrefetchRetries
|
||||
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
|
||||
from sglang.srt.mem_cache.utils import get_hash_str
|
||||
from sglang.srt.mem_cache.unified_cache.cache_action import (
|
||||
BackupKV,
|
||||
CacheAction,
|
||||
@@ -154,6 +156,11 @@ COMPONENT_REGISTRY: dict[ComponentType, type[TreeComponent]] = {
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# DEBUG-ONLY (p2 wedge trap 2026-10-13): env-gated logging for the per-step
|
||||
# hicache event sync. Same SGLANG_AR_SEQ_DEBUG switch as the ARDBG sites.
|
||||
_AR_SEQ_DEBUG = int(os.environ.get("SGLANG_AR_SEQ_DEBUG", "0"))
|
||||
_hc_step = 0
|
||||
|
||||
|
||||
class _OngoingWriteThrough(NamedTuple):
|
||||
"""Tracks an in-flight D→H write-through operation."""
|
||||
@@ -1794,6 +1801,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
for comp in self._components_tuple
|
||||
}
|
||||
success = False
|
||||
_hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
try:
|
||||
success = self._load_back_transfers(
|
||||
node_id=node_id,
|
||||
@@ -1805,6 +1813,14 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
)
|
||||
return success
|
||||
finally:
|
||||
if _hc_timing_enabled():
|
||||
logger.info(
|
||||
"HC_TIMING stage=loadback req=%s node=%d ms=%.1f success=%s",
|
||||
getattr(req, "rid", None) if req is not None else "-",
|
||||
node_id,
|
||||
(time.monotonic() - _hc_t0) * 1e3,
|
||||
success,
|
||||
)
|
||||
for comp in self._components_tuple:
|
||||
comp.finalize_load_back(req, preps[comp.component_type], success)
|
||||
|
||||
@@ -2016,11 +2032,46 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
if len(prefetch_key) < self.prefetch_threshold:
|
||||
return 0
|
||||
|
||||
# DeepSeek V4 keeps no physical KV objects in storage (logical anchor),
|
||||
# so a plain batch_exists on KV keys always misses; the usable prefix is
|
||||
# decided by the side pools. Mirror prefetch_from_storage's transfer
|
||||
# construction so the query goes through batch_exists_v2. Both hooks are
|
||||
# pure sizing/descriptor builders: prepare_prefetch only computes
|
||||
# staging_tokens, and PREFETCH-phase transfers carry placeholder keys
|
||||
# whose length sizes the TRAILING_PAGES window for batch_exists_v2.
|
||||
comp_xfers: dict[ComponentType, list[PoolTransfer]] = {}
|
||||
for ct in self.tree_components:
|
||||
if ct == BASE_COMPONENT_TYPE:
|
||||
continue
|
||||
prep = self.components[ct].prepare_prefetch(
|
||||
last_host_node_id, prefetch_tokens=len(prefetch_key)
|
||||
)
|
||||
if prep.staging_tokens == 0:
|
||||
continue
|
||||
transfers = self.tree_core.build_hicache_transfers(
|
||||
ct,
|
||||
last_host_node_id,
|
||||
CacheTransferPhase.PREFETCH,
|
||||
token_ids=prefetch_key.token_ids,
|
||||
prefetch_tokens=len(prefetch_key),
|
||||
staging_tokens=prep.staging_tokens,
|
||||
last_hash=last_hash,
|
||||
)
|
||||
if transfers:
|
||||
comp_xfers[ct] = transfers
|
||||
kv_xfer = PoolTransfer(name=PoolName.KV, host_indices=None)
|
||||
sidecar_xfers = self._build_sidecar_transfers(
|
||||
CacheTransferPhase.PREFETCH, kv_xfer, comp_xfers
|
||||
)
|
||||
aux_xfers = [x for xfers in comp_xfers.values() for x in xfers]
|
||||
aux_xfers.extend(sidecar_xfers)
|
||||
|
||||
operation = PrefetchOperation(
|
||||
CacheRequestHandle("__storage_hit_query__", 0),
|
||||
prefetch_key,
|
||||
last_hash,
|
||||
prefix_keys,
|
||||
pool_transfers=aux_xfers or None,
|
||||
)
|
||||
_, storage_hit_count = self.cache_controller._storage_hit_query(operation)
|
||||
storage_hit_count_tensor = torch.tensor(storage_hit_count, dtype=torch.int)
|
||||
@@ -2114,6 +2165,15 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
self.storage_prefetch_retries.poll_miss(req_id, storage_hit_end)
|
||||
return
|
||||
|
||||
# Recompute last_hash from the full matched prefix so the hash chain
|
||||
# is correct regardless of which node was selected as anchor. The
|
||||
# anchor's hash_value may be at a different position than matched_len
|
||||
# when the anchor has no host_value (async backup not yet complete).
|
||||
if matched_prefix_tokens and len(matched_prefix_tokens) >= self.page_size:
|
||||
prefix_hashes = get_hash_str(matched_prefix_tokens, None, self.page_size)
|
||||
if prefix_hashes:
|
||||
last_hash = prefix_hashes[-1]
|
||||
|
||||
# Buffer mode holds no tree state during the fetch: buffers are
|
||||
# operation-owned, so the anchor needs no pin.
|
||||
anchor_lock_params = (
|
||||
@@ -2380,10 +2440,21 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
# All PP/TP ranks will get the same `min_completed_tokens`, because `completed_tokens`
|
||||
# and `pool_hits` in their operations are same. No need to sync cross-rank here.
|
||||
if not self._check_hybrid_prefetch_result(
|
||||
request, operation, hash_value, host_indices
|
||||
request,
|
||||
operation,
|
||||
hash_value,
|
||||
host_indices,
|
||||
last_host_node_id,
|
||||
anchor_lock_params,
|
||||
prefetch_key,
|
||||
comp_xfers,
|
||||
):
|
||||
# Hybrid all-or-nothing check failed; result already discarded.
|
||||
return
|
||||
# The check may have degraded the result (fail-soft): re-read the
|
||||
# retained KV prefix length and hash chain off the operation.
|
||||
completed_tokens = operation.completed_tokens
|
||||
hash_value = operation.hash_value
|
||||
|
||||
allocated_tokens = len(host_indices)
|
||||
if completed_tokens < allocated_tokens:
|
||||
@@ -2407,12 +2478,15 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
)
|
||||
|
||||
fetched_key = prefetch_key[:completed_tokens]
|
||||
_hc_tg = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
insert_result = self.tree_core.insert_host(
|
||||
last_host_node_id,
|
||||
fetched_key,
|
||||
host_indices[:completed_tokens],
|
||||
hash_value[: completed_tokens // self.page_size],
|
||||
)
|
||||
if _hc_timing_enabled():
|
||||
operation.graft_ms = (time.monotonic() - _hc_tg) * 1e3
|
||||
|
||||
# Apply the host-insert walk's actions before the transfer commit.
|
||||
self._apply_cache_actions(insert_result.cache_actions)
|
||||
@@ -2468,6 +2542,16 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
loaded_from_storage,
|
||||
self.cache_controller.prefetch_tokens_occupied,
|
||||
)
|
||||
if _hc_timing_enabled():
|
||||
logger.info(
|
||||
"HC_TIMING stage=prefetch_e2e req=%s e2e_ms=%.1f graft_ms=%.1f "
|
||||
"completed=%d loaded=%d",
|
||||
request.rid,
|
||||
(time.monotonic() - operation.start_time) * 1e3,
|
||||
getattr(operation, "graft_ms", -1.0),
|
||||
completed_tokens,
|
||||
loaded_from_storage,
|
||||
)
|
||||
return
|
||||
|
||||
def _check_hybrid_prefetch_result(
|
||||
@@ -2476,6 +2560,10 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
operation: PrefetchOperation,
|
||||
hash_value: list[str],
|
||||
host_indices: torch.Tensor,
|
||||
last_host_node_id: NodeId,
|
||||
anchor_lock_params: DecLockRefParams,
|
||||
prefetch_key: RadixKey,
|
||||
comp_xfers: dict[ComponentType, list[PoolTransfer]],
|
||||
) -> bool:
|
||||
"""Decide the length of usable prefix.
|
||||
|
||||
@@ -2486,11 +2574,16 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
the Full KV pool and every sidecar. A partial prefix is still usable
|
||||
because the sidecar is page-aligned with KV and required for every page.
|
||||
* Everything else (SWA / Mamba components, mixed DeepSeekV4 stacks):
|
||||
*all-or-nothing*. Their pools only cover a window / tail and cannot be
|
||||
truncated page by page, so any shortfall discards the whole prefetch.
|
||||
*fail-soft degrade* when the shortfall is confined to pools with a
|
||||
cheap rebuild path (SWA — the missing window is rebuilt from Full KV
|
||||
at load-back): the fetched KV prefix is retained and only the failed
|
||||
pool is discarded. Falls back to *all-or-nothing* when nothing is
|
||||
usable or a non-rebuildable pool (Mamba) falls short, since those
|
||||
pools cannot be truncated page by page.
|
||||
|
||||
Returns true if prefetch success, or false when an all-or-nothing prefetch
|
||||
was discarded (the caller should then treat the prefetch as finished).
|
||||
Returns true if prefetch success (possibly degraded), or false when an
|
||||
all-or-nothing prefetch was discarded (the caller should then treat the
|
||||
prefetch as finished).
|
||||
"""
|
||||
# Sync completed tokens and per-pool hit pages across ATTN groups, taking
|
||||
# the minimum so every rank agrees on the same usable prefix length.
|
||||
@@ -2516,68 +2609,140 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
for transfer, count in zip(pool_transfers, pool_hit_pages)
|
||||
)
|
||||
if pool_transfers and not all_succeeded:
|
||||
# Drop each pool's beliefs beyond what it served (KV and its
|
||||
# sidecars beyond the completed span); the next insert re-writes
|
||||
# only the missing pools. A deadline is not evidence of absent data.
|
||||
# Identify which non-KV components fell short. KV-derived
|
||||
# sidecars (indices_from_pool == KV) are excluded — they share
|
||||
# the KV hit length and are always covered when KV succeeds.
|
||||
failed_components = [
|
||||
ct
|
||||
for ct, xfers in comp_xfers.items()
|
||||
if any(x.indices_from_pool != PoolName.KV for x in xfers)
|
||||
]
|
||||
# Only SWA has a cheap rebuild path (_translate_full_to_swa);
|
||||
# the missing window is rebuilt at load-back via
|
||||
# SWARebuild/RecoverSWAWithLockedFull. Mamba SSM states require
|
||||
# O(seq_len) recompute — not degradable. A deadline-expired op is
|
||||
# discarded whole as well: its partial prefix can no longer be
|
||||
# grafted onto a request the scheduler already admitted.
|
||||
rebuildable = all(ct != ComponentType.MAMBA for ct in failed_components)
|
||||
|
||||
if (
|
||||
completed_tokens == 0
|
||||
or not rebuildable
|
||||
or operation.deadline_expired
|
||||
):
|
||||
# Full discard — nothing usable from any pool, or a
|
||||
# non-rebuildable pool (Mamba) fell short so the whole
|
||||
# result is unusable.
|
||||
# Drop each pool's beliefs beyond what it served (KV and its
|
||||
# sidecars beyond the completed span); the next insert re-writes
|
||||
# only the missing pools. A deadline is not evidence of absent data.
|
||||
invalidation_hashes = (
|
||||
operation.sidecar_hash_values
|
||||
if operation.sidecar_hash_values is not None
|
||||
else hash_value
|
||||
)
|
||||
trimmed_pages = len(invalidation_hashes) - len(hash_value)
|
||||
keep_pages = trimmed_pages + completed_tokens // self.page_size
|
||||
beliefs = self.storage_existence_cache
|
||||
if not operation.deadline_expired and keep_pages < len(
|
||||
invalidation_hashes
|
||||
):
|
||||
self._belief_policy(
|
||||
PoolName.KV, PoolHitPolicy.ALL_PAGES
|
||||
).heal_from_transfer(
|
||||
beliefs.pool(PoolName.KV), invalidation_hashes, keep_pages
|
||||
)
|
||||
for transfer in operation.pool_transfers or ():
|
||||
if transfer.indices_from_pool == PoolName.KV:
|
||||
self._belief_policy(
|
||||
transfer.name, transfer.hit_policy
|
||||
).heal_from_transfer(
|
||||
beliefs.pool(transfer.name),
|
||||
invalidation_hashes,
|
||||
keep_pages,
|
||||
)
|
||||
for transfer, count in zip(pool_transfers, pool_hit_pages):
|
||||
if (
|
||||
not operation.deadline_expired
|
||||
and transfer.keys is not None
|
||||
and count < len(transfer.keys)
|
||||
):
|
||||
self._belief_policy(
|
||||
transfer.name, transfer.hit_policy
|
||||
).heal_from_transfer(
|
||||
beliefs.pool(transfer.name), transfer.keys, count
|
||||
)
|
||||
reason = (
|
||||
"deadline" if operation.deadline_expired else "storage_transfer"
|
||||
)
|
||||
self._finish_storage_prefetch(
|
||||
request, fulfilled_tokens=0, reason=reason
|
||||
)
|
||||
# The controller's prefetch IO thread already releases the
|
||||
# untransferred tail (host_indices[completed_tokens:])
|
||||
self._retire_ongoing_prefetch(
|
||||
request,
|
||||
host_indices[:completed_tokens],
|
||||
pool_transfers if operation.pool_transfers_done else None,
|
||||
)
|
||||
self.prefetch_loaded_tokens_by_reqid[request] = 0
|
||||
self.prefetch_loaded_storage_start_by_reqid.pop(request, None)
|
||||
logger.warning(
|
||||
"HiCache hybrid prefetch discarded req=%s reason=%s completed=%d "
|
||||
"requested=%d kv_beliefs_kept_pages=%s aux_pool_hit_pages=%s",
|
||||
request.rid,
|
||||
reason,
|
||||
completed_tokens,
|
||||
expected_tokens,
|
||||
None if operation.deadline_expired else keep_pages,
|
||||
{
|
||||
getattr(transfer.name, "value", transfer.name): count
|
||||
for transfer, count in zip(pool_transfers, pool_hit_pages)
|
||||
},
|
||||
)
|
||||
return False
|
||||
|
||||
# Fail-soft degrade: retain the Full KV (+ KV-derived sidecar)
|
||||
# result and discard **only** the failed non-KV independent
|
||||
# pools (SWA) whose storage fetch fell short. Their host
|
||||
# buffers are released here and their component entries are
|
||||
# removed from comp_xfers so the caller's normal commit /
|
||||
# staging path never touches the freed memory. At load-back
|
||||
# time the missing SWA window is rebuilt via the existing
|
||||
# RecoverSWAWithLockedFull / SWARebuild path in swa.py.
|
||||
# Belief self-heal is scoped to the retained KV prefix.
|
||||
invalidation_hashes = (
|
||||
operation.sidecar_hash_values
|
||||
if operation.sidecar_hash_values is not None
|
||||
else hash_value
|
||||
)
|
||||
trimmed_pages = len(invalidation_hashes) - len(hash_value)
|
||||
keep_pages = trimmed_pages + completed_tokens // self.page_size
|
||||
beliefs = self.storage_existence_cache
|
||||
if not operation.deadline_expired and keep_pages < len(invalidation_hashes):
|
||||
self._belief_policy(
|
||||
PoolName.KV, PoolHitPolicy.ALL_PAGES
|
||||
).heal_from_transfer(
|
||||
beliefs.pool(PoolName.KV), invalidation_hashes, keep_pages
|
||||
)
|
||||
for transfer in operation.pool_transfers or ():
|
||||
if transfer.indices_from_pool == PoolName.KV:
|
||||
self._belief_policy(
|
||||
transfer.name, transfer.hit_policy
|
||||
).heal_from_transfer(
|
||||
beliefs.pool(transfer.name),
|
||||
invalidation_hashes,
|
||||
keep_pages,
|
||||
)
|
||||
for transfer, count in zip(pool_transfers, pool_hit_pages):
|
||||
if (
|
||||
not operation.deadline_expired
|
||||
and transfer.keys is not None
|
||||
and count < len(transfer.keys)
|
||||
):
|
||||
self._belief_policy(
|
||||
transfer.name, transfer.hit_policy
|
||||
).heal_from_transfer(
|
||||
beliefs.pool(transfer.name), transfer.keys, count
|
||||
)
|
||||
reason = "deadline" if operation.deadline_expired else "storage_transfer"
|
||||
self._finish_storage_prefetch(request, fulfilled_tokens=0, reason=reason)
|
||||
# The controller's prefetch IO thread already releases the untransferred
|
||||
# tail (host_indices[completed_tokens:])
|
||||
self._retire_ongoing_prefetch(
|
||||
request,
|
||||
host_indices[:completed_tokens],
|
||||
pool_transfers if operation.pool_transfers_done else None,
|
||||
self.storage_existence_cache.pool(PoolName.KV).invalidate_beyond(
|
||||
invalidation_hashes,
|
||||
keep_pages=trimmed_pages + completed_tokens // self.page_size,
|
||||
)
|
||||
self.prefetch_loaded_tokens_by_reqid[request] = 0
|
||||
self.prefetch_loaded_storage_start_by_reqid.pop(request, None)
|
||||
self.cache_controller.append_host_mem_release(
|
||||
extra_pools=pool_transfers if operation.pool_transfers_done else None,
|
||||
)
|
||||
# Remove failed non-KV components from comp_xfers (same dict as
|
||||
# the caller's) so the commit / staging path won't touch freed
|
||||
# host buffers.
|
||||
for ct in failed_components:
|
||||
comp_xfers.pop(ct, None)
|
||||
# Retain only the KV-fetched prefix; the caller re-reads these
|
||||
# off the operation after the check passes.
|
||||
operation.completed_tokens = completed_tokens
|
||||
operation.hash_value = hash_value[: completed_tokens // self.page_size]
|
||||
logger.warning(
|
||||
"HiCache hybrid prefetch discarded req=%s reason=%s completed=%d "
|
||||
"requested=%d kv_beliefs_kept_pages=%s aux_pool_hit_pages=%s",
|
||||
"HiCache hybrid prefetch degraded req=%s completed=%d "
|
||||
"requested=%d retained_kv=%d discarded_components=%s",
|
||||
request.rid,
|
||||
reason,
|
||||
completed_tokens,
|
||||
expected_tokens,
|
||||
None if operation.deadline_expired else keep_pages,
|
||||
{
|
||||
getattr(transfer.name, "value", transfer.name): count
|
||||
for transfer, count in zip(pool_transfers, pool_hit_pages)
|
||||
},
|
||||
completed_tokens,
|
||||
[ct.value for ct in failed_components],
|
||||
)
|
||||
return False
|
||||
return True
|
||||
return True
|
||||
|
||||
def _record_storage_prefetch_hit(
|
||||
@@ -3084,7 +3249,9 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
operation.hash_value = operation.hash_value[: alloc_len // self.page_size]
|
||||
operation.host_indices = host_indices
|
||||
self.ongoing_prefetch[request] = info._replace(host_indices=host_indices)
|
||||
cc.prefetch_buffer.put(operation)
|
||||
if buffer_mode:
|
||||
cc.prefetch_tokens_occupied += alloc_len
|
||||
cc.dispatch_prefetch(operation)
|
||||
return True
|
||||
|
||||
def _drain_and_alloc_storage_hit():
|
||||
@@ -3161,6 +3328,14 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
operation.pool_transfers_done = True
|
||||
if ack.completed_req:
|
||||
if is_current:
|
||||
if _hc_timing_enabled():
|
||||
_hc_enq = getattr(operation, "final_ack_enqueue_ts", None)
|
||||
if _hc_enq is not None:
|
||||
logger.info(
|
||||
"HC_TIMING stage=ack_drain req=%s wait_ms=%.1f",
|
||||
operation.request_id,
|
||||
(time.monotonic() - _hc_enq) * 1e3,
|
||||
)
|
||||
# check_prefetch_progress() is not called for this rid yet.
|
||||
# Let us insert the prefetch result into the radix tree.
|
||||
self._handle_prefetch_result(operation)
|
||||
@@ -3587,6 +3762,16 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
|
||||
def check_hicache_events(self) -> None:
|
||||
"""Called per scheduler step to poll async HiCache events."""
|
||||
global _hc_step
|
||||
if _AR_SEQ_DEBUG:
|
||||
_hc_step += 1
|
||||
if _hc_step & 255 == 0:
|
||||
logger.info(
|
||||
"[ARDBG] hcev rank=%s step=%d linker=%s",
|
||||
torch.distributed.get_rank(),
|
||||
_hc_step,
|
||||
self.linker is not None,
|
||||
)
|
||||
if self.linker is not None:
|
||||
finish_counts = torch.tensor(
|
||||
[self.linker.num_completed_offloads()],
|
||||
@@ -3616,6 +3801,15 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
storage_queue_sizes,
|
||||
extra_pool_names,
|
||||
) = self._sync_hicache_ready_counts()
|
||||
if _AR_SEQ_DEBUG and _hc_step & 255 == 0:
|
||||
logger.info(
|
||||
"[ARDBG] hcsync rank=%s step=%d write=%s load=%s storage=%s",
|
||||
torch.distributed.get_rank(),
|
||||
_hc_step,
|
||||
write_finish_count,
|
||||
load_finish_count,
|
||||
storage_queue_sizes,
|
||||
)
|
||||
self.writing_check(finish_count=write_finish_count)
|
||||
self.loading_check(finish_count=load_finish_count)
|
||||
|
||||
|
||||
@@ -1763,6 +1763,7 @@ class ModelRunner:
|
||||
forward_batch.replace_embeds is not None
|
||||
and forward_batch.replace_positions is not None
|
||||
):
|
||||
misc_utils.validate_replace_embeds_batch(forward_batch)
|
||||
# Token embedding overrides: get base embeddings, scatter replacements
|
||||
if "input_embeds" not in kwargs:
|
||||
embed_layer = self.model.get_input_embeddings()
|
||||
|
||||
@@ -19,6 +19,7 @@ from sglang.srt.server_args import CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACK
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.configs.model_config import ModelConfig
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -114,6 +115,27 @@ def resolve_pp_proxy_dspark_hidden_size(
|
||||
return 0
|
||||
|
||||
|
||||
def validate_replace_embeds_batch(forward_batch: ForwardBatch) -> None:
|
||||
if forward_batch.mm_inputs is None:
|
||||
return
|
||||
for mm_inputs, prefix_len, extend_len in zip(
|
||||
forward_batch.mm_inputs,
|
||||
forward_batch.extend_prefix_lens_cpu,
|
||||
forward_batch.extend_seq_lens_cpu,
|
||||
):
|
||||
if mm_inputs is None:
|
||||
continue
|
||||
chunk_end = prefix_len + extend_len
|
||||
for item in mm_inputs.mm_items:
|
||||
for start, end in item.offsets or ():
|
||||
if start < chunk_end and end >= prefix_len:
|
||||
# Placeholder rows carry hash IDs the base embedding lookup cannot index.
|
||||
raise ValueError(
|
||||
"Token embedding overrides cannot share an extend batch with "
|
||||
"multimodal placeholders"
|
||||
)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class _SupportsPackedAuxHiddenStates(Protocol):
|
||||
def get_aux_hidden_states_width(self) -> int: ...
|
||||
|
||||
@@ -363,14 +363,23 @@ class EagerRunner(BaseRunner):
|
||||
"""
|
||||
model = self.model_runner.model
|
||||
|
||||
input_ids = forward_batch.input_ids
|
||||
input_embeds = kwargs.get("input_embeds")
|
||||
if hasattr(model, "prepare_model_inputs"):
|
||||
# Multimodal offsets are request-global, so the merge and the
|
||||
# placeholder-ID remap must see the full extend layout first.
|
||||
input_ids, input_embeds = model.prepare_model_inputs(
|
||||
input_ids=input_ids,
|
||||
forward_batch=forward_batch,
|
||||
input_embeds=input_embeds,
|
||||
)
|
||||
if prepare_inputs := getattr(model, "prepare_cp_inputs", None):
|
||||
input_embeds, positions, model_kwargs = prepare_inputs(
|
||||
forward_batch, **kwargs
|
||||
)
|
||||
else:
|
||||
input_embeds = kwargs.get("input_embeds")
|
||||
if input_embeds is None:
|
||||
input_embeds = model.get_input_embeddings()(forward_batch.input_ids)
|
||||
input_embeds = model.get_input_embeddings()(input_ids)
|
||||
positions = forward_batch.positions
|
||||
model_kwargs = {}
|
||||
if (pp_proxy_tensors := kwargs.get("pp_proxy_tensors")) is not None:
|
||||
@@ -380,7 +389,7 @@ class EagerRunner(BaseRunner):
|
||||
input_embeds,
|
||||
positions,
|
||||
forward_batch,
|
||||
forward_batch.input_ids,
|
||||
input_ids,
|
||||
) as (sharded_input_embeds, sharded_positions, model_input_ids):
|
||||
model_kwargs["input_embeds"] = sharded_input_embeds
|
||||
hidden_states = model.model(
|
||||
@@ -423,7 +432,7 @@ class EagerRunner(BaseRunner):
|
||||
if aux_hidden_states is None:
|
||||
logits_kwargs["hidden_states_before_norm"] = hidden_states_before_norm
|
||||
return model.logits_processor(
|
||||
forward_batch.input_ids,
|
||||
input_ids,
|
||||
hidden_states,
|
||||
model.lm_head,
|
||||
forward_batch,
|
||||
|
||||
@@ -773,6 +773,9 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
|
||||
def _get_layer_model_positions(self, forward_batch: ForwardBatch) -> torch.Tensor:
|
||||
"""Mirror outer multimodal wrappers when BCG captures layer_model directly."""
|
||||
cp_positions = getattr(forward_batch, "_cp_positions", None)
|
||||
if cp_positions is not None:
|
||||
return cp_positions
|
||||
if forward_batch.mrope_positions is None:
|
||||
return forward_batch.positions
|
||||
|
||||
@@ -858,7 +861,9 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
if self._uses_eager_prefill_tail():
|
||||
# BCG / Full: capture the transformer body only.
|
||||
positions = self._get_layer_model_positions(forward_batch)
|
||||
input_ids = forward_batch.input_ids
|
||||
input_ids = getattr(
|
||||
forward_batch, "_cp_input_ids", forward_batch.input_ids
|
||||
)
|
||||
kwargs = _build_layer_model_forward_kwargs(
|
||||
self.layer_model, forward_batch, pp_proxy_tensors
|
||||
)
|
||||
|
||||
@@ -67,6 +67,7 @@ from sglang.srt.layers.attention.dsv4.dsv41_sparse import (
|
||||
DeepseekV41Indexer,
|
||||
)
|
||||
from sglang.srt.layers.attention.dsv4.indexer import C4Indexer
|
||||
from sglang.srt.layers.cp.base import is_zigzag
|
||||
from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx
|
||||
from sglang.srt.layers.cp.utils import (
|
||||
cp_gather_full_sequence_states,
|
||||
@@ -103,6 +104,11 @@ from sglang.srt.layers.quantization.mxfp8_input import Mxfp8SwizzledInput
|
||||
from sglang.srt.layers.rotary_embedding import get_rope_wrapper
|
||||
from sglang.srt.layers.utils import PPMissingLayer, get_layer_id
|
||||
from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from sglang.srt.managers.mm_owner_embedding import (
|
||||
MmOwnerSession,
|
||||
has_owner_span_work,
|
||||
select_owner_group,
|
||||
)
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
embed_mm_inputs,
|
||||
pad_mm_tokens,
|
||||
@@ -181,6 +187,7 @@ from sglang.srt.multimodal.deepseek_v41_image_processing import (
|
||||
)
|
||||
from sglang.srt.runtime_context import (
|
||||
get_device,
|
||||
get_disagg,
|
||||
get_exec,
|
||||
get_forward,
|
||||
get_parallel,
|
||||
@@ -2324,7 +2331,10 @@ class MQALayer(MqaAttentionBase):
|
||||
if (
|
||||
forward_batch.forward_mode.is_extend()
|
||||
and is_in_breakable_cuda_graph()
|
||||
and not getattr(attn_backend, "low_ratio_prefill_graph", False)
|
||||
and (
|
||||
dsa_use_prefill_cp(forward_batch)
|
||||
or not getattr(attn_backend, "low_ratio_prefill_graph", False)
|
||||
)
|
||||
):
|
||||
bcg_deepseek_v4_low_ratio_sources(self, x, q_lora, positions)
|
||||
else:
|
||||
@@ -3781,11 +3791,18 @@ class DeepseekV4Model(nn.Module):
|
||||
)
|
||||
if self.engram_hasher is not None:
|
||||
if cp_extend:
|
||||
# n-gram hashing needs each token's predecessors: hash the whole prompt
|
||||
# N-gram hashing needs each token's predecessors, so hash the
|
||||
# whole prompt before selecting this CP rank's interleaved rows.
|
||||
# The hasher builds request-to-token indices dynamically; keep
|
||||
# that work at an eager break during breakable graph capture.
|
||||
total = int(forward_batch.attn_cp_metadata.total_seq_lens)
|
||||
hash_ids = self.engram_hasher(
|
||||
forward_batch.input_ids[:total], forward_batch
|
||||
)
|
||||
global_input_ids = forward_batch.input_ids[:total]
|
||||
if is_in_breakable_cuda_graph():
|
||||
hash_ids = bcg_deepseek_v4_engram_hash_ids(
|
||||
self.engram_hasher, global_input_ids
|
||||
)
|
||||
else:
|
||||
hash_ids = self.engram_hasher(global_input_ids, forward_batch)
|
||||
parallel = get_parallel()
|
||||
hash_ids = hash_ids[parallel.attn_cp_rank :: parallel.attn_cp_size]
|
||||
pad_rows = hidden_states.shape[0] - hash_ids.shape[0]
|
||||
@@ -3908,9 +3925,12 @@ class DeepseekV4Model(nn.Module):
|
||||
forward_batch,
|
||||
cp_all_tokens=cp_extend,
|
||||
)
|
||||
# Only multimodal placeholders become image_token_id, so a text-only
|
||||
# batch skips this full-residual where.
|
||||
if (
|
||||
self.config.model_type == "deepseek_v41"
|
||||
and self.config.vision_n_layers > 0
|
||||
and forward_batch.contains_mm_inputs()
|
||||
):
|
||||
hidden_states = torch.where(
|
||||
(input_ids == self.config.image_token_id)[:, None, None],
|
||||
@@ -4302,6 +4322,12 @@ class DeepseekV4Model(nn.Module):
|
||||
return hidden_states, pre_hc_head
|
||||
|
||||
|
||||
def _v41_vision_a2a_supported() -> bool:
|
||||
backend = get_moe_a2a_backend()
|
||||
return backend.is_none() or (
|
||||
backend.is_megamoe() and get_disagg().disaggregation_mode == "decode"
|
||||
)
|
||||
|
||||
# Quark keys its per-layer specs by checkpoint module names (layers.N.attn.wq_a,
|
||||
# mtp.0.ffn.experts.N.w1, ...); this maps them onto the module prefixes built here.
|
||||
_QUARK_CKPT_TO_SGLANG_MAPPER = WeightsMapper(
|
||||
@@ -4366,16 +4392,17 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
and not getattr(config, "language_model_only", False)
|
||||
):
|
||||
if (
|
||||
get_parallel().attn_cp_size != 1
|
||||
or get_parallel().pp_group.world_size != 1
|
||||
or not (
|
||||
get_moe_a2a_backend().is_none()
|
||||
or (_is_cuda and get_moe_a2a_backend().is_megamoe())
|
||||
)
|
||||
get_parallel().pp_group.world_size != 1
|
||||
or not _v41_vision_a2a_supported()
|
||||
):
|
||||
raise ValueError(
|
||||
"V4.1 vision supports TP/EP/DP without CP or PP; "
|
||||
"MoE A2A requires CUDA MegaMoE"
|
||||
"V4.1 vision supports TP/EP/DP without PP; "
|
||||
"MoE A2A is supported only with MegaMoE on a PD decode node"
|
||||
)
|
||||
if get_parallel().attn_cp_size != 1 and (_is_npu or is_zigzag()):
|
||||
raise ValueError(
|
||||
"V4.1 vision context parallelism requires the CUDA interleave "
|
||||
"strategy; NPU and zigzag CP are not supported yet"
|
||||
)
|
||||
|
||||
args = SimpleNamespace(**vars(config), dim=config.hidden_size)
|
||||
@@ -4384,6 +4411,11 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
self.image_start = nn.Parameter(torch.empty(config.hidden_size))
|
||||
self.image_end = nn.Parameter(torch.empty(config.hidden_size))
|
||||
self.image_newline = nn.Parameter(torch.empty(config.hidden_size))
|
||||
self.mm_owner_group = (
|
||||
select_owner_group(get_parallel())
|
||||
if self.vision is not None and _is_cuda
|
||||
else None
|
||||
)
|
||||
self.model = DeepseekV4Model(
|
||||
config, quant_config, prefix=add_prefix("model", prefix)
|
||||
)
|
||||
@@ -4516,7 +4548,42 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
spans.append(span)
|
||||
return spans
|
||||
|
||||
def _prepare_mm_embeddings(self, input_ids, forward_batch):
|
||||
def _image_span_signature(self, item, span_len: int):
|
||||
h, w = int(item.n_vit_h), int(item.n_vit_w)
|
||||
r = self.config.vision_downsample_ratio
|
||||
expected = len(image_token_types((h + r - 1) // r, (w + r - 1) // r))
|
||||
if expected != span_len:
|
||||
raise ValueError(
|
||||
f"image grid {(h, w)} yields {expected} span tokens, "
|
||||
f"placeholder has {span_len}"
|
||||
)
|
||||
plan = item.model_specific_data.get(GPU_PLAN_KEY)
|
||||
feature = item.feature
|
||||
return (
|
||||
h,
|
||||
w,
|
||||
tuple(feature.shape) if isinstance(feature, torch.Tensor) else None,
|
||||
None if plan is None else tuple(sorted(plan.items())),
|
||||
)
|
||||
|
||||
def _mm_owner_session(self, forward_batch) -> Optional[MmOwnerSession]:
|
||||
if self.mm_owner_group is None:
|
||||
return None
|
||||
return MmOwnerSession(
|
||||
group=self.mm_owner_group,
|
||||
device=self.image_start.device,
|
||||
dtype=self.image_start.dtype,
|
||||
width=self.config.hidden_size,
|
||||
rids=list(forward_batch.rids or ()),
|
||||
signature=self._image_span_signature,
|
||||
engaged=has_owner_span_work(
|
||||
forward_batch.mm_inputs,
|
||||
forward_batch.extend_prefix_lens_cpu,
|
||||
forward_batch.extend_seq_lens_cpu,
|
||||
),
|
||||
)
|
||||
|
||||
def _prepare_mm_embeddings(self, input_ids, forward_batch, mm_owner):
|
||||
# Keep scheduler hash IDs intact: the shared embedder clamps its input in place.
|
||||
input_embeds, _ = embed_mm_inputs(
|
||||
mm_inputs_list=[
|
||||
@@ -4528,6 +4595,7 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
input_ids=input_ids.clone(),
|
||||
input_embedding=self.get_input_embeddings(),
|
||||
multimodal_model=self,
|
||||
mm_owner=mm_owner,
|
||||
)
|
||||
forward_batch.mm_input_embeds = input_embeds
|
||||
return input_embeds
|
||||
@@ -4535,6 +4603,41 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
def get_input_embeddings(self) -> nn.Module:
|
||||
return self.model.get_input_embeddings()
|
||||
|
||||
def prepare_model_inputs(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
input_embeds: Optional[torch.Tensor],
|
||||
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
if self.vision is None:
|
||||
return input_ids, input_embeds
|
||||
has_images = (
|
||||
not forward_batch.forward_mode.is_decode()
|
||||
and not forward_batch.forward_mode.is_target_verify()
|
||||
and forward_batch.mm_inputs is not None
|
||||
and any(x is not None for x in forward_batch.mm_inputs)
|
||||
)
|
||||
if has_images and input_embeds is not None:
|
||||
raise ValueError("Cannot combine input_embeds and image inputs")
|
||||
mm_owner = self._mm_owner_session(forward_batch) if has_images else None
|
||||
# Peers may only enter the body or the CP shard once every rank has
|
||||
# finished all of its fallible input preparation, the remap included.
|
||||
with mm_owner.fence() if mm_owner is not None else nullcontext():
|
||||
if has_images:
|
||||
input_embeds = self._prepare_mm_embeddings(
|
||||
input_ids, forward_batch, mm_owner
|
||||
)
|
||||
if not (
|
||||
forward_batch.forward_mode.is_decode_or_idle()
|
||||
or forward_batch.forward_mode.is_target_verify()
|
||||
):
|
||||
# Decode/verify IDs are already vocabulary IDs; remap prompt image
|
||||
# hashes for Engram and routing.
|
||||
input_ids = input_ids.masked_fill(
|
||||
input_ids >= MM_PAD_SHIFT_VALUE, self.config.image_token_id
|
||||
)
|
||||
return input_ids, input_embeds
|
||||
|
||||
def set_dspark_layers_to_capture(self, layer_ids: List[int]) -> None:
|
||||
if not self.pp_group.is_last_rank:
|
||||
return
|
||||
@@ -4585,31 +4688,11 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
input_ids: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
input_embeds: Optional[torch.Tensor] = None,
|
||||
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
"""Prepare full-sequence image embeddings and model IDs before CP splits.
|
||||
|
||||
Scheduler hash IDs stay intact for multimodal cache keys; the language
|
||||
model uses image_token_id for Engram masking and visual MoE routing.
|
||||
"""
|
||||
if (
|
||||
getattr(self, "vision", None) is not None
|
||||
and not forward_batch.forward_mode.is_decode()
|
||||
and not forward_batch.forward_mode.is_target_verify()
|
||||
and forward_batch.mm_inputs is not None
|
||||
and any(x is not None for x in forward_batch.mm_inputs)
|
||||
):
|
||||
if input_embeds is not None:
|
||||
raise ValueError("Cannot combine input_embeds and image inputs")
|
||||
input_embeds = self._prepare_mm_embeddings(input_ids, forward_batch)
|
||||
if getattr(self, "vision", None) is not None and not (
|
||||
forward_batch.forward_mode.is_decode_or_idle()
|
||||
or forward_batch.forward_mode.is_target_verify()
|
||||
):
|
||||
# Decode/verify IDs are already vocabulary IDs; remap prompt image
|
||||
# hashes for Engram and routing.
|
||||
input_ids = input_ids.masked_fill(
|
||||
input_ids >= MM_PAD_SHIFT_VALUE, self.config.image_token_id
|
||||
)
|
||||
pp_proxy_tensors: Optional[PPProxyTensors] = None,
|
||||
) -> torch.Tensor:
|
||||
input_ids, input_embeds = self.prepare_model_inputs(
|
||||
input_ids=input_ids, forward_batch=forward_batch, input_embeds=input_embeds
|
||||
)
|
||||
|
||||
return input_ids, input_embeds
|
||||
|
||||
|
||||
@@ -265,6 +265,7 @@ class DeepseekV4ForCausalLMNextN(DeepseekV4ForCausalLM):
|
||||
self.quant_config = quant_config
|
||||
self.wo_a_fp8 = wo_a_fp8_gemm_enabled(quant_config)
|
||||
self.determine_num_fused_shared_experts()
|
||||
self.vision = None
|
||||
|
||||
self.model = DeepseekV4ModelNextN(
|
||||
config, quant_config, prefix=add_prefix("model", prefix)
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Small CPU tensors; production CP slicing/gather, mocked collective transport."""
|
||||
|
||||
from contextlib import ExitStack, contextmanager, nullcontext
|
||||
from types import SimpleNamespace as NS
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.cp.interleave import InterleaveCPStrategy
|
||||
from sglang.srt.layers.cp.padding import pad_logical_token_to_physical
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
|
||||
CP = "sglang.srt.layers.cp"
|
||||
|
||||
|
||||
@contextmanager
|
||||
def cp_context(size, rank, lengths=(3, 6), prefix_lengths=(7, 13)):
|
||||
"""Keep real interleave indexing/padding; replace only runtime context."""
|
||||
strategy = InterleaveCPStrategy(size)
|
||||
parallel = NS(attn_cp_size=size, attn_cp_rank=rank, attn_cp_group=None)
|
||||
batch = NS(
|
||||
forward_mode=ForwardMode.EXTEND,
|
||||
input_ids=torch.arange(1, sum(lengths) + 1),
|
||||
positions=torch.cat(
|
||||
[
|
||||
torch.arange(prefix, prefix + length)
|
||||
for prefix, length in zip(prefix_lengths, lengths)
|
||||
]
|
||||
),
|
||||
extend_seq_lens_cpu=list(lengths),
|
||||
extend_prefix_lens_cpu=list(prefix_lengths),
|
||||
mm_inputs=None,
|
||||
spec_info=None,
|
||||
)
|
||||
batch.attn_cp_metadata = strategy.build_metadata(
|
||||
sum(lengths), [p + n for p, n in zip(prefix_lengths, lengths)], list(lengths)
|
||||
)
|
||||
with ExitStack() as stack:
|
||||
for module in ("base", "utils", "padding", "interleave"):
|
||||
stack.enter_context(
|
||||
patch(CP + "." + module + ".get_parallel", return_value=parallel)
|
||||
)
|
||||
stack.enter_context(patch(CP + ".utils.get_cp_strategy", return_value=strategy))
|
||||
stack.enter_context(
|
||||
patch(CP + ".padding.get_cp_padding_align_size", return_value=size)
|
||||
)
|
||||
stack.enter_context(
|
||||
patch(
|
||||
CP + ".utils.get_moe_a2a_backend", return_value=NS(is_none=lambda: True)
|
||||
)
|
||||
)
|
||||
pad_logical_token_to_physical(batch.attn_cp_metadata)
|
||||
yield strategy, batch
|
||||
|
||||
|
||||
@contextmanager
|
||||
def simulated_collective(strategy, batch, global_tensor):
|
||||
"""Inject peer buffers into all-gather; retain production unpadding/reordering."""
|
||||
physical = max(batch.attn_cp_metadata.per_rank_actual_token)
|
||||
buffers = []
|
||||
for rank in range(strategy.cp_size):
|
||||
buf = global_tensor.new_zeros((physical, *global_tensor.shape[1:]))
|
||||
local = global_tensor[rank :: strategy.cp_size]
|
||||
buf[: len(local)] = local
|
||||
buffers.append(buf)
|
||||
|
||||
def gather(output, local):
|
||||
torch.testing.assert_close(local, buffers[strategy.cp_rank], rtol=0, atol=0)
|
||||
output.copy_(torch.cat(buffers))
|
||||
|
||||
with (
|
||||
patch(
|
||||
CP + ".interleave.use_symmetric_memory",
|
||||
side_effect=lambda *a, **k: nullcontext(),
|
||||
),
|
||||
patch(CP + ".interleave.is_allocation_symmetric", return_value=False),
|
||||
patch(CP + ".interleave.attn_cp_all_gather_into_tensor", side_effect=gather),
|
||||
):
|
||||
yield
|
||||
@@ -2996,6 +2996,107 @@ fn backup_kv_action_chains_unbacked_ancestors_first() {
|
||||
assert_eq!(action.node_ids, vec![c]);
|
||||
}
|
||||
|
||||
// Mirrors UnifiedRadixCache._execute_and_commit_kv_backup: back up each chain
|
||||
// node and stamp every node its transfers cover with that node's ack.
|
||||
// It skips the inc_lock_ref the Python side takes; the asserts do not depend on it.
|
||||
fn execute_backup_kv_for_test(tc: &mut UnifiedTreeCore<Vec<i64>>, backup: &BackupKV) {
|
||||
for &node_id in &backup.node_ids {
|
||||
let (device_value, mut comp_xfers) = tc.build_backup_spec(node_id).expect("backup spec");
|
||||
if device_value.numel() == 0 && comp_xfers.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let mut publish_node_ids = Vec::new();
|
||||
for transfer in comp_xfers.values_mut().flatten() {
|
||||
transfer.host_indices = transfer.device_indices.as_ref().map(Tensor::copy);
|
||||
publish_node_ids.extend(transfer.nodes_to_load.iter().flatten().copied());
|
||||
}
|
||||
if !publish_node_ids.contains(&node_id) {
|
||||
publish_node_ids.push(node_id);
|
||||
}
|
||||
tc.commit_backup(node_id, device_value.copy(), comp_xfers)
|
||||
.expect("live test node");
|
||||
tc.mark_write_through_pending(publish_node_ids, /* ack_id = */ node_id)
|
||||
.expect("live test nodes");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_back_swa_publish_leaves_an_ancestor_pending_under_another_ack_alone() {
|
||||
// c(Full unbacked) -> b(Full backed) -> t1(Full backed)
|
||||
// -> d(Full unbacked) -> t2(Full backed)
|
||||
// Every node holds a device-only SWA value inside the window.
|
||||
let mut tc = UnifiedTreeCore::new(
|
||||
CacheInitParams {
|
||||
is_write_back: true,
|
||||
enable_hicache: true,
|
||||
has_swa_host_pool: true,
|
||||
swa_sliding_window_size: Some(8),
|
||||
..Default::default()
|
||||
},
|
||||
vec![FULL, SWA],
|
||||
);
|
||||
let (key_t1, key_t2) = (vec![1, 2, 3, 4], vec![1, 2, 5, 6]);
|
||||
tc.insert(&insert_params(&key_t1, &[10, 11, 12, 13]));
|
||||
tc.insert(&insert_params(&key_t2, &[10, 11, 14, 15]));
|
||||
tc.insert(&insert_params(&vec![1, 2, 3], &[10, 11, 12]));
|
||||
tc.insert(&insert_params(&vec![1, 2, 5], &[10, 11, 14]));
|
||||
let child = |parent, page: &[i64]| {
|
||||
tc.arena
|
||||
.child_on_page(parent, /* extra_key = */ None, page)
|
||||
.expect("child on page")
|
||||
};
|
||||
let c_idx = child(tc.arena.root(), &[1]);
|
||||
let b_idx = child(c_idx, &[3]);
|
||||
let t1_idx = child(b_idx, &[4]);
|
||||
let d_idx = child(c_idx, &[5]);
|
||||
let t2_idx = child(d_idx, &[6]);
|
||||
let [c, b, t1, d, t2] = [c_idx, b_idx, t1_idx, d_idx, t2_idx].map(|idx| tc.arena.node(idx).id);
|
||||
for idx in [c_idx, b_idx, t1_idx, d_idx, t2_idx] {
|
||||
let full = tc.arena.node(idx).device_value(FULL).copy();
|
||||
tc.set_component_device_value(tc.arena.node(idx).id, SWA, full.copy())
|
||||
.expect("live test node");
|
||||
if [b_idx, t1_idx, t2_idx].contains(&idx) {
|
||||
tc.arena.set_host_value(idx, FULL, full);
|
||||
}
|
||||
}
|
||||
|
||||
// Two requests in one batch finish at t1 and t2; neither ack drains.
|
||||
for (key, target) in [(&key_t1, t1), (&key_t2, t2)] {
|
||||
let result = tc.insert(&insert_params(key, &[20, 21, 22, 23]));
|
||||
let backups: Vec<&BackupKV> = result
|
||||
.cache_actions
|
||||
.iter()
|
||||
.filter_map(|action| match action {
|
||||
CacheAction::BackupKV(backup) => Some(backup),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(backups.len(), 1);
|
||||
assert_eq!(backups[0].node_ids, vec![target]);
|
||||
execute_backup_kv_for_test(&mut tc, backups[0]);
|
||||
}
|
||||
|
||||
let pending = |node_id| {
|
||||
tc.arena
|
||||
.node(tc.arena.resolve(node_id).expect("live test node"))
|
||||
.write_through_pending_id
|
||||
};
|
||||
for (node_id, ack_id) in [(c, t1), (b, t1), (t1, t1), (d, t2), (t2, t2)] {
|
||||
assert_eq!(pending(node_id), Some(ack_id));
|
||||
}
|
||||
// Write-back defers the unbacked Full prefix to eviction.
|
||||
assert!(
|
||||
!tc.arena
|
||||
.node(tc.arena.resolve(c).expect("live test node"))
|
||||
.backuped()
|
||||
);
|
||||
assert!(
|
||||
!tc.arena
|
||||
.node(tc.arena.resolve(d).expect("live test node"))
|
||||
.backuped()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_back_defers_swa_window_backup_until_eviction() {
|
||||
// c(Full unbacked) -> b(Full backed) -> t1(Full backed)
|
||||
|
||||
@@ -378,6 +378,12 @@ impl RouterConfigBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
/// G9i pool-aware admission control + fast-P routing config (PD mode).
|
||||
pub fn g9i(mut self, g9i: Option<crate::routers::http::g9i::G9iConfig>) -> Self {
|
||||
self.config.g9i = g9i;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn model_path<S: Into<String>>(mut self, path: S) -> Self {
|
||||
self.config.model_path = Some(path.into());
|
||||
self
|
||||
|
||||
@@ -99,6 +99,9 @@ pub struct RouterConfig {
|
||||
/// Enable WASM support
|
||||
#[serde(default)]
|
||||
pub enable_wasm: bool,
|
||||
/// G9i pool-aware admission control + fast-P routing (PD mode, HTTP only).
|
||||
#[serde(default)]
|
||||
pub g9i: Option<crate::routers::http::g9i::G9iConfig>,
|
||||
}
|
||||
|
||||
/// Tokenizer cache configuration
|
||||
@@ -551,6 +554,7 @@ impl Default for RouterConfig {
|
||||
ca_certificates: vec![],
|
||||
mcp_config: None,
|
||||
enable_wasm: false,
|
||||
g9i: None,
|
||||
server_cert: None,
|
||||
server_key: None,
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user