Compare commits

..
Author SHA1 Message Date
sglang-bot 3d8df4b127 chore: update CI test est_time from recent run data 2026-09-28 00:00:30 +00:00
1222 changed files with 1624 additions and 24654 deletions
-57
View File
@@ -1,57 +0,0 @@
# Gitea Actions 自动构建 sglang 镜像(海外节点,原版源)
# 基底 lmsysorg/sglang:dev-dsv41(docker.io),依赖走 pypi.org 默认源。
# 触发:push 到 dsv41-pd 分支。
#
# 前置条件(一次性,在 Gitea 实例上配置):
# 1. 实例已注册 act_runner(Gitea Actions runner,标签含 ubuntu-latest)
# 2. 仓库 Settings → Secrets 添加:
# REGISTRY_USERNAME / REGISTRY_PASSWORD(推送镜像的账号,如 Gitea 访问令牌)
# 3. 可选:Settings → Actions → Variables 添加 REGISTRY(默认 git.agentwithu.com,
# 即 Gitea 自带容器 registry;也可填 docker.io 等)
name: build-sglang-image
on:
push:
branches: [dsv41-pd, dsv41-pd-visioncp]
# compose/部署配置 与 workflow 自身的改动不触发镜像构建(省 runner 与推送带宽)。
# 改了 workflow 想重建镜像时,需伴随任意代码改动或手动触发。
paths-ignore:
- 'deploy/**'
- '.gitea/**'
env:
# 直接写死 Gitea 自带 registry(vars context 在该实例上求值异常会导致回退 docker.io)
REGISTRY: git.agentwithu.com
IMAGE_NAME: minke.yu/sglang
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Resolve tag
id: meta
run: |
SHA9=$(git rev-parse --short=9 HEAD)
echo "tag=${{ github.ref_name }}-${SHA9}-ci-$(date -u +%Y%m%d-%H%M)" >> "$GITHUB_OUTPUT"
- uses: docker/setup-buildx-action@v3
- uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ secrets.REGISTRY_USERNAME }}
password: ${{ secrets.REGISTRY_PASSWORD }}
- uses: docker/build-push-action@v6
with:
context: .
file: Dockerfile.gitea
push: true
# cache-from: type=registry
# cache-to: type=registry,mode=max
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.tag }}
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }}-latest
-24
View File
@@ -1,24 +0,0 @@
# Gitea Actions 用(海外节点):原版 docker.io 基底 + 原版 pypi
# 与 b300 离线版(/data/ymk/build/Dockerfile)的区别:
# - 基底直用 docker.io 的 lmsysorg/sglang:dev-dsv41(不走 umirror/DaoCloud)
# - 不设 PIP_INDEX_URL,用默认 pypi.org
# - 源码由 CI checkout 后经 COPY 进镜像(不用 in-image clone)
FROM lmsysorg/sglang:dev-dsv41
# checkout 含 .git,setuptools-scm 可打戳(main 系分支 version 显示 dev 属正常)
COPY . /sgl-workspace/sglang
# Rust gateway(sgl-model-gateway)编译,与 b300 离线版 deploy/b300-build/Dockerfile 同构。
# 基底自带 rustup stable,RUSTUP_TOOLCHAIN=stable 绕过 rust-toolchain.toml 的 1.90 钉版;
# python 扩展必须显式 -p,产物重命名为 sglang_router_rs.abi3.so 覆盖 python 包内同文件。
# 海外 runner 直连 crates.io 无网络问题;cache mount 海外环境同样无害。
RUN --mount=type=cache,target=/cargo-home \
--mount=type=cache,target=/cargo-target \
cd /sgl-workspace/sglang/sgl-model-gateway \
&& CARGO_HOME=/cargo-home CARGO_TARGET_DIR=/cargo-target RUSTUP_TOOLCHAIN=stable \
cargo build --release -p sgl-model-gateway -p sgl-model-gateway-python \
&& cp /cargo-target/release/sgl-model-gateway /opt/sglang/bin/sgl-model-gateway \
&& cp /cargo-target/release/libsglang_router_rs.so \
/opt/sglang/lib/python3.12/site-packages/sglang_router/sglang_router_rs.abi3.so
RUN pip install --no-cache-dir -e /sgl-workspace/sglang/python
-77
View File
@@ -1,77 +0,0 @@
# 优化版(2026-10-08 v3):torch/nvidia 全家桶永远复用基底,全量路径零重下。
# 要点:
# 1. pyproject build-system.requires 含 torch==2.13.0,pip 隔离构建环境每次都重下 ~6GB。
# 基底镜像已装 torch 2.13 → 用 --no-build-isolation 复用,构建环境零下载。
# 2. 老全量路径让 pip 解析全部依赖:bare `torchvision` 等会让解析器弃用基底已装的
# 本地版本(torch 2.13.0+cu130),改用 PyPI 的 torch==2.13.0,连带重下整套 nvidia
# 依赖(9/23 实测 nvidia-nccl-cu13 等 6GB+,限流下 2h40m);且基底 NCCL 是
# 刻意装高的 2.30.7(torch 元数据钉 ==2.29.7),任何全量解析都会把它降级回去。
# → v3 不再让 pip 碰重型依赖:check_deps.py 在构建期对照 pyproject 与基底
# 已装版本,把增量分三类——重型(torch*/nvidia-*/cuda-*)永不进解析、
# 不满足即让构建失败并提示决策(显式失败好过静默重装);
# kernel wheel(sglang-kernel/flashinfer*/sgl-deep-gemm)要跟随源码版本,
# 钉版变动时以 pip --no-deps 单独装(防止顺藤摸瓜解析 torch);
# 其余轻量依赖只装未满足的增量。
# 3. 基底缺 setuptools-scm(版本打戳要用),单层预装 + pip cache mount,只下载一次。
# 4. RUN 的 pip cache mount 持久化在宿主机,轻量依赖增量不再重复下载。
FROM uhub.service.ucloud.cn/umirror/sglang:dev-dsv41
ARG PIP_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple
ENV PIP_INDEX_URL=${PIP_INDEX}
# 构建后端(pyproject build-system.requires,torch 除外——基底已有)
RUN --mount=type=cache,target=/root/.cache/pip \
pip install "setuptools>=61" "setuptools-scm>=8" "setuptools-rust>=1.11" wheel
# 依赖增量计算脚本(对照 pyproject 与基底已装版本,产出轻量/kernel 增量清单)
# 注意 build context 是 /data/ymk 且 .dockerignore 排除了 build/,只能从 sglang/ 树内 COPY
COPY sglang/deploy/b300-build/check_deps.py /tmp/check_deps.py
# 整个源码树(含 .git,用于 setuptools-scm 版本打戳)
COPY sglang/ /sgl-workspace/sglang/
# Rust gateway(sgl-model-gateway)编译。基底自带 rustup stable(实测 cargo 1.98.1),
# RUSTUP_TOOLCHAIN=stable 绕过仓库 rust-toolchain.toml 钉的 1.90(避免重复下载工具链);
# b300 直连 crates.io(稀疏索引 + static.crates.io 均通),无需任何镜像源。
# cache mount 持久化 registry/target:冷编译 ~15min(二进制 ~11min + python 扩展 ~4min),
# 增量重建秒~分钟级。python 扩展必须显式 -p(workspace 根包默认不编译扩展),
# 产物 libsglang_router_rs.so 需重命名为 sglang_router_rs.abi3.so 覆盖 python 包内同文件
# (与 H20 线上部署方式一致)。2026-10-09 b300 实测验证。
RUN --mount=type=cache,target=/cargo-home \
--mount=type=cache,target=/cargo-target \
cd /sgl-workspace/sglang/sgl-model-gateway \
&& CARGO_HOME=/cargo-home CARGO_TARGET_DIR=/cargo-target RUSTUP_TOOLCHAIN=stable \
cargo build --release -p sgl-model-gateway -p sgl-model-gateway-python \
&& cp /cargo-target/release/sgl-model-gateway /opt/sglang/bin/sgl-model-gateway \
&& cp /cargo-target/release/libsglang_router_rs.so \
/opt/sglang/lib/python3.12/site-packages/sglang_router/sglang_router_rs.abi3.so
# 计算依赖增量(重型 torch/nvidia/cuda 永不进入清单;不满足则 check_deps 非零、构建失败)
RUN python /tmp/check_deps.py /sgl-workspace/sglang/python/pyproject.toml \
/tmp/missing_light.txt /tmp/missing_kernel.txt \
&& echo "== kernel wheel delta (pip --no-deps) ==" && cat /tmp/missing_kernel.txt \
&& echo "== lightweight dep delta ==" && cat /tmp/missing_light.txt
# kernel wheel 增量(如 sglang-kernel 升版):--no-deps 防止 pip 顺藤摸瓜解析 torch
RUN --mount=type=cache,target=/root/.cache/pip \
if grep -q '[^[:space:]]' /tmp/missing_kernel.txt; then \
pip install --no-deps -r /tmp/missing_kernel.txt; \
else \
echo "== no kernel wheel delta"; \
fi
# 轻量依赖增量(通常为空或几个小包),重型全家桶原样复用基底。
# ⚠️ 必须 --no-deps:即使目标只是 xgrammar 这种小包,只要依赖图里出现 torch,
# pip 就会校验 torch+cu130 元数据钉死的 nvidia-nccl-cu13==2.29.7,发现基底装的是
# 2.30.7 便"好心"下载 206MB 降级(10-08 v3 首跑实测踩中)。基底的传递依赖本来就
# 齐全,增量小包的二级依赖也一样;若有漏网,b300 冒烟会在 import 阶段显式暴露。
RUN --mount=type=cache,target=/root/.cache/pip \
if grep -q '[^[:space:]]' /tmp/missing_light.txt; then \
pip install --no-deps -r /tmp/missing_light.txt; \
else \
echo "== no lightweight delta, skipping dependency install"; \
fi
# editable 安装 sglang 本体,--no-deps 保证 pip 完全不碰依赖图
RUN --mount=type=cache,target=/root/.cache/pip \
pip install --no-deps --no-build-isolation -e /sgl-workspace/sglang/python
-65
View File
@@ -1,65 +0,0 @@
# fast 路径(2026-10-08 v3.1):python/pyproject.toml 未变化时使用(build.sh 自动判断)。
# 与 Dockerfile 全量路径同构:同样跑 check_deps.py 增量安装。
# 原因:全量构建若装过 kernel wheel 增量(如 sglang-kernel 0.4.7),随后 pyproject 未变的
# fast 构建若只 --no-deps,会静默退回基底的旧 kernel(0.4.6.post1)——镜像内容漂移。
# check_deps 本地比对 ~0.5s,增量命中 pip cache 秒级,成本可忽略。
FROM uhub.service.ucloud.cn/umirror/sglang:dev-dsv41
ARG PIP_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple
ENV PIP_INDEX_URL=${PIP_INDEX}
# 构建后端(pyproject build-system.requires,torch 除外——基底已有)
RUN --mount=type=cache,target=/root/.cache/pip \
pip install "setuptools>=61" "setuptools-scm>=8" "setuptools-rust>=1.11" wheel
# 依赖增量计算脚本(对照 pyproject 与基底已装版本,产出轻量/kernel 增量清单)
# 注意 build context 是 /data/ymk 且 .dockerignore 排除了 build/,只能从 sglang/ 树内 COPY
COPY sglang/deploy/b300-build/check_deps.py /tmp/check_deps.py
# 整个源码树(含 .git,用于 setuptools-scm 版本打戳)
COPY sglang/ /sgl-workspace/sglang/
# Rust gateway(sgl-model-gateway)编译。基底自带 rustup stable(实测 cargo 1.98.1),
# RUSTUP_TOOLCHAIN=stable 绕过仓库 rust-toolchain.toml 钉的 1.90(避免重复下载工具链);
# b300 直连 crates.io(稀疏索引 + static.crates.io 均通),无需任何镜像源。
# cache mount 持久化 registry/target:冷编译 ~15min,增量重建秒~分钟级。
# python 扩展必须显式 -p(workspace 根包默认不编译扩展),产物 libsglang_router_rs.so
# 需重命名为 sglang_router_rs.abi3.so 覆盖 python 包内同文件(与 H20 线上部署方式一致)。
RUN --mount=type=cache,target=/cargo-home \
--mount=type=cache,target=/cargo-target \
cd /sgl-workspace/sglang/sgl-model-gateway \
&& CARGO_HOME=/cargo-home CARGO_TARGET_DIR=/cargo-target RUSTUP_TOOLCHAIN=stable \
cargo build --release -p sgl-model-gateway -p sgl-model-gateway-python \
&& cp /cargo-target/release/sgl-model-gateway /opt/sglang/bin/sgl-model-gateway \
&& cp /cargo-target/release/libsglang_router_rs.so \
/opt/sglang/lib/python3.12/site-packages/sglang_router/sglang_router_rs.abi3.so
# 计算依赖增量(重型 torch/nvidia/cuda 永不进入清单;不满足则 check_deps 非零、构建失败)
RUN python /tmp/check_deps.py /sgl-workspace/sglang/python/pyproject.toml \
/tmp/missing_light.txt /tmp/missing_kernel.txt \
&& echo "== kernel wheel delta (pip --no-deps) ==" && cat /tmp/missing_kernel.txt \
&& echo "== lightweight dep delta ==" && cat /tmp/missing_light.txt
# kernel wheel 增量(如 sglang-kernel 升版):--no-deps 防止 pip 顺藤摸瓜解析 torch
RUN --mount=type=cache,target=/root/.cache/pip \
if grep -q '[^[:space:]]' /tmp/missing_kernel.txt; then \
pip install --no-deps -r /tmp/missing_kernel.txt; \
else \
echo "== no kernel wheel delta"; \
fi
# 轻量依赖增量(通常为空或几个小包),重型全家桶原样复用基底。
# ⚠️ 必须 --no-deps:即使目标只是 xgrammar 这种小包,只要依赖图里出现 torch,
# pip 就会校验 torch+cu130 元数据钉死的 nvidia-nccl-cu13==2.29.7,发现基底装的是
# 2.30.7 便"好心"下载 206MB 降级(10-08 v3 首跑实测踩中)。基底的传递依赖本来就
# 齐全,增量小包的二级依赖也一样;若有漏网,b300 冒烟会在 import 阶段显式暴露。
RUN --mount=type=cache,target=/root/.cache/pip \
if grep -q '[^[:space:]]' /tmp/missing_light.txt; then \
pip install --no-deps -r /tmp/missing_light.txt; \
else \
echo "== no lightweight delta, skipping dependency install"; \
fi
# editable 安装 sglang 本体,--no-deps 保证 pip 完全不碰依赖图
RUN --mount=type=cache,target=/root/.cache/pip \
pip install --no-deps --no-build-isolation -e /sgl-workspace/sglang/python
-33
View File
@@ -1,33 +0,0 @@
#!/bin/bash
# 用法: /data/ymk/build/build.sh [tag]
# 默认 tag: ymkymx/sglang:<分支>-<sha9>-local-<UTC构建日期时间>(命名规则见 AGENTS.md)
# 另打 <分支>-latest-local 别名。
#
# 依赖路径自动选择:
# python/pyproject.toml 的 sha256 与上次成功构建一致 → Dockerfile.fast(--no-deps,秒级)
# 不一致(或 FULL=1)→ Dockerfile 全量依赖解析,成功后记录新 hash
set -euo pipefail
cd /data/ymk
SHA=$(git -C sglang rev-parse --short=9 HEAD)
BR=$(git -C sglang rev-parse --abbrev-ref HEAD)
DT=$(date -u +%Y%m%d-%H%M)
TAG=${1:-ymkymx/sglang:$BR-$SHA-local-$DT}
HASH_FILE=build/.last_pyproject_sha256
CUR=$(sha256sum sglang/python/pyproject.toml | cut -d' ' -f1)
PREV=$(cat "$HASH_FILE" 2>/dev/null || echo none)
DOCKERFILE=build/Dockerfile
if [ "$CUR" = "$PREV" ] && [ "${FULL:-0}" != "1" ]; then
DOCKERFILE=build/Dockerfile.fast
echo "== pyproject 未变,走 fast 路径(--no-deps)"
else
echo "== pyproject 有变化或 FULL=1,走全量依赖解析"
fi
echo "== building $TAG from sglang@$(git -C sglang log --oneline -1) [$DOCKERFILE]"
docker build -f "$DOCKERFILE" -t "$TAG" .
# 构建成功才记录 hash / 打别名
echo "$CUR" > "$HASH_FILE"
docker tag "$TAG" "ymkymx/sglang:$BR-latest-local"
echo "== done: $TAG (别名 $BR-latest-local)"
-90
View File
@@ -1,90 +0,0 @@
#!/usr/bin/env python3
"""Compute dependency deltas for a b300 sglang image build, in three classes.
Reads pyproject.toml, evaluates every requirement against the *installed*
distributions in the base image, and writes two requirements files:
argv[2] (light): unsatisfied lightweight requirements → `pip install -r`
argv[3] (kernel): unsatisfied kernel wheels (sglang-kernel, flashinfer*,
sgl-deep-gemm) → `pip install --no-deps -r`
These track the sglang source being built and are meant
to be upgraded from the index when pyproject bumps them;
--no-deps keeps pip from walking torch's metadata.
The PyTorch stack (torch/torchvision/torchaudio/torchcodec/
torch_memory_saver), the nvidia-* CUDA family and cuda-* packages are NEVER
emitted. If one of them is missing or its installed version no longer
satisfies the pinned specifier (e.g. upstream bumps nvidia-cutlass-dsl), the
script fails the build loudly instead of letting pip download ~6GB or
"fix" the base image's deliberately newer nvidia-nccl-cu13 2.30.7 back to
torch's metadata pin ==2.29.7 (any pip full-resolution would do both —
2026-09-23/24 builds did, throttled to 2h40m).
"""
import sys
import tomllib
from importlib import metadata as md
from packaging.requirements import Requirement
# Never resolved by pip: reused from the base image or the build fails.
HEAVY_PREFIXES = ("torch", "nvidia-", "cuda-")
# Kernel wheels: upgraded via pip --no-deps when pyproject pins move.
KERNEL_PREFIXES = ("sglang-kernel", "flashinfer", "sgl-deep-gemm")
def norm(name: str) -> str:
return name.lower().replace("_", "-")
def main() -> int:
pyproject, light_out, kernel_out = sys.argv[1], sys.argv[2], sys.argv[3]
with open(pyproject, "rb") as f:
deps = tomllib.load(f)["project"]["dependencies"]
light: list[str] = []
kernel: list[str] = []
heavy_unsat: list[str] = []
for dep in deps:
req = Requirement(dep)
if req.marker is not None and not req.marker.evaluate():
continue
try:
installed = md.version(req.name)
except md.PackageNotFoundError:
installed = None
satisfied = installed is not None and req.specifier.contains(
installed, prereleases=True
)
if satisfied:
continue
name = norm(req.name)
if name.startswith(HEAVY_PREFIXES):
heavy_unsat.append(f"{dep} (installed: {installed})")
elif name.startswith(KERNEL_PREFIXES):
kernel.append(dep)
else:
light.append(dep)
if heavy_unsat:
print("=" * 70, file=sys.stderr)
print("FATAL: heavy base-image deps no longer satisfy pyproject:", file=sys.stderr)
for item in heavy_unsat:
print(f" {item}", file=sys.stderr)
print(
"Refusing to let pip resolve these (would re-download ~6GB and/or\n"
"downgrade NCCL). Decide explicitly: bump the base image, or adjust\n"
"the pin in pyproject.toml.",
file=sys.stderr,
)
return 1
with open(light_out, "w") as f:
f.write("\n".join(light) + ("\n" if light else ""))
with open(kernel_out, "w") as f:
f.write("\n".join(kernel) + ("\n" if kernel else ""))
return 0
if __name__ == "__main__":
sys.exit(main())
-33
View File
@@ -1,33 +0,0 @@
# B300 ds41 部署 compose 档案
b300-01 上 DeepSeek-V4.1-Flash 各部署方案的 docker compose 存档。
**本目录是 source of truth**;b300 上的运行目录 `/data/ymk/ds41/` 是工作副本。
方案说明、参数矩阵、特殊 env、已知坑见飞书部署手册(组内共享):
《B300 sglang 部署手册》 https://u04wb5irxz.feishu.cn/docx/FOi8dbnybodHr0xIfkPcQ6gtnRx
## 同步流程
```bash
# 本机(D:\B300\sglang)
git add deploy/ && git commit -m "deploy: ..."
git push origin dsv41-pd-visioncp # Gitea(paths-ignore,不触发镜像构建)
git push b300 dsv41-pd-visioncp # b300 裸仓库
# b300-01
git -C /data/ymk/sglang pull # deploy/ 出现在 /data/ymk/sglang/deploy/b300-ds41/
```
## CI 说明
`.gitea/workflows/build-image.yaml` 的 push 触发器带 `paths-ignore: ['deploy/**', '.gitea/**']`,
改本目录不会触发镜像构建。注意:workflow 自身的改动也不再自动触发,需要重建镜像时
伴随代码改动 push 或手动触发。
## 目录规则
- 命名:`dockerserve-<拓扑>-<特性>-<卡位>.yml`(如 `-b4` = 后 4 卡)
- 退役文件不要留在主目录:移入 `archive-YYYYMMDD/`(b300 侧同样执行)
- 禁止提交 `.bak` 备份文件
- `mooncake-store.json` 是 hicache L3 mooncake store 配置,被
`dockerserve-pd2-p-r1/r2.yml` 通过 `SGLANG_HICACHE_MOONCAKE_CONFIG_PATH` 引用
-27
View File
@@ -1,27 +0,0 @@
cd /data/ymk/ds41
HDR="Authorization=Bearer deepseek-v4-flash-0731-key"
PORT=30040
MODEL=/data/models/DeepSeek-V4.1-Flash
# 一律 --warmup-requests 0:默认 warmup 会把 prompt#0 种进 L1/L2 污染 L3 测量(已踩过)
run() {
echo "===== [$(date +%H:%M:%S)] RUN: $1 (np=$2) ====="
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
--warmup-requests 0 --num-prompts "$2" --max-concurrency "$2" --seed "$3" --tokenize-prompt \
--random-range-ratio 1.0 --header "$HDR" 2>&1 | grep -E 'Input token throughput|Mean TTFT|P99 TTFT|Successful'
sleep 2
}
flush() { curl -sf -m 10 -X POST http://127.0.0.1:$PORT/flush_cache > /dev/null && echo "[$(date +%H:%M:%S)] flushed"; sleep 3; }
echo "===== PRIME COLD mc8(冷基线 + 写入 L3) ====="
run PRIME_COLD 8 1541391
flush
run MC1_L3 1 1541391
flush
run MC2_L3 2 1541391
flush
run MC4_L3 4 1541391
flush
run MC8_L3 8 1541391
echo "===== [$(date +%H:%M:%S)] ALL DONE ====="
-204
View File
@@ -1,204 +0,0 @@
#!/bin/bash
# bench_suite_v2.sh — DeepSeek-V4.1-Flash on b300-01,2026-10-09 新版基线方法
#
# 相对 v1 的方法变化:
# 1. 服务刚起数据不准 → 每个测试模式自带 warmup 轮(结果丢弃,标记 WARM)
# 2. prefill 冷/热双轮:冷轮写入 hicache(L1/L3),同 seed 热轮测纯 cache-hit TPS
# 3. decode 阶梯并发:步长 32,上限 = cuda-graph-max-bs-decode 的 80%;
# 每档先 warm(灌 L1)再测,prefill 全 cache hit;
# 报告 TPOT + 整实例输出 TPS + 推算实际并发(= 输出TPS × TPOT / 1000)
# 4. l3 模式(3p1d 经 router):大语料溢出 L1 后重放,验证 L3 命中
#
# 在宿主机执行,bench 客户端通过 docker exec 进容器跑(需要 /data 挂载)。
#
# 用法:
# bench_suite_v2.sh warmup <container> <port>
# bench_suite_v2.sh prefill <tag> <container> <port>
# bench_suite_v2.sh decode <tag> <container> <port> [cg_max_bs=256]
# bench_suite_v2.sh l3 <tag> <container> <port> [num_prompts=24]
# bench_suite_v2.sh report <log>
# 产物: /data/ymk/ds41/bench_v2_<tag>.log
set -u
BASE=/data/ymk/ds41
MODEL=${MODEL:-/data/models/DeepSeek-V4.1-Flash}
HDR="Authorization=Bearer deepseek-v4-flash-0731-key"
usage() { sed -n 2,20p "$0" >&2; exit 1; }
# $1=container $2=port $3=tag -> 生成可执行脚本路径并 echo
emit_common() {
cat <<EOF
cd $BASE
HDR="$HDR"
PORT=$2
run() {
local name="\$1"; shift
echo "===== [\$(date +%H:%M:%S)] START: \$name ====="
python3 -m sglang.bench_serving "\$@"
echo "===== [\$(date +%H:%M:%S)] END(exit=\$?): \$name ====="
sleep 5
}
EOF
}
# ---------------- warmup ----------------
cmd_warmup() {
local c=${1:?container} port=${2:?port}
local gen=$BASE/.bench_v2_warmup.sh
{ emit_common "$c" "$port"; cat <<'EOF'
run WARMUP_a --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
--dataset-name random-ids --random-input-len 4096 --random-output-len 16 \
--num-prompts 8 --max-concurrency 8 --seed 777001 --tokenize-prompt \
--random-range-ratio 1.0 --header "$HDR"
run WARMUP_b --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
--dataset-name random-ids --random-input-len 16000 --random-output-len 1 \
--num-prompts 2 --max-concurrency 1 --seed 777002 --tokenize-prompt \
--random-range-ratio 1.0 --header "$HDR"
echo "===== [$(date +%H:%M:%S)] ALL DONE ====="
EOF
} | sed "s|MODELPH|$MODEL|g" > "$gen"
chmod +x "$gen"
docker exec "$c" bash "$gen"
}
# ---------------- prefill(冷/热双轮) ----------------
cmd_prefill() {
local tag=${1:?tag} c=${2:?container} port=${3:?port}
local p=$(date +%H%M%S) gen=$BASE/.bench_v2_gen_${tag}.sh
{ emit_common "$c" "$port"
# warmup(丢弃)
cat <<'EOF'
run WARMUP --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
--dataset-name random-ids --random-input-len 4096 --random-output-len 16 \
--num-prompts 8 --max-concurrency 8 --seed 777000 --tokenize-prompt \
--random-range-ratio 1.0 --header "$HDR"
EOF
# 冷轮
for stage in "bs1_16000 16000 1 4 1" "bs1_160000 160000 1 4 2" "bs16_4096 4096 16 32 3"; do
set -- $stage
cat <<EOF
run COLD_$1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
--dataset-name random-ids --random-input-len $2 --random-output-len 1 \\
--num-prompts $4 --max-concurrency $3 --seed ${p}$5 --tokenize-prompt \\
--random-range-ratio 1.0 --header "\$HDR"
EOF
done
# 热轮(同 seed → cache hit)
for stage in "bs1_16000 16000 1 4 1" "bs1_160000 160000 1 4 2" "bs16_4096 4096 16 32 3"; do
set -- $stage
cat <<EOF
run HIT_$1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
--dataset-name random-ids --random-input-len $2 --random-output-len 1 \\
--num-prompts $4 --max-concurrency $3 --seed ${p}$5 --tokenize-prompt \\
--random-range-ratio 1.0 --header "\$HDR"
EOF
done
echo 'echo "===== [$(date +%H:%M:%S)] ALL DONE ====="'
} | sed "s|MODELPH|$MODEL|g" > "$gen"
chmod +x "$gen"
nohup docker exec "$c" bash "$gen" > "$BASE/bench_v2_${tag}.log" 2>&1 &
echo "launched pid=$! seeds=${p}1..3 log=$BASE/bench_v2_${tag}.log"
}
# ---------------- decode 阶梯 ----------------
cmd_decode() {
local tag=${1:?tag} c=${2:?container} port=${3:?port} cg=${4:-256}
local maxc=$(( cg * 80 / 100 / 32 * 32 )) # 80% 向下取整到 32 的倍数
local p=$(date +%H%M%S) gen=$BASE/.bench_v2_gen_${tag}.sh
echo "ladder: 32..$maxc step 32 (cg_max_bs=$cg)"
{ emit_common "$c" "$port"
cat <<'EOF'
run WARMUP --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
--dataset-name random-ids --random-input-len 2048 --random-output-len 128 \
--num-prompts 16 --max-concurrency 16 --seed 777000 --tokenize-prompt \
--random-range-ratio 1.0 --header "$HDR"
EOF
# bs1 延迟地板(warm + measure)
cat <<EOF
run WARM_bs1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
--num-prompts 4 --max-concurrency 1 --seed ${p}01 --tokenize-prompt \\
--random-range-ratio 1.0 --header "\$HDR"
run DEC_bs1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
--num-prompts 4 --max-concurrency 1 --seed ${p}01 --tokenize-prompt \\
--random-range-ratio 1.0 --header "\$HDR"
EOF
local cc
for (( cc=32; cc<=maxc; cc+=32 )); do
local np=$(( cc * 2 ))
cat <<EOF
run WARM_c${cc} --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
--num-prompts $np --max-concurrency $cc --seed ${p}1${cc} --tokenize-prompt \\
--random-range-ratio 1.0 --header "\$HDR"
run DEC_c${cc} --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
--num-prompts $np --max-concurrency $cc --seed ${p}1${cc} --tokenize-prompt \\
--random-range-ratio 1.0 --header "\$HDR"
EOF
done
echo 'echo "===== [$(date +%H:%M:%S)] ALL DONE ====="'
} | sed "s|MODELPH|$MODEL|g" > "$gen"
chmod +x "$gen"
nohup docker exec "$c" bash "$gen" > "$BASE/bench_v2_${tag}.log" 2>&1 &
echo "launched pid=$! log=$BASE/bench_v2_${tag}.log"
}
# ---------------- L3 语料溢出(3p1d 经 router) ----------------
cmd_l3() {
local tag=${1:?tag} c=${2:?container} port=${3:?port} np=${4:-48}
local p=$(date +%H%M%S) gen=$BASE/.bench_v2_gen_${tag}.sh
{ emit_common "$c" "$port"
cat <<EOF
run WARMUP --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
--dataset-name random-ids --random-input-len 4096 --random-output-len 16 \\
--num-prompts 8 --max-concurrency 8 --seed 777000 --tokenize-prompt \\
--random-range-ratio 1.0 --header "\$HDR"
run L3_COLD_160k --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \\
--num-prompts $np --max-concurrency 8 --seed ${p}1 --tokenize-prompt \\
--random-range-ratio 1.0 --header "\$HDR"
echo "===== [\$(date +%H:%M:%S)] DRAIN: sleep 120 等 L2→L3 备份排空 ====="
sleep 120
run L3_HIT_160k --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \\
--num-prompts $np --max-concurrency 8 --seed ${p}1 --tokenize-prompt \\
--random-range-ratio 1.0 --header "\$HDR"
EOF
echo 'echo "===== [$(date +%H:%M:%S)] ALL DONE ====="'
} > "$gen"
chmod +x "$gen"
nohup docker exec "$c" bash "$gen" > "$BASE/bench_v2_${tag}.log" 2>&1 &
echo "launched pid=$! corpus=${np}x160k log=$BASE/bench_v2_${tag}.log"
}
# ---------------- report ----------------
cmd_report() {
local log=${1:?log required}
awk '
/START:/ { name=$4; sub(/:$/, "", name); tps=""; itps=""; tpot=""; ttft=""; acc="" }
/Input token throughput/ { itps=$NF }
/Output token throughput/ { tps=$NF }
/Mean TPOT/ { tpot=$NF }
/Mean TTFT/ { ttft=$NF }
/[Aa]ccept/ { acc=$0; gsub(/^ +/, "", acc) }
/END\(exit/ {
line=sprintf("%-22s in_tps=%-10s out_tps=%-10s tpot_ms=%-8s ttft_ms=%-8s", name, itps, tps, tpot, ttft)
if (tps != "" && tpot != "" && tpot+0 > 0)
line=line sprintf(" eff_concurrency=%.1f", tps * tpot / 1000)
print line
if (acc != "") print " " acc
}
' "$log"
}
case "${1:-}" in
warmup) shift; cmd_warmup "$@" ;;
prefill) shift; cmd_prefill "$@" ;;
decode) shift; cmd_decode "$@" ;;
l3) shift; cmd_l3 "$@" ;;
report) shift; cmd_report "$@" ;;
*) usage ;;
esac
-61
View File
@@ -1,61 +0,0 @@
cd /data/ymk/ds41
HDR="Authorization=Bearer deepseek-v4-flash-0731-key"
PORT=30040
MODEL=/data/models/DeepSeek-V4.1-Flash
run() {
local name="$1"; shift
echo "===== [$(date +%H:%M:%S)] START: $name ====="
python3 -m sglang.bench_serving "$@" 2>&1 | grep -vE '^Namespace|warn'
echo "===== [$(date +%H:%M:%S)] END(exit=$?): $name ====="
sleep 3
}
r160() { # $1=name $2=seed $3=num_prompts $4=mc
run "$1" --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
--num-prompts "$3" --max-concurrency "$4" --seed "$2" --tokenize-prompt \
--random-range-ratio 1.0 --header "$HDR"
}
# WARMUP
run WARMUP --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
--dataset-name random-ids --random-input-len 4096 --random-output-len 16 \
--num-prompts 8 --max-concurrency 8 --seed 777000 --tokenize-prompt \
--random-range-ratio 1.0 --header "$HDR"
# R0 纯计算极限(集合 A 首次出现,L1/L2/L3 全空)
r160 R0_COMPUTE_COLD 1541391 8 8
# R1 L1 极限(A 全量在 L1 设备池:1.28M < 1.6M)
r160 R1_L1_HIT 1541391 8 8
# R2 填充集 B(12×160k=1.92M > L1 1.6M),把 A 挤出 L1 到 L2
r160 R2_FILLER_EVICT 999000 12 8
# R3 L2 极限(A 已被挤出 L1,从 L2 host 池 load-back)
r160 R3_L2_HIT 1541391 8 8
# R4 flush L1+L2(L3 保留),走 L3 prefetch
echo "===== [$(date +%H:%M:%S)] FLUSH L1+L2 ====="
curl -sf -m 10 -X POST http://127.0.0.1:$PORT/flush_cache && echo "flush ok"
sleep 5
r160 R4_L3_HIT 1541391 8 8
# R5 嫁接后 L2 复核
r160 R5_L2_AFTER_GRAFT 1541391 8 8
# R6 混合负载:新集合 C(纯计算,4×160k mc4)+ 集合 A(命中,8×160k mc8)并发打
echo "===== [$(date +%H:%M:%S)] START: R6_MIX_PARALLEL ====="
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
--num-prompts 4 --max-concurrency 4 --seed 555000 --tokenize-prompt \
--random-range-ratio 1.0 --header "$HDR" > /data/ymk/ds41/.tier_r6_coldclient.log 2>&1 &
COLD_PID=$!
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
--num-prompts 8 --max-concurrency 8 --seed 1541391 --tokenize-prompt \
--random-range-ratio 1.0 --header "$HDR" > /data/ymk/ds41/.tier_r6_hitclient.log 2>&1 &
HIT_PID=$!
wait $COLD_PID $HIT_PID
echo "===== [$(date +%H:%M:%S)] END: R6_MIX_PARALLEL ====="
echo "--- R6 cold client (compute) ---"
grep -E 'Input token throughput|Mean TTFT|Request throughput' /data/ymk/ds41/.tier_r6_coldclient.log
echo "--- R6 hit client (cache) ---"
grep -E 'Input token throughput|Mean TTFT|Request throughput' /data/ymk/ds41/.tier_r6_hitclient.log
echo "===== [$(date +%H:%M:%S)] ALL DONE ====="
-27
View File
@@ -1,27 +0,0 @@
cd /data/ymk/ds41
HDR="Authorization=Bearer deepseek-v4-flash-0731-key"
PORT=30040
MODEL=/data/models/DeepSeek-V4.1-Flash
echo "===== [$(date +%H:%M:%S)] FLUSH L1+L2(A 只剩 L3) ====="
curl -sf -m 10 -X POST http://127.0.0.1:$PORT/flush_cache && echo "flush ok"
sleep 5
echo "===== [$(date +%H:%M:%S)] START: R7_MIX_L3_PLUS_COMPUTE ====="
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
--num-prompts 4 --max-concurrency 4 --seed 666000 --tokenize-prompt \
--random-range-ratio 1.0 --header "$HDR" > /data/ymk/ds41/.tier_r7_coldclient.log 2>&1 &
COLD_PID=$!
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
--num-prompts 8 --max-concurrency 8 --seed 1541391 --tokenize-prompt \
--random-range-ratio 1.0 --header "$HDR" > /data/ymk/ds41/.tier_r7_hitclient.log 2>&1 &
HIT_PID=$!
wait $COLD_PID $HIT_PID
echo "===== [$(date +%H:%M:%S)] END: R7 ====="
echo "--- R7 cold client (compute, 4x160k mc4) ---"
grep -E 'Input token throughput|Mean TTFT|Request throughput' /data/ymk/ds41/.tier_r7_coldclient.log
echo "--- R7 hit client (L3, 8x160k mc8) ---"
grep -E 'Input token throughput|Mean TTFT|Request throughput' /data/ymk/ds41/.tier_r7_hitclient.log
echo "===== [$(date +%H:%M:%S)] ALL DONE ====="
-172
View File
@@ -1,172 +0,0 @@
# 3x cp2-P + 1x dp2-D PD 分离 + L3 mooncake 互联缓存 + engram host table
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
x-sglang-common: &sglang-common
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
shm_size: "32gb"
ipc: host
pid: host
privileged: true
network_mode: host
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
x-p-environment: &p-environment
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
SGLANG_RAGGED_VERIFY_MODE: static
MC_INTRANODE_NVLINK: "true"
MC_INTRA_NVLINK: "true"
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
services:
p1:
<<: *sglang-common
container_name: ds41-cp2-p1
environment:
<<: *p-environment
CUDA_VISIBLE_DEVICES: "0,1"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--enable-prefill-cp --cp-strategy interleave
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 64
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30000
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 2
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
--disaggregation-mode prefill
--disaggregation-transfer-backend mooncake
--disaggregation-bootstrap-port 8998
p2:
<<: *sglang-common
container_name: ds41-cp2-p2
environment:
<<: *p-environment
CUDA_VISIBLE_DEVICES: "2,3"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--enable-prefill-cp --cp-strategy interleave
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 64
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30010
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 2
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
--disaggregation-mode prefill
--disaggregation-transfer-backend mooncake
--disaggregation-bootstrap-port 8999
p3:
<<: *sglang-common
container_name: ds41-cp2-p3
environment:
<<: *p-environment
CUDA_VISIBLE_DEVICES: "4,5"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--enable-prefill-cp --cp-strategy interleave
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 64
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30020
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 2
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
--disaggregation-mode prefill
--disaggregation-transfer-backend mooncake
--disaggregation-bootstrap-port 9000
d:
<<: *sglang-common
container_name: ds41-cp2-d
environment:
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
SGLANG_RAGGED_VERIFY_MODE: static
MC_INTRANODE_NVLINK: "true"
MC_INTRA_NVLINK: "true"
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
CUDA_VISIBLE_DEVICES: "6,7"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2 --dp-size 2
--enable-dp-attention --enable-dp-lm-head
--mem-fraction-static 0.80
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 64
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30001
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--disaggregation-mode decode
--disaggregation-transfer-backend mooncake
--disaggregation-bootstrap-port 9001
@@ -1,225 +0,0 @@
# 【L3 dummy client 变体 2026-10-09 深夜,定型=shm+best_effort】sglang→standalone_storage,数据面经本机 mooncake_client(50052 IPC+进程映射),不碰 TCP 传输引擎
# 实测 24x160k 热/冷:shm+best_effort +26.4%(唯一正收益);wait_complete 因 all-or-nothing discard 等待白等反而 -17.7%。TCP 传输引擎两组皆负。
# 【L3 cache-hit 测试变体 2026-10-09】基于 dockerserve-3p1d-otlp.yml,仅改 P:
# --max-total-tokens 800000(L1 缩到 ~0.8M token)+ --hicache-ratio 1.0(L2=L1=0.8M token。⚠️ 教训:ratio 0.05 时 L2 太小,build_storage_backup_spec 拿不到 host 页会静默跳过 L3 写入,热轮无数据可中;V4 hicache 不支持 --hicache-size)
# 使 ~2.6M token/P 的语料(48×160k)溢出 L1+L2(1.6M),热轮强制走 L3 mooncake 命中。
# 【2026-10-10 L2 放大】hicache-ratio 1.0→4.0(L2=3.2M token/P,宿主可用内存 ~1.9TB 充裕):
# #38348 fail-soft 移植后 bench 显示收益被容量掩盖——24x160k 下 host_capacity 拒绝
# 0.75M~2.97M token/P(L2 装不下,prefetch 未发起即被拒退回重算)。放大后验证:
# ①热轮请求 L3/L2 命中不被容量拒绝;②L3 hit 嫁接进 L2 树后在 L2 长期留存。
# 与主模板同容器名/端口,用前必须先 down 主栈。D 与 router 不变。
# 原注释: 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram + OTLP trace(2026-09-27)
# 基于 dockerserve-3p1d-soak.yml 增加: --enable-trace --trace-modules request,mooncake
# --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name ds41-3p1d-<实例名>
# 依赖: b300-01 k8s cw jaeger-trace(jaegertracing/all-in-one:1.76.0,badger 6h TTL,hostNetwork 4317)
# 安全性: BatchSpanProcessor 导出失败只丢 span;Jaeger 宕机不影响推理服务
# 调优旋钮(默认未开): SGLANG_TRACE_ASYNC=1(异步 span 创建), SGLANG_TRACE_LEVEL(默认 3)
# router(ds41-router-3p1d,rust)无 OTLP 支持,不产生 span,追踪从各 sglang HTTP 层开始
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
x-sglang-common: &sglang-common
image: ymkymx/sglang:dsv41-pd-visioncp-baba18bc7d-local-20261010-2310
shm_size: "32gb"
ipc: host
pid: host
privileged: true
network_mode: host
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
- /data/ymk/sglang/python:/sgl-workspace/sglang/python
- /data/ymk/ds41/fake-cargo:/root/.cargo/bin/cargo:ro
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
x-p-environment: &p-environment
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
RUSTUP_TOOLCHAIN: "1.92-x86_64-unknown-linux-gnu"
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
SGLANG_RAGGED_VERIFY_MODE: static
MC_INTRANODE_NVLINK: "true"
MC_INTRA_NVLINK: "true"
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store-dummy.json
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
services:
p1:
<<: *sglang-common
container_name: ds41-3p1d-p1
environment:
<<: *p-environment
CUDA_VISIBLE_DEVICES: "0,1"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--enable-prefill-cp --cp-strategy interleave
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 256
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30000
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 4.0
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy best_effort
--hicache-size 0
--max-total-tokens 800000
--disaggregation-mode prefill
--disaggregation-transfer-backend mooncake
--disaggregation-bootstrap-port 8998
--enable-trace
--trace-modules request,mooncake
--otlp-traces-endpoint 127.0.0.1:4317
--otlp-service-name ds41-3p1d-p1
p2:
<<: *sglang-common
container_name: ds41-3p1d-p2
environment:
<<: *p-environment
CUDA_VISIBLE_DEVICES: "2,3"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--enable-prefill-cp --cp-strategy interleave
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 256
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30010
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 4.0
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy best_effort
--hicache-size 0
--max-total-tokens 800000
--disaggregation-mode prefill
--disaggregation-transfer-backend mooncake
--disaggregation-bootstrap-port 8999
--enable-trace
--trace-modules request,mooncake
--otlp-traces-endpoint 127.0.0.1:4317
--otlp-service-name ds41-3p1d-p2
p3:
<<: *sglang-common
container_name: ds41-3p1d-p3
environment:
<<: *p-environment
CUDA_VISIBLE_DEVICES: "4,5"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--enable-prefill-cp --cp-strategy interleave
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 256
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30020
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 4.0
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy best_effort
--hicache-size 0
--max-total-tokens 800000
--disaggregation-mode prefill
--disaggregation-transfer-backend mooncake
--disaggregation-bootstrap-port 9000
--enable-trace
--trace-modules request,mooncake
--otlp-traces-endpoint 127.0.0.1:4317
--otlp-service-name ds41-3p1d-p3
d:
<<: *sglang-common
# G9 P0 验证(2026-10-11):D 单独升镜像拿 /query_cache_hit 修复(71c6d8bc5),
# 并开 decode radix + hicache L3 客户端(复用 dummy store,只读查询)。
image: ymkymx/sglang:dsv41-pd-visioncp-6612af9e0-local-20261011-0740
container_name: ds41-3p1d-d1
environment:
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
RUSTUP_TOOLCHAIN: "1.92-x86_64-unknown-linux-gnu"
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
SGLANG_RAGGED_VERIFY_MODE: static
MC_INTRANODE_NVLINK: "true"
MC_INTRA_NVLINK: "true"
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store-dummy.json
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
CUDA_VISIBLE_DEVICES: "6,7"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2 --dp-size 2
--enable-dp-attention --enable-dp-lm-head
--mem-fraction-static 0.80
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 256
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 256
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30001
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--disaggregation-mode decode
--disaggregation-transfer-backend mooncake
--disaggregation-bootstrap-port 9001
--disaggregation-decode-enable-radix-cache
--enable-hierarchical-cache
--hicache-ratio 1.0
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy best_effort
--enable-trace
--trace-modules request,mooncake
--otlp-traces-endpoint 127.0.0.1:4317
--otlp-service-name ds41-3p1d-d1
@@ -1,239 +0,0 @@
# 【G9 P0 验证栈:真实 store 3p1d 2026-10-11】基于 dockerserve-3p1d-otlp.yml。
# 用途:验证 P0 改动(/v1/loads 排队 token + drain rate 字段、P 侧 /query_cache_hit)。
# 与主模板同容器名/端口,用前必须先 down 主栈;router(ds41-router-3p1d)独立不在本文件。
# 2026-10-11 改造要点(vs 原 l3test):
# - 镜像升 6612af9e0(含 G9 P0 + /query_cache_hit KeyError 修复)+ scheduler.py
# 热补丁(L3 查询门限从 enable_decode_hicache 放宽到 enable_hicache_storage,
# 否则 P 节点永远查不到 L3)。
# - 每实例独立 MOONCAKE_LOCAL_HOSTNAME(a/b/c/d),extra_hosts 互配全部 rank 别名
# (真实 store 跨实例副本写要求对端 rank 主机名可解析,见 realstore-x2 注释;
# rank 命名规则 {hostname}-d{dp}t{tp}p{pp}c{attn_cp},commit 84fc632088;
# D 为 dp2 dp-attention(attn_tp=1),后缀推断为 -d0t0p0c0/-d1t1p0c0,未实证)。
# - ⚠️ D 侧 cache 方案实证否决(2026-10-11):SWA 模型下
# --disaggregation-decode-enable-radix-cache 先要 SGLANG_ENABLE_UNIFIED_RADIX_TREE=1,
# 开了又报 "supports only device-resident cache and is incompatible with
# --enable-hierarchical-cache"(kv_cache_builder.py 硬 ValueError)——D 挂不了 L3。
# 结论:/query_cache_hit 放 P(L3 全局统一,P 有完整 hicache),D 保持生产配置。
# - D mem-fraction 0.80:曾因崩溃 d1 泄漏 ~61GiB 临时降 0.75,全栈 down 掉旧容器后
# 驱动即回收(GPU 全零),已恢复 0.80。D max-running 保持 256(512 时 SWA 池挤爆 KV 池)。
# P 侧保持 l3test 原状:--max-total-tokens 800000 + ratio 1.0(L1/L2 各 ~0.8M token),
# write_through 保证 prefill 后 L3 有数据,供 /query_cache_hit 对拍。
# 原注释: 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram + OTLP trace(2026-09-27)
# 依赖: mooncake-master 容器(50051,起栈前 restart 清旧 segment 注册);
# b300-01 k8s cw jaeger-trace(4317,导出失败只丢 span 不影响服务)
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
x-sglang-common: &sglang-common
image: ymkymx/sglang:dsv41-pd-visioncp-e7dc175e9-local-20261011-0940
shm_size: "32gb"
ipc: host
pid: host
privileged: true
network_mode: host
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
x-rank-hosts: &rank-hosts
# mooncake rank 主机名别名:P(tp2/cp2 interleave)-d0t0p0c0/-d0t1p0c1;
# D(dp2 dp-attention)-d0t0p0c0/-d1t1p0c0(推断,起栈后核对 master 注册名)
- "localhost-a-d0t0p0c0:127.0.0.1"
- "localhost-a-d0t1p0c1:127.0.0.1"
- "localhost-b-d0t0p0c0:127.0.0.1"
- "localhost-b-d0t1p0c1:127.0.0.1"
- "localhost-c-d0t0p0c0:127.0.0.1"
- "localhost-c-d0t1p0c1:127.0.0.1"
- "localhost-d-d0t0p0c0:127.0.0.1"
- "localhost-d-d1t1p0c0:127.0.0.1"
x-p-environment: &p-environment
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
SGLANG_RAGGED_VERIFY_MODE: static
MC_INTRANODE_NVLINK: "true"
MC_INTRA_NVLINK: "true"
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
services:
p1:
<<: *sglang-common
container_name: ds41-3p1d-p1
extra_hosts: *rank-hosts
environment:
<<: *p-environment
MOONCAKE_LOCAL_HOSTNAME: localhost-a
CUDA_VISIBLE_DEVICES: "0,1"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--enable-prefill-cp --cp-strategy interleave
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 256
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30000
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 1.0
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy best_effort
--hicache-size 0
--max-total-tokens 800000
--disaggregation-mode prefill
--disaggregation-transfer-backend mooncake
--disaggregation-bootstrap-port 8998
--enable-trace
--trace-modules request,mooncake
--otlp-traces-endpoint 127.0.0.1:4317
--otlp-service-name ds41-3p1d-p1
p2:
<<: *sglang-common
container_name: ds41-3p1d-p2
extra_hosts: *rank-hosts
environment:
<<: *p-environment
MOONCAKE_LOCAL_HOSTNAME: localhost-b
CUDA_VISIBLE_DEVICES: "2,3"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--enable-prefill-cp --cp-strategy interleave
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 256
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30010
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 1.0
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy best_effort
--hicache-size 0
--max-total-tokens 800000
--disaggregation-mode prefill
--disaggregation-transfer-backend mooncake
--disaggregation-bootstrap-port 8999
--enable-trace
--trace-modules request,mooncake
--otlp-traces-endpoint 127.0.0.1:4317
--otlp-service-name ds41-3p1d-p2
p3:
<<: *sglang-common
container_name: ds41-3p1d-p3
extra_hosts: *rank-hosts
environment:
<<: *p-environment
MOONCAKE_LOCAL_HOSTNAME: localhost-c
CUDA_VISIBLE_DEVICES: "4,5"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--enable-prefill-cp --cp-strategy interleave
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 256
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30020
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 1.0
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy best_effort
--hicache-size 0
--max-total-tokens 800000
--disaggregation-mode prefill
--disaggregation-transfer-backend mooncake
--disaggregation-bootstrap-port 9000
--enable-trace
--trace-modules request,mooncake
--otlp-traces-endpoint 127.0.0.1:4317
--otlp-service-name ds41-3p1d-p3
d:
<<: *sglang-common
container_name: ds41-3p1d-d1
extra_hosts: *rank-hosts
# ⚠️ D 保持生产配置(2026-10-11 实证):SWA 模型下
# --disaggregation-decode-enable-radix-cache 与 --enable-hierarchical-cache 互斥
# (kv_cache_builder.py 硬 ValueError,"supports only device-resident cache"),
# D 无法挂 L3 —— /query_cache_hit 改由 P 节点回答(L3 全局统一,P 有完整 hicache)。
# MOONCAKE_LOCAL_HOSTNAME 保留备用(无害)。
environment:
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
SGLANG_RAGGED_VERIFY_MODE: static
MC_INTRANODE_NVLINK: "true"
MC_INTRA_NVLINK: "true"
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
MOONCAKE_LOCAL_HOSTNAME: localhost-d
CUDA_VISIBLE_DEVICES: "6,7"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2 --dp-size 2
--enable-dp-attention --enable-dp-lm-head
--mem-fraction-static 0.80
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 256
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 256
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30001
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--disaggregation-mode decode
--disaggregation-transfer-backend mooncake
--disaggregation-bootstrap-port 9001
--enable-trace
--trace-modules request,mooncake
--otlp-traces-endpoint 127.0.0.1:4317
--otlp-service-name ds41-3p1d-d1
@@ -1,196 +0,0 @@
# 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram + OTLP trace(2026-09-27)
# 基于 dockerserve-3p1d-soak.yml 增加: --enable-trace --trace-modules request,mooncake
# --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name ds41-3p1d-<实例名>
# 依赖: b300-01 k8s cw jaeger-trace(jaegertracing/all-in-one:1.76.0,badger 6h TTL,hostNetwork 4317)
# 安全性: BatchSpanProcessor 导出失败只丢 span;Jaeger 宕机不影响推理服务
# 调优旋钮(默认未开): SGLANG_TRACE_ASYNC=1(异步 span 创建), SGLANG_TRACE_LEVEL(默认 3)
# router(ds41-router-3p1d,rust)无 OTLP 支持,不产生 span,追踪从各 sglang HTTP 层开始
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
x-sglang-common: &sglang-common
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
shm_size: "32gb"
ipc: host
pid: host
privileged: true
network_mode: host
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
x-p-environment: &p-environment
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
SGLANG_RAGGED_VERIFY_MODE: static
MC_INTRANODE_NVLINK: "true"
MC_INTRA_NVLINK: "true"
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
services:
p1:
<<: *sglang-common
container_name: ds41-3p1d-p1
environment:
<<: *p-environment
CUDA_VISIBLE_DEVICES: "0,1"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--enable-prefill-cp --cp-strategy interleave --chunked-prefill-size 32768 --max-prefill-tokens 32768
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 256
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30000
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 2
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
--disaggregation-mode prefill
--disaggregation-transfer-backend mooncake
--disaggregation-bootstrap-port 8998
--enable-trace
--trace-modules request,mooncake
--otlp-traces-endpoint 127.0.0.1:4317
--otlp-service-name ds41-3p1d-p1
p2:
<<: *sglang-common
container_name: ds41-3p1d-p2
environment:
<<: *p-environment
CUDA_VISIBLE_DEVICES: "2,3"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--enable-prefill-cp --cp-strategy interleave --chunked-prefill-size 32768 --max-prefill-tokens 32768
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 256
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30010
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 2
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
--disaggregation-mode prefill
--disaggregation-transfer-backend mooncake
--disaggregation-bootstrap-port 8999
--enable-trace
--trace-modules request,mooncake
--otlp-traces-endpoint 127.0.0.1:4317
--otlp-service-name ds41-3p1d-p2
p3:
<<: *sglang-common
container_name: ds41-3p1d-p3
environment:
<<: *p-environment
CUDA_VISIBLE_DEVICES: "4,5"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--enable-prefill-cp --cp-strategy interleave --chunked-prefill-size 32768 --max-prefill-tokens 32768
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 256
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30020
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 2
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
--disaggregation-mode prefill
--disaggregation-transfer-backend mooncake
--disaggregation-bootstrap-port 9000
--enable-trace
--trace-modules request,mooncake
--otlp-traces-endpoint 127.0.0.1:4317
--otlp-service-name ds41-3p1d-p3
d:
<<: *sglang-common
container_name: ds41-3p1d-d1
environment:
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
SGLANG_RAGGED_VERIFY_MODE: static
MC_INTRANODE_NVLINK: "true"
MC_INTRA_NVLINK: "true"
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
CUDA_VISIBLE_DEVICES: "6,7"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2 --dp-size 2
--enable-dp-attention --enable-dp-lm-head
--mem-fraction-static 0.80
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 256
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 256
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30001
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--disaggregation-mode decode
--disaggregation-transfer-backend mooncake
--disaggregation-bootstrap-port 9001
--enable-trace
--trace-modules request,mooncake
--otlp-traces-endpoint 127.0.0.1:4317
--otlp-service-name ds41-3p1d-d1
-196
View File
@@ -1,196 +0,0 @@
# 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram + OTLP trace(2026-09-27)
# 基于 dockerserve-3p1d-soak.yml 增加: --enable-trace --trace-modules request,mooncake
# --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name ds41-3p1d-<实例名>
# 依赖: b300-01 k8s cw jaeger-trace(jaegertracing/all-in-one:1.76.0,badger 6h TTL,hostNetwork 4317)
# 安全性: BatchSpanProcessor 导出失败只丢 span;Jaeger 宕机不影响推理服务
# 调优旋钮(默认未开): SGLANG_TRACE_ASYNC=1(异步 span 创建), SGLANG_TRACE_LEVEL(默认 3)
# router(ds41-router-3p1d,rust)无 OTLP 支持,不产生 span,追踪从各 sglang HTTP 层开始
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
x-sglang-common: &sglang-common
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
shm_size: "32gb"
ipc: host
pid: host
privileged: true
network_mode: host
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
x-p-environment: &p-environment
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
SGLANG_RAGGED_VERIFY_MODE: static
MC_INTRANODE_NVLINK: "true"
MC_INTRA_NVLINK: "true"
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
services:
p1:
<<: *sglang-common
container_name: ds41-3p1d-p1
environment:
<<: *p-environment
CUDA_VISIBLE_DEVICES: "0,1"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--enable-prefill-cp --cp-strategy interleave
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 256
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30000
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 2
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
--disaggregation-mode prefill
--disaggregation-transfer-backend mooncake
--disaggregation-bootstrap-port 8998
--enable-trace
--trace-modules request,mooncake
--otlp-traces-endpoint 127.0.0.1:4317
--otlp-service-name ds41-3p1d-p1
p2:
<<: *sglang-common
container_name: ds41-3p1d-p2
environment:
<<: *p-environment
CUDA_VISIBLE_DEVICES: "2,3"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--enable-prefill-cp --cp-strategy interleave
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 256
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30010
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 2
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
--disaggregation-mode prefill
--disaggregation-transfer-backend mooncake
--disaggregation-bootstrap-port 8999
--enable-trace
--trace-modules request,mooncake
--otlp-traces-endpoint 127.0.0.1:4317
--otlp-service-name ds41-3p1d-p2
p3:
<<: *sglang-common
container_name: ds41-3p1d-p3
environment:
<<: *p-environment
CUDA_VISIBLE_DEVICES: "4,5"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--enable-prefill-cp --cp-strategy interleave
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 256
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30020
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 2
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
--disaggregation-mode prefill
--disaggregation-transfer-backend mooncake
--disaggregation-bootstrap-port 9000
--enable-trace
--trace-modules request,mooncake
--otlp-traces-endpoint 127.0.0.1:4317
--otlp-service-name ds41-3p1d-p3
d:
<<: *sglang-common
container_name: ds41-3p1d-d1
environment:
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
SGLANG_RAGGED_VERIFY_MODE: static
MC_INTRANODE_NVLINK: "true"
MC_INTRA_NVLINK: "true"
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
CUDA_VISIBLE_DEVICES: "6,7"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2 --dp-size 2
--enable-dp-attention --enable-dp-lm-head
--mem-fraction-static 0.80
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 256
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 256
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30001
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--disaggregation-mode decode
--disaggregation-transfer-backend mooncake
--disaggregation-bootstrap-port 9001
--enable-trace
--trace-modules request,mooncake
--otlp-traces-endpoint 127.0.0.1:4317
--otlp-service-name ds41-3p1d-d1
-174
View File
@@ -1,174 +0,0 @@
# 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram(2026-09-25,mr256,nsys on p1/d1)
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
x-sglang-common: &sglang-common
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
shm_size: "32gb"
ipc: host
pid: host
privileged: true
network_mode: host
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
x-p-environment: &p-environment
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
SGLANG_RAGGED_VERIFY_MODE: static
MC_INTRANODE_NVLINK: "true"
MC_INTRA_NVLINK: "true"
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
services:
p1:
<<: *sglang-common
container_name: ds41-3p1d-prof-p1
environment:
<<: *p-environment
CUDA_VISIBLE_DEVICES: "0,1"
command: >
nsys launch --session-new=p1prof --cuda-graph-trace=node sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--enable-prefill-cp --cp-strategy interleave
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 256
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30000
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 2
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
--disaggregation-mode prefill
--disaggregation-transfer-backend mooncake
--disaggregation-bootstrap-port 8998
p2:
<<: *sglang-common
container_name: ds41-3p1d-prof-p2
environment:
<<: *p-environment
CUDA_VISIBLE_DEVICES: "2,3"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--enable-prefill-cp --cp-strategy interleave
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 256
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30010
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 2
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
--disaggregation-mode prefill
--disaggregation-transfer-backend mooncake
--disaggregation-bootstrap-port 8999
p3:
<<: *sglang-common
container_name: ds41-3p1d-prof-p3
environment:
<<: *p-environment
CUDA_VISIBLE_DEVICES: "4,5"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--enable-prefill-cp --cp-strategy interleave
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 256
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30020
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 2
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
--disaggregation-mode prefill
--disaggregation-transfer-backend mooncake
--disaggregation-bootstrap-port 9000
d:
<<: *sglang-common
container_name: ds41-3p1d-prof-d1
environment:
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
SGLANG_RAGGED_VERIFY_MODE: static
MC_INTRANODE_NVLINK: "true"
MC_INTRA_NVLINK: "true"
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
CUDA_VISIBLE_DEVICES: "6,7"
command: >
nsys launch --session-new=d1prof --cuda-graph-trace=node sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2 --dp-size 2
--enable-dp-attention --enable-dp-lm-head
--mem-fraction-static 0.80
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 256
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 256
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30001
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--disaggregation-mode decode
--disaggregation-transfer-backend mooncake
--disaggregation-bootstrap-port 9001
-174
View File
@@ -1,174 +0,0 @@
# 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram(2026-09-25,mr256,nsys on p1/d1)
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
x-sglang-common: &sglang-common
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
shm_size: "32gb"
ipc: host
pid: host
privileged: true
network_mode: host
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
x-p-environment: &p-environment
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
SGLANG_RAGGED_VERIFY_MODE: static
MC_INTRANODE_NVLINK: "true"
MC_INTRA_NVLINK: "true"
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
services:
p1:
<<: *sglang-common
container_name: ds41-3p1d-p1
environment:
<<: *p-environment
CUDA_VISIBLE_DEVICES: "0,1"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--enable-prefill-cp --cp-strategy interleave
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 256
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30000
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 2
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
--disaggregation-mode prefill
--disaggregation-transfer-backend mooncake
--disaggregation-bootstrap-port 8998
p2:
<<: *sglang-common
container_name: ds41-3p1d-p2
environment:
<<: *p-environment
CUDA_VISIBLE_DEVICES: "2,3"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--enable-prefill-cp --cp-strategy interleave
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 256
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30010
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 2
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
--disaggregation-mode prefill
--disaggregation-transfer-backend mooncake
--disaggregation-bootstrap-port 8999
p3:
<<: *sglang-common
container_name: ds41-3p1d-p3
environment:
<<: *p-environment
CUDA_VISIBLE_DEVICES: "4,5"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--enable-prefill-cp --cp-strategy interleave
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 256
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30020
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 2
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
--disaggregation-mode prefill
--disaggregation-transfer-backend mooncake
--disaggregation-bootstrap-port 9000
d:
<<: *sglang-common
container_name: ds41-3p1d-d1
environment:
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
SGLANG_RAGGED_VERIFY_MODE: static
MC_INTRANODE_NVLINK: "true"
MC_INTRA_NVLINK: "true"
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
CUDA_VISIBLE_DEVICES: "6,7"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2 --dp-size 2
--enable-dp-attention --enable-dp-lm-head
--mem-fraction-static 0.80
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 256
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 256
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30001
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--disaggregation-mode decode
--disaggregation-transfer-backend mooncake
--disaggregation-bootstrap-port 9001
-43
View File
@@ -1,43 +0,0 @@
services:
sglang:
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
container_name: ds41-sglang-b4
shm_size: "32gb"
ipc: host
privileged: true
ports:
- "30001:30000"
environment:
- CUDA_VISIBLE_DEVICES=4,5,6,7
- PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
command: >-
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 4
--ep-size 4
--mem-fraction-static 0.60
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 64
--reasoning-parser auto
--tool-call-parser auto
--max-running-requests 64
--host 0.0.0.0
--port 30000
--dp-size 4
--enable-dp-attention
--enable-cache-report
--enable-metrics
--json-model-override-args '{"vision_n_layers": 0}'
-43
View File
@@ -1,43 +0,0 @@
services:
sglang:
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
container_name: ds41-sglang-b4
shm_size: "32gb"
ipc: host
privileged: true
ports:
- "30001:30000"
environment:
- CUDA_VISIBLE_DEVICES=4,5,6,7
- PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
command: >-
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 4
--ep-size 4
--mem-fraction-static 0.60
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--enable-decoder-swa-bounded-replay
--cuda-graph-max-bs-decode 64
--reasoning-parser auto
--tool-call-parser auto
--max-running-requests 64
--host 0.0.0.0
--port 30000
--enable-cache-report
--enable-metrics
--json-model-override-args '{"vision_n_layers": 0}'
-42
View File
@@ -1,42 +0,0 @@
services:
sglang:
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
container_name: ds41-sglang-b4
shm_size: "32gb"
ipc: host
privileged: true
ports:
- "30001:30000"
environment:
- CUDA_VISIBLE_DEVICES=4,5,6,7
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
command: >-
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 4
--ep-size 4
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--enable-decoder-swa-bounded-replay
--cuda-graph-max-bs-decode 64
--reasoning-parser auto
--tool-call-parser auto
--max-running-requests 64
--host 0.0.0.0
--port 30000
--enable-prefill-cp
--cp-strategy interleave
--enable-cache-report
--enable-metrics
--json-model-override-args '{"vision_n_layers": 0}'
-160
View File
@@ -1,160 +0,0 @@
# 4×cp2 独立实例(非 PD)+ hicache L3(mooncake) + dspark + engram host table 卸载(2026-09-23 晚)
# 用户指定组合:4 cp2 hicache l3 dspark roundrobin loadbalance engram offload
# 每实例:tp2 ep2 + interleave CP2 + dspark b5 + hicache L3 write_through(size 0) + engram host table
# (tp2 权重必须靠 engram 卸载才放得下,见 goals-track G1.9 反转)
# GPU: a=0,1 b=2,3 c=4,5 d=6,7;端口: 30000/30010/30020/30030;rr router=30002
# 参考:dockerserve-cpdspark-l3.yml(hicache 参数)、dockerserve-3cp2-pd.yml(cp2/engram 参数)
x-common: &common
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
shm_size: "32gb"
ipc: host
privileged: true
network_mode: host
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
# 热补丁(2026-09-23):CP 对称 idle 检查,修 health-check × hicache drain 死锁
# 源文件:/data/ymk/sglang(dsv41-pd-visioncp 分支工作区已改);补丁脚本见
# D:\B300\experiments\cp2x4\patch_healthcheck_cp_idle.py
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
environment: &env
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
SGLANG_RAGGED_VERIFY_MODE: static
MC_MS_AUTO_DISC: "0"
MOONCAKE_MASTER: 127.0.0.1:50051
MOONCAKE_TE_META_DATA_SERVER: P2PHANDSHAKE
MOONCAKE_PROTOCOL: tcp
MOONCAKE_GLOBAL_SEGMENT_SIZE: 300gb
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
services:
a:
<<: *common
container_name: ds41-cp2-a
environment:
<<: *env
CUDA_VISIBLE_DEVICES: "0,1"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--enable-prefill-cp --cp-strategy interleave
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 64
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30000
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 2
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
b:
<<: *common
container_name: ds41-cp2-b
environment:
<<: *env
CUDA_VISIBLE_DEVICES: "2,3"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--enable-prefill-cp --cp-strategy interleave
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 64
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30010
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 2
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
c:
<<: *common
container_name: ds41-cp2-c
environment:
<<: *env
CUDA_VISIBLE_DEVICES: "4,5"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--enable-prefill-cp --cp-strategy interleave
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 64
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30020
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 2
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
d:
<<: *common
container_name: ds41-cp2-d2
environment:
<<: *env
CUDA_VISIBLE_DEVICES: "6,7"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--enable-prefill-cp --cp-strategy interleave
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 64
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30030
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 2
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
-44
View File
@@ -1,44 +0,0 @@
# cp8 单机非 PD 测试(2026-09-23 晚)
# 背景:dp8 单机 dspark 健康但长上下文 prefill 结构性慢(单请求只落 1 rank),
# 换 cp8(tp8 + interleave prefill CP8,单请求 prefill 切到 8 卡)对照。
# 关 engram、无 hicache L3、无 PD。cp+dspark 无 replay 是已知健康组合
# (cp4+dspark+replay 三元组必崩,本配置不开 replay)。
# GPU 0-7,端口 30000,容器 ds41-cp8,project cp8
services:
cp8:
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
container_name: ds41-cp8
shm_size: "32gb"
ipc: host
privileged: true
network_mode: host
environment:
SGLANG_RAGGED_VERIFY_MODE: "static"
CUDA_VISIBLE_DEVICES: "0,1,2,3,4,5,6,7"
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 8 --ep-size 8
--enable-prefill-cp --cp-strategy interleave
--mem-fraction-static 0.80
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 64
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 64
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30000
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
@@ -1,50 +0,0 @@
services:
sglang:
image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536
container_name: ds41-cpdspark-l3-b
shm_size: "32gb"
ipc: host
privileged: true
network_mode: host
environment:
- CUDA_VISIBLE_DEVICES=4,5,6,7
- MC_MS_AUTO_DISC=0
- MOONCAKE_MASTER=127.0.0.1:50051
- MOONCAKE_TE_META_DATA_SERVER=P2PHANDSHAKE
- MOONCAKE_PROTOCOL=tcp
- MOONCAKE_GLOBAL_SEGMENT_SIZE=400gb
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
command: >-
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 4 --ep-size 4
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 64
--reasoning-parser auto --tool-call-parser auto
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30001
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK
--speculative-dspark-block-size 5
--enable-prefill-cp --cp-strategy interleave
--enable-hierarchical-cache
--hicache-ratio 2
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
--json-model-override-args '{"vision_n_layers": 0}'
@@ -1,51 +0,0 @@
services:
sglang:
image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536
container_name: ds41-cpdspark-l3-b
shm_size: "32gb"
ipc: host
privileged: true
network_mode: host
environment:
- CUDA_VISIBLE_DEVICES=4,5,6,7
- MC_MS_AUTO_DISC=0
- MOONCAKE_MASTER=127.0.0.1:50051
- MOONCAKE_TE_META_DATA_SERVER=P2PHANDSHAKE
- MOONCAKE_PROTOCOL=tcp
- MOONCAKE_GLOBAL_SEGMENT_SIZE=400gb
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
command: >-
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 4 --ep-size 4
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 64
--reasoning-parser auto --tool-call-parser auto
--max-running-requests 64
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30001
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK
--speculative-dspark-block-size 5
--enable-prefill-cp --cp-strategy interleave
--enable-hierarchical-cache
--hicache-ratio 2
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
--json-model-override-args '{"vision_n_layers": 0}'
@@ -1,50 +0,0 @@
services:
sglang:
image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536
container_name: ds41-cpdspark-l3-a
shm_size: "32gb"
ipc: host
privileged: true
network_mode: host
environment:
- CUDA_VISIBLE_DEVICES=0,1,2,3
- MC_MS_AUTO_DISC=0
- MOONCAKE_MASTER=127.0.0.1:50051
- MOONCAKE_TE_META_DATA_SERVER=P2PHANDSHAKE
- MOONCAKE_PROTOCOL=tcp
- MOONCAKE_GLOBAL_SEGMENT_SIZE=400gb
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
command: >-
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 4 --ep-size 4
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 64
--reasoning-parser auto --tool-call-parser auto
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30000
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK
--speculative-dspark-block-size 5
--enable-prefill-cp --cp-strategy interleave
--enable-hierarchical-cache
--hicache-ratio 2
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
--json-model-override-args '{"vision_n_layers": 0}'
@@ -1,51 +0,0 @@
services:
sglang:
image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536
container_name: ds41-cpdspark-l3-a
shm_size: "32gb"
ipc: host
privileged: true
network_mode: host
environment:
- CUDA_VISIBLE_DEVICES=0,1,2,3
- MC_MS_AUTO_DISC=0
- MOONCAKE_MASTER=127.0.0.1:50051
- MOONCAKE_TE_META_DATA_SERVER=P2PHANDSHAKE
- MOONCAKE_PROTOCOL=tcp
- MOONCAKE_GLOBAL_SEGMENT_SIZE=400gb
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
command: >-
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 4 --ep-size 4
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 64
--reasoning-parser auto --tool-call-parser auto
--max-running-requests 64
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30000
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK
--speculative-dspark-block-size 5
--enable-prefill-cp --cp-strategy interleave
--enable-hierarchical-cache
--hicache-ratio 2
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
--json-model-override-args '{"vision_n_layers": 0}'
@@ -1,44 +0,0 @@
services:
sglang:
image: ymkymx/sglang:dsv41-pd-92632a60b-20260922-2100
container_name: ds41-cpdspark-nr-b4
shm_size: "32gb"
ipc: host
privileged: true
ports:
- "30001:30001"
environment:
- CUDA_VISIBLE_DEVICES=4,5,6,7
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
command: >-
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 4
--ep-size 4
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 64
--reasoning-parser auto
--tool-call-parser auto
--max-running-requests 64
--host 0.0.0.0
--port 30000
--enable-cache-report
--enable-metrics
--speculative-algorithm DSPARK
--speculative-dspark-block-size 5
--enable-prefill-cp
--cp-strategy interleave
--json-model-override-args '{"vision_n_layers": 0}'
@@ -1,44 +0,0 @@
services:
sglang:
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
container_name: ds41-cpdspark-nr
shm_size: "32gb"
ipc: host
privileged: true
ports:
- "30000:30000"
environment:
- CUDA_VISIBLE_DEVICES=0,1,2,3
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
command: >-
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 4
--ep-size 4
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 64
--reasoning-parser auto
--tool-call-parser auto
--max-running-requests 64
--host 0.0.0.0
--port 30000
--enable-cache-report
--enable-metrics
--speculative-algorithm DSPARK
--speculative-dspark-block-size 5
--enable-prefill-cp
--cp-strategy interleave
--json-model-override-args '{"vision_n_layers": 0}'
-44
View File
@@ -1,44 +0,0 @@
services:
sglang:
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
container_name: ds41-cpdspark
shm_size: "32gb"
ipc: host
privileged: true
ports:
- "30000:30000"
environment:
- CUDA_VISIBLE_DEVICES=0,1,2,3
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
command: >-
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 4
--ep-size 4
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 64
--reasoning-parser auto
--tool-call-parser auto
--max-running-requests 64
--host 0.0.0.0
--port 30000
--enable-cache-report
--enable-metrics
--speculative-algorithm DSPARK
--speculative-dspark-block-size 5
--enable-prefill-cp
--cp-strategy interleave
--enable-decoder-swa-bounded-replay
--json-model-override-args '{"vision_n_layers": 0}'
@@ -1,49 +0,0 @@
# 单独 D 节点基线(非 PD role):3p1d 的 d1 配置去掉 disaggregation,直接对外服务
# 用途:测单 D 节点 decode 基线(阶梯并发 32..80%*cuda-graph-max-bs-decode,
# 报 TPOT / 实际并发 / 整实例输出 TPS;prefill 走 L1 radix cache-hit)
# GPU 6,7;端口 30001;无 hicache 依赖(L1 即可支撑重复 prompt 命中)
# 2026-10-09 建:配合 bench_suite_v2.sh decode
x-sglang-common: &sglang-common
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
shm_size: "32gb"
ipc: host
pid: host
privileged: true
network_mode: host
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
services:
d1s:
<<: *sglang-common
container_name: ds41-d1-standalone
environment:
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
SGLANG_RAGGED_VERIFY_MODE: static
CUDA_VISIBLE_DEVICES: "6,7"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2 --dp-size 2
--enable-dp-attention --enable-dp-lm-head
--mem-fraction-static 0.80
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 256
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 256
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30001
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
-62
View File
@@ -1,62 +0,0 @@
# dp2 对照(2026-09-24 上午):验证 dp2(dp-attention)下 dspark 是否正常。
# 与 dockerserve-tp2.yml 逐参数对齐,唯一差异 = 加 --dp-size 2 --enable-dp-attention --enable-dp-lm-head。
# GPU 6-7,端口 30030 直连。镜像/补丁挂载/L3/engram/tok8 全部相同。
# 注:dp+dspark 不能开 --enable-decoder-swa-bounded-replay(本配置本来就没开)。
x-common: &common
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
shm_size: "32gb"
ipc: host
privileged: true
network_mode: host
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
environment: &env
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
SGLANG_RAGGED_VERIFY_MODE: static
MC_MS_AUTO_DISC: "0"
MOONCAKE_MASTER: 127.0.0.1:50051
MOONCAKE_TE_META_DATA_SERVER: P2PHANDSHAKE
MOONCAKE_PROTOCOL: tcp
MOONCAKE_GLOBAL_SEGMENT_SIZE: 300gb
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
services:
dp2:
<<: *common
container_name: ds41-dp2
environment:
<<: *env
CUDA_VISIBLE_DEVICES: "6,7"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--dp-size 2 --enable-dp-attention --enable-dp-lm-head
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 64
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30030
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 2
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
-42
View File
@@ -1,42 +0,0 @@
services:
sglang:
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
container_name: ds41-dp4test
shm_size: "32gb"
ipc: host
privileged: true
ports:
- "30000:30000"
environment:
- CUDA_VISIBLE_DEVICES=0,1,2,3
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
command: >-
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 4
--ep-size 4
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--enable-decoder-swa-bounded-replay
--cuda-graph-max-bs-decode 64
--reasoning-parser auto
--tool-call-parser auto
--max-running-requests 64
--host 0.0.0.0
--port 30000
--enable-prefill-cp
--cp-strategy interleave
--enable-cache-report
--enable-metrics
--json-model-override-args '{"vision_n_layers": 0}'
-44
View File
@@ -1,44 +0,0 @@
# dp8+tp8 单机非 PD 对照测试(2026-09-23 晚)
# 背景:3cp2+dp2 PD 部署 dspark accept 回归(32k ctx accept len 3.14→1.31,
# 见 experiments/3cp2-pd/bench-20260923.md「未决问题」)。本配置退回单机验证
# dspark 健康度:engram host table 关闭、无 hicache L3、无 PD、无 mooncake。
# 参数基准:dockerserve-3cp2-pd.yml 的 d 服务(dp 路径),去掉 PD/L3/engram 相关。
# GPU 0-7,端口 30000,容器 ds41-dp8,project dp8
services:
dp8:
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
container_name: ds41-dp8
shm_size: "32gb"
ipc: host
privileged: true
network_mode: host
environment:
SGLANG_RAGGED_VERIFY_MODE: "static"
CUDA_VISIBLE_DEVICES: "0,1,2,3,4,5,6,7"
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 8 --ep-size 8 --dp-size 8
--enable-dp-attention --enable-dp-lm-head
--mem-fraction-static 0.80
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 64
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30000
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
@@ -1,61 +0,0 @@
# 单独 P 节点基线(非 PD role):3p1d 的 p1 配置去掉 disaggregation,直接对外服务
# 用途:测单 P 节点 prefill 基线吞吐(冷)+ hicache cache-hit TPS(热)
# GPU 0,1;端口 30000;依赖 mooncake-master(hicache L3,127.0.0.1:50051)
# 2026-10-09 建:配合 bench_suite_v2.sh prefill
x-sglang-common: &sglang-common
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
shm_size: "32gb"
ipc: host
pid: host
privileged: true
network_mode: host
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
services:
p1s:
<<: *sglang-common
container_name: ds41-p1-standalone
environment:
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
SGLANG_RAGGED_VERIFY_MODE: static
MC_INTRANODE_NVLINK: "true"
MC_INTRA_NVLINK: "true"
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
CUDA_VISIBLE_DEVICES: "0,1"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--enable-prefill-cp --cp-strategy interleave
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 256
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30000
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 2
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
-47
View File
@@ -1,47 +0,0 @@
services:
sglang:
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
container_name: ds41-prefill
shm_size: "32gb"
ipc: host
privileged: true
ports:
- "30000:30000"
environment:
- CUDA_VISIBLE_DEVICES=0,1,2,3
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
command: >-
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 4
--ep-size 4
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 64
--reasoning-parser auto
--tool-call-parser auto
--max-running-requests 64
--host 0.0.0.0
--port 30000
--enable-cache-report
--enable-metrics
--speculative-algorithm DSPARK
--speculative-dspark-block-size 5
--enable-decoder-swa-bounded-replay
--enable-hierarchical-cache
--hicache-ratio 2.5
--hicache-write-policy write_back
--disaggregation-mode prefill
--optimistic-prefill-attempts 4
--json-model-override-args '{"vision_n_layers": 0}'
@@ -1,51 +0,0 @@
services:
sglang:
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
container_name: ds41-pd-decode
shm_size: "32gb"
ipc: host
pid: host
privileged: true
network_mode: host
environment:
- CUDA_VISIBLE_DEVICES=4,5,6,7
- SGLANG_RAGGED_VERIFY_MODE=static
- MC_INTRANODE_NVLINK=true
- MC_INTRA_NVLINK=true
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
command: >-
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 4
--ep-size 4
--dp-size 4
--enable-dp-attention
--enable-dp-lm-head
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 64
--reasoning-parser auto
--tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--tokenizer-worker-num 8
--host 0.0.0.0
--port 30001
--enable-cache-report
--enable-metrics
--speculative-algorithm DSPARK
--speculative-dspark-block-size 5
--disaggregation-mode decode
--disaggregation-transfer-backend mooncake
-50
View File
@@ -1,50 +0,0 @@
services:
sglang:
image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536
container_name: ds41-pd-decode
shm_size: "32gb"
ipc: host
pid: host
privileged: true
network_mode: host
environment:
- CUDA_VISIBLE_DEVICES=4,5,6,7
- SGLANG_RAGGED_VERIFY_MODE=static
- MC_INTRANODE_NVLINK=true
- MC_INTRA_NVLINK=true
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
command: >-
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 4
--ep-size 4
--dp-size 4
--enable-dp-attention
--enable-dp-lm-head
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 64
--reasoning-parser auto
--tool-call-parser auto
--tokenizer-worker-num 8
--host 0.0.0.0
--port 30001
--enable-cache-report
--speculative-algorithm DSPARK
--speculative-dspark-block-size 5
--disaggregation-mode decode
--disaggregation-transfer-backend mooncake
--json-model-override-args '{"vision_n_layers": 0}'
-55
View File
@@ -1,55 +0,0 @@
services:
sglang:
image: ymkymx/sglang:dsv41-pd-c74a4037f-20260921-1300
container_name: ds41-pd-prefill-dp
shm_size: "32gb"
ipc: host
pid: host
privileged: true
network_mode: host
environment:
- CUDA_VISIBLE_DEVICES=0,1,2,3
- SGLANG_RAGGED_VERIFY_MODE=static
- MC_INTRANODE_NVLINK=true
- MC_INTRA_NVLINK=true
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
command: >-
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 4
--ep-size 4
--dp-size 4
--enable-dp-attention
--enable-dp-lm-head
--load-balance-method total_tokens
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 64
--reasoning-parser auto
--tool-call-parser auto
--max-running-requests 64
--host 0.0.0.0
--port 30010
--enable-cache-report
--speculative-algorithm DSPARK
--speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 2.5
--hicache-write-policy write_back
--disaggregation-mode prefill
--disaggregation-bootstrap-port 8918
--disaggregation-transfer-backend mooncake
--json-model-override-args '{"vision_n_layers": 0}'
@@ -1,53 +0,0 @@
services:
sglang:
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
container_name: ds41-pd-prefill
shm_size: "32gb"
ipc: host
pid: host
privileged: true
network_mode: host
environment:
- CUDA_VISIBLE_DEVICES=0,1,2,3
- SGLANG_RAGGED_VERIFY_MODE=static
- MC_INTRANODE_NVLINK=true
- MC_INTRA_NVLINK=true
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
command: >-
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 4
--ep-size 4
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 64
--reasoning-parser auto
--tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--tokenizer-worker-num 8
--host 0.0.0.0
--port 30000
--enable-cache-report
--enable-metrics
--speculative-algorithm DSPARK
--speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 2.5
--hicache-write-policy write_back
--enable-prefill-cp
--cp-strategy interleave
--disaggregation-mode prefill
--disaggregation-transfer-backend mooncake
-52
View File
@@ -1,52 +0,0 @@
services:
sglang:
image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536
container_name: ds41-pd-prefill
shm_size: "32gb"
ipc: host
pid: host
privileged: true
network_mode: host
environment:
- CUDA_VISIBLE_DEVICES=0,1,2,3
- SGLANG_RAGGED_VERIFY_MODE=static
- MC_INTRANODE_NVLINK=true
- MC_INTRA_NVLINK=true
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
command: >-
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 4
--ep-size 4
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 64
--reasoning-parser auto
--tool-call-parser auto
--tokenizer-worker-num 8
--host 0.0.0.0
--port 30000
--enable-cache-report
--speculative-algorithm DSPARK
--speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 2.5
--hicache-write-policy write_back
--enable-prefill-cp
--cp-strategy interleave
--disaggregation-mode prefill
--disaggregation-transfer-backend mooncake
--json-model-override-args '{"vision_n_layers": 0}'
-57
View File
@@ -1,57 +0,0 @@
# PD2 二分 Round 2 D 侧(2026-09-24):干净 D + 嫌疑项②
# SGLANG_DISAGGREGATION_QUEUE_SIZE=16 + SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=32
services:
sglang:
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
container_name: ds41-pd2-decode
shm_size: "32gb"
ipc: host
pid: host
privileged: true
network_mode: host
environment:
- CUDA_VISIBLE_DEVICES=6,7
- SGLANG_RAGGED_VERIFY_MODE=static
- MC_INTRANODE_NVLINK=true
- MC_INTRA_NVLINK=true
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
- SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1
- SGLANG_DISAGGREGATION_QUEUE_SIZE=16
- SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=32
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
command: >-
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2
--ep-size 2
--dp-size 2
--enable-dp-attention
--enable-dp-lm-head
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto
--tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--tokenizer-worker-num 8
--host 0.0.0.0
--port 30030
--enable-cache-report
--enable-metrics
--speculative-algorithm DSPARK
--speculative-dspark-block-size 5
--disaggregation-mode decode
--disaggregation-transfer-backend mooncake
-54
View File
@@ -1,54 +0,0 @@
# PD 干净对照实验 D 节点(2026-09-24 上午):dp2 decode,配 dockerserve-pd2-p.yml 使用。
services:
sglang:
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
container_name: ds41-pd2-decode
shm_size: "32gb"
ipc: host
pid: host
privileged: true
network_mode: host
environment:
- CUDA_VISIBLE_DEVICES=6,7
- SGLANG_RAGGED_VERIFY_MODE=static
- MC_INTRANODE_NVLINK=true
- MC_INTRA_NVLINK=true
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
- SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
command: >-
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2
--ep-size 2
--dp-size 2
--enable-dp-attention
--enable-dp-lm-head
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto
--tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--tokenizer-worker-num 8
--host 0.0.0.0
--port 30030
--enable-cache-report
--enable-metrics
--speculative-algorithm DSPARK
--speculative-dspark-block-size 5
--disaggregation-mode decode
--disaggregation-transfer-backend mooncake
-68
View File
@@ -1,68 +0,0 @@
# PD2 二分 Round 1(2026-09-24):干净基线 + 嫌疑项①「P 侧 L3 hicache 块 + mooncake-store.json」
# 相对 dockerserve-pd2-p.yml 的差异:
# env 加 SGLANG_HICACHE_MOONCAKE_CONFIG_PATH=/data/ymk/ds41/mooncake-store.json
# hicache 参数块换成坏部署同款:page_first_direct + direct + write_through + mooncake +
# wait_complete + size 0(替代 write_back L2)
# D 侧不变(坏部署 D 本无 L3)。探针:bs16 low_entropy decode 看 accept len。
services:
sglang:
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
container_name: ds41-pd2-prefill
shm_size: "32gb"
ipc: host
pid: host
privileged: true
network_mode: host
environment:
- CUDA_VISIBLE_DEVICES=4,5
- SGLANG_RAGGED_VERIFY_MODE=static
- MC_INTRANODE_NVLINK=true
- MC_INTRA_NVLINK=true
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
- SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1
- SGLANG_HICACHE_MOONCAKE_CONFIG_PATH=/data/ymk/ds41/mooncake-store.json
- SGLANG_HICACHE_PREFETCH_IO_THREADS=4
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
command: >-
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2
--ep-size 2
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto
--tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--tokenizer-worker-num 8
--host 0.0.0.0
--port 30020
--enable-cache-report
--enable-metrics
--speculative-algorithm DSPARK
--speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 2
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
--enable-prefill-cp
--cp-strategy interleave
--disaggregation-mode prefill
--disaggregation-transfer-backend mooncake
-66
View File
@@ -1,66 +0,0 @@
# PD2 二分 Round 2 P 侧(2026-09-24):R1(L3 块)之上再加嫌疑项②
# SGLANG_DISAGGREGATION_QUEUE_SIZE=16 + SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=32
services:
sglang:
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
container_name: ds41-pd2-prefill
shm_size: "32gb"
ipc: host
pid: host
privileged: true
network_mode: host
environment:
- CUDA_VISIBLE_DEVICES=4,5
- SGLANG_RAGGED_VERIFY_MODE=static
- MC_INTRANODE_NVLINK=true
- MC_INTRA_NVLINK=true
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
- SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1
- SGLANG_HICACHE_MOONCAKE_CONFIG_PATH=/data/ymk/ds41/mooncake-store.json
- SGLANG_HICACHE_PREFETCH_IO_THREADS=4
- SGLANG_DISAGGREGATION_QUEUE_SIZE=16
- SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=32
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
command: >-
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2
--ep-size 2
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto
--tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--tokenizer-worker-num 8
--host 0.0.0.0
--port 30020
--enable-cache-report
--enable-metrics
--speculative-algorithm DSPARK
--speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 2
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
--enable-prefill-cp
--cp-strategy interleave
--disaggregation-mode prefill
--disaggregation-transfer-backend mooncake
-59
View File
@@ -1,59 +0,0 @@
# PD 干净对照实验(2026-09-24 上午):验证「cp2-P → dp2-D」传输路径下 dspark 是否正常。
# 用户疑问:单机六形态健康不等于 PD 链路健康,可能两边传输没对齐。
# 以已验证可用的 dockerserve-pd-{p,d}-vision.yml 为底,缩到 2+2 卡;P 加 engram host table
# (2 卡权重放不下,见 G1.9)。P=GPU4-5 端口 30020,D=GPU6-7 端口 30030,mini_lb=30004。
services:
sglang:
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
container_name: ds41-pd2-prefill
shm_size: "32gb"
ipc: host
pid: host
privileged: true
network_mode: host
environment:
- CUDA_VISIBLE_DEVICES=4,5
- SGLANG_RAGGED_VERIFY_MODE=static
- MC_INTRANODE_NVLINK=true
- MC_INTRA_NVLINK=true
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
- SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
command: >-
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2
--ep-size 2
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto
--tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--tokenizer-worker-num 8
--host 0.0.0.0
--port 30020
--enable-cache-report
--enable-metrics
--speculative-algorithm DSPARK
--speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 2.5
--hicache-write-policy write_back
--enable-prefill-cp
--cp-strategy interleave
--disaggregation-mode prefill
--disaggregation-transfer-backend mooncake
@@ -1,42 +0,0 @@
services:
sglang:
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
container_name: ds41-replay-off
shm_size: "32gb"
ipc: host
privileged: true
ports:
- "30000:30000"
environment:
- CUDA_VISIBLE_DEVICES=0,1,2,3
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
command: >-
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 4
--ep-size 4
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 64
--reasoning-parser auto
--tool-call-parser auto
--max-running-requests 64
--host 0.0.0.0
--port 30000
--enable-cache-report
--enable-metrics
--speculative-algorithm DSPARK
--speculative-dspark-block-size 5
--json-model-override-args '{"vision_n_layers": 0}'
@@ -1,42 +0,0 @@
services:
sglang:
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
container_name: ds41-replay-on
shm_size: "32gb"
ipc: host
privileged: true
ports:
- "30000:30000"
environment:
- CUDA_VISIBLE_DEVICES=0,1,2,3
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
command: >-
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 4
--ep-size 4
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 64
--reasoning-parser auto
--tool-call-parser auto
--max-running-requests 64
--host 0.0.0.0
--port 30000
--enable-cache-report
--enable-metrics
--speculative-algorithm DSPARK
--speculative-dspark-block-size 5
--enable-decoder-swa-bounded-replay
--json-model-override-args '{"vision_n_layers": 0}'
@@ -1,76 +0,0 @@
# 【跨 P 实例 L3 读写验证 2026-10-10】dockerserve-tier-cp2-realstore.yml 的第二实例:
# 与 ds41-tier-t2(A:GPU 0,1,端口 30040)同时跑,共用 mooncake master 50051。
# 关键差异:
# - GPU 2,3,端口 30041,容器名 ds41-tier-t3
# - MOONCAKE_LOCAL_HOSTNAME=localhost-b:mooncake 客户端 hostname 前缀按实例区分
# (rank 后缀 -d0t0p0c0/-d0t1p0c1 由 84fc632088 拼在后面的),否则两实例 rank 同名
# → master preferred-segment 分不出段归属 → 副本落错进程段 → 跨进程回退 TCP(实例内)
# 且跨实例段地址冲突。extra_hosts 别名同步加 -b 前缀(P2P handshake 要求可解析,
# ac11e9162d)。
# 验证路径:A prime 写 L3 → flush B 的 L1/L2 → B 读同 seed prompts = 跨实例 TCP P2P 命中。
x-common: &common
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
shm_size: 32gb
ipc: host
pid: host
privileged: true
network_mode: host
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
- /data/ymk/sglang/python:/sgl-workspace/sglang/python
- /data/ymk/ds41/fake-cargo:/root/.cargo/bin/cargo:ro
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
services:
t3:
<<: *common
container_name: ds41-tier-t3
extra_hosts:
- "localhost-b-d0t0p0c0:127.0.0.1"
- "localhost-b-d0t1p0c1:127.0.0.1"
# 跨实例 P2P:对端实例(realstore compose 的 t2)rank 主机名也要能解析
- "localhost-d0t0p0c0:127.0.0.1"
- "localhost-d0t1p0c1:127.0.0.1"
environment:
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: 1
SGLANG_RAGGED_VERIFY_MODE: static
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
SGLANG_HICACHE_TIMING: 1
MOONCAKE_LOCAL_HOSTNAME: localhost-b
SGLANG_HICACHE_PREFETCH_IO_THREADS: 4
CUDA_VISIBLE_DEVICES: 2,3
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--enable-prefill-cp --cp-strategy interleave
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 256
--chunked-prefill-size 32768
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30041
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 4.0
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
--max-total-tokens 1600000
@@ -1,73 +0,0 @@
# 【L3 真实 store A/B 2026-10-10】同 dockerserve-tier-cp2.yml,唯一差异:
# L3 从 dummy standalone(mooncake-store-dummy.json,50052 守护进程)
# 切到真实 mooncake store(mooncake-store.json:master 50051 + P2PHANDSHAKE + tcp,
# 客户端注册 300GB 自身内存段)——与 k8s 线上 ds41-3p1d 的 L3 模式一致。
# 目的:区分 "sglang L3 路径本身慢" 还是 "dummy 后端 per-key 路径慢"(mc 扫描 A/B)。
x-common: &common
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
shm_size: 32gb
ipc: host
pid: host
privileged: true
network_mode: host
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
- /data/ymk/sglang/python:/sgl-workspace/sglang/python
- /data/ymk/ds41/fake-cargo:/root/.cargo/bin/cargo:ro
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
services:
t2:
<<: *common
container_name: ds41-tier-t2
extra_hosts:
- "localhost-d0t0p0c0:127.0.0.1"
- "localhost-d0t1p0c1:127.0.0.1"
# 跨实例 P2P:对端实例(x2 compose 的 t3)rank 主机名也要能解析,
# 否则 master 把副本分配到对端段后本端 open segment 失败(TRANSFER_FAIL)
- "localhost-b-d0t0p0c0:127.0.0.1"
- "localhost-b-d0t1p0c1:127.0.0.1"
environment:
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: 1
SGLANG_RAGGED_VERIFY_MODE: static
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
SGLANG_HICACHE_TIMING: 1
# L3 预取传输并行 lane 数(crc32(rid) 分 lane,每 lane 独立 gloo 组;
# 实例内所有 rank 必须一致)。2026-10-10 A/B 见 l3-hicache-deep-dive 文档。
SGLANG_HICACHE_PREFETCH_IO_THREADS: 4
CUDA_VISIBLE_DEVICES: 0,1
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--enable-prefill-cp --cp-strategy interleave
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 256
--chunked-prefill-size 32768
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30040
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 4.0
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
--max-total-tokens 1600000
-65
View File
@@ -1,65 +0,0 @@
# 【prefill 分层极限测试 2026-10-10】单实例 cp2(tp2+ep2+prefill-cp interleave)+ dspark b5
# + hicache L2(ratio 4.0=6.4M token) + L3(shm dummy) + best_effort
# 目的:测 纯计算 / L1 / L2 / L3 四档 prefill 极限吞吐 + 混合负载异步聚合验证
# L1=1.6M token(装得下 8×160k=1.28M 工作集);L2 6.4M;L3 shm 64GB
# GPU 0,1(用前停 p3d1 栈);端口 30040;共用 mooncake-store(50052)/master(50051)
x-common: &common
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
shm_size: 32gb
ipc: host
pid: host
privileged: true
network_mode: host
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
- /data/ymk/sglang/python:/sgl-workspace/sglang/python
- /data/ymk/ds41/fake-cargo:/root/.cargo/bin/cargo:ro
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
services:
t2:
<<: *common
container_name: ds41-tier-t2
environment:
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: 1
SGLANG_RAGGED_VERIFY_MODE: static
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store-dummy.json
SGLANG_HICACHE_PREFETCH_IO_THREADS: 4
SGLANG_HICACHE_TIMING: 1
SGLANG_HICACHE_STORAGE_BATCH_SIZE: 512
CUDA_VISIBLE_DEVICES: 0,1
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--enable-prefill-cp --cp-strategy interleave
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 256
--chunked-prefill-size 32768
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30040
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 4.0
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
--max-total-tokens 1600000
-92
View File
@@ -1,92 +0,0 @@
# 纯 tp2 单实例对照(2026-09-24 早):与 dockerserve-cp2x4.yml 逐参数对齐,唯一差异 = 去掉
# --enable-prefill-cp --cp-strategy interleave(即无 CP),用于测「tp2 vs cp2」的 prefill 效率差。
# GPU 4-5(a)/6-7(b),端口 30020/30030;2×tp2 聚合经 rr router 30003。
# 镜像/补丁挂载/L3/engram/tok8 全部与 cp2x4 相同。
x-common: &common
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
shm_size: "32gb"
ipc: host
privileged: true
network_mode: host
volumes:
- /data:/data
- /data/ymk/cache/sglang:/root/.cache/sglang
# 与 cp2x4 保持同一份 scheduler.py(含未 commit 的 CP idle 补丁),保证唯一变量是 CP 开关
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
environment: &env
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
SGLANG_RAGGED_VERIFY_MODE: static
MC_MS_AUTO_DISC: "0"
MOONCAKE_MASTER: 127.0.0.1:50051
MOONCAKE_TE_META_DATA_SERVER: P2PHANDSHAKE
MOONCAKE_PROTOCOL: tcp
MOONCAKE_GLOBAL_SEGMENT_SIZE: 300gb
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
services:
tp2:
<<: *common
container_name: ds41-tp2
environment:
<<: *env
CUDA_VISIBLE_DEVICES: "4,5"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 64
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30020
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 2
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
b:
<<: *common
container_name: ds41-tp2-b
environment:
<<: *env
CUDA_VISIBLE_DEVICES: "6,7"
command: >
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 2 --ep-size 2
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--cuda-graph-max-bs-decode 32
--reasoning-parser auto --tool-call-parser auto
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
--max-running-requests 64
--tokenizer-worker-num 8
--host 0.0.0.0 --port 30030
--enable-cache-report --enable-metrics
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
--enable-hierarchical-cache
--hicache-ratio 2
--hicache-mem-layout page_first_direct
--hicache-io-backend direct
--hicache-write-policy write_through
--hicache-storage-backend mooncake
--hicache-storage-prefetch-policy wait_complete
--hicache-size 0
-43
View File
@@ -1,43 +0,0 @@
services:
sglang:
image: uhub.service.ucloud.cn/umirror/sglang:dev-dsv41
shm_size: "32gb"
ipc: host
privileged: true
ports:
- "30000:30000"
volumes:
- /data:/data
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
command: >-
sglang serve
--trust-remote-code
--model-path /data/models/DeepSeek-V4.1-Flash
--tp 4
--ep-size 4
--mem-fraction-static 0.75
--attention-backend dsv4
--moe-runner-backend flashinfer_mxfp4
--enable-decoder-swa-bounded-replay
--cuda-graph-max-bs-decode 64
--reasoning-parser auto
--tool-call-parser auto
--max-running-requests 64
--host 0.0.0.0
--port 30000
--enable-cache-report
--enable-metrics
--enable-prefill-cp
--cp-strategy interleave
--json-model-override-args '{"vision_n_layers": 0}'
-454
View File
@@ -1,454 +0,0 @@
# ds41-3p1d: DeepSeek-V4.1-Flash 3P1D PD 分离(cs32k + OTLP trace),RBG 版
# 拓扑: 3×prefill(tp2+ep2+cp2, cs32k, dspark, hicache L3) + 1×decode(tp2+ep2+dp2, cg256 mr320, dspark,
# load-balance-method=total_requests 按请求数分流) + mooncake-master(hicache L3 store) + router(mini-lb PD)
# 全部钉 b300-01(单机 8 卡,hostNetwork + hostPID——mooncake NVLink-intra 跨 Pod CUDA IPC 必需,
# 等同 docker compose 的 pid: host;参考 AGENTS.md「部署四要素」)
# 镜像已侧载 containerd: docker.io/ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831(IfNotPresent)
# 参数来源: deploy/b300-ds41/dockerserve-3p1d-otlp-cs32k.yml(docker compose 版)
# 对外: 仅 ClusterIP Service ds41-3p1d:50002(router 用普通 Pod 网络,不占宿主机端口,
# 天然不对宿主机/外网暴露);new-api-gw 用集群 DNS http://ds41-3p1d:50002/v1 访问
# trace: OTLP → 127.0.0.1:4317(同节点 jaeger-trace cw,badger 6h 轮窗)
apiVersion: workloads.x-k8s.io/v1alpha2
kind: RoleBasedGroup
metadata:
name: ds41-3p1d
labels:
app: ds41-3p1d
sglang-model: ds-v41-flash
spec:
roleTemplates:
- name: prefill-base
template:
metadata:
labels:
sglang-model: ds-v41-flash
sglang-role: prefill
spec:
hostNetwork: true
hostPID: true
dnsPolicy: ClusterFirstWithHostNet
nodeSelector:
kubernetes.io/hostname: b300-01
containers:
- name: sglang
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
imagePullPolicy: IfNotPresent
command:
- /bin/bash
- -c
- |
exec sglang serve \
--trust-remote-code \
--model-path /data/models/DeepSeek-V4.1-Flash \
--tp 2 --ep-size 2 \
--enable-prefill-cp --cp-strategy interleave \
--chunked-prefill-size 32768 --max-prefill-tokens 32768 \
--mem-fraction-static 0.75 \
--attention-backend dsv4 \
--moe-runner-backend flashinfer_mxfp4 \
--cuda-graph-max-bs-decode 32 \
--reasoning-parser auto --tool-call-parser auto \
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' \
--max-running-requests 256 \
--prefill-max-requests 8 \
--tokenizer-worker-num 8 \
--host 0.0.0.0 --port "$SGLANG_PORT" \
--enable-cache-report --enable-metrics \
--speculative-algorithm DSPARK --speculative-dspark-block-size 5 \
--enable-hierarchical-cache \
--hicache-ratio 2 \
--hicache-mem-layout page_first_direct \
--hicache-io-backend direct \
--hicache-write-policy write_through \
--hicache-storage-backend mooncake \
--hicache-storage-prefetch-policy wait_complete \
--hicache-size 0 \
--disaggregation-mode prefill \
--disaggregation-transfer-backend mooncake \
--disaggregation-bootstrap-port "$SGLANG_BOOTSTRAP_PORT" \
--enable-trace \
--trace-modules request,mooncake \
--otlp-traces-endpoint 127.0.0.1:4317 \
--otlp-service-name "$OTLP_NAME"
env:
- name: SGLANG_TORCH_PROFILER_DIR
value: /data/ymk/prof/torch
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
value: "1"
- name: SGLANG_RAGGED_VERIFY_MODE
value: static
- name: MC_INTRANODE_NVLINK
value: "true"
- name: MC_INTRA_NVLINK
value: "true"
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
value: "1"
- name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH
value: /data/ymk/ds41/mooncake-store.json
- name: SGLANG_HICACHE_PREFETCH_IO_THREADS
value: "4"
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
value: "16"
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
value: "32"
- name: TZ
value: Asia/Shanghai
readinessProbe:
httpGet:
path: /health
port: metrics
initialDelaySeconds: 60
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 90
resources:
requests:
cpu: "32"
memory: 220Gi
nvidia.com/gpu: "2"
limits:
cpu: "96"
memory: 950Gi
nvidia.com/gpu: "2"
securityContext:
privileged: true
volumeMounts:
- name: data
mountPath: /data
- name: sglang-cache
mountPath: /root/.cache/sglang
- name: dshm
mountPath: /dev/shm
volumes:
- name: data
hostPath:
path: /data
type: Directory
- name: sglang-cache
hostPath:
path: /data/ymk/cache/sglang
type: Directory
- name: dshm
emptyDir:
medium: Memory
sizeLimit: 32Gi
- name: decode-base
template:
metadata:
labels:
sglang-model: ds-v41-flash
sglang-role: decode
spec:
hostNetwork: true
hostPID: true
dnsPolicy: ClusterFirstWithHostNet
nodeSelector:
kubernetes.io/hostname: b300-01
containers:
- name: sglang
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
imagePullPolicy: IfNotPresent
command:
- /bin/bash
- -c
- |
exec sglang serve \
--trust-remote-code \
--model-path /data/models/DeepSeek-V4.1-Flash \
--tp 2 --ep-size 2 --dp-size 2 \
--enable-dp-attention --enable-dp-lm-head \
--mem-fraction-static 0.83 \
--attention-backend dsv4 \
--moe-runner-backend flashinfer_mxfp4 \
--cuda-graph-max-bs-decode 256 \
--load-balance-method total_requests \
--reasoning-parser auto --tool-call-parser auto \
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' \
--max-running-requests 320 \
--tokenizer-worker-num 8 \
--host 0.0.0.0 --port 30030 \
--enable-cache-report --enable-metrics \
--speculative-algorithm DSPARK --speculative-dspark-block-size 5 \
--disaggregation-mode decode \
--disaggregation-transfer-backend mooncake \
--disaggregation-bootstrap-port 9001 \
--enable-trace \
--trace-modules request,mooncake \
--otlp-traces-endpoint 127.0.0.1:4317 \
--otlp-service-name ds41-3p1d-d1
env:
- name: SGLANG_TORCH_PROFILER_DIR
value: /data/ymk/prof/torch
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
value: "1"
- name: SGLANG_RAGGED_VERIFY_MODE
value: static
- name: MC_INTRANODE_NVLINK
value: "true"
- name: MC_INTRA_NVLINK
value: "true"
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
value: "1"
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
value: "16"
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
value: "32"
- name: TZ
value: Asia/Shanghai
ports:
- name: metrics
containerPort: 30030
readinessProbe:
httpGet:
path: /health
port: metrics
initialDelaySeconds: 60
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 90
resources:
requests:
cpu: "32"
memory: 100Gi
nvidia.com/gpu: "2"
limits:
cpu: "96"
memory: 600Gi
nvidia.com/gpu: "2"
securityContext:
privileged: true
volumeMounts:
- name: data
mountPath: /data
- name: sglang-cache
mountPath: /root/.cache/sglang
- name: dshm
mountPath: /dev/shm
volumes:
- name: data
hostPath:
path: /data
type: Directory
- name: sglang-cache
hostPath:
path: /data/ymk/cache/sglang
type: Directory
- name: dshm
emptyDir:
medium: Memory
sizeLimit: 32Gi
roles:
- name: mooncake-master
replicas: 1
standalonePattern:
template:
metadata:
labels:
app: ds41-3p1d-mooncake-master
spec:
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
nodeSelector:
kubernetes.io/hostname: b300-01
containers:
- name: mooncake-master
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
imagePullPolicy: IfNotPresent
command:
- /bin/bash
- -c
- |
exec /opt/sglang/bin/mooncake_master \
--port 50051 \
--enable_http_metadata_server=true \
--http_metadata_server_port=18080 \
--eviction_high_watermark_ratio=0.90
readinessProbe:
tcpSocket:
port: 50051
initialDelaySeconds: 5
periodSeconds: 5
failureThreshold: 60
resources:
requests:
cpu: "1"
memory: 8Gi
limits:
cpu: "4"
memory: 64Gi
- name: p1
replicas: 1
dependencies:
- mooncake-master
standalonePattern:
templateRef:
name: prefill-base
patch:
metadata:
labels:
app: ds41-3p1d-p1
spec:
containers:
- name: sglang
env:
- name: CUDA_VISIBLE_DEVICES
value: "0,1"
- name: SGLANG_PORT
value: "30000"
- name: SGLANG_BOOTSTRAP_PORT
value: "8998"
- name: OTLP_NAME
value: ds41-3p1d-p1
ports:
- name: metrics
containerPort: 30000
- name: p2
replicas: 1
dependencies:
- mooncake-master
standalonePattern:
templateRef:
name: prefill-base
patch:
metadata:
labels:
app: ds41-3p1d-p2
spec:
containers:
- name: sglang
env:
- name: CUDA_VISIBLE_DEVICES
value: "2,3"
- name: SGLANG_PORT
value: "30010"
- name: SGLANG_BOOTSTRAP_PORT
value: "8999"
- name: OTLP_NAME
value: ds41-3p1d-p2
ports:
- name: metrics
containerPort: 30010
- name: p3
replicas: 1
dependencies:
- mooncake-master
standalonePattern:
templateRef:
name: prefill-base
patch:
metadata:
labels:
app: ds41-3p1d-p3
spec:
containers:
- name: sglang
env:
- name: CUDA_VISIBLE_DEVICES
value: "4,5"
- name: SGLANG_PORT
value: "30020"
- name: SGLANG_BOOTSTRAP_PORT
value: "9000"
- name: OTLP_NAME
value: ds41-3p1d-p3
ports:
- name: metrics
containerPort: 30020
- name: d1
replicas: 1
dependencies:
- mooncake-master
standalonePattern:
templateRef:
name: decode-base
patch:
metadata:
labels:
app: ds41-3p1d-d1
spec:
containers:
- name: sglang
env:
- name: CUDA_VISIBLE_DEVICES
value: "6,7"
- name: router
replicas: 1
dependencies:
- p1
- p2
- p3
- d1
standalonePattern:
template:
metadata:
labels:
app: ds41-3p1d-router
spec:
# 入口不走 hostNetwork:普通 Pod 网络 + ClusterIP,天然不对宿主机/外网暴露。
# P/D 地址用节点内网 IP(hostNetwork 监听在 192.168.1.243 上,Calico 可达)。
nodeSelector:
kubernetes.io/hostname: b300-01
containers:
- name: router
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
imagePullPolicy: IfNotPresent
command:
- /bin/bash
- -c
- |
exec sglang-router launch \
--mini-lb --pd-disaggregation \
--host 0.0.0.0 --port 50002 \
--prefill http://192.168.1.243:30000 8998 \
--prefill http://192.168.1.243:30010 8999 \
--prefill http://192.168.1.243:30020 9000 \
--decode http://192.168.1.243:30030
ports:
- name: http
containerPort: 50002
readinessProbe:
httpGet:
path: /health
port: http
initialDelaySeconds: 5
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 30
resources:
requests:
cpu: "1"
memory: 4Gi
limits:
cpu: "8"
memory: 16Gi
---
apiVersion: v1
kind: Service
metadata:
name: ds41-3p1d
labels:
app: ds41-3p1d
spec:
type: ClusterIP
selector:
rbg.workloads.x-k8s.io/group-name: ds41-3p1d
rbg.workloads.x-k8s.io/role-name: router
ports:
- name: http
port: 50002
targetPort: 50002
---
apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
metadata:
name: ds41-3p1d
labels:
release: kube-prometheus-stack
spec:
selector:
matchLabels:
sglang-model: ds-v41-flash
podMetricsEndpoints:
- interval: 15s
path: /metrics
port: metrics
-2
View File
@@ -1,2 +0,0 @@
#!/bin/sh
exit 1
-451
View File
@@ -1,451 +0,0 @@
# ds41-3p1d: DeepSeek-V4.1-Flash 3P1D PD 分离(cs32k + OTLP trace),RBG 版
# 拓扑: 3×prefill(tp2+ep2+cp2, cs32k, dspark, hicache L3) + 1×decode(tp2+ep2+dp2, cg256 mr320, dspark,
# load-balance-method=total_requests 按请求数分流) + mooncake-master(hicache L3 store) + router(mini-lb PD)
# 全部钉 b300-01(单机 8 卡,hostNetwork + hostPID——mooncake NVLink-intra 跨 Pod CUDA IPC 必需,
# 等同 docker compose 的 pid: host;参考 AGENTS.md「部署四要素」)
# 镜像已侧载 containerd: docker.io/ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix(IfNotPresent)
# 参数来源: deploy/b300-ds41/dockerserve-3p1d-otlp-cs32k.yml(docker compose 版)
# 对外: 仅 ClusterIP Service ds41-3p1d:50002(router 用普通 Pod 网络,不占宿主机端口,
# 天然不对宿主机/外网暴露);new-api-gw 用集群 DNS http://ds41-3p1d:50002/v1 访问
# trace: OTLP → 127.0.0.1:4317(同节点 jaeger-trace cw,badger 6h 轮窗)
apiVersion: workloads.x-k8s.io/v1alpha2
kind: RoleBasedGroup
metadata:
name: ds41-3p1d
labels:
app: ds41-3p1d
sglang-model: ds-v41-flash
spec:
roleTemplates:
- name: prefill-base
template:
metadata:
labels:
sglang-model: ds-v41-flash
sglang-role: prefill
spec:
hostNetwork: true
hostPID: true
dnsPolicy: ClusterFirstWithHostNet
nodeSelector:
kubernetes.io/hostname: b300-01
containers:
- name: sglang
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
imagePullPolicy: IfNotPresent
command:
- /bin/bash
- -c
- |
exec sglang serve \
--trust-remote-code \
--model-path /data/models/DeepSeek-V4.1-Flash \
--tp 2 --ep-size 2 \
--enable-prefill-cp --cp-strategy interleave \
--chunked-prefill-size 32768 --max-prefill-tokens 32768 \
--mem-fraction-static 0.75 \
--attention-backend dsv4 \
--moe-runner-backend flashinfer_mxfp4 \
--cuda-graph-max-bs-decode 32 \
--reasoning-parser auto --tool-call-parser auto \
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' \
--max-running-requests 256 \
--tokenizer-worker-num 8 \
--host 0.0.0.0 --port "$SGLANG_PORT" \
--enable-cache-report --enable-metrics \
--speculative-algorithm DSPARK --speculative-dspark-block-size 5 \
--enable-hierarchical-cache \
--hicache-ratio 2 \
--hicache-mem-layout page_first_direct \
--hicache-io-backend direct \
--hicache-write-policy write_through \
--hicache-storage-backend mooncake \
--hicache-storage-prefetch-policy wait_complete \
--hicache-size 0 \
--disaggregation-mode prefill \
--disaggregation-transfer-backend mooncake \
--disaggregation-bootstrap-port "$SGLANG_BOOTSTRAP_PORT" \
--enable-trace \
--trace-modules request,mooncake \
--otlp-traces-endpoint 127.0.0.1:4317 \
--otlp-service-name "$OTLP_NAME"
env:
- name: SGLANG_TORCH_PROFILER_DIR
value: /data/ymk/prof/torch
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
value: "1"
- name: SGLANG_RAGGED_VERIFY_MODE
value: static
- name: MC_INTRANODE_NVLINK
value: "true"
- name: MC_INTRA_NVLINK
value: "true"
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
value: "1"
- name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH
value: /data/ymk/ds41/mooncake-store.json
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
value: "16"
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
value: "32"
- name: TZ
value: Asia/Shanghai
readinessProbe:
httpGet:
path: /health
port: metrics
initialDelaySeconds: 60
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 90
resources:
requests:
cpu: "32"
memory: 220Gi
nvidia.com/gpu: "2"
limits:
cpu: "96"
memory: 950Gi
nvidia.com/gpu: "2"
securityContext:
privileged: true
volumeMounts:
- name: data
mountPath: /data
- name: sglang-cache
mountPath: /root/.cache/sglang
- name: dshm
mountPath: /dev/shm
volumes:
- name: data
hostPath:
path: /data
type: Directory
- name: sglang-cache
hostPath:
path: /data/ymk/cache/sglang
type: Directory
- name: dshm
emptyDir:
medium: Memory
sizeLimit: 32Gi
- name: decode-base
template:
metadata:
labels:
sglang-model: ds-v41-flash
sglang-role: decode
spec:
hostNetwork: true
hostPID: true
dnsPolicy: ClusterFirstWithHostNet
nodeSelector:
kubernetes.io/hostname: b300-01
containers:
- name: sglang
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
imagePullPolicy: IfNotPresent
command:
- /bin/bash
- -c
- |
exec sglang serve \
--trust-remote-code \
--model-path /data/models/DeepSeek-V4.1-Flash \
--tp 2 --ep-size 2 --dp-size 2 \
--enable-dp-attention --enable-dp-lm-head \
--mem-fraction-static 0.83 \
--attention-backend dsv4 \
--moe-runner-backend flashinfer_mxfp4 \
--cuda-graph-max-bs-decode 256 \
--load-balance-method total_requests \
--reasoning-parser auto --tool-call-parser auto \
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' \
--max-running-requests 320 \
--tokenizer-worker-num 8 \
--host 0.0.0.0 --port 30030 \
--enable-cache-report --enable-metrics \
--speculative-algorithm DSPARK --speculative-dspark-block-size 5 \
--disaggregation-mode decode \
--disaggregation-transfer-backend mooncake \
--disaggregation-bootstrap-port 9001 \
--enable-trace \
--trace-modules request,mooncake \
--otlp-traces-endpoint 127.0.0.1:4317 \
--otlp-service-name ds41-3p1d-d1
env:
- name: SGLANG_TORCH_PROFILER_DIR
value: /data/ymk/prof/torch
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
value: "1"
- name: SGLANG_RAGGED_VERIFY_MODE
value: static
- name: MC_INTRANODE_NVLINK
value: "true"
- name: MC_INTRA_NVLINK
value: "true"
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
value: "1"
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
value: "16"
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
value: "32"
- name: TZ
value: Asia/Shanghai
ports:
- name: metrics
containerPort: 30030
readinessProbe:
httpGet:
path: /health
port: metrics
initialDelaySeconds: 60
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 90
resources:
requests:
cpu: "32"
memory: 100Gi
nvidia.com/gpu: "2"
limits:
cpu: "96"
memory: 600Gi
nvidia.com/gpu: "2"
securityContext:
privileged: true
volumeMounts:
- name: data
mountPath: /data
- name: sglang-cache
mountPath: /root/.cache/sglang
- name: dshm
mountPath: /dev/shm
volumes:
- name: data
hostPath:
path: /data
type: Directory
- name: sglang-cache
hostPath:
path: /data/ymk/cache/sglang
type: Directory
- name: dshm
emptyDir:
medium: Memory
sizeLimit: 32Gi
roles:
- name: mooncake-master
replicas: 1
standalonePattern:
template:
metadata:
labels:
app: ds41-3p1d-mooncake-master
spec:
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
nodeSelector:
kubernetes.io/hostname: b300-01
containers:
- name: mooncake-master
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
imagePullPolicy: IfNotPresent
command:
- /bin/bash
- -c
- |
exec /opt/sglang/bin/mooncake_master \
--port 50051 \
--enable_http_metadata_server=true \
--http_metadata_server_port=18080 \
--eviction_high_watermark_ratio=0.90
readinessProbe:
tcpSocket:
port: 50051
initialDelaySeconds: 5
periodSeconds: 5
failureThreshold: 60
resources:
requests:
cpu: "1"
memory: 8Gi
limits:
cpu: "4"
memory: 64Gi
- name: p1
replicas: 1
dependencies:
- mooncake-master
standalonePattern:
templateRef:
name: prefill-base
patch:
metadata:
labels:
app: ds41-3p1d-p1
spec:
containers:
- name: sglang
env:
- name: CUDA_VISIBLE_DEVICES
value: "0,1"
- name: SGLANG_PORT
value: "30000"
- name: SGLANG_BOOTSTRAP_PORT
value: "8998"
- name: OTLP_NAME
value: ds41-3p1d-p1
ports:
- name: metrics
containerPort: 30000
- name: p2
replicas: 1
dependencies:
- mooncake-master
standalonePattern:
templateRef:
name: prefill-base
patch:
metadata:
labels:
app: ds41-3p1d-p2
spec:
containers:
- name: sglang
env:
- name: CUDA_VISIBLE_DEVICES
value: "2,3"
- name: SGLANG_PORT
value: "30010"
- name: SGLANG_BOOTSTRAP_PORT
value: "8999"
- name: OTLP_NAME
value: ds41-3p1d-p2
ports:
- name: metrics
containerPort: 30010
- name: p3
replicas: 1
dependencies:
- mooncake-master
standalonePattern:
templateRef:
name: prefill-base
patch:
metadata:
labels:
app: ds41-3p1d-p3
spec:
containers:
- name: sglang
env:
- name: CUDA_VISIBLE_DEVICES
value: "4,5"
- name: SGLANG_PORT
value: "30020"
- name: SGLANG_BOOTSTRAP_PORT
value: "9000"
- name: OTLP_NAME
value: ds41-3p1d-p3
ports:
- name: metrics
containerPort: 30020
- name: d1
replicas: 1
dependencies:
- mooncake-master
standalonePattern:
templateRef:
name: decode-base
patch:
metadata:
labels:
app: ds41-3p1d-d1
spec:
containers:
- name: sglang
env:
- name: CUDA_VISIBLE_DEVICES
value: "6,7"
- name: router
replicas: 1
dependencies:
- p1
- p2
- p3
- d1
standalonePattern:
template:
metadata:
labels:
app: ds41-3p1d-router
spec:
# 入口不走 hostNetwork:普通 Pod 网络 + ClusterIP,天然不对宿主机/外网暴露。
# P/D 地址用节点内网 IP(hostNetwork 监听在 192.168.1.243 上,Calico 可达)。
nodeSelector:
kubernetes.io/hostname: b300-01
containers:
- name: router
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
imagePullPolicy: IfNotPresent
command:
- /bin/bash
- -c
- |
exec sglang-router launch \
--mini-lb --pd-disaggregation \
--host 0.0.0.0 --port 50002 \
--prefill http://192.168.1.243:30000 8998 \
--prefill http://192.168.1.243:30010 8999 \
--prefill http://192.168.1.243:30020 9000 \
--decode http://192.168.1.243:30030
ports:
- name: http
containerPort: 50002
readinessProbe:
httpGet:
path: /health
port: http
initialDelaySeconds: 5
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 30
resources:
requests:
cpu: "1"
memory: 4Gi
limits:
cpu: "8"
memory: 16Gi
---
apiVersion: v1
kind: Service
metadata:
name: ds41-3p1d
labels:
app: ds41-3p1d
spec:
type: ClusterIP
selector:
rbg.workloads.x-k8s.io/group-name: ds41-3p1d
rbg.workloads.x-k8s.io/role-name: router
ports:
- name: http
port: 50002
targetPort: 50002
---
apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
metadata:
name: ds41-3p1d
labels:
release: kube-prometheus-stack
spec:
selector:
matchLabels:
sglang-model: ds-v41-flash
podMetricsEndpoints:
- interval: 15s
path: /metrics
port: metrics
@@ -1,4 +0,0 @@
{
"standalone_storage": true,
"client_server_address": "127.0.0.1:50052"
}
-6
View File
@@ -1,6 +0,0 @@
{
"master_server_address": "127.0.0.1:50051",
"metadata_server": "P2PHANDSHAKE",
"global_segment_size": "300gb",
"protocol": "tcp"
}
+38 -172
View File
@@ -18,8 +18,6 @@
namespace sglang {
using device::math::fast_mod_div_u32_t;
/// \brief Ratio-2 decode compressor: pair-pool, RMSNorm and the main-KV write.
///
/// `kv_input` and `kv_state` rows are `2 * kHeadDim` floats, kv then score.
@@ -36,7 +34,8 @@ struct Compress2DecodeParams {
const int64_t* __restrict__ req; // [num_tokens], req_pool_idx per token
const void* __restrict__ raw_out_loc; // [num_tokens] LocT, the FULL slot; 0 marks a padded row
uint8_t* __restrict__ kvcache; // [npages, kPageBytes] uint8
fast_mod_div_u32_t ring_size; // Positions per request slot in the pair-state ring.
/// Positions per request slot in the pair-state ring.
uint32_t ring_size;
float eps;
};
@@ -44,32 +43,21 @@ struct Compress2DecodeParams {
/// At (512, 64) it also keeps the nope/rope split warp-aligned, as the fp8 amax reduction requires.
constexpr uint32_t kC2VecSize = 2;
/// \brief Where a completing row finds its partner, and who parks a pending row.
enum class C2Mode {
/// One row per request: the partner comes from the ring, a pending row parks there.
Decode,
/// `draft_len` consecutive positions per request: `blockIdx.x` is the position inside the
/// block, `blockIdx.y` the request, and every row but the first takes its partner from the
/// previous `kv_input` row instead of the ring.
Verify,
/// Arbitrary rows, paired as `_low_ratio_pair_partners` pairs them: the partner is the previous
/// row when it is the same request's previous position, else the ring. A pending row does
/// nothing here; `c2_prefill_store_kernel` parks the ring rows after this kernel read them.
Prefill,
};
/// \brief grid = num_tokens (Decode, Prefill) or `draft_len x batch` (Verify),
/// block = kHeadDim / kC2VecSize.
/// \brief grid = num_tokens, block = kHeadDim / kC2VecSize.
///
/// An odd position completes a group with its even predecessor; an even one parks in the state.
///
/// Under target-verify, `draft_len` consecutive positions per request: `blockIdx.x` is the
/// position inside the block, `blockIdx.y` the request, and every row but the first takes its
/// partner from the previous `kv_input` row instead of the ring.
///
/// The three reductions have different widths and are not interchangeable: the RMSNorm
/// statistic spans the row, an fp8 store scale 64 elements, an fp4 block 16.
///
/// kLayout is the cache's page format: V4 and V41 store the fake-quantized value; V41_FP4
/// stores the e2m1 codes and their e4m3 scales directly, so the fp4 rounding happens once.
template <
C2Mode kMode,
bool kVerify,
int64_t kHeadDim,
int64_t kRopeDim,
int32_t kPageBits,
@@ -103,21 +91,16 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
const auto tx = threadIdx.x;
// Verify gives each request a CTA column; decode a flat grid of one row each.
const auto row = kMode == C2Mode::Verify ? blockIdx.y * gridDim.x + blockIdx.x : blockIdx.x;
const auto row = kVerify ? blockIdx.y * gridDim.x + blockIdx.x : blockIdx.x;
PDLWaitPrimary<kUsePDL>();
const auto* loc_ptr = static_cast<const LocT*>(params.raw_out_loc);
const auto* pos_ptr = static_cast<const PosT*>(params.positions);
const auto raw_out_loc = loc_ptr[row];
// Prefill rows are many, so a CUDA graph padding row exits before any load.
// Decode and verify rows are few: every row issues its loads to keep more in flight.
if constexpr (kMode == C2Mode::Prefill) {
if (raw_out_loc == 0) return PDLTriggerSecondary<kUsePDL>();
}
const auto pos = pos_ptr[row];
// Prefill leaves a pending row's latent unwritten; its index-K slot is -1, so nothing reads it.
if constexpr (kMode == C2Mode::Prefill) {
if ((pos & 1) == 0) return PDLTriggerSecondary<kUsePDL>();
}
// Slots fit in int32 whatever width the scheduler hands them in.
const auto raw_out_loc = static_cast<int32_t>(static_cast<const LocT*>(params.raw_out_loc)[row]);
// CUDA graph padding is a completely inert row: do not read schedule, input,
// state, or RoPE data, and do not publish output or update the cache.
if (raw_out_loc == 0) return PDLTriggerSecondary<kUsePDL>();
const auto pos = static_cast<const PosT*>(params.positions)[row];
// A completing row reads the slot left by `pos - 1`;
// a pending row writes its own slot, so reads and writes stay disjoint.
const auto rid = params.req[row];
fp32_vec_t kv_new, score_new;
@@ -125,32 +108,21 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
score_new.load(params.kv_input + row * kStride, tx + kCTASize);
const auto ring = static_cast<int64_t>(rid) * params.ring_size;
const auto write_offset = static_cast<uint32_t>(pos) % params.ring_size;
const auto read_offset = write_offset == 0 ? params.ring_size - 1 : write_offset - 1;
const auto write_row = ring + write_offset;
const auto read_row = ring + read_offset;
const auto read_row = ring + (pos - 1 + params.ring_size) % params.ring_size;
const auto write_row = ring + pos % params.ring_size;
fp32_vec_t kv_old, score_old;
const float* partner = params.kv_state + read_row * kStride;
if constexpr (kMode == C2Mode::Verify) {
if constexpr (kVerify) {
if (blockIdx.x != 0) partner = params.kv_input + static_cast<int64_t>(row - 1) * kStride;
}
if constexpr (kMode == C2Mode::Prefill) {
if (row > 0 && params.req[row - 1] == rid && pos_ptr[row - 1] == pos - 1 && loc_ptr[row - 1] != 0) {
partner = params.kv_input + static_cast<int64_t>(row - 1) * kStride;
}
}
kv_old.load(partner, tx);
score_old.load(partner, tx + kCTASize);
if constexpr (kMode != C2Mode::Prefill) {
// CUDA graph padding case
if (raw_out_loc == 0) return PDLTriggerSecondary<kUsePDL>();
if ((pos & 1) == 0) {
kv_new.store(params.kv_state + write_row * kStride, tx);
score_new.store(params.kv_state + write_row * kStride, tx + kCTASize);
return PDLTriggerSecondary<kUsePDL>();
}
if ((pos & 1) == 0) {
kv_new.store(params.kv_state + write_row * kStride, tx);
score_new.store(params.kv_state + write_row * kStride, tx + kCTASize);
return PDLTriggerSecondary<kUsePDL>();
}
constexpr uint32_t kNumWarps = kCTASize / kWarpThreads;
@@ -231,9 +203,9 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
}
}
const int32_t out_loc = static_cast<uint32_t>(raw_out_loc) >> 1;
if constexpr (kLayout == KVLayout::V41_FP4) {
// The fp4 cache takes the rotated bf16 value as is: its row quantizer is the fake quant, minus the dequant.
const int32_t out_loc = raw_out_loc >> 1;
const auto kv_row = Paged::row(params.kvcache, out_loc);
return deepseek_v4::v41::store_row<kLayout>(kv_row.data, kv_row.scale, tx, staged);
}
@@ -256,6 +228,7 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
}
// `raw_out_loc / ratio`; ratio 2 makes it a shift.
const int32_t out_loc = raw_out_loc >> 1;
const auto kv_row = Paged::row(params.kvcache, out_loc);
if constexpr (kLayout == KVLayout::V41) {
@@ -291,63 +264,21 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
}
}
/// \brief Prefill ring write-back, launched after `flash_c2_decode_kernel<C2Mode::Prefill>`.
///
/// Each request's last `ring_size` rows park their `| kv | score |` at `req * ring_size +
/// pos % ring_size` for a later step's first partner. It cannot run inside the pooling kernel:
/// a chunk whose first row is odd and whose last row is even reads and writes the same ring
/// slot. grid = `ring_size x num_reqs`, CTA (k, i) parks the k-th row from the end of request i,
/// one float4 per thread; those rows are consecutive positions, so their slots are distinct.
template <int64_t kHeadDim, typename PosT, typename LocT, bool kUsePDL>
__global__ __launch_bounds__(kHeadDim / 2) void c2_prefill_store_kernel(
const Compress2DecodeParams params,
const int32_t* __restrict__ extend_start_loc,
const int32_t* __restrict__ extend_seq_lens) {
using namespace device;
constexpr int64_t kStride = kHeadDim * 2;
using vec_t = AlignedVector<float, 4>;
static_assert(kStride == 4 * (kHeadDim / 2));
// The pooling kernel reads the ring rows this one overwrites.
PDLWaitPrimary<kUsePDL>();
const auto bx = static_cast<int32_t>(blockIdx.x);
const auto len = extend_seq_lens[blockIdx.y];
const auto start_loc = extend_start_loc[blockIdx.y];
if (bx >= len) return;
const auto row = start_loc + (len - 1 - bx);
const auto rid = params.req[row];
const auto out = static_cast<const LocT*>(params.raw_out_loc)[row];
const auto pos = static_cast<uint32_t>(static_cast<const PosT*>(params.positions)[row]);
const auto dst_row = rid * params.ring_size + pos % params.ring_size;
if (out == 0) return;
vec_t buf;
buf.load(params.kv_input + row * kStride, threadIdx.x);
PDLTriggerSecondary<kUsePDL>();
buf.store(params.kv_state + dst_row * kStride, threadIdx.x);
}
template <int64_t kHeadDim, int64_t kRopeDim, uint32_t kPageSize, deepseek_v4::KVLayout kLayout, bool kUsePDL>
struct FlashCompress2Kernel {
static constexpr uint32_t kBlockSize = kHeadDim / kC2VecSize;
static constexpr int32_t kPageBits = std::bit_width(kPageSize) - 1;
static constexpr int64_t kPageBytes = deepseek_v4::kv_page_bytes<kLayout>(kPageSize);
static_assert(kLayout != deepseek_v4::KVLayout::V4 || kPageBytes == host::div_ceil(584ll * kPageSize, 576) * 576);
template <C2Mode kMode, typename PosT, typename LocT>
template <bool kVerify, typename PosT, typename LocT>
static constexpr auto kernel =
flash_c2_decode_kernel<kMode, kHeadDim, kRopeDim, kPageBits, PosT, LocT, kLayout, kUsePDL>;
template <typename PosT, typename LocT>
static constexpr auto store_kernel = c2_prefill_store_kernel<kHeadDim, PosT, LocT, kUsePDL>;
flash_c2_decode_kernel<kVerify, kHeadDim, kRopeDim, kPageBits, PosT, LocT, kLayout, kUsePDL>;
/// \brief The (`positions`, `raw_out_loc`) dtype pair, resolved at run time.
template <C2Mode kMode>
template <bool kVerify>
static auto select(const bool pos_i32, const bool loc_i32) {
if (pos_i32) return loc_i32 ? kernel<kMode, int32_t, int32_t> : kernel<kMode, int32_t, int64_t>;
return loc_i32 ? kernel<kMode, int64_t, int32_t> : kernel<kMode, int64_t, int64_t>;
}
static auto select_store(const bool pos_i32, const bool loc_i32) {
if (pos_i32) return loc_i32 ? store_kernel<int32_t, int32_t> : store_kernel<int32_t, int64_t>;
return loc_i32 ? store_kernel<int64_t, int32_t> : store_kernel<int64_t, int64_t>;
if (pos_i32) return loc_i32 ? kernel<kVerify, int32_t, int32_t> : kernel<kVerify, int32_t, int64_t>;
return loc_i32 ? kernel<kVerify, int64_t, int32_t> : kernel<kVerify, int64_t, int64_t>;
}
// The sum of squares reduces through a fixed-size shared array, so the CTA must be whole warps.
@@ -371,7 +302,7 @@ struct FlashCompress2Kernel {
const tvm::ffi::TensorView kvcache,
const int64_t ring_size,
const int64_t draft_len) {
launch<false>(
launch(
kv_input,
kv_state,
kv_output,
@@ -386,62 +317,7 @@ struct FlashCompress2Kernel {
draft_len);
}
/// \brief The extend-batch pooling kernel, then the ring write-back.
///
/// Rows may mix requests, prefix offsets and padding; a pending (even) row's `kv_output`
/// row is left unwritten. `extend_start_loc` / `extend_seq_lens` are int32 `[num_reqs]`:
/// request i owns rows `[start[i], start[i] + len[i])`; zero-length requests are skipped.
static void run_prefill_fusion(
const tvm::ffi::TensorView kv_input,
const tvm::ffi::TensorView kv_state,
const tvm::ffi::TensorView kv_output,
const tvm::ffi::TensorView norm_weight,
const tvm::ffi::TensorView positions,
const tvm::ffi::TensorView req,
const tvm::ffi::TensorView raw_out_loc,
const float eps,
const tvm::ffi::TensorView freqs_cis,
const tvm::ffi::TensorView kvcache,
const int64_t ring_size,
const tvm::ffi::TensorView extend_start_loc,
const tvm::ffi::TensorView extend_seq_lens) {
using namespace host;
auto B = SymbolicSize{"num_reqs"};
auto device_ = SymbolicDevice{};
device_.set_options<kDLGPU>();
TensorMatcher({B}).with_dtype<int32_t>().with_device(device_).verify(extend_start_loc);
TensorMatcher({B}).with_dtype<int32_t>().with_device(device_).verify(extend_seq_lens);
const auto requests = StoreRequests{
.extend_start_loc = static_cast<const int32_t*>(extend_start_loc.data_ptr()),
.extend_seq_lens = static_cast<const int32_t*>(extend_seq_lens.data_ptr()),
.num_reqs = static_cast<uint32_t>(B.unwrap()),
};
launch<true>(
kv_input,
kv_state,
kv_output,
norm_weight,
positions,
req,
raw_out_loc,
eps,
ring_size,
freqs_cis,
kvcache,
1,
&requests);
}
private:
struct StoreRequests {
const int32_t* extend_start_loc;
const int32_t* extend_seq_lens;
uint32_t num_reqs;
};
/// \tparam kPrefill `C2Mode::Prefill` plus the ring write-back of `requests`; otherwise
/// Decode, or Verify when `draft_len > 1`.
template <bool kPrefill>
static void launch(
const tvm::ffi::TensorView kv_input,
const tvm::ffi::TensorView kv_state,
@@ -454,8 +330,7 @@ struct FlashCompress2Kernel {
const int64_t ring_size,
const tvm::ffi::TensorView freqs_cis,
const tvm::ffi::TensorView kvcache,
const int64_t draft_len,
const StoreRequests* requests = nullptr) {
const int64_t draft_len) {
using namespace host;
auto N = SymbolicSize{"num_tokens"};
@@ -483,7 +358,7 @@ struct FlashCompress2Kernel {
const auto num_tokens = static_cast<uint32_t>(N.unwrap());
if (num_tokens == 0) return;
const auto is_verify = !kPrefill && draft_len > 1;
const auto is_verify = draft_len > 1;
CHECK_HOST(ring_size > 0 && draft_len >= 1);
CHECK_HOST(!is_verify || num_tokens % draft_len == 0);
CHECK_HOST(!is_verify || ring_size > draft_len)
@@ -498,28 +373,19 @@ struct FlashCompress2Kernel {
.req = static_cast<const int64_t*>(req.data_ptr()),
.raw_out_loc = raw_out_loc.data_ptr(),
.kvcache = static_cast<uint8_t*>(kvcache.data_ptr()),
.ring_size = static_cast<fast_mod_div_u32_t>(ring_size),
.ring_size = static_cast<uint32_t>(ring_size),
.eps = eps,
};
// `LaunchKernel` is move-only, so each arm builds its own.
const auto pos_i32 = pos_dtype.is_type<int32_t>();
const auto loc_i32 = loc_dtype.is_type<int32_t>();
if constexpr (kPrefill) {
const auto k = select<C2Mode::Prefill>(pos_i32, loc_i32);
LaunchKernel(num_tokens, kBlockSize, device_.unwrap()) //
.enable_pdl(kUsePDL)(k, params);
if (requests->num_reqs == 0) return;
const auto store = select_store(pos_i32, loc_i32);
const auto grid = dim3{static_cast<uint32_t>(ring_size), requests->num_reqs};
LaunchKernel(grid, kHeadDim / 2, device_.unwrap()) //
.enable_pdl(kUsePDL)(store, params, requests->extend_start_loc, requests->extend_seq_lens);
} else if (is_verify) {
if (is_verify) {
const auto block = static_cast<uint32_t>(draft_len);
const auto k = select<C2Mode::Verify>(pos_i32, loc_i32);
const auto k = select<true>(pos_i32, loc_i32);
LaunchKernel(dim3{block, num_tokens / block}, kBlockSize, device_.unwrap()) //
.enable_pdl(kUsePDL)(k, params);
} else {
const auto k = select<C2Mode::Decode>(pos_i32, loc_i32);
const auto k = select<false>(pos_i32, loc_i32);
LaunchKernel(num_tokens, kBlockSize, device_.unwrap()) //
.enable_pdl(kUsePDL)(k, params);
}
@@ -2,7 +2,6 @@
#include <sgl_kernel/tensor.h>
#include <sgl_kernel/utils.h>
#include <sgl_kernel/math.cuh>
#include <sgl_kernel/runtime.cuh>
#include <sgl_kernel/type.cuh>
#include <sgl_kernel/utils.cuh>
@@ -22,7 +21,31 @@ namespace sglang {
using device::distributed::PushWorkSpace;
using device::distributed::Semaphore;
using device::math::fast_mod_div_u32_t;
// Runtime uint32 division as a multiply-high and a shift (round-up magic,
// exact below 2^31); cuda::fast_mod_div needs a newer CCCL than CUDA 13 bundles.
struct fast_mod_div_u32_t {
uint32_t divisor;
uint32_t magic;
uint32_t shift;
__host__ explicit fast_mod_div_u32_t(uint32_t d) : divisor(d), magic(0), shift(0) {
if (d > 1) {
const uint32_t log2_ceil = 32 - std::countl_zero(d - 1);
const uint32_t p = 31 + log2_ceil;
magic = static_cast<uint32_t>(((uint64_t{1} << p) + d - 1) / d);
shift = p - 32;
}
}
__device__ friend uint32_t operator/(uint32_t n, const fast_mod_div_u32_t& fd) {
return fd.divisor == 1 ? n : __umulhi(n, fd.magic) >> fd.shift;
}
__device__ friend uint32_t operator%(uint32_t n, const fast_mod_div_u32_t& fd) {
return n - (n / fd) * fd.divisor;
}
};
template <uint32_t kWorldSize>
struct NVLinkCommPushParams {
@@ -145,7 +168,8 @@ PUSH_KERNEL void nvlink_push_kernel(const __grid_constant__ NVLinkCommPushParams
ptx::st_multimem_16B(vec, dst_ptr_mc, vid);
}
} else /* reduce-scatter only */ {
const auto [token_id, offset] = math::div(vid, params.vecs_per_token_div);
const auto token_id = vid / params.vecs_per_token_div;
const auto offset = vid % params.vecs_per_token_div;
// Both by a compile-time constant, so this is a mask and a shift.
const auto dst_rank = token_id % kWorldSize;
const auto dst_token_id = token_id / kWorldSize;
@@ -112,53 +112,6 @@ SGL_DEVICE float fma_f32_bf16(bf16_t a, float b, float acc) {
return fmaf(cast<fp32_t>(a), b, acc);
}
/**
* \brief Fast uint32 division and modulo by a compile-time constant divisor.
*
* This struct is only exact when the divisor is less than `2^31`. Larger dividends
* will lead to incorrect results. Be careful when using this struct.
*/
struct fast_mod_div_u32_t {
public:
explicit constexpr fast_mod_div_u32_t(uint32_t d) : m_divisor(d) {
if (d > 1) {
const uint32_t log2_ceil = 32 - std::countl_zero(d - 1);
const uint32_t p = 31 + log2_ceil;
m_magic = static_cast<uint32_t>(((uint64_t{1} << p) + d - 1) / d);
m_shift = p - 32;
} else {
m_magic = 0;
m_shift = 0;
}
}
template <typename T>
SGL_DEVICE friend uint32_t operator/(T n, const fast_mod_div_u32_t& d) {
static_assert(std::is_unsigned_v<T> && sizeof(T) <= sizeof(uint32_t));
return d.m_divisor == 1 ? n : __umulhi(n, d.m_magic) >> d.m_shift;
}
template <typename T>
SGL_DEVICE friend uint32_t operator%(T n, const fast_mod_div_u32_t& d) {
static_assert(std::is_unsigned_v<T> && sizeof(T) <= sizeof(uint32_t));
return n - (n / d) * d.m_divisor;
}
constexpr operator uint32_t() const {
return m_divisor;
}
private:
uint32_t m_divisor;
uint32_t m_magic;
uint32_t m_shift;
};
template <typename T>
SGL_DEVICE uint2 div(T n, const fast_mod_div_u32_t& d) {
return {n / d, n % d};
}
} // namespace device::math
} // namespace sglang
@@ -273,67 +273,6 @@ def store_fp4_index_k_cache(
)
@triton.jit
def _gather_fp4_index_k_kernel(
cache,
cache_stride,
slots,
payload,
scales,
n,
page_size: tl.constexpr,
BLOCK_ROWS: tl.constexpr,
):
# int32 view of a page: page_size * 16 payload words, then page_size scale words.
rows = tl.program_id(0) * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS)
valid = rows < n
slot = tl.load(slots + rows, valid, 0).to(tl.int64)
page = slot // page_size
offset = slot - page * page_size
page_base = cache + page * cache_stride
words = tl.arange(0, 16)
payload_words = tl.load(
page_base[:, None] + offset[:, None] * 16 + words[None, :], valid[:, None]
)
tl.store(
payload + rows[:, None].to(tl.int64) * 16 + words[None, :],
payload_words,
valid[:, None],
)
tl.store(scales + rows, tl.load(page_base + page_size * 16 + offset, valid), valid)
def gather_fp4_index_k(
cache: torch.Tensor, slots: torch.Tensor, *, page_size: int
) -> tuple[torch.Tensor, torch.Tensor]:
"""Packed fp4 rows at ``slots`` of the index-K ``cache`` (``[pages, page_size *
68]`` uint8, ``store_fp4_index_k_cache``'s layout): (payload int8 ``[n, 64]``,
ue8m0 scales packed int32 ``[n]``), the layout ``quantize_fp4_indexer_tensor``
returns."""
assert cache.shape[1] == page_size * INDEX_K_SLOT_BYTES and cache.stride(1) == 1
# The kernel addresses slots as `slots + rows`, so a strided view would be read
# as if its elements were packed.
assert slots.is_contiguous()
words = cache.view(torch.int32)
n = slots.shape[0]
payload = torch.empty((n, 16), device=cache.device, dtype=torch.int32)
scales = torch.empty((n,), device=cache.device, dtype=torch.int32)
if n > 0:
block_rows = 64
_gather_fp4_index_k_kernel[(triton.cdiv(n, block_rows),)](
words,
words.stride(0),
slots,
payload,
scales,
n,
page_size,
BLOCK_ROWS=block_rows,
num_warps=4,
)
return payload.view(torch.int8), scales
@triton.jit
def _index_k_rope_pack_kernel(
X,
@@ -1,6 +1,5 @@
"""Fused ratio-1 and ratio-2 compressors: RMSNorm, RoPE and the FlashMLA cache
write in one launch. The ratio-1 kernel is per token, so decode and extend share
it; ratio 2 has a decode/verify entry point and an extend one.
"""Fused ratio-1 and ratio-2 decode compressors: RMSNorm, RoPE and the FlashMLA
cache write in one launch.
Ratio 1 takes the bf16 ``wkv`` projection as is: RoPE uses the token's own
position and the compressed slot equals the FULL slot. Ratio 2 pair-pools the
@@ -18,7 +17,6 @@ import torch
from sglang.kernels.jit.utils import (
cache_once,
is_arch_support_pdl,
is_hip_runtime,
load_jit,
make_cpp_args,
)
@@ -121,12 +119,11 @@ def _jit_c2_module(
is_arch_support_pdl(),
)
return load_jit(
make_name("c2"),
make_name("c2_decode"),
*args,
cuda_files=["deepseek_v4/c2.cuh"],
cuda_wrappers=[
("decode_fusion", f"FlashCompress2Kernel<{args}>::run_decode_fusion"),
("prefill_fusion", f"FlashCompress2Kernel<{args}>::run_prefill_fusion"),
],
)
@@ -152,12 +149,6 @@ def c2_decode_norm_rope_store(
The cache store uses ``raw_out_loc // 2`` as its slot.
A padded graph row is marked by ``raw_out_loc == 0`` and writes nothing, but
decode and verify rows issue their ring load before that check, so a padded
row's ``req`` and ``positions`` must still address an in-bounds ring slot.
The decode graph buffers zero both tails (``PaddingPolicy.ZERO``); a runner
that left stale request indices there would read out of the ring pool.
:param freqs_cis: ``[max_pos, rope_dim]`` fp32, real/imag interleaved --
``torch.view_as_real(freqs).flatten(-2)``. Indexed
in-kernel at ``positions - 1``, the position the latent
@@ -190,60 +181,3 @@ def c2_decode_norm_rope_store(
draft_len,
)
return out
def c2_prefill_norm_rope_store(
kv_input: torch.Tensor,
kv_state: torch.Tensor,
norm_weight: torch.Tensor,
positions: torch.Tensor,
req: torch.Tensor,
raw_out_loc: torch.Tensor,
eps: float,
freqs_cis: torch.Tensor,
k_cache: torch.Tensor,
extend_start_loc: torch.Tensor,
extend_seq_lens: torch.Tensor,
*,
page_size: int,
ring_size: int,
layout: Union[KVLayout, str] = KVLayout.V4,
out: Optional[torch.Tensor] = None,
) -> torch.Tensor:
""":func:`c2_decode_norm_rope_store` for an extend batch, then the ring write-back.
Rows may mix requests, prefix offsets and CUDA graph padding (``raw_out_loc
== 0``). An odd position pairs with the previous row when that row is the
same request's previous position, else with its ring slot. Afterwards each
request's last ``ring_size`` rows are saved to the ring, which a later step
reads for its first partner.
:param extend_start_loc: int32 ``[num_reqs]``, each request's first row.
:param extend_seq_lens: int32 ``[num_reqs]``, each request's row count.
:return: ``out``, ``[num_tokens, head_dim]`` bf16. Only the rows of odd
positions are written; the rest are left as they were.
"""
num_tokens, fused_dim = kv_input.shape
head_dim = fused_dim // 2
if out is None:
out = kv_input.new_empty((num_tokens, head_dim), dtype=torch.bfloat16)
layout = KVLayout.parse(layout)
module = _jit_c2_module(head_dim, freqs_cis.shape[-1], page_size, layout)
module.prefill_fusion(
kv_input,
kv_state,
out,
norm_weight,
positions,
req,
raw_out_loc,
eps,
freqs_cis,
# HIP's fp8_e4m3_t is uint8_t, so the kernel matches the fp8 pool as bytes
k_cache.view(torch.uint8) if is_hip_runtime() else k_cache,
ring_size,
extend_start_loc,
extend_seq_lens,
)
return out
@@ -36,7 +36,7 @@ def _build_swa_token_ids_kernel(
tl.store(out_ptr + out_off + i, swa_id)
@triton.jit(do_not_specialize=["num_tokens", "num_reqs", "top_k"])
@triton.jit(do_not_specialize=["top_k"])
def _combine_topk_swa_indices_kernel(
combined_indices_ptr,
combined_indices_stride,
@@ -49,76 +49,59 @@ def _combine_topk_swa_indices_kernel(
gather_lens_ptr,
compressed_base_ptr,
swa_base_ptr,
num_tokens,
num_reqs,
top_k,
COMPRESS_RATIO: tl.constexpr,
WINDOW_SIZE: tl.constexpr,
WIDTH: tl.constexpr,
PADDED_WIDTH: tl.constexpr,
BLOCK_T: tl.constexpr,
SEARCH_STEPS: tl.constexpr,
PADDED_TOP_K: tl.constexpr,
):
# BLOCK_T consecutive tokens per program, so the grid scales with tokens, not requests.
token = tl.program_id(0) * BLOCK_T + tl.arange(0, BLOCK_T)
valid = token < num_tokens
batch_idx = tl.program_id(0)
worker_id = tl.program_id(1)
num_workers = tl.num_programs(1)
# query_start_loc may be a global tensor; rebase to chunk-local offsets
# by subtracting the chunk's starting value.
base = tl.load(query_start_loc_ptr)
# Each token's request: the last r with query_start_loc[r] - base <= token.
lo = tl.zeros([BLOCK_T], dtype=tl.int32)
hi = tl.full([BLOCK_T], 0, dtype=tl.int32) + num_reqs
for _ in tl.static_range(SEARCH_STEPS):
mid = (lo + hi + 1) // 2
start = tl.load(query_start_loc_ptr + mid, mask=lo < hi, other=0) - base
go_right = (lo < hi) & (start <= token)
lo = tl.where(go_right, mid, lo)
hi = tl.where((lo < hi) & ~go_right, mid - 1, hi)
batch_idx = tl.minimum(lo, num_reqs - 1)
# Rows past the last request's queries belong to no request: -1 rows, length 0.
owned = valid & (token < tl.load(query_start_loc_ptr + num_reqs) - base)
pos = tl.load(query_pos_ptr + token, mask=owned, other=0)
seq_len = tl.load(seq_lens_ptr + batch_idx, mask=owned, other=0)
gather_len = tl.load(gather_lens_ptr + batch_idx, mask=owned, other=0)
compressed_base = tl.load(compressed_base_ptr + batch_idx, mask=owned, other=0)
swa_base = tl.load(swa_base_ptr + batch_idx, mask=owned, other=0)
query_start = tl.load(query_start_loc_ptr + batch_idx) - base
query_end = tl.load(query_start_loc_ptr + batch_idx + 1) - base
seq_len = tl.load(seq_lens_ptr + batch_idx)
gather_len = tl.load(gather_lens_ptr + batch_idx)
compressed_base = tl.load(compressed_base_ptr + batch_idx)
swa_base = tl.load(swa_base_ptr + batch_idx)
# SWA portion of the gathered buffer starts from position
# (seq_len - gather_len), not 0. The +pos-gather_start formula maps a
# query's window back into the workspace's SWA region.
gather_start = seq_len - gather_len
# -1 entries inside the top-k span stay -1 (attention skips them).
# top_k=0 disables the compressed portion for SWA-only layers.
topk_len = tl.where(owned, tl.minimum((pos + 1) // COMPRESS_RATIO, top_k), 0)
swa_len = tl.where(owned, tl.minimum(pos + 1, WINDOW_SIZE), 0)
for token_idx in range(query_start + worker_id, query_end, num_workers):
pos = tl.load(query_pos_ptr + token_idx)
# -1 entries inside the top-k span stay -1 (attention skips them).
# top_k=0 disables the compressed portion for SWA-only layers.
topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, top_k)
swa_len = tl.minimum(pos + 1, WINDOW_SIZE)
combined_row = token.to(tl.int64)[:, None] * combined_indices_stride
topk_row = token.to(tl.int64)[:, None] * topk_indices_stride
combined_row = token_idx.to(tl.int64) * combined_indices_stride
topk_row = token_idx.to(tl.int64) * topk_indices_stride
# Whole rows, -1 padded, so the output needs no fill pass:
# [top-k (-1 holes kept) | window | -1 up to WIDTH].
offset = tl.arange(0, PADDED_WIDTH)[None, :]
in_topk = offset < topk_len[:, None]
topk_vals = tl.load(
topk_indices_ptr + topk_row + offset, mask=valid[:, None] & in_topk, other=-1
)
# Workspace SWA index: swa_base[r] + (gather_offset_in_buffer).
# For positions [pos - swa_len + 1, pos], the buffer offsets are
# [pos - swa_len + 1 - gather_start, pos - gather_start].
swa_start = swa_base + pos - swa_len + 1 - gather_start
swa_vals = swa_start[:, None] + (offset - topk_len[:, None])
in_swa = offset < (topk_len + swa_len)[:, None]
vals = tl.where(
in_topk,
tl.where(topk_vals >= 0, topk_vals + compressed_base[:, None], -1),
tl.where(in_swa, swa_vals, -1),
)
tl.store(
combined_indices_ptr + combined_row + offset,
vals,
mask=valid[:, None] & (offset < WIDTH),
)
offset = tl.arange(0, PADDED_TOP_K)
mask = offset < topk_len
topk_vals = tl.load(
topk_indices_ptr + topk_row + offset,
mask=mask,
)
tl.store(
combined_indices_ptr + combined_row + offset,
tl.where(topk_vals >= 0, topk_vals + compressed_base, -1),
mask=mask,
)
tl.store(combined_lens_ptr + token, topk_len + swa_len, mask=valid)
offset = tl.arange(0, WINDOW_SIZE)
# Workspace SWA index: swa_base[r] + (gather_offset_in_buffer).
# For positions [pos - swa_len + 1, pos], the buffer offsets are
# [pos - swa_len + 1 - gather_start, pos - gather_start].
tl.store(
combined_indices_ptr + combined_row + topk_len + offset,
swa_base + offset + pos - swa_len + 1 - gather_start,
mask=offset < swa_len,
)
tl.store(combined_lens_ptr + token_idx, topk_len + swa_len)
@@ -1882,34 +1882,6 @@ def hc_expand(x: torch.Tensor, n: int) -> torch.Tensor:
return x.repeat(1, n)
@triton.jit
def _hc_broadcast_kernel(
x_ptr, y_ptr, H, x_stride_m, HC: tl.constexpr, BLOCK_H: tl.constexpr
):
row = tl.program_id(0).to(tl.int64)
offs = tl.program_id(1) * BLOCK_H + tl.arange(0, BLOCK_H)
mask = offs < H
x = tl.load(x_ptr + row * x_stride_m + offs, mask=mask)
y_row = y_ptr + row * (HC * H)
for k in tl.static_range(HC):
tl.store(y_row + k * H + offs, x, mask=mask)
def hc_broadcast(x: torch.Tensor, hc_mult: int) -> torch.Tensor:
"""[T, H] -> [T, hc_mult, H], every copy equal to x: the initial mHC residual."""
if not x.is_cuda or x.stride(-1) != 1:
return x.unsqueeze(1).repeat(1, hc_mult, 1)
num_tokens, hidden = x.shape
y = x.new_empty(num_tokens, hc_mult, hidden)
if num_tokens == 0:
return y
block_h = 1024
_hc_broadcast_kernel[(num_tokens, triton.cdiv(hidden, block_h))](
x, y, hidden, x.stride(0), HC=hc_mult, BLOCK_H=block_h, num_warps=4
)
return y
def hc_contract(x: torch.Tensor, n: int) -> torch.Tensor:
return x.unflatten(-1, (n, -1)).mean(dim=-2)
@@ -181,15 +181,12 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
)
cfg = resolving_view(server_args)
hf_config = model_config_of(server_args).hf_config
if hf_config.model_type != "deepseek_v41":
if model_config_of(server_args).hf_config.model_type != "deepseek_v41":
if cfg.enable_encoder_swa_bounded_replay:
raise ValueError(
"--enable-encoder-swa-bounded-replay requires DeepSeek-V4.1"
)
return
if hf_config.vision_n_layers > 0 and cfg.enable_prefill_cp:
_validate_deepseek_v41_vision_prefill_cp(server_args)
if cfg.enable_encoder_swa_bounded_replay:
from sglang.srt.model_executor.cuda_graph_config import Backend
@@ -200,8 +197,7 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
cfg.cuda_graph_config.prefill.backend != Backend.DISABLED,
),
("DP attention", cfg.enable_dp_attention),
# Prefill CP declares attn_cp_size and DP attention only later.
("context parallelism", cfg.attn_cp_size > 1 or cfg.enable_prefill_cp),
("context parallelism", cfg.attn_cp_size > 1),
("external cache linker", cfg.enable_unified_cache_external_linker),
("unified memory", cfg.enable_unified_memory),
("PD disaggregation", cfg.disaggregation_mode != "null"),
@@ -253,33 +249,23 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
if (
read_ragged_verify_mode() is not RaggedVerifyMode.STATIC
or cfg.disaggregation_transfer_backend != "mooncake"
or cfg.dp_size != 1
or cfg.enable_dp_attention
or cfg.attn_cp_size != 1
or cfg.dcp_size != 1
):
raise ValueError(
"DeepSeek-V4.1 DSpark PD requires static verify, Mooncake, "
"and CP=1 on both servers. DP attention is supported when "
"both servers use the same block size and target/draft KV layout."
"DP=1 and CP=1. Both servers must enable DSpark with the same "
"block size and TP size."
)
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
prefill_graph = cfg.cuda_graph_config.prefill
cp_breakable_prefill = (
cfg.enable_prefill_cp
and cfg.cp_strategy == "interleave"
and cfg.tp_size > 1
and prefill_graph.backend == Backend.BREAKABLE
)
if (
prefill_graph.backend != Backend.DISABLED
and prefill_graph.max_seq_len is None
and not cp_breakable_prefill
):
# The non-CP captured low-ratio indexer scores a static context width.
# CP BCG runs these sources eagerly with live prefix metadata, so this
# default would only force long-prefix CP batches back to eager.
# Explicit max_seq_len values still constrain both paths.
if prefill_graph.backend != Backend.DISABLED and prefill_graph.max_seq_len is None:
# The captured low-ratio indexer scores a static context width; 16k
# keeps it inside the candidate window at under 1 ms per layer.
declare_resolution(
server_args,
"validate_deepseek_v41_features",
@@ -310,40 +296,3 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
"--enable-decoder-swa-bounded-replay cannot be combined with "
f"{feature} yet; disable one of them."
)
def _validate_deepseek_v41_vision_prefill_cp(server_args: ServerArgs) -> None:
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
cfg = resolving_view(server_args)
if cfg.cp_strategy != "interleave":
raise ValueError(
"DeepSeek-V4.1 vision with prefill CP requires --cp-strategy "
f"interleave; got {cfg.cp_strategy!r}."
)
if cfg.cuda_graph_config.prefill.backend != Backend.DISABLED:
# The CP runner merges image features eagerly; no capture path replays it.
locked = getattr(server_args, "_cuda_graph_config_locked", set())
if (Phase.PREFILL, "backend") in locked:
raise ValueError(
"DeepSeek-V4.1 vision with prefill CP runs eager prefill; remove "
"the explicit prefill CUDA graph backend."
)
declare_resolution(
server_args,
"validate_deepseek_v41_features",
cuda_graph_config=with_phase(
cfg.cuda_graph_config, Phase.PREFILL, backend=Backend.DISABLED
),
)
logger.warning(
"Disabling the prefill CUDA graph for DeepSeek-V4.1 vision with prefill CP."
)
if (
str(cfg.speculative_algorithm).upper() == "DSPARK"
and cfg.enable_decoder_swa_bounded_replay
):
raise ValueError(
"DeepSeek-V4.1 vision with prefill CP does not support DSpark together "
"with --enable-decoder-swa-bounded-replay yet."
)
+2 -4
View File
@@ -966,14 +966,12 @@ def handle_language_model_only(server_args: Any):
):
if flag:
raise ValueError(f"--language-model-only cannot be combined with {name}")
hf_config = model_config_of(server_args).hf_config
# V4.1 text-only workers use the standard PD KV transfer path.
if cfg.disaggregation_mode != "null" and hf_config.model_type != "deepseek_v41":
if cfg.disaggregation_mode != "null":
raise ValueError(
"--language-model-only is incompatible with --disaggregation-mode "
"prefill/decode"
)
architectures = hf_config.architectures
architectures = model_config_of(server_args).hf_config.architectures
if not any(
a in server_args.LANGUAGE_MODEL_ONLY_ARCHITECTURES for a in architectures
):
@@ -945,33 +945,9 @@ class CommonKVManager(BaseKVManager):
"enable DSpark with the same block size and target/draft KV "
"layout. Upgrade both servers together."
)
same_tp_with_prefill_cp = (
info.attn_cp_size > 1
and (self.is_mla_backend or self.is_hybrid_mla_backend)
and self.attn_cp_size == 1
and info.attn_tp_size * info.attn_cp_size == self.attn_tp_size
)
# Combined branch (40323-series + 40177): prefill CP can also pair
# with a DP-attention decode server. MLA KV is replicated across
# prefill CP ranks, so per-rank layouts match when attn_tp matches.
dp_decode_with_prefill_cp = (
info.attn_cp_size > 1
and self.attn_cp_size == 1
and (self.is_mla_backend or self.is_hybrid_mla_backend)
and info.attn_tp_size == self.attn_tp_size
)
non_cp_mla_layout = info.attn_cp_size == self.attn_cp_size == 1 and (
self.is_mla_backend or self.is_hybrid_mla_backend
)
if info.attn_tp_size != self.attn_tp_size and not (
same_tp_with_prefill_cp
or dp_decode_with_prefill_cp
or non_cp_mla_layout
):
if info.attn_tp_size != self.attn_tp_size:
raise RuntimeError(
"DeepSeek-V4.1 DSpark PD requires matching attention TP "
"unless both servers use CP=1 with an MLA KV layout, "
"or prefill runs CP with an MLA KV layout"
"DeepSeek-V4.1 DSpark PD requires the same TP size on both servers"
)
if self.dcp_size > 1:
@@ -32,7 +32,7 @@ class TransferKVChunk:
# Set when the staging worker first counts this chunk toward the per-room
# outstanding count; stays set across re-enqueue on a watermark defer.
staging_counted: bool = False
# Early-send: GPU event to synchronize before reading KV (optional).
# Mori early-send: CUDA event to synchronize before RDMA (optional).
wait_event: Optional[object] = None
@@ -10,7 +10,6 @@ from sglang.srt.managers.overlap_utils import RelayPayload
from sglang.srt.mem_cache.common import maybe_cache_unfinished_req
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo
from sglang.srt.utils.common import is_pin_memory_available
logger = logging.getLogger(__name__)
@@ -141,14 +140,9 @@ class ScheduleBatchDisaggregationDecodeMixin:
error_message, HTTPStatus.INTERNAL_SERVER_ERROR
)
req.grammar.finished = req.finished()
# Non-blocking H2D: with overlap this runs on the schedule stream after
# its wait on the in-flight forward, so a blocking copy would stall the
# host until that forward ends and leave the GPU idle.
last_tokens_tensor = torch.tensor(
last_tokens,
dtype=torch.int64,
pin_memory=is_pin_memory_available(self.device),
).to(self.device, non_blocking=True)
last_tokens, dtype=torch.int64, device=self.device
)
spec_info = self.spec_algorithm.build_disagg_draft_input(
self,
+10 -354
View File
@@ -82,115 +82,6 @@ FAILED_SESSION_RECOVERIES = Counter(
)
# ---------------------------------------------------------------------------
# Intra-node NVLink transport helpers.
#
# Mooncake's IntraNodeNvlinkTransport can only register and reach *device*
# memory (it IPC-opens the remote cudaMalloc segments). Host-resident regions
# (aux buffers, some state components) cannot be registered: one host region
# makes the whole registerLocalMemoryBatch fail, and the engine then rolls
# back *every* region, leaving the segment descriptor empty and all KV
# transfers failing with "Requested address ... not found". When the
# intra-node NVLink transport is active we therefore
# 1. register only device-memory regions, and
# 2. route blocks whose source is host memory over the ordered zmq channel
# (same ordering guarantee the aux TCP path relies on) instead of the
# transfer engine.
# ---------------------------------------------------------------------------
import ctypes as _ctypes
_CUDA_MEMORY_TYPE_DEVICE = 2
try:
from cuda.bindings import runtime as _cudart
except ImportError: # pragma: no cover - cuda-python is always present in images
_cudart = None
def _is_device_pointer(ptr: int) -> bool:
"""Probe a *local* pointer with cudaPointerGetAttributes.
Only valid for pointers owned by this process (never probe remote
segment addresses). Returns False on any error so the caller falls back
to the safe host path.
"""
if _cudart is None:
# Cannot tell; assume device so behavior stays unchanged.
return True
err, attr = _cudart.cudaPointerGetAttributes(int(ptr))
if int(err) != 0:
# Clear the error so subsequent CUDA calls are not poisoned.
_cudart.cudaGetLastError()
return False
return int(attr.type) == _CUDA_MEMORY_TYPE_DEVICE
def _read_bytes_from_address(addr: int, length: int) -> Optional[bytes]:
if length <= 0:
return b""
if _is_device_pointer(addr):
buf = bytearray(length)
# cudaMemcpyDeviceToHost = 2; synchronous default-stream copy.
err, = _cudart.cudaMemcpy(
_ctypes.addressof((_ctypes.c_char * length).from_buffer(buf)),
int(addr),
length,
2,
)
if int(err) != 0:
logger.error(
f"cudaMemcpy D2H failed (err={err}) for addr {hex(addr)} len {length}"
)
return None
return bytes(buf)
return _ctypes.string_at(int(addr), length)
def _write_bytes_to_address(addr: int, data: bytes) -> bool:
if not data:
return True
if _is_device_pointer(addr):
buf = _ctypes.create_string_buffer(data, len(data))
# cudaMemcpyHostToDevice = 1; synchronous default-stream copy.
err, = _cudart.cudaMemcpy(
int(addr), _ctypes.addressof(buf), len(data), 1
)
if int(err) != 0:
logger.error(
f"cudaMemcpy H2D failed (err={err}) for addr {hex(addr)} "
f"len {len(data)}"
)
return False
return True
_ctypes.memmove(int(addr), data, len(data))
return True
_NVLINK_INTRA_ACTIVE = None
def _nvlink_intra_transport_active() -> bool:
"""Whether mooncake installed the intra-node NVLink transport.
Mirrors the env probing in mooncake's transfer_engine_impl.cpp: the
transport is installed iff MC_INTRANODE_NVLINK is set (any value), or an
equivalent protocol selection was made.
"""
global _NVLINK_INTRA_ACTIVE
if _NVLINK_INTRA_ACTIVE is None:
active = bool(
os.environ.get("MC_INTRANODE_NVLINK")
or os.environ.get("MC_INTRA_NVLINK")
)
if not active:
proto = (os.environ.get("MOONCAKE_PROTOCOL") or "").strip().lower()
active = proto in ("nvlink_intra", "nvlink-intra", "intra_nvlink")
_NVLINK_INTRA_ACTIVE = active
return _NVLINK_INTRA_ACTIVE
# decode
@dataclasses.dataclass
class TransferInfo:
@@ -323,7 +214,6 @@ class KVArgsRegisterInfo:
class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
AUX_DATA_HEADER = b"AUX_DATA"
STATE_DATA_HEADER = b"STATE_DATA"
# Implements teardown() below, so runtime PD role switching is supported.
supports_role_switch = True
@@ -337,10 +227,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
super().__init__(args, disaggregation_mode, server_args, is_mla_backend)
self.init_engine()
self.register_buffer_to_engine()
# session_id -> (endpoint, dst_port, room), used to route host-memory
# transfer blocks over zmq when the intra-node NVLink transport is
# active (it cannot reach host memory). Populated on bootstrap.
self._session_endpoint_map = {}
self.enable_staging = envs.SGLANG_DISAGG_STAGING_BUFFER.get()
self.max_transfer_batch_indices = (
envs.SGLANG_MOONCAKE_MAX_TRANSFER_BATCH_INDICES.get()
@@ -436,13 +322,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
Deduped because the unified memory pool reports one raw buffer as both
its KV and its mamba state component, and double registration fails in
the engine.
When the intra-node NVLink transport is active, host-memory regions
(aux buffers, some state components) are skipped: the transport only
accepts device memory, and a single host region fails the whole batch
and triggers a full engine-side rollback that would unregister the KV
pools too. Host-resident payloads are instead exchanged over the
ordered zmq channel (see _transfer_data / send_aux).
"""
regions: List[Tuple[int, int]] = []
seen: Set[Tuple[int, int]] = set()
@@ -459,24 +338,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
self.kv_args.state_data_ptrs, self.kv_args.state_data_lens
):
add(ptrs, lens)
if _nvlink_intra_transport_active():
device_regions = []
skipped = []
for ptr, length in regions:
if _is_device_pointer(ptr):
device_regions.append((ptr, length))
else:
skipped.append((ptr, length))
if skipped:
logger.info(
"Intra-node NVLink transport: skipping %d host-memory "
"regions from engine registration (they will be exchanged "
"over the zmq channel instead): %s",
len(skipped),
[(hex(p), l) for p, l in skipped[:8]],
)
regions = device_regions
return regions
def register_buffer_to_engine(self):
@@ -887,63 +748,10 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
if not transfer_blocks:
return 0
if not _nvlink_intra_transport_active():
src_addrs, dst_addrs, lengths = zip(*transfer_blocks)
return self.engine.batch_transfer_sync(
mooncake_session_id, list(src_addrs), list(dst_addrs), list(lengths)
)
# Intra-node NVLink transport can only move device memory. Partition
# blocks by the *local source* pointer (probing a local pointer is
# safe; the remote dst is never probed): device-sourced blocks go
# through the engine as usual, host-sourced blocks are shipped over
# the ordered zmq channel and written into the peer's buffer by the
# receiver (see _handle_state_data). This mirrors the aux TCP path.
device_blocks = []
host_blocks = []
for src, dst, length in transfer_blocks:
if _is_device_pointer(src):
device_blocks.append((src, dst, length))
else:
host_blocks.append((src, dst, length))
rc = 0
if device_blocks:
src_addrs, dst_addrs, lengths = zip(*device_blocks)
rc = self.engine.batch_transfer_sync(
mooncake_session_id, list(src_addrs), list(dst_addrs), list(lengths)
)
if rc == 0 and host_blocks:
rc = self._send_host_blocks_tcp(mooncake_session_id, host_blocks)
return rc
def _send_host_blocks_tcp(self, mooncake_session_id, host_blocks):
target = self._session_endpoint_map.get(mooncake_session_id)
if target is None:
logger.error(
f"No zmq endpoint known for mooncake session "
f"{mooncake_session_id}; cannot deliver {len(host_blocks)} "
"host-memory transfer blocks"
)
return -1
endpoint, dst_port, room = target
na = NetworkAddress(endpoint, dst_port)
for src, dst, length in host_blocks:
data = _read_bytes_from_address(src, length)
if data is None:
return -1
self._send_multipart_locked(
na.to_tcp(),
[
MooncakeKVManager.STATE_DATA_HEADER,
str(room).encode("ascii"),
str(int(dst)).encode("ascii"),
struct.pack(">I", len(data)),
data,
],
is_ipv6=na.is_ipv6,
)
return 0
src_addrs, dst_addrs, lengths = zip(*transfer_blocks)
return self.engine.batch_transfer_sync(
mooncake_session_id, list(src_addrs), list(dst_addrs), list(lengths)
)
def _send_kvcache_generic(
self,
@@ -960,8 +768,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
dst_layer_ids: Optional[List[int]] = None,
dst_device_data_indices: Optional[npt.NDArray[np.int32]] = None,
dst_device_data_ptrs: Optional[set[int]] = None,
dst_item_lens: list[int] | None = None,
bootstrap_room: Optional[int] = None,
) -> int:
"""
Generic KV cache transfer supporting both MHA and MLA architectures.
@@ -1013,21 +819,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
len(dst_data_ptrs),
allow_positional_fallback=self.pp_size == 1,
)
if dst_item_lens is not None:
for i, j in pairs:
if item_lens[i] != dst_item_lens[j]:
assert bootstrap_room is not None
failure_reason = (
f"{state_type} item length mismatch for paired "
f"entries src[{i}]={item_lens[i]} "
f"dst[{j}]={dst_item_lens[j]}"
)
logger.error(failure_reason)
self.conclude_failure(
bootstrap_room=bootstrap_room,
failure_reason=failure_reason,
)
return -1
layers_params = [
(src_data_ptrs[i], dst_data_ptrs[j], item_lens[i]) for i, j in pairs
]
@@ -1037,24 +828,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
src_data_ptrs, dst_data_ptrs, state_type
)
)
if dst_item_lens is not None:
_, mapped_dst_lens, _ = self.get_mla_kv_ptrs_with_pp(
item_lens, dst_item_lens, state_type
)
for layer_id in range(layers_current_pp_stage):
if item_lens[layer_id] != mapped_dst_lens[layer_id]:
assert bootstrap_room is not None
failure_reason = (
f"{state_type} item length mismatch for positional "
f"entry {layer_id}: prefill={item_lens[layer_id]} "
f"decode={mapped_dst_lens[layer_id]}"
)
logger.error(failure_reason)
self.conclude_failure(
bootstrap_room=bootstrap_room,
failure_reason=failure_reason,
)
return -1
layers_params = [
(
src_kv_ptrs[layer_id],
@@ -1709,10 +1482,8 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
):
# TODO(shangming): Fix me when nvlink_transport of Mooncake is bug-free
if (
(self.enable_custom_mem_pool and self.custom_mem_pool_type == "NVLINK")
or envs.SGLANG_MOONCAKE_SEND_AUX_TCP.get()
or _nvlink_intra_transport_active()
):
self.enable_custom_mem_pool and self.custom_mem_pool_type == "NVLINK"
) or envs.SGLANG_MOONCAKE_SEND_AUX_TCP.get():
return self.send_aux_tcp(req, prefill_aux_index, dst_aux_ptrs)
transfer_blocks = []
@@ -1795,71 +1566,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
f"Received AUX_DATA for bootstrap_room {room} with length:{len(data)}"
)
def _host_transfer_regions(self):
"""Address ranges this process published as transfer targets.
Used to validate STATE_DATA writes. Built lazily because kv_args is
fully populated only after registration.
"""
regions = getattr(self, "_host_transfer_regions_cache", None)
if regions is None:
regions = []
for ptr, length in zip(
self.kv_args.kv_data_ptrs or [], self.kv_args.kv_data_lens or []
):
regions.append((int(ptr), int(ptr) + int(length)))
for ptr, length in zip(
self.kv_args.aux_data_ptrs or [], self.kv_args.aux_data_lens or []
):
regions.append((int(ptr), int(ptr) + int(length)))
for ptrs, lens in zip(
self.kv_args.state_data_ptrs or [], self.kv_args.state_data_lens or []
):
for ptr, length in zip(ptrs or [], lens or []):
regions.append((int(ptr), int(ptr) + int(length)))
self._host_transfer_regions_cache = regions
return regions
def _handle_state_data(self, msg: List[bytes]):
"""Handle STATE_DATA messages received by the decode thread.
Carries one host-memory transfer block that could not go through the
intra-node NVLink transport. Written directly into the local buffer at
the destination address; ordering against the final status message is
guaranteed by the shared per-endpoint zmq socket.
"""
room = int(msg[1].decode("ascii"))
dst_addr = int(msg[2].decode("ascii"))
data_length = struct.unpack(">I", msg[3])[0]
data = msg[4]
if len(data) != data_length:
logger.error(f"STATE_DATA length mismatch for bootstrap_room {room}")
return
in_region = any(
start <= dst_addr and dst_addr + len(data) <= end
for start, end in self._host_transfer_regions()
)
if not in_region:
logger.error(
f"STATE_DATA for bootstrap_room {room} targets unknown region "
f"{hex(dst_addr)}..{hex(dst_addr + len(data))}; dropping"
)
return
if not _write_bytes_to_address(dst_addr, data):
logger.error(
f"STATE_DATA write failed for bootstrap_room {room} at "
f"{hex(dst_addr)} len {len(data)}"
)
return
logger.debug(
f"Received STATE_DATA for bootstrap_room {room} at {hex(dst_addr)} "
f"with length:{len(data)}"
)
def _get_dsa_cache_transfer_skip_flags(
self, info: Optional[KVArgsRegisterInfo]
) -> Tuple[bool, bool]:
@@ -2145,13 +1851,9 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
),
src_layer_ids=src_state_layer_ids,
dst_layer_ids=dst_state_layer_ids,
dst_item_lens=dst_item_lens,
bootstrap_room=req.room,
)
or rc
)
if self.request_status.get(req.room, KVPoll.Failed) == KVPoll.Failed:
return rc
elif st in (StateType.MINIMAX_INDEX_K, StateType.MINIMAX_DENSE_KV):
# Compacted layer lists require equal TP and PP=1 on both peers.
if self.pp_size is not None and self.pp_size > 1:
@@ -2415,19 +2117,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
self._staging_outstanding[kv_chunk.room] += 1
kv_chunk.staging_counted = True
# Under overlap scheduling an early cached-prefix send can
# precede completion of the previous prefill forward. Wait on
# the worker, not the scheduler, before reading source KV.
# A room that already failed skips the wait; the status check
# below then drops the chunk, including on an abort during it.
if kv_chunk.wait_event is not None:
if (
kv_chunk.room in self.request_status
and self.check_status(kv_chunk.room) != KVPoll.Failed
):
kv_chunk.wait_event.synchronize()
kv_chunk.wait_event = None
if (
kv_chunk.room not in self.request_status
or self.check_status(kv_chunk.room) == KVPoll.Failed
@@ -2640,13 +2329,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
executor,
target_rank_registration_info,
)
if (
self.request_status.get(
kv_chunk.room, KVPoll.Failed
)
== KVPoll.Failed
):
break
if state_rc != 0:
with self.session_lock:
self.session_failures[
@@ -2730,8 +2412,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
):
self._staging_outstanding.pop(kv_chunk.room, None)
if kv_chunk.room in self.transfer_infos:
for sid in self.transfer_infos[kv_chunk.room]:
self._session_endpoint_map.pop(sid, None)
self.transfer_infos.pop(kv_chunk.room)
self.req_to_decode_prefix_len.pop(kv_chunk.room, None)
if self.enable_staging:
@@ -2877,11 +2557,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
self.transfer_infos[room][mooncake_session_id] = (
TransferInfo.from_zmq(waiting_req_bytes)
)
self._session_endpoint_map[mooncake_session_id] = (
self.transfer_infos[room][mooncake_session_id].endpoint,
self.transfer_infos[room][mooncake_session_id].dst_port,
room,
)
# NOTE: after bootstrapping we can mark the req as waiting for input
if len(self.transfer_infos[room]) == required_dst_info_num:
self.resolve_kv_replica_factor(self.transfer_infos[room])
@@ -2910,9 +2585,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
if msg[0] == MooncakeKVManager.AUX_DATA_HEADER:
self._handle_aux_data(msg)
continue
if msg[0] == MooncakeKVManager.STATE_DATA_HEADER:
self._handle_state_data(msg)
continue
# Staging: prefill notifies a chunk written to staging buffer
if msg[0] == b"CHUNK_READY":
@@ -2978,7 +2650,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
state_indices: Optional[List] = None,
num_kv_tokens: Optional[int] = None,
trace_ctx: Optional[Union[TraceReqContext, TraceNullContext]] = None,
wait_event: Optional[object] = None,
):
assert self.disaggregation_mode == DisaggregationMode.PREFILL
assert not is_last_chunk or (is_last_chunk and aux_index is not None)
@@ -2998,22 +2669,12 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
# add further chunks into the transfer queue.
return
# Shard by dst session ports AND bootstrap_room. The port term alone
# degenerates to a constant when all requests target a single decode
# instance (its session ports are fixed), collapsing every transfer
# onto 1/16 of the lanes and throttling P->D intake (measured at
# client concurrency 300: #transfer-req piled at 100+ while decode
# itself stayed healthy). Mixing in bootstrap_room keeps all chunks
# of one request on the same lane (per-request FIFO order preserved)
# while spreading distinct rooms across all lanes. Abort/cleanup is
# unaffected: chunks are dropped per-room at dequeue via
# request_status (see the worker loop) and per-request via
# failed_sessions before send — no lane-level purge exists, and the
# deferred abort ack is room-scoped (_staging_outstanding). Same
# trade-off as mori/conn.py which shards by bootstrap_room alone.
# NOTE(shangming): sharding according to the dst_infos to make sure
# requests with the same dst_sessions will be added into the same
# queue, which enables early abort with failed sessions.
dst_infos = self.transfer_infos[bootstrap_room].keys()
session_port_sum = sum(int(session.rsplit(":", 1)[1]) for session in dst_infos)
shard_idx = (session_port_sum + bootstrap_room) % len(self.transfer_queues)
shard_idx = session_port_sum % len(self.transfer_queues)
if trace_ctx is None:
trace_ctx = TraceNullContext()
@@ -3028,7 +2689,6 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
state_indices=state_indices,
num_kv_tokens=num_kv_tokens,
trace_ctx=trace_ctx,
wait_event=wait_event,
)
)
@@ -3135,8 +2795,6 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
if should_skip:
return
wait_event = getattr(self, "_early_send_wait_event", None)
self._early_send_wait_event = None
if not is_last_chunk:
self.kv_mgr.add_transfer_request(
self.bootstrap_room,
@@ -3145,7 +2803,6 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
False,
num_kv_tokens=num_kv_tokens,
trace_ctx=self.trace_ctx.copy_for_thread(),
wait_event=wait_event,
)
else:
self.kv_mgr.add_transfer_request(
@@ -3157,7 +2814,6 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
state_indices=state_indices,
num_kv_tokens=num_kv_tokens,
trace_ctx=self.trace_ctx.copy_for_thread(),
wait_event=wait_event,
)
self._record_transfer_indices(kv_indices, state_indices)
+7 -23
View File
@@ -851,11 +851,7 @@ class SchedulerDisaggregationPrefillMixin:
advance_logprob_pt(i, req)
continue
self.checkpoint_disagg_prefill(req)
# G9: the disagg-prefill path bypasses batch_result_processor's
# prefill loop, so record the drain event here — final chunk
# done, request heading to the inflight (KV transfer) queue.
self.record_prefill_drain(req.seqlen)
maybe_cache_unfinished_req(req, self.tree_cache)
self.disagg_prefill_inflight_queue.append(req)
if self.spec_algorithm.is_eagle() and draft_input is not None:
req.output_topk_p = draft_input.topk_p[i]
@@ -1212,7 +1208,7 @@ class SchedulerDisaggregationPrefillMixin:
chunked_req_to_exclude = set()
if (req := self.chunked_req) is not None:
chunked_req_to_exclude.add(req)
self.checkpoint_disagg_prefill(req, chunked=True)
maybe_cache_unfinished_req(req, self.tree_cache, chunked=True)
if not self.check_bootstrap(req):
if is_aborted(req):
@@ -1507,21 +1503,10 @@ class SchedulerDisaggregationPrefillMixin:
else:
self.disagg_prefill_pending_chunk_rids.add(req.rid)
def checkpoint_disagg_prefill(
self: Scheduler, req: Req, chunked: bool = False
) -> None:
"""Publish the request's computed prefix to the tree cache.
Name shim for the #42035-era call structure: this branch predates
#41520's ``checkpoint_kv_cache`` / ``insert_req`` cache API, so the
equivalent here is ``maybe_cache_unfinished_req``.
"""
maybe_cache_unfinished_req(req, self.tree_cache, chunked=chunked)
def optimistic_release_and_requeue(self: Scheduler, req: Req) -> None:
"""Release KV cache and requeue an optimistic prefill request."""
max_attempts = get_disagg().optimistic_prefill_attempts
self.checkpoint_disagg_prefill(req)
maybe_cache_unfinished_req(req, self.tree_cache)
# The cached prefix is evictable once the KV is released. Its length
# (capped at what a retry can match) seeds the retry's storage baseline,
# so an evicted prefix is looked up in L3 once before it is recomputed.
@@ -1534,11 +1519,10 @@ class SchedulerDisaggregationPrefillMixin:
)
)
self._release_aborted_request(req)
# The checkpoint above (maybe_cache_unfinished_req) already handed the
# prefill KV to the tree; the request is not finished, so the release
# only frees the rest. (#42194 semantics; our tree predates #41520's
# mamba-donation rework, mamba models are not run on this branch.)
release_kv_cache(req, self.tree_cache, is_insert=False)
# Mamba insertion donates the checkpoint and clears its sequence marker.
release_kv_cache(
req, self.tree_cache, is_insert=not self.tree_cache.supports_mamba()
)
req.reset_for_retract()
req.output_ids = array("q")
req.start_send_idx = 0
@@ -1776,18 +1776,6 @@ async def openai_v1_chat_completions(
)
@app.post("/query_cache_hit", dependencies=[Depends(validate_json_request)])
async def query_cache_hit(request: ChatCompletionRequest, raw_request: Request):
"""G9: probe prefix-cache hit (L1/L2 + L3 storage) for a chat request.
Deployed on decode engines; the PD router calls it before dispatch to
decide conditional bypass. Accepts a normal chat body or `input_ids`.
"""
return await raw_request.app.state.openai_serving_chat.query_cache_hit(
request, raw_request
)
@app.post(
"/v1/embeddings",
response_class=ORJSONResponse,
@@ -1140,73 +1140,6 @@ class OpenAIServingChat(OpenAIServingBase):
return "text", processed_messages.prompt_ids
return "input_ids", processed_messages.prompt_ids
async def query_cache_hit(
self, request: ChatCompletionRequest, raw_request: Request = None
) -> ORJSONResponse:
"""G9 probe: report how much of this chat request's prompt is already
cached (per-rank-best L1/L2 radix + shared L3 storage) without running
any prefill. The router uses it for conditional PD-bypass decisions.
Multimodal requests are rejected (v1): their sentinel-token expansion
costs ~10-100ms per image for only ~2% of traffic; the router skips
them for bypass anyway.
"""
try:
internal_req, _ = self._convert_to_internal_request(request, raw_request)
except Exception as e:
return ORJSONResponse(
status_code=400,
content={"error": f"failed to convert chat request: {e}"},
)
if internal_req.contains_mm_input():
return ORJSONResponse(
status_code=400,
content={"error": "multimodal requests are not supported by /query_cache_hit"},
)
try:
# 只取 input_ids,不走 _tokenize_one_request——它要求 rid_to_state
# 里已有 ReqState(正常生成路径在 tokenize 前注册),纯查询没有。
# dsv41 系 chat encoding 直接渲染 prompt_ids;否则对渲染后的
# text 走 tokenizer(_tokenize_texts 与真实路径同一入口)。
if internal_req.input_ids is not None:
input_ids = list(internal_req.input_ids)
elif internal_req.text:
input_ids, _ = await self.tokenizer_manager._tokenize_texts(
internal_req.text, False
)
else:
return ORJSONResponse(
status_code=400,
content={"error": "request produced neither input_ids nor text"},
)
from sglang.srt.managers.io_struct import QueryCacheHitReqInput
result = await self.tokenizer_manager.query_cache_hit(
QueryCacheHitReqInput(
input_ids=list(input_ids),
extra_key=getattr(internal_req, "extra_key", None),
cache_salt=getattr(internal_req, "cache_salt", None),
)
)
except Exception as e:
return ORJSONResponse(
status_code=500, content={"error": f"cache-hit query failed: {e}"}
)
if result.error is not None:
return ORJSONResponse(status_code=500, content={"error": result.error})
hit = result.l1_hit_tokens + result.l2_hit_tokens + result.l3_hit_tokens
return ORJSONResponse(
content={
"total_tokens": result.total_tokens,
"l1_hit_tokens": result.l1_hit_tokens,
"l2_hit_tokens": result.l2_hit_tokens,
"l3_hit_tokens": result.l3_hit_tokens,
"hit_tokens": hit,
"new_tokens": max(0, result.total_tokens - hit),
"dp_rank": result.dp_rank,
}
)
def _convert_to_internal_request(
self,
request: ChatCompletionRequest,
-10
View File
@@ -758,16 +758,6 @@ class Envs:
SGLANG_HICACHE_HF3FS_CONFIG_PATH = EnvStr(None)
SGLANG_HICACHE_DECODE_OFFLOAD_STRIDE = EnvInt(None)
SGLANG_HICACHE_SKIP_HOST_DUPLICATE_RECLAIM = EnvBool(False)
# Per-stage timing logs for the L3 storage prefetch path (query / transfer /
# ack reduce / drain / graft / load-back). Emits "HC_TIMING ..." lines.
SGLANG_HICACHE_TIMING = EnvBool(False)
# Pages per batched storage IO call (query / get / put chunking).
SGLANG_HICACHE_STORAGE_BATCH_SIZE = EnvInt(128)
# Number of parallel prefetch IO lanes (worker threads) per controller.
# Operations are sharded by crc32(request_id) so every rank maps a request
# to the same lane; each lane owns dedicated completion sync groups.
# MUST be identical across all TP/CP/PP ranks of an instance.
SGLANG_HICACHE_PREFETCH_IO_THREADS = EnvInt(4)
SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR = EnvStr(None)
# File-backend LRU eviction (opt-in; sizes accept SI/IEC suffixes, "0" disables).
SGLANG_HICACHE_FILE_BACKEND_MAX_SIZE = EnvStr(None)
@@ -206,7 +206,6 @@ def _create_flashmla_metadata():
# only for that shape and go stale silently if FlashMLA retunes it.
_FLASHMLA_SCHED_BLOCK_SIZE_N = 64
_FLASHMLA_SCHED_FIXED_OVERHEAD = 5
_FLASHMLA_SCHED_MAX_SHARED_BYTES = 48 << 10
@functools.lru_cache(maxsize=None)
@@ -243,10 +242,6 @@ def _maybe_precompute_flashmla_sched_meta(
b, s_q = q.shape[0], q.shape[1]
num_sm_parts = max(_num_sms(q.device.index) // s_q, 1)
# Skip when shared memory exceeds the limit.
smem_bytes = 4 * (b * 5 + 1 + num_sm_parts * 8)
if smem_bytes > _FLASHMLA_SCHED_MAX_SHARED_BYTES:
return
meta = torch.empty((num_sm_parts, META_INTS), dtype=torch.int32, device=q.device)
num_splits = torch.empty((b + 1,), dtype=torch.int32, device=q.device)
flashmla_sched_meta(
@@ -741,14 +736,7 @@ class DSV4AttnMetadata:
if src_val is None and dst_val is None:
continue
assert dst_val is not None, f"{field_name=} {src_val=} {dst_val=}"
shape_mismatch = dst_val.shape != src_val.shape
assert not shape_mismatch or field_name in self._CP_GLOBAL_FIELDS, (
f"Only CP-global replay metadata may use a shorter live prefix, "
f"got {field_name=} {src_val.shape=} {dst_val.shape=}"
)
_copy_tensor_allowing_storage_alias(
dst_val, src_val, pad_value=0 if shape_mismatch else None
)
dst_val.copy_(src_val)
# These fields are safe to replace because captured kernels only need
# the current per-replay objects, or the field is produced inside the
@@ -1000,27 +988,6 @@ def _prefill_graph_max_seq_len() -> Optional[int]:
return get_exec().graph.cuda_graph_config.prefill.max_seq_len
def _copy_tensor_allowing_storage_alias(
dst: torch.Tensor, src: torch.Tensor, *, pad_value: Optional[int] = None
) -> None:
"""Copy replay metadata while preserving capture-stable destination addresses."""
if dst is src:
return
if dst.untyped_storage().data_ptr() == src.untyped_storage().data_ptr():
src = src.clone()
if dst.shape == src.shape:
dst.copy_(src)
return
assert (
pad_value is not None
and dst.ndim == src.ndim
and dst.shape[0] >= src.shape[0]
and dst.shape[1:] == src.shape[1:]
), f"Cannot copy replay metadata from {src.shape=} to {dst.shape=}"
dst.fill_(pad_value)
dst[: src.shape[0]].copy_(src)
@dataclass
class DSV4Metadata:
core_attn_metadata: DSV4AttnMetadata
@@ -1287,11 +1254,6 @@ class DeepseekV4AttnBackend(
] = None
self.online_c128_mtp = OnlineC128MTPController(self)
self.sparse_prefill_workspace = SparsePrefillWorkspace(self.device)
# CP V4.1 consumers share compressed KV across layers. Separate ratio
# workspaces keep those prefixes intact while each layer refreshes SWA.
self.shared_compressed_prefill_workspaces = {
ratio: SparsePrefillWorkspace(self.device) for ratio in (1, 2)
}
spec_alg = model_runner.spec_algorithm
self.needs_cpu_seq_lens = not spec_alg.is_dspark() and (
not _is_cuda or self.online_c128_mtp.enabled()
@@ -1604,12 +1566,8 @@ class DeepseekV4AttnBackend(
@property
def low_ratio_prefill_graph(self) -> bool:
"""Whether ratio-1/2 sources use captured projections and indexer metadata."""
return (
bool(self.low_ratios)
and _has_dense_fp4_indexer()
and _is_sm100_or_newer()
and get_parallel().attn_cp_size == 1
bool(self.low_ratios) and _has_dense_fp4_indexer() and _is_sm100_or_newer()
)
def can_run_prefill_cuda_graph(self, forward_batch: ForwardBatch) -> bool:
@@ -2548,7 +2506,7 @@ class DeepseekV4AttnBackend(
req_to_token=self.req_to_token,
full_to_swa=self.token_to_kv_pool.full_to_swa_index_mapping,
swa_window_size=SWA_WINDOW,
swa_page_size=self.token_to_kv_pool.get_swa_key_page_size(),
swa_page_size=self.token_to_kv_pool.swa_kv_pool.page_size,
num_qo_tokens=num_qo_tokens,
max_seq_len=max(seq_lens_cpu_list),
total_swa=total_swa,
@@ -2912,7 +2870,7 @@ class DeepseekV4AttnBackend(
q_lora[:num_local],
positions[:num_local].to(torch.int64),
forward_batch,
self._move_to_device(q_lens_cpu),
torch.tensor(q_lens_cpu, dtype=torch.int32, device=x.device),
q_lens_cpu,
)
@@ -2935,22 +2893,6 @@ class DeepseekV4AttnBackend(
self._low_ratio_compress_fused(
layer, x, req, pos, draft_len=self.speculative_num_draft_tokens
)
elif (
layer.compressor.use_fused_compress
and forward_batch.forward_mode.is_extend_without_speculative()
and forward_batch.extend_start_loc is not None
):
# DP padding extends extend_seq_lens but not extend_start_loc, whose
# length is the request count; padded rows belong to no request.
start_loc = forward_batch.extend_start_loc
seq_lens = forward_batch.extend_seq_lens[: start_loc.shape[0]]
self._low_ratio_compress_fused(
layer,
x,
req,
pos,
extend_offsets=(start_loc.to(torch.int32), seq_lens.to(torch.int32)),
)
else:
self._low_ratio_compress_torch(
layer,
@@ -3022,18 +2964,13 @@ class DeepseekV4AttnBackend(
fuse_index_store=_is_sm100_or_newer(),
)
def _low_ratio_compress_fused(
self, layer, x, req, pos, *, draft_len=1, extend_offsets=None
) -> None:
"""Decode or target-verify; an extend batch passes ``extend_offsets``, the int32
(first row, row count) of each request."""
def _low_ratio_compress_fused(self, layer, x, req, pos, *, draft_len=1) -> None:
from sglang.kernels.ops.attention.dsv4.fp4_indexer_rope import (
index_k_norm_rope_pack_store,
)
from sglang.kernels.ops.attention.dsv4.low_ratio_compress import (
c1_decode_norm_rope_store,
c2_decode_norm_rope_store,
c2_prefill_norm_rope_store,
)
pool = self.token_to_kv_pool
@@ -3066,7 +3003,7 @@ class DeepseekV4AttnBackend(
# CompressStatePool stores each request's pending pairs in a position ring.
# KVAndScore rows use | kv | score |, addressed as req * ring_size + pos % ring_size.
state = pool.get_attention_compress_states(layer_id)
args = (
latent = c2_decode_norm_rope_store(
compressor.project_fused(x),
state.kv_score_buffer.kv_score,
compressor.norm.weight.data,
@@ -3076,23 +3013,11 @@ class DeepseekV4AttnBackend(
compressor.norm.eps,
freqs_cis,
kv_cache,
page_size=page_size,
ring_size=state.ring_size,
draft_len=draft_len,
layout=kv_layout,
)
if extend_offsets is None:
latent = c2_decode_norm_rope_store(
*args,
page_size=page_size,
ring_size=state.ring_size,
draft_len=draft_len,
layout=kv_layout,
)
else:
latent = c2_prefill_norm_rope_store(
*args,
*extend_offsets,
page_size=page_size,
ring_size=state.ring_size,
layout=kv_layout,
)
out_loc = core.c2_out_loc
indexer = layer.indexer
@@ -3311,9 +3236,7 @@ class DeepseekV4AttnBackend(
continue
j = torch.arange(lc, device=device)
slot_chunks.append(
self.req_to_token[req_pool_indices[r : r + 1], j * ratio].to(
torch.int64
)
self.req_to_token[req_pool_indices[r], j * ratio].to(torch.int64)
// ratio
)
start += lc
@@ -3338,7 +3261,7 @@ class DeepseekV4AttnBackend(
weights = indexer.head_weights(x).float()
compress_lens = ((pos + 1) // ratio).to(torch.int32)
ks = torch.repeat_interleave(
self._move_to_device(starts),
torch.tensor(starts, dtype=torch.int32, device=device),
q_lens.to(torch.int64),
output_size=num_tokens,
)
@@ -3524,44 +3447,17 @@ class DeepseekV4AttnBackend(
self.candidate_indexer.publish_decode(inputs, page_indices, raw_indices)
)
return
if isinstance(metadata.deep_gemm_metadata, list):
topk_plans = metadata.topk_metadata_chunks
assert not metadata.use_topk_v2 or topk_plans is not None
for chunk_idx, (rows, plan) in enumerate(metadata.row_chunks()):
logits = deep_gemm_fp4_paged_mqa_logits(
(q_fp4[rows], q_sf[rows]),
k_cache,
weights[rows],
metadata.compressed_seq_lens[rows],
metadata.page_table[rows],
plan,
metadata.max_compressed_seq_len,
)
# TODO(dark): add bf16 topk
topk_transform_paged_from_metadata(
logits,
metadata,
page_indices,
raw_indices,
rows=rows,
topk_metadata=(
topk_plans[chunk_idx] if topk_plans is not None else None
),
)
else:
logits = deep_gemm_fp4_paged_mqa_logits(
(q_fp4, q_sf),
k_cache,
weights,
metadata.compressed_seq_lens,
metadata.page_table,
metadata.deep_gemm_metadata,
metadata.max_compressed_seq_len,
)
# TODO(dark): add bf16 topk
topk_transform_paged_from_metadata(
logits, metadata, page_indices, raw_indices
)
logits = deep_gemm_fp4_paged_mqa_logits(
(q_fp4, q_sf),
k_cache,
weights,
metadata.compressed_seq_lens,
metadata.page_table,
metadata.deep_gemm_metadata,
metadata.max_compressed_seq_len,
)
# TODO(dark): add bf16 topk
topk_transform_paged_from_metadata(logits, metadata, page_indices, raw_indices)
# TODO(candidate): Hopper decode still publishes / consumes masks inline (torch
# top-k); move into the candidate indexer with the prefill paths.
@@ -3798,7 +3694,7 @@ class DeepseekV4AttnBackend(
compress_ratio
)
swa_kv_page_size = token_to_kv_pool.get_swa_key_page_size()
swa_kv_page_size = token_to_kv_pool.swa_kv_pool.page_size
assert swa_k_cache.ndim == 2
# The kernel detects each cache's format from the last dim of this
# view: 584 (V4), 528 (V4.1 fp8) or 288 (V4.1 fp4, extra cache only).
@@ -4049,38 +3945,20 @@ class DeepseekV4AttnBackend(
compress_ratio, core_attn_metadata, extra_page_size
)
n_compressed = flat_token_ids.shape[0]
reuse_compressed = compress_ratio in (1, 2) and is_cp_active(forward_batch)
workspace_pool = (
self.shared_compressed_prefill_workspaces[compress_ratio]
if reuse_compressed
else self.sparse_prefill_workspace
workspace = self.sparse_prefill_workspace.get(
n_compressed + cache.swa_token_ids.shape[0]
)
workspace = workspace_pool.get(n_compressed + cache.swa_token_ids.shape[0])
compressed_slice = workspace[:n_compressed]
swa_slice = workspace[n_compressed:]
if compressed_slice is not None:
source_key = None
if reuse_compressed:
source_layer = token_to_kv_pool.source_layer_of(layer_id)
source_key = (source_layer, workspace.data_ptr())
gather = cache.compressed[compress_ratio]
# A source layer may have just updated its cache in place. Consumer
# layers only reuse the compressed prefix; their top-k and SWA stay live.
if (
source_key is None
or layer_id == source_key[0]
or gather.dequantized_source != source_key
):
dequantize_k_cache_paged(
extra_k_cache,
flat_token_ids,
page_size=extra_page_size,
out=compressed_slice,
layout=token_to_kv_pool.get_extra_key_layout(layer_id),
)
if source_key is not None:
gather.dequantized_source = source_key
dequantize_k_cache_paged(
extra_k_cache,
flat_token_ids,
page_size=extra_page_size,
out=compressed_slice,
layout=token_to_kv_pool.get_extra_key_layout(layer_id),
)
dequantize_k_cache_paged(
token_to_kv_pool.get_swa_key_buffer_radix(layer_id),
cache.swa_token_ids,
@@ -1472,10 +1472,9 @@ class IndexerKPool(MultiPlatformOp):
if metadata is None:
return None
# Spec-v2 decode/verify batches may carry no CPU seq_lens mirror (the
# backend opts out of the D2H); test emptiness on the device shape.
assert forward_batch.seq_lens_cpu is not None
mode = forward_batch.forward_mode
if mode.is_idle() or forward_batch.seq_lens.shape[0] == 0:
if mode.is_idle() or len(forward_batch.seq_lens_cpu) == 0:
return torch.full(
(x.shape[0], self.index_topk + self.index_kpool - 1),
-1,
@@ -306,8 +306,7 @@ class DeepseekSparseAttnBackend(
extend_dummy_seqs_capped_by_req_pool: bool = True
# Decode/verify/draft graph replay rebuilds metadata from static buffers
# (page-table width) and never reads seq_lens_cpu / seq_lens_sum; opt out of
# the D2H sync. The eager fallback derives lengths from GPU seq_lens, and the
# KPool indexer reads the host mirror only for extend, which always has it.
# the D2H sync. The eager fallback derives lengths from GPU seq_lens.
needs_cpu_seq_lens: bool = False
# init_cuda_graph_state sizes this for every backend, but only the TRT-LLM
# branch of __init__ allocates one.
@@ -338,6 +337,7 @@ class DeepseekSparseAttnBackend(
)
self.dsa_index_topk = get_dsa_index_topk(hf_config)
self.dsa_index_kpool = get_dsa_index_kpool(hf_config)
self.needs_cpu_seq_lens = self.dsa_index_kpool > 1
self._init_kpool_metadata_fusion()
self.max_context_len = model_runner.model_config.context_len
self.num_q_heads = (
@@ -21,7 +21,6 @@ from sglang.srt.layers.attention.dsv4.candidate_indexer import (
)
from sglang.srt.layers.attention.dsv4.indexer import (
deep_gemm_fp4_paged_mqa_logits,
topk_transform_paged_from_metadata,
)
CANDIDATE_BLOCK_SIZE = 8 # positions per block; DeepGEMM accepts 8 or 16
@@ -177,10 +176,6 @@ class DeepGemmCandidateIndexer:
metadata."""
metadata = inputs.metadata
seq_lens = metadata.compressed_seq_lens.reshape(-1)
if isinstance(metadata.deep_gemm_metadata, list):
return self._publish_decode_chunked(
inputs, page_indices, raw_indices, seq_lens
)
logits = deep_gemm_fp4_paged_mqa_logits(
(inputs.q_fp4, inputs.q_sf),
inputs.k_cache,
@@ -236,83 +231,6 @@ class DeepGemmCandidateIndexer:
ready=ready,
)
def _publish_decode_chunked(
self,
inputs: IndexerInputs,
page_indices: torch.Tensor,
raw_indices: Optional[torch.Tensor],
seq_lens: torch.Tensor,
) -> SparseBlockTable:
"""Publish an eager forward whose dense logits are bounded by row chunks.
CUDA-graph metadata always carries one tensor schedule and keeps using the
asynchronous fast path above. The exceptional eager path stays on the
current stream so each chunk's full logits can be released before the next.
"""
metadata = inputs.metadata
block_chunks = []
phys_block_chunks = []
valid_len_chunks = []
topk_plans = metadata.topk_metadata_chunks
assert not metadata.use_topk_v2 or topk_plans is not None
for chunk_idx, (rows, plan) in enumerate(metadata.row_chunks()):
logits = deep_gemm_fp4_paged_mqa_logits(
(inputs.q_fp4[rows], inputs.q_sf[rows]),
inputs.k_cache,
inputs.weights[rows],
metadata.compressed_seq_lens[rows],
metadata.page_table[rows],
plan,
metadata.max_compressed_seq_len,
)
topk_transform_paged_from_metadata(
logits,
metadata,
page_indices,
raw_indices,
rows=rows,
topk_metadata=(
topk_plans[chunk_idx] if topk_plans is not None else None
),
)
chunk_seq_lens = seq_lens[rows]
nblocks, row_valid_lens = candidate_row_lens(
chunk_seq_lens, self.topk_blocks
)
blocks = amax_topk_blocks(logits, chunk_seq_lens, nblocks, self.topk_blocks)
phys_blocks = sort_candidate_blocks(
blocks,
chunk_seq_lens,
metadata.page_table[rows],
metadata.compressed_page_size,
)
block_chunks.append(blocks)
phys_block_chunks.append(phys_blocks)
valid_len_chunks.append(row_valid_lens)
blocks = torch.cat(block_chunks)
phys_blocks = torch.cat(phys_block_chunks)
row_valid_lens = torch.cat(valid_len_chunks)
schedule = build_sparse_indexer_schedule(
blocks,
seq_lens,
metadata.page_table,
metadata.compressed_page_size,
inputs.q_fp4.dtype,
self._request_ids(inputs.request_ids, inputs.num_rows, blocks.device),
)
ready = torch.cuda.Event()
ready.record(torch.cuda.current_stream())
return SparseBlockTable(
blocks=blocks,
schedule=schedule,
phys_blocks=phys_blocks,
valid_lens=row_valid_lens,
ready=ready,
)
def _scores(self, table: SparseBlockTable, inputs: IndexerInputs) -> torch.Tensor:
return sparse_logits(
inputs.q_fp4,
@@ -475,40 +475,27 @@ def topk_transform_paged_from_metadata(
metadata,
page_indices: torch.Tensor,
raw_indices: Optional[torch.Tensor] = None,
*,
rows: Optional[slice] = None,
topk_metadata: Optional[torch.Tensor] = None,
) -> None:
"""Pool slots into ``page_indices`` (``-1`` past the valid count) and, when given,
positions into ``raw_indices``; ``metadata`` is a ``PagedIndexerMetadata``."""
if rows is None:
seq_lens = metadata.compressed_seq_lens
page_table = metadata.page_table
out_page_indices = page_indices
out_raw_indices = raw_indices
else:
seq_lens = metadata.compressed_seq_lens[rows]
page_table = metadata.page_table[rows]
out_page_indices = page_indices[rows]
out_raw_indices = raw_indices[rows] if raw_indices is not None else None
if metadata.use_topk_v2:
topk_transform_paged_v2(
logits,
seq_lens,
page_table,
out_page_indices,
metadata.compressed_seq_lens,
metadata.page_table,
page_indices,
metadata.compressed_page_size,
metadata.topk_metadata if topk_metadata is None else topk_metadata,
out_raw_indices,
metadata.topk_metadata,
raw_indices,
)
else:
topk_transform_paged(
logits,
seq_lens,
page_table,
out_page_indices,
metadata.compressed_seq_lens,
metadata.page_table,
page_indices,
metadata.compressed_page_size,
out_raw_indices,
raw_indices,
)
@@ -21,7 +21,6 @@ from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.context
from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph import (
is_in_tc_piecewise_cuda_graph,
)
from sglang.srt.model_executor.runner_utils.capture_mode import get_is_capture_mode
from sglang.srt.utils import is_hip, is_sm120_supported, is_xpu
logger = logging.getLogger(__name__)
@@ -307,8 +306,7 @@ class PagedIndexerMetadata:
):
return None
if (
get_is_capture_mode()
or torch.cuda.is_current_stream_capturing()
torch.cuda.is_current_stream_capturing()
or is_in_breakable_cuda_graph()
or is_in_tc_piecewise_cuda_graph()
):
@@ -327,27 +325,14 @@ class PagedIndexerMetadata:
def row_chunks(self):
num_rows = self.compressed_seq_lens.shape[0]
if self.row_chunk > 0:
rows_per_chunk = self.row_chunk
elif isinstance(self.deep_gemm_metadata, list):
assert self.rows_per_chunk is not None, (
"chunked DeepGEMM metadata requires rows_per_chunk"
)
rows_per_chunk = self.rows_per_chunk
else:
if self.row_chunk <= 0:
return [(slice(0, num_rows), self.deep_gemm_metadata)]
chunks = [
(slice(start, min(start + rows_per_chunk, num_rows)), plan)
return [
(slice(start, min(start + self.row_chunk, num_rows)), plan)
for start, plan in zip(
range(0, num_rows, rows_per_chunk), self.deep_gemm_metadata
range(0, num_rows, self.row_chunk), self.deep_gemm_metadata
)
]
assert chunks and chunks[-1][0].stop == num_rows, (
f"chunk schedules do not cover all rows: {num_rows=} {rows_per_chunk=} "
f"{len(chunks)=}"
)
return chunks
def copy_(self, other: PagedIndexerMetadata):
# A chunked schedule list has no in-place copy; rebind it instead.
@@ -78,11 +78,10 @@ def use_dsv4_q8kv8_sparse_prefill(dsv4_prefill_backend: str = "auto") -> bool:
class SparsePrefillWorkspace:
"""Backend-owned scratch storage for sparse prefill KV dequantization.
Callers normally overwrite the entire workspace. Shared compressed-KV
callers keep separate workspaces per ratio and track prefix validity in the
per-forward gather cache, including the allocation address. Sparse prefill
executes eagerly and serially on the supported paths, so the allocation can
be replaced when a larger extent is needed.
The workspace contents are fully overwritten before every attention call,
so token buckets and compression ratios can safely share one buffer. Sparse
prefill executes eagerly and serially on the supported paths, which makes it
safe to replace the scratch allocation when a larger extent is needed.
"""
def __init__(self, device: torch.device):
@@ -136,8 +135,8 @@ def combine_topk_swa_indices(
Returns int32 indices [num_tokens, padded_topk_swa] and per-token scanned-prefix
lengths, including -1 entries skipped by attention. Width is padded to 128.
Rows are written whole and -1 padded, so preallocated out_indices / out_lens
are overwritten entirely; rows past the last request are -1 with length 0.
Preallocated out_indices must contain -1 outside the written prefix;
reuse requires chunk-invariant scanned-prefix lengths. out_lens is overwritten.
"""
assert topk_indices.dtype == torch.int32
assert query_start_loc.dtype == torch.int32
@@ -155,9 +154,9 @@ def combine_topk_swa_indices(
num_reqs = seq_lens.shape[0]
combined_topk = combined_topk_width(topk, window_size)
if out_indices is None:
# The kernel writes whole -1 padded rows.
combined_indices = torch.empty(
combined_indices = torch.full(
(num_tokens, combined_topk),
-1,
dtype=torch.int32,
device=topk_indices.device,
)
@@ -166,7 +165,7 @@ def combine_topk_swa_indices(
assert out_indices.dtype == torch.int32
combined_indices = out_indices
if out_lens is None:
combined_lens = torch.empty(
combined_lens = torch.zeros(
num_tokens, dtype=torch.int32, device=topk_indices.device
)
else:
@@ -174,10 +173,8 @@ def combine_topk_swa_indices(
assert out_lens.dtype == torch.int32
combined_lens = out_lens
if num_tokens == 0:
return combined_indices, combined_lens
block_t = 2 # Measured on B200 with num_warps=2, 16384 tokens, bs 1 to 64.
_combine_topk_swa_indices_kernel[(triton.cdiv(num_tokens, block_t),)](
NUM_WORKERS = 128
_combine_topk_swa_indices_kernel[(num_reqs, NUM_WORKERS)](
combined_indices,
combined_indices.stride(0),
combined_lens,
@@ -189,16 +186,10 @@ def combine_topk_swa_indices(
gather_lens,
compressed_base,
swa_base,
num_tokens,
num_reqs,
top_k=topk,
COMPRESS_RATIO=compress_ratio,
WINDOW_SIZE=window_size,
WIDTH=combined_topk,
PADDED_WIDTH=triton.next_power_of_2(combined_topk),
BLOCK_T=block_t,
SEARCH_STEPS=max(num_reqs, 1).bit_length(),
num_warps=2,
PADDED_TOP_K=triton.next_power_of_2(topk_indices.shape[-1]),
)
return combined_indices, combined_lens
@@ -284,18 +275,14 @@ class CompressedGather:
# chunk-invariant per request; subsequent layers only overwrite that prefix.
combined_indices: Optional[torch.Tensor] = None
combined_lens: Optional[torch.Tensor] = None
# Valid only for this forward's gather layout. Each ratio has its own
# workspace; its compressed prefix survives consumer layers' SWA writes.
dequantized_source: Optional[tuple[int, int]] = None
@dataclass
class SparsePrefillChunkCache:
"""Cache prefill-chunk metadata shared across layers.
Gather layouts depend on request/token mappings and compressed page tables.
Shared-source dequantization keys live only for this forward; per-layer
top-k combinations are recomputed into reused
Fields depend on request/token mappings and compressed page tables, not
per-layer k_cache; per-layer top-k combinations are recomputed into reused
buffers.
"""
+10 -97
View File
@@ -23,22 +23,17 @@ import torch
from sglang.srt.arg_groups.overrides import (
attention_backends_of,
model_config_of,
resolved_view,
resolving_view,
)
from sglang.srt.configs.model_config import is_deepseek_v4
from sglang.srt.layers.cp.base import get_cp_strategy
from sglang.srt.layers.cp.interleave import InterleaveCPStrategy
from sglang.srt.layers.cp.padding import get_cp_padding_align_size
from sglang.srt.layers.cp.utils import (
cp_gather_after_forward,
cp_shard_hidden_states,
cp_split_before_forward,
prepare_cp_forward,
)
from sglang.srt.layers.cp.zigzag import ZigzagCPStrategy
from sglang.srt.layers.logits_processor import LogitsMetadata
from sglang.srt.model_executor.forward_batch_info import PPProxyTensors
if TYPE_CHECKING:
@@ -55,18 +50,12 @@ def supports_prefill_cp_bcg(server_args: ServerArgs) -> bool:
cfg = resolving_view(server_args)
resolved = resolved_view(server_args)
prefill_attention_backend, _ = attention_backends_of(resolved_view(server_args))
supports_layout = (
cfg.cp_strategy == "zigzag" and prefill_attention_backend == "trtllm_mha"
) or (
cfg.cp_strategy == "interleave"
and prefill_attention_backend == "dsv4"
and is_deepseek_v4(model_config_of(server_args).hf_config)
)
return (
cfg.enable_prefill_cp
and cfg.pp_size == 1
and resolved.attn_cp_size == cfg.tp_size
and supports_layout
and cfg.cp_strategy == "zigzag"
and prefill_attention_backend == "trtllm_mha"
)
@@ -78,12 +67,8 @@ def enable_cp_bcg_capture(server_args: ServerArgs) -> bool:
def filter_prefill_cp_bcg_capture_num_tokens(
capture_num_tokens: list[int], server_args: ServerArgs
) -> list[int]:
"""Keep only token buckets where the configured CP strategy can run."""
cfg = resolving_view(server_args)
cp_segments_per_token_block = 2 if cfg.cp_strategy == "zigzag" else 1
min_num_tokens = (
resolved_view(server_args).attn_cp_size * cp_segments_per_token_block
)
"""Keep only token buckets where the zigzag CP strategy can run."""
min_num_tokens = resolved_view(server_args).attn_cp_size * 2
filtered = [size for size in capture_num_tokens if size >= min_num_tokens]
if not filtered:
raise ValueError(
@@ -111,8 +96,6 @@ class PrefillCPBCGInput:
input_embeds: torch.Tensor
positions: torch.Tensor
input_ids: Optional[torch.Tensor] = None
num_token_non_padded: Optional[torch.Tensor] = None
bucket_local_tokens: Dict[int, int] = field(default_factory=dict)
live_local_tokens: int = 0
@@ -131,22 +114,12 @@ class PrefillCPBCGInput:
(runner.max_num_tokens,),
dtype=torch.int64,
),
input_ids=torch.zeros((runner.max_num_tokens,), dtype=torch.int64),
num_token_non_padded=torch.zeros((), dtype=torch.int32),
)
def required_local_tokens(self, extend_seq_lens: Any) -> Optional[int]:
"""Return the aligned CP-local rows required by the active layout."""
"""Return the aligned CP-local rows required by a live zigzag layout."""
strategy = get_cp_strategy()
if extend_seq_lens is None:
return None
if isinstance(strategy, InterleaveCPStrategy):
logical_tokens = (
sum(int(length) for length in extend_seq_lens) + strategy.cp_size - 1
) // strategy.cp_size
align_size = get_cp_padding_align_size()
return (logical_tokens + align_size - 1) // align_size * align_size
if not isinstance(strategy, ZigzagCPStrategy):
if not isinstance(strategy, ZigzagCPStrategy) or extend_seq_lens is None:
return None
cp_segment_num = strategy.cp_size * 2
@@ -246,7 +219,6 @@ class PrefillCPBCGInput:
raw_tokens = int(forward_batch.extend_num_tokens)
global_input_ids = forward_batch.input_ids[:raw_tokens]
global_positions = forward_batch.positions[:raw_tokens]
local_input_ids = cp_shard_hidden_states(global_input_ids, forward_batch)
global_input_embeds = runner.model_runner.model.get_input_embeddings()(
global_input_ids
)
@@ -277,31 +249,12 @@ class PrefillCPBCGInput:
input_embeds = self.input_embeds[:captured_local_tokens]
positions = self.positions[:captured_local_tokens]
assert self.input_ids is not None
input_ids = self.input_ids[:captured_local_tokens]
input_embeds.zero_()
positions.zero_()
input_ids.zero_()
input_embeds[:live_local_tokens].copy_(local_input_embeds)
positions[:live_local_tokens].copy_(local_positions)
input_ids[:live_local_tokens].copy_(local_input_ids)
forward_batch.input_embeds = input_embeds
forward_batch._cp_positions = positions
# Keep the global input_ids field intact: the runner uses its length to
# select the global capture bucket. The DSV4 body consumes this fixed,
# rank-local view for hash routing and MegaMoE.
forward_batch._cp_input_ids = input_ids
forward_batch.input_ids_global = input_ids
if forward_batch.num_token_non_padded is not None:
assert self.num_token_non_padded is not None
metadata = forward_batch.attn_cp_metadata
logical_tokens = (
metadata.per_rank_logical_token or metadata.per_rank_actual_token
)
strategy = get_cp_strategy()
assert strategy is not None
self.num_token_non_padded.fill_(logical_tokens[strategy.cp_rank])
forward_batch.num_token_non_padded = self.num_token_non_padded
forward_batch.positions = positions
self.live_local_tokens = live_local_tokens
@@ -354,50 +307,10 @@ def execute_prefill_cp_bcg(
static_forward_batch,
torch.cuda.current_stream(),
)
if aux_hidden_states is not None:
if torch.is_tensor(aux_hidden_states):
aux_hidden_states = cp_gather_after_forward(
aux_hidden_states, static_forward_batch, torch.cuda.current_stream()
)
else:
aux_hidden_states = [
cp_gather_after_forward(
aux, static_forward_batch, torch.cuda.current_stream()
)
for aux in aux_hidden_states
]
hidden_states_before_norm = None
if isinstance(hidden_states, tuple):
assert len(hidden_states) == 2
hidden_states, hidden_states_before_norm = hidden_states
input_ids = forward_batch.input_ids
logits_metadata = forward_batch
tail = None
language_model = getattr(model, "model", None)
if (
capture_aux_hidden_states
and getattr(language_model, "late_layer_start", None) is not None
and forward_batch.forward_mode.is_extend_without_speculative()
):
tail_metadata = runner.model_runner.attn_backend.tail_forward_metadata
tail = tail_metadata.late_layer_tail
input_ids = tail.rows(input_ids)
logits_metadata = LogitsMetadata.from_forward_batch(forward_batch)
logits_metadata.extend_seq_lens = tail.extend_seq_lens
logits_metadata.extend_seq_lens_cpu = tail.extend_seq_lens_cpu
logits_metadata.extend_logprob_start_lens_cpu = tail.extend_seq_lens_cpu
output = model.logits_processor(
input_ids,
return model.logits_processor(
forward_batch.input_ids,
hidden_states,
model.lm_head,
logits_metadata,
forward_batch,
aux_hidden_states,
hidden_states_before_norm=(
None if aux_hidden_states is not None else hidden_states_before_norm
),
)
if tail is not None:
output.hidden_states_token_indices = tail.token_indices
return output
-5
View File
@@ -239,11 +239,6 @@ class EngramHasher(nn.Module):
self.image_token_id: Optional[int] = None
self.history: Optional[torch.Tensor] = None
self.pad_row = 0
# typing only
self.token_map: torch.Tensor
self.multipliers: torch.Tensor
self.primes: torch.Tensor
self.offsets: torch.Tensor
def init_history(self, num_req_slots: int, device) -> None:
"""Allocate oldest-first history with a spare row for graph padding."""
+13 -26
View File
@@ -216,33 +216,20 @@ def _run_mega_routed(
if num_tokens > 0:
router_logits = moe.gate(hidden_states, forward_batch=forward_batch)
num_token_non_padded = (
forward_batch.num_token_non_padded if forward_batch is not None else None
topk_kwargs = {"input_ids": input_ids_global} if moe.is_hash else {}
topk_output = moe.topk(
hidden_states,
router_logits,
num_token_non_padded=(
forward_batch.num_token_non_padded
if forward_batch is not None
else None
),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=moe.layer_id,
),
**topk_kwargs,
)
if isinstance(
getattr(moe.gate, "e_score_correction_bias_vl", None), torch.Tensor
):
# V4.1 uses a different correction bias for image-token rows. The
# MegaMoE transport consumes the same routed ids/weights as TopK.
from sglang.srt.multimodal.dsv41.vl_routing import vision_topk
topk_output = vision_topk(
moe,
router_logits,
input_ids_global,
num_token_non_padded=num_token_non_padded,
)
else:
topk_kwargs = {"input_ids": input_ids_global} if moe.is_hash else {}
topk_output = moe.topk(
hidden_states,
router_logits,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=moe.layer_id,
),
**topk_kwargs,
)
topk_ids = topk_output.topk_ids
topk_weights = topk_output.topk_weights
else:
+53 -181
View File
@@ -17,7 +17,6 @@ limitations under the License.
import logging
import threading
import time
import zlib
from dataclasses import dataclass
from queue import Empty, Queue
from typing import TYPE_CHECKING, Callable, List, NamedTuple, Optional
@@ -33,17 +32,6 @@ from sglang.srt.mem_cache.hicache_storage import (
count_pool_hits,
)
from sglang.srt.environ import envs as _envs
_HC_TIMING = None
def _hc_timing_enabled() -> bool:
global _HC_TIMING
if _HC_TIMING is None:
_HC_TIMING = _envs.SGLANG_HICACHE_TIMING.get()
return _HC_TIMING
if TYPE_CHECKING:
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
from sglang.srt.mem_cache.pool_host import HostKVCache
@@ -318,18 +306,7 @@ class HiCacheController:
self.attn_tp_group = attn_tp_group
self.pp_group = pp_group
self.prefetch_hits_sync_groups: List[torch.distributed.ProcessGroup] = []
# One set of completion sync groups per prefetch IO lane. Each lane's
# worker/sync threads form a serial pipeline, so the collective
# sequence on a lane's groups is identical across ranks as long as
# lane assignment is deterministic (see dispatch_prefetch).
self.prefetch_completion_sync_groups_by_lane: List[
List[torch.distributed.ProcessGroup]
] = []
# Read once at construction; attach/reset reuse the stored value so
# lane count never changes underneath live sync groups.
self.num_prefetch_io_lanes = max(
1, _envs.SGLANG_HICACHE_PREFETCH_IO_THREADS.get()
)
self.prefetch_completion_sync_groups: List[torch.distributed.ProcessGroup] = []
self.mem_pool_device_allocator = token_to_kv_pool_allocator
mem_pool_device = token_to_kv_pool_allocator.get_kvcache()
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool
@@ -358,8 +335,8 @@ class HiCacheController:
self.storage_stop_event = threading.Event()
# Storage control queues, (re)created whenever the storage threads start.
self.prefetch_io_queues: List[Queue[PrefetchOperation]] = []
self.prefetch_sync_queues: List[Queue[PrefetchAck]] = []
self.prefetch_buffer: Optional[Queue[PrefetchOperation]] = None
self.prefetch_sync_queue: Optional[Queue[PrefetchAck]] = None
self.prefetch_hit_queue: Optional[Queue[StorageOperation]] = None
self.ack_prefetch_queue = Queue[PrefetchAck]()
self.ack_backup_queue: Optional[Queue[StorageOperation]] = None
@@ -467,39 +444,27 @@ class HiCacheController:
self.prefetch_thread = threading.Thread(
target=self.prefetch_thread_func, daemon=True
)
self.prefetch_io_aux_threads = [
threading.Thread(
target=self.prefetch_io_aux_func, args=(lane,), daemon=True
)
for lane in range(self.num_prefetch_io_lanes)
]
self.prefetch_sync_threads = [
threading.Thread(
target=self.prefetch_sync_thread_func, args=(lane,), daemon=True
)
for lane in range(self.num_prefetch_io_lanes)
]
self.prefetch_io_aux_thread = threading.Thread(
target=self.prefetch_io_aux_func, daemon=True
)
self.prefetch_sync_thread = threading.Thread(
target=self.prefetch_sync_thread_func, daemon=True
)
self.backup_thread = threading.Thread(
target=self.backup_thread_func, daemon=True
)
self.prefetch_queue = Queue()
self.backup_queue = Queue()
self.prefetch_io_queues = [
Queue() for _ in range(self.num_prefetch_io_lanes)
]
self.prefetch_sync_queues = [
Queue() for _ in range(self.num_prefetch_io_lanes)
]
self.prefetch_buffer = Queue()
self.prefetch_sync_queue = Queue()
self.prefetch_hit_queue = Queue()
self.ack_prefetch_queue = Queue()
self.ack_backup_queue = Queue()
self.host_mem_release_queue = Queue()
self.prefetch_thread.start()
for thread in self.prefetch_io_aux_threads:
thread.start()
for thread in self.prefetch_sync_threads:
thread.start()
self.prefetch_io_aux_thread.start()
self.prefetch_sync_thread.start()
self.backup_thread.start()
def _stop_storage_threads(self):
@@ -520,10 +485,10 @@ class HiCacheController:
self.prefetch_queue.put_nowait(None)
if hasattr(self, "backup_queue"):
self.backup_queue.put_nowait(None)
for queue in getattr(self, "prefetch_io_queues", []) or []:
queue.put_nowait(None)
for queue in getattr(self, "prefetch_sync_queues", []) or []:
queue.put_nowait(None)
if hasattr(self, "prefetch_buffer"):
self.prefetch_buffer.put_nowait(None)
if hasattr(self, "prefetch_sync_queue"):
self.prefetch_sync_queue.put_nowait(None)
except Exception:
pass
@@ -533,8 +498,10 @@ class HiCacheController:
threads.append(self.prefetch_thread)
if hasattr(self, "backup_thread"):
threads.append(self.backup_thread)
threads.extend(getattr(self, "prefetch_io_aux_threads", []) or [])
threads.extend(getattr(self, "prefetch_sync_threads", []) or [])
if hasattr(self, "prefetch_io_aux_thread"):
threads.append(self.prefetch_io_aux_thread)
if hasattr(self, "prefetch_sync_thread"):
threads.append(self.prefetch_sync_thread)
for t in threads:
try:
@@ -618,13 +585,7 @@ class HiCacheController:
# Use dedicated gloo groups so storage prefetch sync is isolated
# from other collectives and consistent across CPxTP participants.
self.prefetch_hits_sync_groups = self._create_sync_groups()
# Each IO lane gets its own completion groups: per-lane collectives
# stay ordered because a lane is a serial pipeline and dispatch is
# deterministic (crc32(request_id) % num_prefetch_io_lanes).
self.prefetch_completion_sync_groups_by_lane = [
self._create_sync_groups()
for _ in range(self.num_prefetch_io_lanes)
]
self.prefetch_completion_sync_groups = self._create_sync_groups()
# Select the get and set functions
self.page_get_func = self._generic_page_get
@@ -659,10 +620,9 @@ class HiCacheController:
except Exception:
pass
self._destroy_sync_groups(self.prefetch_hits_sync_groups)
for lane_groups in self.prefetch_completion_sync_groups_by_lane:
self._destroy_sync_groups(lane_groups)
self._destroy_sync_groups(self.prefetch_completion_sync_groups)
self.prefetch_hits_sync_groups = []
self.prefetch_completion_sync_groups_by_lane = []
self.prefetch_completion_sync_groups = []
try:
if (
hasattr(self, "storage_backend")
@@ -700,11 +660,11 @@ class HiCacheController:
raise RuntimeError("Stop storage threads failed; detach aborted.") from e
# Best-effort destroy process groups created for storage ops.
self._destroy_sync_groups(self.prefetch_hits_sync_groups)
for lane_groups in self.prefetch_completion_sync_groups_by_lane:
self._destroy_sync_groups(lane_groups)
self._destroy_sync_groups(
self.prefetch_hits_sync_groups + self.prefetch_completion_sync_groups
)
self.prefetch_hits_sync_groups = []
self.prefetch_completion_sync_groups_by_lane = []
self.prefetch_completion_sync_groups = []
# Best-effort close (some backends rely on GC/destructor).
try:
@@ -798,17 +758,13 @@ class HiCacheController:
self.ack_load_queue.clear()
if self.enable_storage:
self.prefetch_thread.join()
for thread in self.prefetch_io_aux_threads:
thread.join()
for thread in self.prefetch_sync_threads:
thread.join()
self.prefetch_io_aux_thread.join()
self.prefetch_sync_thread.join()
self.backup_thread.join()
self.prefetch_queue.queue.clear()
self.backup_queue.queue.clear()
for queue in self.prefetch_io_queues:
queue.queue.clear()
for queue in self.prefetch_sync_queues:
queue.queue.clear()
self.prefetch_buffer.queue.clear()
self.prefetch_sync_queue.queue.clear()
self.prefetch_hit_queue.queue.clear()
self.ack_prefetch_queue.queue.clear()
self.ack_backup_queue.queue.clear()
@@ -821,26 +777,18 @@ class HiCacheController:
self.prefetch_thread = threading.Thread(
target=self.prefetch_thread_func, daemon=True
)
self.prefetch_io_aux_threads = [
threading.Thread(
target=self.prefetch_io_aux_func, args=(lane,), daemon=True
)
for lane in range(self.num_prefetch_io_lanes)
]
self.prefetch_sync_threads = [
threading.Thread(
target=self.prefetch_sync_thread_func, args=(lane,), daemon=True
)
for lane in range(self.num_prefetch_io_lanes)
]
self.prefetch_io_aux_thread = threading.Thread(
target=self.prefetch_io_aux_func, daemon=True
)
self.prefetch_sync_thread = threading.Thread(
target=self.prefetch_sync_thread_func, daemon=True
)
self.backup_thread = threading.Thread(
target=self.backup_thread_func, daemon=True
)
self.prefetch_thread.start()
for thread in self.prefetch_io_aux_threads:
thread.start()
for thread in self.prefetch_sync_threads:
thread.start()
self.prefetch_io_aux_thread.start()
self.prefetch_sync_thread.start()
self.backup_thread.start()
def has_inflight_device_transfers(self) -> bool:
@@ -1127,23 +1075,7 @@ class HiCacheController:
count += 1
return count
def dispatch_prefetch(self, operation: PrefetchOperation) -> None:
"""Route a prefetch operation to its IO lane.
Lane assignment must be deterministic and identical on every rank:
each lane owns dedicated completion sync groups, and the collective
sequence on a lane is the filtered dispatch order. crc32 (not the
per-process salted builtin hash) keeps the mapping stable across
processes. All chunks of one operation stay in one lane, preserving
the per-operation ack order the sync thread reduces over.
"""
lane = (
zlib.crc32(str(operation.request_id).encode())
% self.num_prefetch_io_lanes
)
self.prefetch_io_queues[lane].put(operation)
def _page_transfer(self, operation: PrefetchOperation, sync_queue: Queue) -> int:
def _page_transfer(self, operation: PrefetchOperation) -> int:
# Transfer batch by batch
prefix_keys = operation.prefix_keys
kv_derived_transfers = [
@@ -1155,8 +1087,8 @@ class HiCacheController:
completed_pages = 0
for i in range(0, len(operation.hash_value), STORAGE_BATCH_SIZE):
# When an error is occurred, we should keep looping and produce the same number of
# PrefetchAck as other ranks do, because the prefetch sync thread (i.e. consumer of
# the lane sync queue) perform reduce on the results. This is so tricky.
# PrefetchAck as other ranks do, because prefetch_sync_thread (i.e. consumer of
# prefetch_sync_queue) perform reduce on the results. This is so tricky.
if all_success and operation.is_terminated():
all_success = False
if all_success:
@@ -1168,7 +1100,6 @@ class HiCacheController:
# Get one batch token, and update the completed_tokens if succeed
extra_info = HiCacheStorageExtraInfo(prefix_keys=prefix_keys)
_hc_tb = time.monotonic() if _hc_timing_enabled() else 0.0
hit_pages = self._page_transfer_kv_batch(
operation,
batch_hashes,
@@ -1176,14 +1107,6 @@ class HiCacheController:
extra_info,
kv_derived_transfers,
)
if _hc_timing_enabled():
logger.info(
"HC_TIMING stage=xfer_batch req=%s keys=%d ms=%.1f hit=%d",
operation.request_id,
len(batch_hashes),
(time.monotonic() - _hc_tb) * 1e3,
hit_pages,
)
# Check termination
if hit_pages != len(batch_hashes):
all_success = False
@@ -1195,7 +1118,7 @@ class HiCacheController:
completed_tokens=completed_pages * self.page_size,
operation=operation,
)
sync_queue.put(ack)
self.prefetch_sync_queue.put(ack)
return completed_pages
def _page_transfer_kv_batch(
@@ -1237,33 +1160,18 @@ class HiCacheController:
# Clamp to minimal number of hits.
return min([kv_hits, *sidecar_hits.values()])
def prefetch_io_aux_func(self, lane: int = 0):
def prefetch_io_aux_func(self):
"""
Auxiliary function conducting IO operations for prefetching.
Each lane has its own worker thread and queues; an operation's whole
chunk sequence stays on one lane so the sync thread sees it in order.
"""
io_queue = self.prefetch_io_queues[lane]
sync_queue = self.prefetch_sync_queues[lane]
while not self.storage_stop_event.is_set():
try:
operation = io_queue.get(block=True, timeout=1)
operation = self.prefetch_buffer.get(block=True, timeout=1)
if operation is None:
continue
_hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0
self._page_transfer(operation, sync_queue)
if _hc_timing_enabled():
logger.info(
"HC_TIMING stage=xfer req=%s lane=%d wait_ms=%.1f xfer_ms=%.1f",
operation.request_id,
lane,
(_hc_t0 - getattr(operation, "query_done_ts", operation.start_time))
* 1e3,
(time.monotonic() - _hc_t0) * 1e3,
)
self._page_transfer(operation)
sync_queue.put(
self.prefetch_sync_queue.put(
PrefetchAck(
rid=operation.request_id,
completed_req=True,
@@ -1308,16 +1216,7 @@ class HiCacheController:
for start in range(0, len(page_hashes), STORAGE_BATCH_SIZE):
batch_hashes = page_hashes[start : start + STORAGE_BATCH_SIZE]
extra_info = HiCacheStorageExtraInfo(prefix_keys=prefix_keys)
_hc_tb = time.monotonic() if _hc_timing_enabled() else 0.0
hit_page_num = self.storage_backend.batch_exists(batch_hashes, extra_info)
if _hc_timing_enabled():
logger.info(
"HC_TIMING stage=query_batch req=%s keys=%d ms=%.1f hit=%d",
operation.request_id,
len(batch_hashes),
(time.monotonic() - _hc_tb) * 1e3,
hit_page_num,
)
hash_value.extend(batch_hashes[:hit_page_num])
storage_query_count += hit_page_num * self.page_size
if hit_page_num < len(batch_hashes):
@@ -1336,12 +1235,10 @@ class HiCacheController:
operation = self.prefetch_queue.get(block=True, timeout=1)
if operation is None:
continue
_hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0
if operation.is_terminated():
hash_value, storage_hit_count = [], 0
else:
hash_value, storage_hit_count = self._storage_hit_query(operation)
_hc_t1 = time.monotonic() if _hc_timing_enabled() else 0.0
storage_hit_count_tensor = torch.tensor(
storage_hit_count, dtype=torch.int
)
@@ -1351,18 +1248,6 @@ class HiCacheController:
self.prefetch_hits_sync_groups,
)
storage_hit_count = storage_hit_count_tensor.item()
if _hc_timing_enabled():
_hc_t2 = time.monotonic()
operation.query_done_ts = _hc_t2
logger.info(
"HC_TIMING stage=query req=%s wait_ms=%.1f query_ms=%.1f "
"hit_reduce_ms=%.1f hit_tokens=%d",
operation.request_id,
(_hc_t0 - operation.start_time) * 1e3,
(_hc_t1 - _hc_t0) * 1e3,
(_hc_t2 - _hc_t1) * 1e3,
storage_hit_count,
)
# Record the TP-synced hit count; the scheduler thread decides
# at drain time whether to revoke (below threshold) or allocate.
@@ -1444,41 +1329,28 @@ class HiCacheController:
except Empty:
continue
def prefetch_sync_thread_func(self, lane: int = 0):
def prefetch_sync_thread_func(self):
"""Synchronize prefetch results across all PP and TP ranks."""
sync_queue = self.prefetch_sync_queues[lane]
groups = self.prefetch_completion_sync_groups_by_lane[lane]
while not self.storage_stop_event.is_set():
try:
ack = sync_queue.get(block=True, timeout=1)
ack = self.prefetch_sync_queue.get(block=True, timeout=1)
if ack is None:
continue
self._reduce_prefetch_ack(ack, groups)
if _hc_timing_enabled() and ack.completed_req:
ack.operation.final_ack_enqueue_ts = time.monotonic()
self._reduce_prefetch_ack(ack)
self.ack_prefetch_queue.put(ack)
except Empty:
continue
def _reduce_prefetch_ack(
self, ack: PrefetchAck, groups: List[torch.distributed.ProcessGroup]
) -> None:
def _reduce_prefetch_ack(self, ack: PrefetchAck) -> None:
"""Synchronize all ranks to agree on a PrefetchAck."""
if ack.completed_tokens is not None:
# Determine the minimal successful prefix of tokens.
_hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0
completed_tokens_tensor = torch.tensor(
ack.completed_tokens, dtype=torch.int
)
self._all_reduce(
completed_tokens_tensor,
torch.distributed.ReduceOp.MIN,
groups,
self.prefetch_completion_sync_groups,
)
ack.completed_tokens = completed_tokens_tensor.item()
if _hc_timing_enabled():
logger.info(
"HC_TIMING stage=ack_reduce req=%s ms=%.1f",
ack.operation.request_id,
(time.monotonic() - _hc_t0) * 1e3,
)
-21
View File
@@ -2141,27 +2141,6 @@ class SetInternalStateReqOutput(BaseReq, kw_only=True):
updated: bool
class QueryCacheHitReqInput(BaseReq, kw_only=True):
"""G9: read-only probe of prefix-cache hit for a token sequence.
Sent to every DP rank of a (decode) engine; each rank answers with its own
L1/L2 radix match plus the shared L3 storage hit continuing from it.
"""
input_ids: List[int]
extra_key: Optional[str] = None
cache_salt: Optional[str] = None
class QueryCacheHitReqOutput(BaseReq, kw_only=True):
total_tokens: int = 0
l1_hit_tokens: int = 0
l2_hit_tokens: int = 0
l3_hit_tokens: int = 0
dp_rank: int = 0
error: Optional[str] = None
class ProfileReqType(Enum):
START_PROFILE = 1
STOP_PROFILE = 2
@@ -197,12 +197,6 @@ class LoadSnapshot(msgspec.Struct, omit_defaults=True):
num_running_reqs: int = 0
num_waiting_reqs: int = 0
num_waiting_uncached_tokens: int = 0
# G9 queue-time estimation (总输入长度口径, self-consistent numerator/denominator):
# sum of total input lengths (req.seqlen) over all pending queues, and the
# self-learned drain rate in the same unit — completed prefills' total
# input tokens / wall second over a sliding window.
queued_total_input_tokens: int = 0
drain_rate_tokens_per_s: float = 0.0
num_used_tokens: int = 0
num_total_tokens: int = 0
# num_total_tokens minus tokens still awaiting a KV transfer (equal to it
@@ -1,486 +0,0 @@
"""One owner rank encodes each image span and broadcasts it to the ranks that
run the same prefill chunk; every agreement precedes the payload it guards."""
from __future__ import annotations
import logging
from contextlib import contextmanager
from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence, Tuple
import msgspec
import torch
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
disable_symmetric_memory_context,
restore_symmetric_memory_context,
)
from sglang.srt.mem_cache.multimodal_cache import EmbeddingResult, MultiModalStaticCache
logger = logging.getLogger(__name__)
SpanKey = Tuple[Optional[int], int]
SpanEncoder = Callable[[List[Any]], torch.Tensor | List[torch.Tensor]]
SpanSignature = Callable[[Any, int], Tuple[Any, ...]]
LOCAL_HIT = 0
OWNER_CACHE_BROADCAST = 1
OWNER_ENCODE_BROADCAST = 2
PHASE_PREPARE = "prepare"
PHASE_FEATURES = "features"
PHASE_FINALIZE = "finalize"
class MmOwnerProtocolError(RuntimeError):
"""Raised with identical text on every group member after a group-agreed failure."""
class ImageSpanRequest(msgspec.Struct, frozen=True):
hash: Optional[int]
span_len: int
item: Any
inside_chunk: bool
duplicates: List[Any] = []
class ImageSpanKey(msgspec.Struct, frozen=True):
hash: Optional[int]
span_len: int
geometry: Optional[Tuple[Any, ...]]
class RankManifest(msgspec.Struct, frozen=True):
rank: int
keys: List[ImageSpanKey]
cached: List[bool]
dtype: str
width: int
rids: List[str]
error: Optional[str] = None
class OwnerPlan(msgspec.Struct, frozen=True):
actions: List[int]
owners: List[int]
error: Optional[str] = None
class RankStatus(msgspec.Struct, frozen=True):
rank: int
error: Optional[str] = None
def select_owner_group(parallel) -> Optional[Any]:
"""The group whose members all execute the same requests, or None when a
single rank already encodes every image it sees."""
replication = parallel.tp_size // parallel.attn_dp_size
if replication <= 1:
return None
if parallel.attn_cp_size == 1:
group = parallel.attn_tp_group
elif parallel.attn_dp_size == 1 and parallel.attn_cp_size == parallel.tp_size:
group = parallel.attn_cp_group
else:
return None
return group if group.world_size == replication else None
def has_owner_span_work(
mm_inputs: Sequence[Any],
extend_prefix_lens: Sequence[int],
extend_seq_lens: Sequence[int],
) -> bool:
"""Host-side mirror of the per-image scheduling path: does any raw
single-span image overlap the chunk on every rank of the group."""
for mm_input, prefix_len, extend_len in zip(
mm_inputs, extend_prefix_lens, extend_seq_lens
):
if mm_input is None or extend_len <= 0:
continue
items = [item for item in mm_input.mm_items if item is not None]
if not items or any(
item.precomputed_embeddings is not None or len(item.offsets) != 1
for item in items
):
continue
for item in items:
start, end = item.offsets[0]
if end >= prefix_len and start < prefix_len + extend_len:
return True
return False
class MmOwnerSession(msgspec.Struct):
group: Any
device: Any
dtype: Any
width: int
rids: List[str]
signature: Any
engaged: bool
phase: str = PHASE_PREPARE
in_collective: bool = False
def resolve(
self,
requests: Sequence[ImageSpanRequest],
cache: MultiModalStaticCache,
encode: SpanEncoder,
) -> Dict[SpanKey, torch.Tensor]:
if not self.engaged:
raise RuntimeError(
"owner protocol reached for a chunk whose host metadata has no image span"
)
# Owners allocate different amounts than receivers, so none of these
# buffers may come out of a symmetric pool.
saved_context = disable_symmetric_memory_context()
try:
return _resolve_owner_features(self, requests, cache, encode)
finally:
restore_symmetric_memory_context(saved_context)
def features_ready(self) -> None:
self._complete()
self.phase = PHASE_FINALIZE
@contextmanager
def uncaptured(self) -> Iterator[None]:
# A failure inside a collective leaves the group in an unknown state;
# no later exchange may try to agree on it.
self.in_collective = True
yield
self.in_collective = False
@contextmanager
def fence(self) -> Iterator[None]:
try:
yield
except Exception as exc:
self._fail(exc)
raise
self._complete()
def _fail(self, exc: BaseException) -> None:
if (
not self.engaged
or self.in_collective
or isinstance(exc, MmOwnerProtocolError)
):
raise exc
text = _describe(self, self.phase, exc)
if self.phase == PHASE_PREPARE:
try:
_exchange_manifest(self, _manifest(self, [], [], error=text))
except MmOwnerProtocolError as agreed:
raise agreed from exc
_exchange_status(self, text, exc)
def _complete(self) -> None:
if not self.engaged:
return
if self.phase == PHASE_PREPARE:
raise RuntimeError(
f"owner protocol {self.phase} completed without a manifest exchange"
)
error = None
cause = None
try:
_synchronize(self.device)
except Exception as exc:
cause = exc
error = _describe(self, self.phase, exc)
_exchange_status(self, error, cause)
def _manifest(
session: MmOwnerSession,
keys: List[ImageSpanKey],
cached: List[bool],
error: Optional[str] = None,
) -> RankManifest:
return RankManifest(
rank=session.group.rank_in_group,
keys=keys,
cached=cached,
dtype=str(session.dtype),
width=session.width,
rids=list(session.rids),
error=error,
)
def _exchange_manifest(session: MmOwnerSession, manifest: RankManifest) -> OwnerPlan:
group = session.group
with session.uncaptured():
manifests = group.all_gather_object(manifest)
plan = _plan_or_error(session, manifests) if group.rank_in_group == 0 else None
plan = group.broadcast_object(plan, src=0)
session.phase = PHASE_FEATURES
if plan.error is not None:
raise MmOwnerProtocolError(plan.error)
return plan
def _plan_or_error(session: MmOwnerSession, manifests: List[RankManifest]) -> OwnerPlan:
try:
return _make_plan(manifests)
except Exception as exc:
return OwnerPlan(actions=[], owners=[], error=_describe(session, "plan", exc))
def _exchange_status(
session: MmOwnerSession, error: Optional[str], cause: Optional[BaseException]
) -> None:
with session.uncaptured():
statuses = session.group.all_gather_object(
RankStatus(rank=session.group.rank_in_group, error=error)
)
_raise_first_error(statuses, cause)
def _resolve_owner_features(
session: MmOwnerSession,
requests: Sequence[ImageSpanRequest],
cache: MultiModalStaticCache,
encode: SpanEncoder,
) -> Dict[SpanKey, torch.Tensor]:
group = session.group
features: Dict[SpanKey, torch.Tensor] = {}
keys: List[ImageSpanKey] = []
cached: List[bool] = []
error = None
try:
keys, cached = _pin_local_cache(session, requests, cache, features)
except Exception as exc:
error = _describe(session, "manifest", exc)
plan = _exchange_manifest(session, _manifest(session, keys, cached, error))
if all(action == LOCAL_HIT for action in plan.actions):
return features
buffers: Dict[int, torch.Tensor] = {}
error = None
try:
buffers = _prepare_transfers(session, requests, keys, plan, features, encode)
_synchronize(session.device)
except Exception as exc:
error = _describe(session, "encode", exc)
_exchange_status(session, error, None)
with session.uncaptured():
for index, (action, owner) in enumerate(zip(plan.actions, plan.owners)):
if action != LOCAL_HIT:
group.broadcast(buffers[index], src=owner)
for index, key in enumerate(keys):
if plan.actions[index] == LOCAL_HIT:
continue
span = buffers[index]
features[(key.hash, key.span_len)] = span
cache.set(key.hash, EmbeddingResult(embedding=span))
return features
def _pin_local_cache(
session: MmOwnerSession,
requests: Sequence[ImageSpanRequest],
cache: MultiModalStaticCache,
features: Dict[SpanKey, torch.Tensor],
) -> Tuple[List[ImageSpanKey], List[bool]]:
keys: List[ImageSpanKey] = []
cached: List[bool] = []
for request in requests:
if request.hash is None:
raise ValueError(
f"image span of {request.span_len} tokens has no content hash"
)
geometry = session.signature(request.item, request.span_len)
for duplicate in request.duplicates:
other = session.signature(duplicate, request.span_len)
if other != geometry:
raise ValueError(
f"image hash {request.hash} ({request.span_len} tokens) occurs "
f"with different geometry: {geometry} vs {other}"
)
keys.append(
ImageSpanKey(
hash=request.hash, span_len=request.span_len, geometry=geometry
)
)
span = _valid_cached_span(session, cache, request)
if span is not None:
features[(request.hash, request.span_len)] = span
cached.append(span is not None)
return keys, cached
def _valid_cached_span(
session: MmOwnerSession,
cache: MultiModalStaticCache,
request: ImageSpanRequest,
) -> Optional[torch.Tensor]:
entry = cache.get_single(request.hash)
if entry is None:
return None
span = entry.embedding
if (
span.dim() == 2
and span.shape[0] == request.span_len
and span.shape[1] == session.width
and span.dtype == session.dtype
and span.device == session.device
):
return span
logger.warning(
"Discarding cached multimodal embedding that cannot serve the current "
"image span: cache_key=%s expected=(%d, %d, %s) cached=(%s, %s).",
request.hash,
request.span_len,
session.width,
session.dtype,
tuple(span.shape),
span.dtype,
)
cache.free(request.hash, None)
return None
def _make_plan(manifests: List[RankManifest]) -> OwnerPlan:
for manifest in manifests:
if manifest.error is not None:
return OwnerPlan(actions=[], owners=[], error=manifest.error)
lead = manifests[0]
for manifest in manifests[1:]:
if (manifest.keys, manifest.dtype, manifest.width, manifest.rids) != (
lead.keys,
lead.dtype,
lead.width,
lead.rids,
):
return OwnerPlan(
actions=[],
owners=[],
error=(
"image manifest mismatch between group ranks 0 and "
f"{manifest.rank}: rids={lead.rids} vs {manifest.rids}, "
f"keys={lead.keys} vs {manifest.keys}, "
f"dtype={lead.dtype} vs {manifest.dtype}, "
f"width={lead.width} vs {manifest.width}"
),
)
replication = len(manifests)
actions: List[int] = []
owners: List[int] = []
for index, key in enumerate(lead.keys):
owner = key.hash % replication
if all(manifest.cached[index] for manifest in manifests):
action = LOCAL_HIT
elif manifests[owner].cached[index]:
action = OWNER_CACHE_BROADCAST
else:
action = OWNER_ENCODE_BROADCAST
actions.append(action)
owners.append(owner)
return OwnerPlan(actions=actions, owners=owners)
def _prepare_transfers(
session: MmOwnerSession,
requests: Sequence[ImageSpanRequest],
keys: List[ImageSpanKey],
plan: OwnerPlan,
features: Dict[SpanKey, torch.Tensor],
encode: SpanEncoder,
) -> Dict[int, torch.Tensor]:
rank = session.group.rank_in_group
buffers: Dict[int, torch.Tensor] = {}
owned: List[int] = []
for index, (action, owner) in enumerate(zip(plan.actions, plan.owners)):
if action == LOCAL_HIT:
continue
if owner != rank:
try:
buffers[index] = _new_span_buffer(session, keys[index])
except Exception as exc:
raise RuntimeError(
f"receive buffer for image hash {keys[index].hash} shape "
f"{(keys[index].span_len, session.width)} {session.dtype} "
f"failed: {type(exc).__name__}: {exc}"
) from exc
elif action == OWNER_CACHE_BROADCAST:
key = (keys[index].hash, keys[index].span_len)
buffers[index] = features[key].contiguous()
else:
owned.append(index)
if owned:
owned_hashes = [keys[index].hash for index in owned]
try:
encoded = encode([requests[index].item for index in owned])
except Exception as exc:
raise RuntimeError(
f"owner encode of image hashes {owned_hashes} failed: "
f"{type(exc).__name__}: {exc}"
) from exc
spans = _split_spans(encoded, [keys[index].span_len for index in owned])
for index, span in zip(owned, spans):
buffers[index] = _validated_span(session, keys[index], span)
return buffers
def _new_span_buffer(session: MmOwnerSession, key: ImageSpanKey) -> torch.Tensor:
return torch.empty(
(key.span_len, session.width), device=session.device, dtype=session.dtype
)
def _split_spans(
encoded: torch.Tensor | List[torch.Tensor], span_lens: List[int]
) -> List[torch.Tensor]:
if isinstance(encoded, list):
if len(encoded) != len(span_lens):
raise ValueError(
f"encoder returned {len(encoded)} spans for {len(span_lens)} images"
)
return [span.reshape(-1, span.shape[-1]) for span in encoded]
encoded = encoded.reshape(-1, encoded.shape[-1])
if encoded.shape[0] != sum(span_lens):
raise ValueError(
f"encoder returned {encoded.shape[0]} rows for spans of {span_lens}"
)
return list(torch.split(encoded, span_lens, dim=0))
def _validated_span(
session: MmOwnerSession, key: ImageSpanKey, span: torch.Tensor
) -> torch.Tensor:
expected = (key.span_len, session.width)
if tuple(span.shape) != expected or span.dtype != session.dtype:
raise ValueError(
f"encoded span for hash={key.hash} has shape {tuple(span.shape)} "
f"dtype {span.dtype}; expected {expected} {session.dtype}"
)
if span.device != session.device:
span = span.to(session.device)
return span.contiguous()
def _synchronize(device) -> None:
if device.type == "cuda":
torch.cuda.current_stream(device).synchronize()
def _describe(session: MmOwnerSession, stage: str, exc: BaseException) -> str:
return (
f"multimodal owner protocol failed during {stage} on group rank "
f"{session.group.rank_in_group} (global rank "
f"{session.group.ranks[session.group.rank_in_group]}, rids={list(session.rids)}): "
f"{type(exc).__name__}: {exc}"
)
def _raise_first_error(
statuses: List[RankStatus], cause: Optional[BaseException]
) -> None:
for status in statuses:
if status.error is not None:
raise MmOwnerProtocolError(status.error) from cause
+37 -83
View File
@@ -5,7 +5,6 @@ from typing import Callable, Dict, List, Optional, Tuple
import torch
from sglang.srt.managers.mm_owner_embedding import ImageSpanRequest, MmOwnerSession
from sglang.srt.managers.schedule_batch import MultimodalDataItem
from sglang.srt.mem_cache.multimodal_cache import EmbeddingResult, MultiModalStaticCache
from sglang.srt.multimodal.evs import EVSEmbeddingResult
@@ -340,24 +339,43 @@ def _batch_encode_per_image_misses(
unique_misses: Dict[Tuple[Optional[int], int], Tuple[MultimodalDataItem, int]] = {}
hash_to_embedding: Dict[Tuple[Optional[int], int], torch.Tensor] = {}
# Phase 1a: collect cache misses over the unique overlapping spans
for span in _collect_image_span_requests(per_image_requests):
cache_key = (span.hash, span.span_len)
cached = embedding_cache.get_single(span.hash)
if cached is not None:
cached_embedding = cached.embedding
cached_token_count = _embedding_token_count(cached_embedding)
if cached_token_count == span.span_len:
hash_to_embedding[cache_key] = cached_embedding
# Phase 1a: find overlapping items per request and collect cache misses
for req_info in per_image_requests:
chunk_start = req_info.extend_prefix_len
chunk_end = chunk_start + req_info.extend_seq_len # exclusive
overlapping = []
if req_info.extend_seq_len > 0:
for idx, (item, (start, end)) in enumerate(
zip(req_info.items, req_info.items_offset)
):
if end >= chunk_start and start < chunk_end:
overlapping.append((idx, item, start, end))
req_info.overlapping = overlapping
for _idx, item, start, end in overlapping:
expected_token_count = end - start + 1
cache_key = (item.hash, expected_token_count)
if cache_key in hash_to_embedding:
continue
_discard_mismatched_cached_embedding(
span.hash, span.span_len, cached_token_count
)
elif (
span.inside_chunk and span.item.can_defer_cuda_ipc_feature_reconstruction()
):
span.item.model_specific_data[BORROW_CUDA_IPC_FEATURE_KEY] = True
unique_misses[cache_key] = (span.item, span.span_len)
cached = embedding_cache.get_single(item.hash)
if cached is not None:
cached_embedding = cached.embedding
cached_token_count = _embedding_token_count(cached_embedding)
if cached_token_count == expected_token_count:
hash_to_embedding[cache_key] = cached_embedding
else:
_discard_mismatched_cached_embedding(
item.hash, expected_token_count, cached_token_count
)
unique_misses[cache_key] = (item, expected_token_count)
elif cache_key not in unique_misses:
if (
start >= chunk_start
and end < chunk_end
and item.can_defer_cuda_ipc_feature_reconstruction()
):
item.model_specific_data[BORROW_CUDA_IPC_FEATURE_KEY] = True
unique_misses[cache_key] = (item, expected_token_count)
# Phase 1b: single ViT call for all unique cache misses
if unique_misses:
@@ -394,52 +412,6 @@ def _batch_encode_per_image_misses(
return hash_to_embedding
def _collect_image_span_requests(
per_image_requests: List[PerImageRequestInfo],
) -> List[ImageSpanRequest]:
spans: Dict[
Tuple[Optional[int], int],
Tuple[MultimodalDataItem, bool, List[MultimodalDataItem]],
] = {}
for req_info in per_image_requests:
chunk_start = req_info.extend_prefix_len
chunk_end = chunk_start + req_info.extend_seq_len # exclusive
overlapping = []
if req_info.extend_seq_len > 0:
for idx, (item, (start, end)) in enumerate(
zip(req_info.items, req_info.items_offset)
):
if end >= chunk_start and start < chunk_end:
overlapping.append((idx, item, start, end))
req_info.overlapping = overlapping
for _idx, item, start, end in overlapping:
cache_key = (item.hash, end - start + 1)
if cache_key in spans:
spans[cache_key][2].append(item)
continue
spans[cache_key] = (item, start >= chunk_start and end < chunk_end, [])
return [
ImageSpanRequest(
hash=item_hash,
span_len=span_len,
item=item,
inside_chunk=inside_chunk,
duplicates=duplicates,
)
for (item_hash, span_len), (item, inside_chunk, duplicates) in spans.items()
]
def _owner_span_encoder(data_embedding_func: DataEmbeddingFunc, device: torch.device):
def encode(items: List[MultimodalDataItem]):
if not _can_skip_pre_embed_feature_move(data_embedding_func):
_move_items_to_device(items, device)
return data_embedding_func(items)
return encode
def _get_chunked_embedding_by_item(
data_embedding_func: DataEmbeddingFunc,
embedding_items_per_req: List[MultimodalDataItem],
@@ -565,7 +537,6 @@ def _get_chunked_prefill_embedding(
extend_length: List[int],
items_offset_list: List[List[Tuple[int, int]]],
input_ids: torch.Tensor,
mm_owner: Optional[MmOwnerSession] = None,
) -> tuple[torch.Tensor | None, torch.Tensor]:
"""
Chunked prefill embedding: encode items across all requests and extract
@@ -627,22 +598,7 @@ def _get_chunked_prefill_embedding(
# Phase 1: batch encode all per-image cache misses in ONE ViT call
hash_to_embedding: Dict[Tuple[Optional[int], int], torch.Tensor] = {}
if per_image_requests and mm_owner is not None:
# The owner protocol must see every overlapping span before any local
# cache filtering: a rank-local hit can never skip a group collective.
span_requests = _collect_image_span_requests(per_image_requests)
if mm_owner.engaged:
hash_to_embedding = mm_owner.resolve(
span_requests,
cache=embedding_cache,
encode=_owner_span_encoder(data_embedding_func, device),
)
elif span_requests:
raise RuntimeError(
"owner eligibility saw no image span in this chunk, but "
f"scheduling found {len(span_requests)}"
)
elif per_image_requests:
if per_image_requests:
hash_to_embedding = _batch_encode_per_image_misses(
data_embedding_func, per_image_requests, device
)
@@ -745,7 +701,6 @@ def get_embedding_and_mask(
prefix_length: List[int],
extend_length: List[int],
items_offset_list: List[List[Tuple[int, int]]],
mm_owner: Optional[MmOwnerSession] = None,
) -> Tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor]:
"""
Generate multimodal embeddings and create a mask for identifying their positions in the input sequence.
@@ -786,7 +741,6 @@ def get_embedding_and_mask(
extend_length,
items_offset_list,
input_ids,
mm_owner=mm_owner,
)
if embedding is None:
return None, None, input_ids
+2 -13
View File
@@ -10,7 +10,6 @@ import pickle
import sys
from abc import abstractmethod
from collections import defaultdict
from contextlib import nullcontext
from multiprocessing import shared_memory
from typing import Any, Dict, List, Optional, Tuple
@@ -25,7 +24,6 @@ from sglang.srt.managers.io_struct import (
TokenizedEmbeddingReqInput,
TokenizedGenerateReqInput,
)
from sglang.srt.managers.mm_owner_embedding import MmOwnerSession
# Preserve the existing initialization import for downstream callers.
from sglang.srt.managers.mm_schedule import (
@@ -399,7 +397,6 @@ def embed_mm_inputs(
data_embedding_func_mapping: Dict[Modality, DataEmbeddingFunc] = None,
placeholder_tokens: dict[Modality, List[int]] = None,
use_deepstack: Dict[Modality, bool] = {},
mm_owner: Optional[MmOwnerSession] = None,
) -> Optional[torch.Tensor]:
"""
Embed multimodal inputs and integrate them with text token embeddings.
@@ -481,7 +478,6 @@ def embed_mm_inputs(
prefix_length=extend_prefix_lens,
extend_length=extend_seq_lens,
items_offset_list=items_offsets,
mm_owner=mm_owner,
)
if use_deepstack.get(modality, None) and embedding is not None:
@@ -502,12 +498,7 @@ def embed_mm_inputs(
# filled with the hash values of the multimodal for the prefix matching in the radix attention.
# There values are useless because their embeddings will be replaced by vision embeddings anyway.
input_ids.clamp_(min=0, max=vocab_size - 1)
if mm_owner is not None:
# The text embedding may all-reduce across TP; a rank-local failure in
# feature preparation has to be agreed on before any rank enters it.
mm_owner.features_ready()
with mm_owner.uncaptured() if mm_owner is not None else nullcontext():
input_embeds = input_embedding(input_ids)
input_embeds = input_embedding(input_ids)
# deepstack embedding
if use_deepstack:
@@ -534,9 +525,7 @@ def embed_mm_inputs(
_scatter_mm_embedding(dest=input_embeds, mask=mask, src=embedding)
if use_deepstack.get(modality, None):
_scatter_mm_embedding(
dest=input_deepstack_embeds,
mask=mask,
src=deepstack_embeddings[i],
dest=input_deepstack_embeds, mask=mask, src=deepstack_embeddings[i]
)
return input_embeds, other_info
+6 -6
View File
@@ -1034,12 +1034,12 @@ class Req(ReqDllmMixin):
else self.origin_input_ids
) # Before image padding
# Each decode stage's output ids. Append-only by contract:
# refresh_fill_ids infers how many output tokens are already in
# _refresh_fill_ids infers how many output tokens are already in
# full_untruncated_fill_ids from lengths alone, so in-place rewrites
# that preserve length would silently corrupt fill_ids.
self.output_ids = array("q")
# Full untruncated sequence: origin + output (+ DLLM mask block).
# Kept in sync by refresh_fill_ids; admission only updates
# Kept in sync by _refresh_fill_ids; admission only updates
# extend_range, never mutates this array's length.
self.full_untruncated_fill_ids = array("q")
self.extend_range: Optional[Range] = None
@@ -1533,7 +1533,7 @@ class Req(ReqDllmMixin):
def get_fill_ids(self) -> array:
return self.full_untruncated_fill_ids[: self.extend_range.end]
def refresh_fill_ids(self) -> None:
def _refresh_fill_ids(self) -> None:
"""Keep full_untruncated_fill_ids == origin_input_ids + output_ids by
appending only the new output tokens.
@@ -1562,7 +1562,7 @@ class Req(ReqDllmMixin):
self._init_fill_ids_for_dllm()
self.determine_dllm_phase()
else:
self.refresh_fill_ids()
self._refresh_fill_ids()
input_len = len(self.full_untruncated_fill_ids)
@@ -3107,7 +3107,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
for req, prefix_len in zip(
running_batch.reqs, running_prefix_lens, strict=True
):
req.refresh_fill_ids()
req._refresh_fill_ids()
req.set_extend_range(prefix_len, prefix_len + 1)
# Decode tokens of the running portion live in future_map.output_tokens_buf.
@@ -3185,7 +3185,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
# cannot stand in for it. Rows past bs are a beam tail, not requests.
seq_lens = self.seq_lens_cpu[:bs].tolist()
for req, seq_len in zip(self.reqs, seq_lens, strict=True):
req.refresh_fill_ids()
req._refresh_fill_ids()
# end runs one past full_untruncated_fill_ids while output_ids
# trails; safe only while decoding_reqs suppresses cache_unfinished_req.
req.set_extend_range(seq_len - 1, seq_len)
@@ -1019,15 +1019,6 @@ class PrefillAdder:
else AddReqResult.CONTINUE
)
def can_share_extend_batch(self, req: Req) -> bool:
# Token embedding overrides embed the batch's raw input_ids before the
# model runs, and that lookup cannot index multimodal placeholder hash IDs.
if req.positional_embed_overrides is not None:
return all(r.multimodal_inputs is None for r in self.can_run_list)
if req.multimodal_inputs is not None:
return all(r.positional_embed_overrides is None for r in self.can_run_list)
return True
def add_chunked_req(self, req: Req):
if self.dllm_config is not None:
_rem_tokens = self._get_dllm_remain_tokens()
+3 -150
View File
@@ -161,8 +161,6 @@ from sglang.srt.managers.io_struct import (
PauseGenerationReqInput,
PdRoleSwitchReqInput,
ProfileReq,
QueryCacheHitReqInput,
QueryCacheHitReqOutput,
ReleaseMemoryOccupationReqInput,
RemoveExternalCorpusReqInput,
RemoveExternalCorpusReqOutput,
@@ -1808,7 +1806,6 @@ class Scheduler(
(ShutdownReq, self.handle_shutdown),
(GetInternalStateReq, self.get_internal_state),
(SetInternalStateReq, self.set_internal_state),
(QueryCacheHitReqInput, self.handle_query_cache_hit),
(RpcReqInput, self.handle_rpc_request),
(ExpertDistributionReq, self.expert_distribution_handle),
(LoadLoRAAdapterReqInput, self.load_lora_adapter),
@@ -2100,13 +2097,9 @@ class Scheduler(
vmm_errors = self._materialize_cuda_vmm_inputs(recv_req)
# Skip health check when server is busy — ongoing requests already carry health info.
# NOTE: the admit/skip decision must be identical on every CP/TP rank.
# is_fully_idle() includes rank-local hicache drain queues, which diverge
# across ranks right after activity; a divergent decision lets one rank
# dispatch the health-check generate while others piggyback-skip, breaking
# collective ordering (deadlock: one rank blocks in the hicache drain
# all_reduce while another waits in the CP request broadcast).
if is_health_check_generate_req(recv_req) and not self.is_sched_idle_cp_symmetric():
if is_health_check_generate_req(recv_req) and not self.is_fully_idle(
for_health_check=True
):
self.return_health_check_ipcs.append(
getattr(recv_req, "http_worker_ipc", None)
)
@@ -2434,11 +2427,6 @@ class Scheduler(
self._prev_step: Optional[Tuple[int, float, bool]] = None
self._prev_prefill_end_ts: Optional[float] = None
self._sched_idled = False
# G9: sliding-window drain events (monotonic_ts, total_input_tokens) for
# the queue-time estimator. Recorded once per request when its final
# prefill chunk completes (batch_result_processor), in the same
# total-input-length unit as LoadSnapshot.queued_total_input_tokens.
self.prefill_drain_window: deque = deque()
self.load_inquirer = SchedulerLoadInquirer(
disaggregation_mode=self.disaggregation_mode,
server_args=self.server_args,
@@ -2477,35 +2465,8 @@ class Scheduler(
),
get_total_prefill_busy_us=lambda: self.total_prefill_busy_us,
get_decode_moment_totals=lambda: self.decode_moment_totals,
get_prefill_drain_rate=self.get_prefill_drain_rate,
)
PREFILL_DRAIN_WINDOW_S = 60.0
def record_prefill_drain(self, total_input_tokens: int) -> None:
"""G9: one drain event per request whose final prefill chunk completed.
Tokens are the request's total input length (seqlen), matching the
queued_total_input_tokens numerator so the router's
wait = queued_tokens / drain_rate is unit-consistent (决策②).
"""
now = time.monotonic()
self.prefill_drain_window.append((now, total_input_tokens))
cutoff = now - self.PREFILL_DRAIN_WINDOW_S
while self.prefill_drain_window and self.prefill_drain_window[0][0] < cutoff:
self.prefill_drain_window.popleft()
def get_prefill_drain_rate(self) -> float:
"""Completed prefill input tokens per wall second over the window."""
now = time.monotonic()
cutoff = now - self.PREFILL_DRAIN_WINDOW_S
while self.prefill_drain_window and self.prefill_drain_window[0][0] < cutoff:
self.prefill_drain_window.popleft()
if not self.prefill_drain_window:
return 0.0
span = max(now - self.prefill_drain_window[0][0], 1.0)
return sum(tokens for _, tokens in self.prefill_drain_window) / span
def init_output_streamer(self) -> None:
self.output_streamer = self.get_output_streamer_class()(
send_to_detokenizer=self.ipc_channels.send_to_detokenizer,
@@ -2556,7 +2517,6 @@ class Scheduler(
output_streamer=self.output_streamer,
beam_coordinator=self.beam_coordinator,
abort_request=self.abort_request,
record_prefill_drain=self.record_prefill_drain,
)
def init_req_max_new_tokens(self, req):
@@ -3980,8 +3940,6 @@ class Scheduler(
for req in self.waiting_queue:
if self.enable_lora and not self.can_schedule_lora_req(req, running_loras):
continue
if not adder.can_share_extend_batch(req):
break
running_bs = len(running_batch.reqs)
candidate_beam_width = (
@@ -5022,31 +4980,6 @@ class Scheduler(
else:
self.metrics_reporter.record_scheduler_active()
def is_sched_idle_cp_symmetric(self) -> bool:
"""Idle check using only state that is identical across CP/TP ranks.
Request/batch/queue state is collectively maintained (requests arrive
via broadcast, batches are collectively scheduled), so every rank
computes the same result. Rank-local hicache drain and disagg transfer
queues are deliberately excluded: those are exactly the terms that
diverge across ranks and caused the CP health-check deadlock
(hicache drain all_reduce vs CP request broadcast cross-collective
wait, seen on cp2/cp4 + hicache L3 right after router health checks).
Used only for health-check admission; all other idle logic keeps using
is_fully_idle().
"""
return (
self.running_batch.is_empty()
and self.chunked_req is None
and not self.dllm_manager.any_staging_reqs()
and (self.last_batch is None or self.last_batch.is_empty())
and (not self.enable_overlap or len(self.result_queue) == 0)
and self._pp_microbatches_drained()
and len(self.waiting_queue) == 0
and len(self.grammar_manager.grammar_queue) == 0
)
def is_fully_idle(self, for_health_check=False) -> bool:
# Health check piggybacks on running requests in process_output.
# Only running_batch + waiting_queue guarantee active GPU processing;
@@ -5251,86 +5184,6 @@ class Scheduler(
success = False
return success
def handle_query_cache_hit(self, recv_req: QueryCacheHitReqInput):
"""G9: read-only prefix-hit probe (L1/L2 radix + L3 storage).
The router calls this (via /query_cache_hit on the decode engine) to
decide conditional PD-bypass. Runs inside the scheduler event loop so
radix/storage state is consistent, and reuses the exact admission-time
match + L3 query logic (mirrors DecodeHiCachePreallocMixin
._build_decode_prefix_match) without creating a Req or mutating
request state. Node access metadata (LRU) may be touched; a hit found
here can still be evicted before the real request arrives (inherent
TOCTOU, acceptable for an estimate).
"""
from sglang.srt.mem_cache.base_prefix_cache import MatchPrefixParams
from sglang.srt.mem_cache.radix_cache import RadixKey
total = len(recv_req.input_ids)
# 树节点键是 array("q"),RadixKey.match_at 断言两侧类型一致——
# 查询侧必须同样用 array,否则树非空时匹配直接炸(2026-10-11 实测)。
from array import array
query_ids = array("q", recv_req.input_ids)
try:
match_result = self.tree_cache.match_prefix(
MatchPrefixParams(
key=RadixKey(
token_ids=query_ids,
extra_key=recv_req.extra_key,
cache_salt=recv_req.cache_salt,
)
)
)
l1_hit = len(match_result.device_indices)
l2_hit = getattr(match_result, "host_hit_length", 0) or 0
l3_hit = 0
last_host_node = getattr(match_result, "last_host_node", None)
# L3 查询条件:本调度器挂了 hicache 存储后端即可(P 节点 HiCache
# write_through/write_back 场景),不必是 decode-hicache 模式。
# SWA 模型的 decode radix 与 hicache 互斥(kv_cache_builder 硬限制),
# 若只认 enable_decode_hicache,P 节点永远查不到 L3。
can_query_l3 = getattr(self, "enable_decode_hicache", False) or getattr(
self, "enable_hicache_storage", False
)
if (
can_query_l3
and last_host_node is not None
and hasattr(self.tree_cache, "query_storage_hit_length")
and (
self.tree_cache.is_backuped(last_host_node)
or self.tree_cache.is_root(last_host_node)
)
):
matched_len = l1_hit + l2_hit
suffix_tokens = query_ids[matched_len:]
last_hash = self.tree_cache.get_last_hash_value(last_host_node)
prefix_keys = (
self.tree_cache.get_prefix_hash_values(last_host_node)
if getattr(
self.tree_cache, "hicache_storage_pass_prefix_keys", False
)
else None
)
l3_hit = self.tree_cache.query_storage_hit_length(
last_host_node,
suffix_tokens,
last_hash,
prefix_keys,
extra_key=recv_req.extra_key,
cache_salt=recv_req.cache_salt,
)
return QueryCacheHitReqOutput(
total_tokens=total,
l1_hit_tokens=l1_hit,
l2_hit_tokens=l2_hit,
l3_hit_tokens=l3_hit,
dp_rank=int(get_parallel().dp_rank or 0),
)
except Exception as e:
logger.warning(f"handle_query_cache_hit failed: {e}")
return QueryCacheHitReqOutput(total_tokens=total, error=str(e))
def get_internal_state(self, recv_req: GetInternalStateReq):
# Resolved config (pristine server_args + post-publish overrides) so a
# readback reflects values changed via /set_internal_state, not startup.
@@ -112,8 +112,6 @@ class SchedulerBatchResultProcessor:
output_streamer: SchedulerOutputStreamer
beam_coordinator: BeamCoordinator
abort_request: Callable
# G9: scheduler-side drain-event recorder for the queue-time estimator.
record_prefill_drain: Callable
def process_batch_result_prebuilt(self, batch: ScheduleBatch):
assert self.disaggregation_mode == DisaggregationMode.DECODE
@@ -345,9 +343,6 @@ class SchedulerBatchResultProcessor:
if req.inflight_middle_chunks <= 0:
req.time_stats.set_prefill_finished_time()
# G9: exactly once per request (final chunk only) — feed the
# queue-time estimator with the drained total input length.
self.record_prefill_drain(req.seqlen)
if sampling_mask_finish_reason is not None:
req.to_finish = sampling_mask_finish_reason
@@ -54,7 +54,6 @@ class SchedulerLoadInquirer:
get_total_prefill_uncached_tokens: Callable
get_total_prefill_busy_us: Callable
get_decode_moment_totals: Callable
get_prefill_drain_rate: Callable
def _get_num_pending_tokens(self, chunk_deduct: int = 0) -> int:
"""Get the total number of tokens pending prefill.
@@ -129,10 +128,9 @@ class SchedulerLoadInquirer:
num_used_tokens, kv_token_usage = (
self.pool_stats_observer.get_pool_stats().get_kv_token_stats()
)
queued_total_input_tokens = sum(
num_total_tokens = num_used_tokens + sum(
req.seqlen for queue in pending_token_queues for req in queue
)
num_total_tokens = num_used_tokens + queued_total_input_tokens
num_active_tokens = max(0, num_total_tokens - awaiting_kv_tokens)
memory = None
@@ -222,8 +220,6 @@ class SchedulerLoadInquirer:
num_running_reqs=num_running_reqs,
num_waiting_reqs=num_waiting_reqs,
num_waiting_uncached_tokens=self.get_num_waiting_uncached_tokens(),
queued_total_input_tokens=queued_total_input_tokens,
drain_rate_tokens_per_s=round(self.get_prefill_drain_rate(), 2),
num_used_tokens=num_used_tokens,
num_total_tokens=num_total_tokens,
num_active_tokens=num_active_tokens,
@@ -54,8 +54,6 @@ from sglang.srt.managers.io_struct import (
ProfileReq,
ProfileReqOutput,
ProfileReqType,
QueryCacheHitReqInput,
QueryCacheHitReqOutput,
ReleaseMemoryOccupationReqInput,
ReleaseMemoryOccupationReqOutput,
RemoveExternalCorpusReqInput,
@@ -131,7 +129,6 @@ _COMMUNICATOR_SPECS = [
("detach_hicache_storage", DetachHiCacheStorageReqOutput),
("profile", ProfileReqOutput),
("get_internal_state", GetInternalStateReqOutput),
("query_cache_hit", QueryCacheHitReqOutput),
("set_internal_state", SetInternalStateReqOutput),
("expert_distribution", ExpertDistributionReqOutput),
("update_lora_adapter", LoRAUpdateOutput),
@@ -908,37 +905,6 @@ class TokenizerControlMixin:
self.auto_create_handle_loop()
return await self.dumper_control_communicator(obj)
async def query_cache_hit(
self: TokenizerManager, obj: QueryCacheHitReqInput
) -> QueryCacheHitReqOutput:
"""G9: fan the probe out to all DP ranks and keep the best hit.
The decode node's DP controller routes new requests to any rank, so
the bypass decision uses the best per-rank L1/L2 (L3 is shared); the
real request still goes through normal dispatch afterwards.
"""
self.auto_create_handle_loop()
results: List[QueryCacheHitReqOutput] = await self.query_cache_hit_communicator(
obj
)
ok = [r for r in results if r.error is None]
if not ok:
msg = (
"; ".join(r.error or "empty response" for r in results)
or "no scheduler responded"
)
return QueryCacheHitReqOutput(total_tokens=len(obj.input_ids), error=msg)
best = max(
ok, key=lambda r: r.l1_hit_tokens + r.l2_hit_tokens + r.l3_hit_tokens
)
return QueryCacheHitReqOutput(
total_tokens=best.total_tokens,
l1_hit_tokens=best.l1_hit_tokens,
l2_hit_tokens=best.l2_hit_tokens,
l3_hit_tokens=best.l3_hit_tokens,
dp_rank=best.dp_rank,
)
async def get_loads(
self: TokenizerManager,
include: Optional[List[str]] = None,
@@ -1279,16 +1279,6 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
raise ValueError(
"encoder SWA replay cannot return cached prompt logprobs"
)
requests_embed_overrides = obj.positional_embed_overrides is not None or (
isinstance(obj, EmbeddingReqInput)
and obj.embed_overrides is not None
and obj.embed_override_token_id is not None
)
if requests_embed_overrides and obj.contains_mm_input():
raise ValueError(
"embedding overrides cannot be combined with image, video, or audio "
"inputs"
)
_max_req_len = self.context_len
input_token_num = len(input_ids) if input_ids is not None else 0
input_token_num += self.num_reserved_tokens
@@ -866,15 +866,3 @@ class PureSWATokenToKVPoolAllocator(SWATokenToKVPoolAllocator):
def is_swa_req_ring(allocator) -> bool:
return isinstance(allocator, SWATokenToKVPoolAllocator) and allocator.swa_req_ring
def swa_kept_out_of_tree(allocator) -> bool:
"""Whether radix-tree nodes do not own the SWA state: a per-request SWA ring,
or the DeepSeek-V4.1 encoder-replay request window, which replay rebuilds on a
prefix hit. SWA bookkeeping must not gate prefix matching for either."""
if not isinstance(allocator, SWATokenToKVPoolAllocator):
return False
if allocator.swa_req_ring:
return True
kvcache = getattr(allocator, "_kvcache", None)
return getattr(kvcache, "request_window", None) is not None
-4
View File
@@ -155,10 +155,6 @@ def free_kv_row_segments(
def maybe_cache_unfinished_req(req: Req, tree_cache: BasePrefixCache, **kwargs):
if getattr(req, "skip_radix_cache_insert", False):
kv_indices = tree_cache.req_to_token_pool.req_to_token[
req.kv.req_pool_idx, : len(req.get_fill_ids())
]
req.prefix_indices = kv_indices.to(dtype=torch.int64, copy=True)
return
tree_cache.cache_unfinished_req(req, **kwargs)

Some files were not shown because too many files have changed in this diff Show More