Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c8d06424fa | ||
|
|
97c7eabb33 | ||
|
|
e9b436dc2d | ||
|
|
fc63ae5392 | ||
|
|
9df2d40324 | ||
|
|
6a8a6e399f | ||
|
|
e6c0737541 | ||
|
|
9374e03f13 | ||
|
|
86883ae89f | ||
|
|
43c2844c18 | ||
|
|
9e288079e4 | ||
|
|
738d5a50d0 | ||
|
|
e80c8608dc | ||
|
|
5351d5491b | ||
|
|
8a2e1403a3 | ||
|
|
1910eccb8e | ||
|
|
6217b9de46 | ||
|
|
9dd446b5c1 | ||
|
|
f76eaf0f92 | ||
|
|
45fdc97082 | ||
|
|
f690861265 | ||
|
|
e4bfdc949b | ||
|
|
59e3aee6d9 | ||
|
|
406081d3e3 | ||
|
|
1975265315 | ||
|
|
534417b82e | ||
|
|
ed0a454730 | ||
|
|
04ee2d03d7 | ||
|
|
9d5db13e53 | ||
|
|
7e20f0af79 | ||
|
|
0b74ab2ac4 | ||
|
|
3c7ec33d26 | ||
|
|
09ca4957ee | ||
|
|
7921442a5c | ||
|
|
05ef104208 | ||
|
|
d44620f285 | ||
|
|
5df4f2c036 | ||
|
|
db1e8cdbe3 | ||
|
|
f3ac513d92 | ||
|
|
0054c5b8fe | ||
|
|
6da0a40b02 | ||
|
|
caf3403086 | ||
|
|
617f19ba61 | ||
|
|
ba3318ffbd | ||
|
|
8279d3d6df | ||
|
|
ef4cccb6a0 | ||
|
|
98b950366d | ||
|
|
87ab6c0155 | ||
|
|
0308920bdb | ||
|
|
57db2a4ae9 | ||
|
|
7fd7fd58a5 | ||
|
|
71a72b8fea | ||
|
|
6c9518fcd8 | ||
|
|
3a4ab451ac | ||
|
|
5d38fc96a9 | ||
|
|
261ebbc55b | ||
|
|
53fb9be7ee | ||
|
|
e5be84f795 | ||
|
|
39ceba8a3d | ||
|
|
b9d259441c | ||
|
|
e7dc175e94 | ||
|
|
6612af9e0e | ||
|
|
71c6d8bc55 | ||
|
|
dbc661602e | ||
|
|
054dcee896 | ||
|
|
73a25267ac | ||
|
|
bcad628d30 | ||
|
|
baba18bc7d | ||
|
|
0afcb334a4 | ||
|
|
33e6381b7d | ||
|
|
9c4ff32562 | ||
|
|
7d5759b2d9 | ||
|
|
ac11e9162d | ||
|
|
84fc632088 | ||
|
|
8d9bce30a7 | ||
|
|
5f2ffa70b8 | ||
|
|
f4ef648b16 | ||
|
|
6469cf4197 | ||
|
|
3f8751c6ed | ||
|
|
e4f1310d6f | ||
|
|
a42ab6fce0 | ||
|
|
fa8ad18b5c | ||
|
|
7f1edbba80 | ||
|
|
99c317b131 | ||
|
|
2b2ede8211 | ||
|
|
b4dbcebbec | ||
|
|
30ba5818e7 | ||
|
|
c9bfb24bcc | ||
|
|
3d109d50be | ||
|
|
4eabf5e228 | ||
|
|
7d67d57804 | ||
|
|
491989284f | ||
|
|
53ea8fb2a7 | ||
|
|
efdbfd2cc3 | ||
|
|
8342b89e85 | ||
|
|
c92938d718 | ||
|
|
7578cdd218 | ||
|
|
21f7ce37dd | ||
|
|
236686454c | ||
|
|
1f6ca2cf68 | ||
|
|
bde1fcbf51 | ||
|
|
c4f3e3be33 | ||
|
|
9ef65b51c2 | ||
|
|
6ba34507f1 | ||
|
|
93d610c1fd | ||
|
|
4d0efcc921 | ||
|
|
f71fafa684 | ||
|
|
1068926cd6 | ||
|
|
48fb49baef | ||
|
|
2372cdc06e | ||
|
|
7b815cc98b | ||
|
|
d06c5dde42 | ||
|
|
8d0ee5e51e | ||
|
|
75758cb0a0 | ||
|
|
60e80ac7d1 | ||
|
|
4ef799aaa6 | ||
|
|
1be762368b | ||
|
|
f577b61b52 | ||
|
|
4792b61332 | ||
|
|
62f1ed099e | ||
|
|
bfd9bb2e03 | ||
|
|
6deac260bf | ||
|
|
842ad7c4d4 | ||
|
|
b605d2d370 | ||
|
|
e403cede0a | ||
|
|
735434fcde | ||
|
|
e17fbc9a2d | ||
|
|
5507fff133 | ||
|
|
4f0ca95477 | ||
|
|
b91137ab98 | ||
|
|
67d8368a84 | ||
|
|
db7d2cb7db | ||
|
|
f23179ce99 | ||
|
|
8266769b2d | ||
|
|
3b671d6086 | ||
|
|
7925735a3e | ||
|
|
6833498646 | ||
|
|
b48e2cb1eb | ||
|
|
bfeb7cd9b2 |
@@ -12,7 +12,12 @@ name: build-sglang-image
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [dsv41-pd]
|
||||
branches: [dsv41-pd, dsv41-pd-visioncp]
|
||||
# compose/部署配置 与 workflow 自身的改动不触发镜像构建(省 runner 与推送带宽)。
|
||||
# 改了 workflow 想重建镜像时,需伴随任意代码改动或手动触发。
|
||||
paths-ignore:
|
||||
- 'deploy/**'
|
||||
- '.gitea/**'
|
||||
|
||||
env:
|
||||
# 直接写死 Gitea 自带 registry(vars context 在该实例上求值异常会导致回退 docker.io)
|
||||
@@ -44,7 +49,9 @@ jobs:
|
||||
context: .
|
||||
file: Dockerfile.gitea
|
||||
push: true
|
||||
# cache-from: type=registry
|
||||
# cache-to: type=registry,mode=max
|
||||
tags: |
|
||||
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.tag }}
|
||||
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }}-latest
|
||||
# 注意:type=gha 缓存在本实例的 runner 上会 404(cache server 未配),勿加回
|
||||
|
||||
|
||||
@@ -8,4 +8,17 @@ FROM lmsysorg/sglang:dev-dsv41
|
||||
# checkout 含 .git,setuptools-scm 可打戳(main 系分支 version 显示 dev 属正常)
|
||||
COPY . /sgl-workspace/sglang
|
||||
|
||||
# Rust gateway(sgl-model-gateway)编译,与 b300 离线版 deploy/b300-build/Dockerfile 同构。
|
||||
# 基底自带 rustup stable,RUSTUP_TOOLCHAIN=stable 绕过 rust-toolchain.toml 的 1.90 钉版;
|
||||
# python 扩展必须显式 -p,产物重命名为 sglang_router_rs.abi3.so 覆盖 python 包内同文件。
|
||||
# 海外 runner 直连 crates.io 无网络问题;cache mount 海外环境同样无害。
|
||||
RUN --mount=type=cache,target=/cargo-home \
|
||||
--mount=type=cache,target=/cargo-target \
|
||||
cd /sgl-workspace/sglang/sgl-model-gateway \
|
||||
&& CARGO_HOME=/cargo-home CARGO_TARGET_DIR=/cargo-target RUSTUP_TOOLCHAIN=stable \
|
||||
cargo build --release -p sgl-model-gateway -p sgl-model-gateway-python \
|
||||
&& cp /cargo-target/release/sgl-model-gateway /opt/sglang/bin/sgl-model-gateway \
|
||||
&& cp /cargo-target/release/libsglang_router_rs.so \
|
||||
/opt/sglang/lib/python3.12/site-packages/sglang_router/sglang_router_rs.abi3.so
|
||||
|
||||
RUN pip install --no-cache-dir -e /sgl-workspace/sglang/python
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
# 优化版(2026-10-08 v3):torch/nvidia 全家桶永远复用基底,全量路径零重下。
|
||||
# 要点:
|
||||
# 1. pyproject build-system.requires 含 torch==2.13.0,pip 隔离构建环境每次都重下 ~6GB。
|
||||
# 基底镜像已装 torch 2.13 → 用 --no-build-isolation 复用,构建环境零下载。
|
||||
# 2. 老全量路径让 pip 解析全部依赖:bare `torchvision` 等会让解析器弃用基底已装的
|
||||
# 本地版本(torch 2.13.0+cu130),改用 PyPI 的 torch==2.13.0,连带重下整套 nvidia
|
||||
# 依赖(9/23 实测 nvidia-nccl-cu13 等 6GB+,限流下 2h40m);且基底 NCCL 是
|
||||
# 刻意装高的 2.30.7(torch 元数据钉 ==2.29.7),任何全量解析都会把它降级回去。
|
||||
# → v3 不再让 pip 碰重型依赖:check_deps.py 在构建期对照 pyproject 与基底
|
||||
# 已装版本,把增量分三类——重型(torch*/nvidia-*/cuda-*)永不进解析、
|
||||
# 不满足即让构建失败并提示决策(显式失败好过静默重装);
|
||||
# kernel wheel(sglang-kernel/flashinfer*/sgl-deep-gemm)要跟随源码版本,
|
||||
# 钉版变动时以 pip --no-deps 单独装(防止顺藤摸瓜解析 torch);
|
||||
# 其余轻量依赖只装未满足的增量。
|
||||
# 3. 基底缺 setuptools-scm(版本打戳要用),单层预装 + pip cache mount,只下载一次。
|
||||
# 4. RUN 的 pip cache mount 持久化在宿主机,轻量依赖增量不再重复下载。
|
||||
FROM uhub.service.ucloud.cn/umirror/sglang:dev-dsv41
|
||||
|
||||
ARG PIP_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
ENV PIP_INDEX_URL=${PIP_INDEX}
|
||||
|
||||
# 构建后端(pyproject build-system.requires,torch 除外——基底已有)
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install "setuptools>=61" "setuptools-scm>=8" "setuptools-rust>=1.11" wheel
|
||||
|
||||
# 依赖增量计算脚本(对照 pyproject 与基底已装版本,产出轻量/kernel 增量清单)
|
||||
# 注意 build context 是 /data/ymk 且 .dockerignore 排除了 build/,只能从 sglang/ 树内 COPY
|
||||
COPY sglang/deploy/b300-build/check_deps.py /tmp/check_deps.py
|
||||
|
||||
# 整个源码树(含 .git,用于 setuptools-scm 版本打戳)
|
||||
COPY sglang/ /sgl-workspace/sglang/
|
||||
|
||||
# Rust gateway(sgl-model-gateway)编译。基底自带 rustup stable(实测 cargo 1.98.1),
|
||||
# RUSTUP_TOOLCHAIN=stable 绕过仓库 rust-toolchain.toml 钉的 1.90(避免重复下载工具链);
|
||||
# b300 直连 crates.io(稀疏索引 + static.crates.io 均通),无需任何镜像源。
|
||||
# cache mount 持久化 registry/target:冷编译 ~15min(二进制 ~11min + python 扩展 ~4min),
|
||||
# 增量重建秒~分钟级。python 扩展必须显式 -p(workspace 根包默认不编译扩展),
|
||||
# 产物 libsglang_router_rs.so 需重命名为 sglang_router_rs.abi3.so 覆盖 python 包内同文件
|
||||
# (与 H20 线上部署方式一致)。2026-10-09 b300 实测验证。
|
||||
RUN --mount=type=cache,target=/cargo-home \
|
||||
--mount=type=cache,target=/cargo-target \
|
||||
cd /sgl-workspace/sglang/sgl-model-gateway \
|
||||
&& CARGO_HOME=/cargo-home CARGO_TARGET_DIR=/cargo-target RUSTUP_TOOLCHAIN=stable \
|
||||
cargo build --release -p sgl-model-gateway -p sgl-model-gateway-python \
|
||||
&& cp /cargo-target/release/sgl-model-gateway /opt/sglang/bin/sgl-model-gateway \
|
||||
&& cp /cargo-target/release/libsglang_router_rs.so \
|
||||
/opt/sglang/lib/python3.12/site-packages/sglang_router/sglang_router_rs.abi3.so
|
||||
|
||||
# 计算依赖增量(重型 torch/nvidia/cuda 永不进入清单;不满足则 check_deps 非零、构建失败)
|
||||
RUN python /tmp/check_deps.py /sgl-workspace/sglang/python/pyproject.toml \
|
||||
/tmp/missing_light.txt /tmp/missing_kernel.txt \
|
||||
&& echo "== kernel wheel delta (pip --no-deps) ==" && cat /tmp/missing_kernel.txt \
|
||||
&& echo "== lightweight dep delta ==" && cat /tmp/missing_light.txt
|
||||
|
||||
# kernel wheel 增量(如 sglang-kernel 升版):--no-deps 防止 pip 顺藤摸瓜解析 torch
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
if grep -q '[^[:space:]]' /tmp/missing_kernel.txt; then \
|
||||
pip install --no-deps -r /tmp/missing_kernel.txt; \
|
||||
else \
|
||||
echo "== no kernel wheel delta"; \
|
||||
fi
|
||||
|
||||
# 轻量依赖增量(通常为空或几个小包),重型全家桶原样复用基底。
|
||||
# ⚠️ 必须 --no-deps:即使目标只是 xgrammar 这种小包,只要依赖图里出现 torch,
|
||||
# pip 就会校验 torch+cu130 元数据钉死的 nvidia-nccl-cu13==2.29.7,发现基底装的是
|
||||
# 2.30.7 便"好心"下载 206MB 降级(10-08 v3 首跑实测踩中)。基底的传递依赖本来就
|
||||
# 齐全,增量小包的二级依赖也一样;若有漏网,b300 冒烟会在 import 阶段显式暴露。
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
if grep -q '[^[:space:]]' /tmp/missing_light.txt; then \
|
||||
pip install --no-deps -r /tmp/missing_light.txt; \
|
||||
else \
|
||||
echo "== no lightweight delta, skipping dependency install"; \
|
||||
fi
|
||||
|
||||
# editable 安装 sglang 本体,--no-deps 保证 pip 完全不碰依赖图
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install --no-deps --no-build-isolation -e /sgl-workspace/sglang/python
|
||||
@@ -0,0 +1,65 @@
|
||||
# fast 路径(2026-10-08 v3.1):python/pyproject.toml 未变化时使用(build.sh 自动判断)。
|
||||
# 与 Dockerfile 全量路径同构:同样跑 check_deps.py 增量安装。
|
||||
# 原因:全量构建若装过 kernel wheel 增量(如 sglang-kernel 0.4.7),随后 pyproject 未变的
|
||||
# fast 构建若只 --no-deps,会静默退回基底的旧 kernel(0.4.6.post1)——镜像内容漂移。
|
||||
# check_deps 本地比对 ~0.5s,增量命中 pip cache 秒级,成本可忽略。
|
||||
FROM uhub.service.ucloud.cn/umirror/sglang:dev-dsv41
|
||||
|
||||
ARG PIP_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
ENV PIP_INDEX_URL=${PIP_INDEX}
|
||||
|
||||
# 构建后端(pyproject build-system.requires,torch 除外——基底已有)
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install "setuptools>=61" "setuptools-scm>=8" "setuptools-rust>=1.11" wheel
|
||||
|
||||
# 依赖增量计算脚本(对照 pyproject 与基底已装版本,产出轻量/kernel 增量清单)
|
||||
# 注意 build context 是 /data/ymk 且 .dockerignore 排除了 build/,只能从 sglang/ 树内 COPY
|
||||
COPY sglang/deploy/b300-build/check_deps.py /tmp/check_deps.py
|
||||
|
||||
# 整个源码树(含 .git,用于 setuptools-scm 版本打戳)
|
||||
COPY sglang/ /sgl-workspace/sglang/
|
||||
|
||||
# Rust gateway(sgl-model-gateway)编译。基底自带 rustup stable(实测 cargo 1.98.1),
|
||||
# RUSTUP_TOOLCHAIN=stable 绕过仓库 rust-toolchain.toml 钉的 1.90(避免重复下载工具链);
|
||||
# b300 直连 crates.io(稀疏索引 + static.crates.io 均通),无需任何镜像源。
|
||||
# cache mount 持久化 registry/target:冷编译 ~15min,增量重建秒~分钟级。
|
||||
# python 扩展必须显式 -p(workspace 根包默认不编译扩展),产物 libsglang_router_rs.so
|
||||
# 需重命名为 sglang_router_rs.abi3.so 覆盖 python 包内同文件(与 H20 线上部署方式一致)。
|
||||
RUN --mount=type=cache,target=/cargo-home \
|
||||
--mount=type=cache,target=/cargo-target \
|
||||
cd /sgl-workspace/sglang/sgl-model-gateway \
|
||||
&& CARGO_HOME=/cargo-home CARGO_TARGET_DIR=/cargo-target RUSTUP_TOOLCHAIN=stable \
|
||||
cargo build --release -p sgl-model-gateway -p sgl-model-gateway-python \
|
||||
&& cp /cargo-target/release/sgl-model-gateway /opt/sglang/bin/sgl-model-gateway \
|
||||
&& cp /cargo-target/release/libsglang_router_rs.so \
|
||||
/opt/sglang/lib/python3.12/site-packages/sglang_router/sglang_router_rs.abi3.so
|
||||
|
||||
# 计算依赖增量(重型 torch/nvidia/cuda 永不进入清单;不满足则 check_deps 非零、构建失败)
|
||||
RUN python /tmp/check_deps.py /sgl-workspace/sglang/python/pyproject.toml \
|
||||
/tmp/missing_light.txt /tmp/missing_kernel.txt \
|
||||
&& echo "== kernel wheel delta (pip --no-deps) ==" && cat /tmp/missing_kernel.txt \
|
||||
&& echo "== lightweight dep delta ==" && cat /tmp/missing_light.txt
|
||||
|
||||
# kernel wheel 增量(如 sglang-kernel 升版):--no-deps 防止 pip 顺藤摸瓜解析 torch
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
if grep -q '[^[:space:]]' /tmp/missing_kernel.txt; then \
|
||||
pip install --no-deps -r /tmp/missing_kernel.txt; \
|
||||
else \
|
||||
echo "== no kernel wheel delta"; \
|
||||
fi
|
||||
|
||||
# 轻量依赖增量(通常为空或几个小包),重型全家桶原样复用基底。
|
||||
# ⚠️ 必须 --no-deps:即使目标只是 xgrammar 这种小包,只要依赖图里出现 torch,
|
||||
# pip 就会校验 torch+cu130 元数据钉死的 nvidia-nccl-cu13==2.29.7,发现基底装的是
|
||||
# 2.30.7 便"好心"下载 206MB 降级(10-08 v3 首跑实测踩中)。基底的传递依赖本来就
|
||||
# 齐全,增量小包的二级依赖也一样;若有漏网,b300 冒烟会在 import 阶段显式暴露。
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
if grep -q '[^[:space:]]' /tmp/missing_light.txt; then \
|
||||
pip install --no-deps -r /tmp/missing_light.txt; \
|
||||
else \
|
||||
echo "== no lightweight delta, skipping dependency install"; \
|
||||
fi
|
||||
|
||||
# editable 安装 sglang 本体,--no-deps 保证 pip 完全不碰依赖图
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install --no-deps --no-build-isolation -e /sgl-workspace/sglang/python
|
||||
@@ -0,0 +1,33 @@
|
||||
#!/bin/bash
|
||||
# 用法: /data/ymk/build/build.sh [tag]
|
||||
# 默认 tag: ymkymx/sglang:<分支>-<sha9>-local-<UTC构建日期时间>(命名规则见 AGENTS.md)
|
||||
# 另打 <分支>-latest-local 别名。
|
||||
#
|
||||
# 依赖路径自动选择:
|
||||
# python/pyproject.toml 的 sha256 与上次成功构建一致 → Dockerfile.fast(--no-deps,秒级)
|
||||
# 不一致(或 FULL=1)→ Dockerfile 全量依赖解析,成功后记录新 hash
|
||||
set -euo pipefail
|
||||
cd /data/ymk
|
||||
SHA=$(git -C sglang rev-parse --short=9 HEAD)
|
||||
BR=$(git -C sglang rev-parse --abbrev-ref HEAD)
|
||||
DT=$(date -u +%Y%m%d-%H%M)
|
||||
TAG=${1:-ymkymx/sglang:$BR-$SHA-local-$DT}
|
||||
|
||||
HASH_FILE=build/.last_pyproject_sha256
|
||||
CUR=$(sha256sum sglang/python/pyproject.toml | cut -d' ' -f1)
|
||||
PREV=$(cat "$HASH_FILE" 2>/dev/null || echo none)
|
||||
|
||||
DOCKERFILE=build/Dockerfile
|
||||
if [ "$CUR" = "$PREV" ] && [ "${FULL:-0}" != "1" ]; then
|
||||
DOCKERFILE=build/Dockerfile.fast
|
||||
echo "== pyproject 未变,走 fast 路径(--no-deps)"
|
||||
else
|
||||
echo "== pyproject 有变化或 FULL=1,走全量依赖解析"
|
||||
fi
|
||||
|
||||
echo "== building $TAG from sglang@$(git -C sglang log --oneline -1) [$DOCKERFILE]"
|
||||
docker build -f "$DOCKERFILE" -t "$TAG" .
|
||||
# 构建成功才记录 hash / 打别名
|
||||
echo "$CUR" > "$HASH_FILE"
|
||||
docker tag "$TAG" "ymkymx/sglang:$BR-latest-local"
|
||||
echo "== done: $TAG (别名 $BR-latest-local)"
|
||||
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compute dependency deltas for a b300 sglang image build, in three classes.
|
||||
|
||||
Reads pyproject.toml, evaluates every requirement against the *installed*
|
||||
distributions in the base image, and writes two requirements files:
|
||||
|
||||
argv[2] (light): unsatisfied lightweight requirements → `pip install -r`
|
||||
argv[3] (kernel): unsatisfied kernel wheels (sglang-kernel, flashinfer*,
|
||||
sgl-deep-gemm) → `pip install --no-deps -r`
|
||||
These track the sglang source being built and are meant
|
||||
to be upgraded from the index when pyproject bumps them;
|
||||
--no-deps keeps pip from walking torch's metadata.
|
||||
|
||||
The PyTorch stack (torch/torchvision/torchaudio/torchcodec/
|
||||
torch_memory_saver), the nvidia-* CUDA family and cuda-* packages are NEVER
|
||||
emitted. If one of them is missing or its installed version no longer
|
||||
satisfies the pinned specifier (e.g. upstream bumps nvidia-cutlass-dsl), the
|
||||
script fails the build loudly instead of letting pip download ~6GB or
|
||||
"fix" the base image's deliberately newer nvidia-nccl-cu13 2.30.7 back to
|
||||
torch's metadata pin ==2.29.7 (any pip full-resolution would do both —
|
||||
2026-09-23/24 builds did, throttled to 2h40m).
|
||||
"""
|
||||
|
||||
import sys
|
||||
import tomllib
|
||||
from importlib import metadata as md
|
||||
|
||||
from packaging.requirements import Requirement
|
||||
|
||||
# Never resolved by pip: reused from the base image or the build fails.
|
||||
HEAVY_PREFIXES = ("torch", "nvidia-", "cuda-")
|
||||
# Kernel wheels: upgraded via pip --no-deps when pyproject pins move.
|
||||
KERNEL_PREFIXES = ("sglang-kernel", "flashinfer", "sgl-deep-gemm")
|
||||
|
||||
|
||||
def norm(name: str) -> str:
|
||||
return name.lower().replace("_", "-")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
pyproject, light_out, kernel_out = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
with open(pyproject, "rb") as f:
|
||||
deps = tomllib.load(f)["project"]["dependencies"]
|
||||
|
||||
light: list[str] = []
|
||||
kernel: list[str] = []
|
||||
heavy_unsat: list[str] = []
|
||||
for dep in deps:
|
||||
req = Requirement(dep)
|
||||
if req.marker is not None and not req.marker.evaluate():
|
||||
continue
|
||||
try:
|
||||
installed = md.version(req.name)
|
||||
except md.PackageNotFoundError:
|
||||
installed = None
|
||||
satisfied = installed is not None and req.specifier.contains(
|
||||
installed, prereleases=True
|
||||
)
|
||||
if satisfied:
|
||||
continue
|
||||
name = norm(req.name)
|
||||
if name.startswith(HEAVY_PREFIXES):
|
||||
heavy_unsat.append(f"{dep} (installed: {installed})")
|
||||
elif name.startswith(KERNEL_PREFIXES):
|
||||
kernel.append(dep)
|
||||
else:
|
||||
light.append(dep)
|
||||
|
||||
if heavy_unsat:
|
||||
print("=" * 70, file=sys.stderr)
|
||||
print("FATAL: heavy base-image deps no longer satisfy pyproject:", file=sys.stderr)
|
||||
for item in heavy_unsat:
|
||||
print(f" {item}", file=sys.stderr)
|
||||
print(
|
||||
"Refusing to let pip resolve these (would re-download ~6GB and/or\n"
|
||||
"downgrade NCCL). Decide explicitly: bump the base image, or adjust\n"
|
||||
"the pin in pyproject.toml.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
with open(light_out, "w") as f:
|
||||
f.write("\n".join(light) + ("\n" if light else ""))
|
||||
with open(kernel_out, "w") as f:
|
||||
f.write("\n".join(kernel) + ("\n" if kernel else ""))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,33 @@
|
||||
# B300 ds41 部署 compose 档案
|
||||
|
||||
b300-01 上 DeepSeek-V4.1-Flash 各部署方案的 docker compose 存档。
|
||||
**本目录是 source of truth**;b300 上的运行目录 `/data/ymk/ds41/` 是工作副本。
|
||||
|
||||
方案说明、参数矩阵、特殊 env、已知坑见飞书部署手册(组内共享):
|
||||
《B300 sglang 部署手册》 https://u04wb5irxz.feishu.cn/docx/FOi8dbnybodHr0xIfkPcQ6gtnRx
|
||||
|
||||
## 同步流程
|
||||
|
||||
```bash
|
||||
# 本机(D:\B300\sglang)
|
||||
git add deploy/ && git commit -m "deploy: ..."
|
||||
git push origin dsv41-pd-visioncp # Gitea(paths-ignore,不触发镜像构建)
|
||||
git push b300 dsv41-pd-visioncp # b300 裸仓库
|
||||
|
||||
# b300-01
|
||||
git -C /data/ymk/sglang pull # deploy/ 出现在 /data/ymk/sglang/deploy/b300-ds41/
|
||||
```
|
||||
|
||||
## CI 说明
|
||||
|
||||
`.gitea/workflows/build-image.yaml` 的 push 触发器带 `paths-ignore: ['deploy/**', '.gitea/**']`,
|
||||
改本目录不会触发镜像构建。注意:workflow 自身的改动也不再自动触发,需要重建镜像时
|
||||
伴随代码改动 push 或手动触发。
|
||||
|
||||
## 目录规则
|
||||
|
||||
- 命名:`dockerserve-<拓扑>-<特性>-<卡位>.yml`(如 `-b4` = 后 4 卡)
|
||||
- 退役文件不要留在主目录:移入 `archive-YYYYMMDD/`(b300 侧同样执行)
|
||||
- 禁止提交 `.bak` 备份文件
|
||||
- `mooncake-store.json` 是 hicache L3 mooncake store 配置,被
|
||||
`dockerserve-pd2-p-r1/r2.yml` 通过 `SGLANG_HICACHE_MOONCAKE_CONFIG_PATH` 引用
|
||||
@@ -0,0 +1,27 @@
|
||||
cd /data/ymk/ds41
|
||||
HDR="Authorization=Bearer deepseek-v4-flash-0731-key"
|
||||
PORT=30040
|
||||
MODEL=/data/models/DeepSeek-V4.1-Flash
|
||||
|
||||
# 一律 --warmup-requests 0:默认 warmup 会把 prompt#0 种进 L1/L2 污染 L3 测量(已踩过)
|
||||
run() {
|
||||
echo "===== [$(date +%H:%M:%S)] RUN: $1 (np=$2) ====="
|
||||
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--warmup-requests 0 --num-prompts "$2" --max-concurrency "$2" --seed "$3" --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR" 2>&1 | grep -E 'Input token throughput|Mean TTFT|P99 TTFT|Successful'
|
||||
sleep 2
|
||||
}
|
||||
flush() { curl -sf -m 10 -X POST http://127.0.0.1:$PORT/flush_cache > /dev/null && echo "[$(date +%H:%M:%S)] flushed"; sleep 3; }
|
||||
|
||||
echo "===== PRIME COLD mc8(冷基线 + 写入 L3) ====="
|
||||
run PRIME_COLD 8 1541391
|
||||
flush
|
||||
run MC1_L3 1 1541391
|
||||
flush
|
||||
run MC2_L3 2 1541391
|
||||
flush
|
||||
run MC4_L3 4 1541391
|
||||
flush
|
||||
run MC8_L3 8 1541391
|
||||
echo "===== [$(date +%H:%M:%S)] ALL DONE ====="
|
||||
@@ -0,0 +1,204 @@
|
||||
#!/bin/bash
|
||||
# bench_suite_v2.sh — DeepSeek-V4.1-Flash on b300-01,2026-10-09 新版基线方法
|
||||
#
|
||||
# 相对 v1 的方法变化:
|
||||
# 1. 服务刚起数据不准 → 每个测试模式自带 warmup 轮(结果丢弃,标记 WARM)
|
||||
# 2. prefill 冷/热双轮:冷轮写入 hicache(L1/L3),同 seed 热轮测纯 cache-hit TPS
|
||||
# 3. decode 阶梯并发:步长 32,上限 = cuda-graph-max-bs-decode 的 80%;
|
||||
# 每档先 warm(灌 L1)再测,prefill 全 cache hit;
|
||||
# 报告 TPOT + 整实例输出 TPS + 推算实际并发(= 输出TPS × TPOT / 1000)
|
||||
# 4. l3 模式(3p1d 经 router):大语料溢出 L1 后重放,验证 L3 命中
|
||||
#
|
||||
# 在宿主机执行,bench 客户端通过 docker exec 进容器跑(需要 /data 挂载)。
|
||||
#
|
||||
# 用法:
|
||||
# bench_suite_v2.sh warmup <container> <port>
|
||||
# bench_suite_v2.sh prefill <tag> <container> <port>
|
||||
# bench_suite_v2.sh decode <tag> <container> <port> [cg_max_bs=256]
|
||||
# bench_suite_v2.sh l3 <tag> <container> <port> [num_prompts=24]
|
||||
# bench_suite_v2.sh report <log>
|
||||
# 产物: /data/ymk/ds41/bench_v2_<tag>.log
|
||||
set -u
|
||||
|
||||
BASE=/data/ymk/ds41
|
||||
MODEL=${MODEL:-/data/models/DeepSeek-V4.1-Flash}
|
||||
HDR="Authorization=Bearer deepseek-v4-flash-0731-key"
|
||||
|
||||
usage() { sed -n 2,20p "$0" >&2; exit 1; }
|
||||
|
||||
# $1=container $2=port $3=tag -> 生成可执行脚本路径并 echo
|
||||
emit_common() {
|
||||
cat <<EOF
|
||||
cd $BASE
|
||||
HDR="$HDR"
|
||||
PORT=$2
|
||||
run() {
|
||||
local name="\$1"; shift
|
||||
echo "===== [\$(date +%H:%M:%S)] START: \$name ====="
|
||||
python3 -m sglang.bench_serving "\$@"
|
||||
echo "===== [\$(date +%H:%M:%S)] END(exit=\$?): \$name ====="
|
||||
sleep 5
|
||||
}
|
||||
EOF
|
||||
}
|
||||
|
||||
# ---------------- warmup ----------------
|
||||
cmd_warmup() {
|
||||
local c=${1:?container} port=${2:?port}
|
||||
local gen=$BASE/.bench_v2_warmup.sh
|
||||
{ emit_common "$c" "$port"; cat <<'EOF'
|
||||
run WARMUP_a --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
|
||||
--dataset-name random-ids --random-input-len 4096 --random-output-len 16 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 777001 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
run WARMUP_b --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
|
||||
--dataset-name random-ids --random-input-len 16000 --random-output-len 1 \
|
||||
--num-prompts 2 --max-concurrency 1 --seed 777002 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
echo "===== [$(date +%H:%M:%S)] ALL DONE ====="
|
||||
EOF
|
||||
} | sed "s|MODELPH|$MODEL|g" > "$gen"
|
||||
chmod +x "$gen"
|
||||
docker exec "$c" bash "$gen"
|
||||
}
|
||||
|
||||
# ---------------- prefill(冷/热双轮) ----------------
|
||||
cmd_prefill() {
|
||||
local tag=${1:?tag} c=${2:?container} port=${3:?port}
|
||||
local p=$(date +%H%M%S) gen=$BASE/.bench_v2_gen_${tag}.sh
|
||||
{ emit_common "$c" "$port"
|
||||
# warmup(丢弃)
|
||||
cat <<'EOF'
|
||||
run WARMUP --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
|
||||
--dataset-name random-ids --random-input-len 4096 --random-output-len 16 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 777000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
EOF
|
||||
# 冷轮
|
||||
for stage in "bs1_16000 16000 1 4 1" "bs1_160000 160000 1 4 2" "bs16_4096 4096 16 32 3"; do
|
||||
set -- $stage
|
||||
cat <<EOF
|
||||
run COLD_$1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len $2 --random-output-len 1 \\
|
||||
--num-prompts $4 --max-concurrency $3 --seed ${p}$5 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
done
|
||||
# 热轮(同 seed → cache hit)
|
||||
for stage in "bs1_16000 16000 1 4 1" "bs1_160000 160000 1 4 2" "bs16_4096 4096 16 32 3"; do
|
||||
set -- $stage
|
||||
cat <<EOF
|
||||
run HIT_$1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len $2 --random-output-len 1 \\
|
||||
--num-prompts $4 --max-concurrency $3 --seed ${p}$5 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
done
|
||||
echo 'echo "===== [$(date +%H:%M:%S)] ALL DONE ====="'
|
||||
} | sed "s|MODELPH|$MODEL|g" > "$gen"
|
||||
chmod +x "$gen"
|
||||
nohup docker exec "$c" bash "$gen" > "$BASE/bench_v2_${tag}.log" 2>&1 &
|
||||
echo "launched pid=$! seeds=${p}1..3 log=$BASE/bench_v2_${tag}.log"
|
||||
}
|
||||
|
||||
# ---------------- decode 阶梯 ----------------
|
||||
cmd_decode() {
|
||||
local tag=${1:?tag} c=${2:?container} port=${3:?port} cg=${4:-256}
|
||||
local maxc=$(( cg * 80 / 100 / 32 * 32 )) # 80% 向下取整到 32 的倍数
|
||||
local p=$(date +%H%M%S) gen=$BASE/.bench_v2_gen_${tag}.sh
|
||||
echo "ladder: 32..$maxc step 32 (cg_max_bs=$cg)"
|
||||
{ emit_common "$c" "$port"
|
||||
cat <<'EOF'
|
||||
run WARMUP --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 128 \
|
||||
--num-prompts 16 --max-concurrency 16 --seed 777000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
EOF
|
||||
# bs1 延迟地板(warm + measure)
|
||||
cat <<EOF
|
||||
run WARM_bs1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
|
||||
--num-prompts 4 --max-concurrency 1 --seed ${p}01 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
run DEC_bs1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
|
||||
--num-prompts 4 --max-concurrency 1 --seed ${p}01 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
local cc
|
||||
for (( cc=32; cc<=maxc; cc+=32 )); do
|
||||
local np=$(( cc * 2 ))
|
||||
cat <<EOF
|
||||
run WARM_c${cc} --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
|
||||
--num-prompts $np --max-concurrency $cc --seed ${p}1${cc} --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
run DEC_c${cc} --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
|
||||
--num-prompts $np --max-concurrency $cc --seed ${p}1${cc} --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
done
|
||||
echo 'echo "===== [$(date +%H:%M:%S)] ALL DONE ====="'
|
||||
} | sed "s|MODELPH|$MODEL|g" > "$gen"
|
||||
chmod +x "$gen"
|
||||
nohup docker exec "$c" bash "$gen" > "$BASE/bench_v2_${tag}.log" 2>&1 &
|
||||
echo "launched pid=$! log=$BASE/bench_v2_${tag}.log"
|
||||
}
|
||||
|
||||
# ---------------- L3 语料溢出(3p1d 经 router) ----------------
|
||||
cmd_l3() {
|
||||
local tag=${1:?tag} c=${2:?container} port=${3:?port} np=${4:-48}
|
||||
local p=$(date +%H%M%S) gen=$BASE/.bench_v2_gen_${tag}.sh
|
||||
{ emit_common "$c" "$port"
|
||||
cat <<EOF
|
||||
run WARMUP --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 4096 --random-output-len 16 \\
|
||||
--num-prompts 8 --max-concurrency 8 --seed 777000 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
run L3_COLD_160k --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \\
|
||||
--num-prompts $np --max-concurrency 8 --seed ${p}1 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
echo "===== [\$(date +%H:%M:%S)] DRAIN: sleep 120 等 L2→L3 备份排空 ====="
|
||||
sleep 120
|
||||
run L3_HIT_160k --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \\
|
||||
--num-prompts $np --max-concurrency 8 --seed ${p}1 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
echo 'echo "===== [$(date +%H:%M:%S)] ALL DONE ====="'
|
||||
} > "$gen"
|
||||
chmod +x "$gen"
|
||||
nohup docker exec "$c" bash "$gen" > "$BASE/bench_v2_${tag}.log" 2>&1 &
|
||||
echo "launched pid=$! corpus=${np}x160k log=$BASE/bench_v2_${tag}.log"
|
||||
}
|
||||
|
||||
# ---------------- report ----------------
|
||||
cmd_report() {
|
||||
local log=${1:?log required}
|
||||
awk '
|
||||
/START:/ { name=$4; sub(/:$/, "", name); tps=""; itps=""; tpot=""; ttft=""; acc="" }
|
||||
/Input token throughput/ { itps=$NF }
|
||||
/Output token throughput/ { tps=$NF }
|
||||
/Mean TPOT/ { tpot=$NF }
|
||||
/Mean TTFT/ { ttft=$NF }
|
||||
/[Aa]ccept/ { acc=$0; gsub(/^ +/, "", acc) }
|
||||
/END\(exit/ {
|
||||
line=sprintf("%-22s in_tps=%-10s out_tps=%-10s tpot_ms=%-8s ttft_ms=%-8s", name, itps, tps, tpot, ttft)
|
||||
if (tps != "" && tpot != "" && tpot+0 > 0)
|
||||
line=line sprintf(" eff_concurrency=%.1f", tps * tpot / 1000)
|
||||
print line
|
||||
if (acc != "") print " " acc
|
||||
}
|
||||
' "$log"
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
warmup) shift; cmd_warmup "$@" ;;
|
||||
prefill) shift; cmd_prefill "$@" ;;
|
||||
decode) shift; cmd_decode "$@" ;;
|
||||
l3) shift; cmd_l3 "$@" ;;
|
||||
report) shift; cmd_report "$@" ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
@@ -0,0 +1,61 @@
|
||||
cd /data/ymk/ds41
|
||||
HDR="Authorization=Bearer deepseek-v4-flash-0731-key"
|
||||
PORT=30040
|
||||
MODEL=/data/models/DeepSeek-V4.1-Flash
|
||||
|
||||
run() {
|
||||
local name="$1"; shift
|
||||
echo "===== [$(date +%H:%M:%S)] START: $name ====="
|
||||
python3 -m sglang.bench_serving "$@" 2>&1 | grep -vE '^Namespace|warn'
|
||||
echo "===== [$(date +%H:%M:%S)] END(exit=$?): $name ====="
|
||||
sleep 3
|
||||
}
|
||||
|
||||
r160() { # $1=name $2=seed $3=num_prompts $4=mc
|
||||
run "$1" --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--num-prompts "$3" --max-concurrency "$4" --seed "$2" --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
}
|
||||
|
||||
# WARMUP
|
||||
run WARMUP --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 4096 --random-output-len 16 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 777000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
|
||||
# R0 纯计算极限(集合 A 首次出现,L1/L2/L3 全空)
|
||||
r160 R0_COMPUTE_COLD 1541391 8 8
|
||||
# R1 L1 极限(A 全量在 L1 设备池:1.28M < 1.6M)
|
||||
r160 R1_L1_HIT 1541391 8 8
|
||||
# R2 填充集 B(12×160k=1.92M > L1 1.6M),把 A 挤出 L1 到 L2
|
||||
r160 R2_FILLER_EVICT 999000 12 8
|
||||
# R3 L2 极限(A 已被挤出 L1,从 L2 host 池 load-back)
|
||||
r160 R3_L2_HIT 1541391 8 8
|
||||
# R4 flush L1+L2(L3 保留),走 L3 prefetch
|
||||
echo "===== [$(date +%H:%M:%S)] FLUSH L1+L2 ====="
|
||||
curl -sf -m 10 -X POST http://127.0.0.1:$PORT/flush_cache && echo "flush ok"
|
||||
sleep 5
|
||||
r160 R4_L3_HIT 1541391 8 8
|
||||
# R5 嫁接后 L2 复核
|
||||
r160 R5_L2_AFTER_GRAFT 1541391 8 8
|
||||
|
||||
# R6 混合负载:新集合 C(纯计算,4×160k mc4)+ 集合 A(命中,8×160k mc8)并发打
|
||||
echo "===== [$(date +%H:%M:%S)] START: R6_MIX_PARALLEL ====="
|
||||
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--num-prompts 4 --max-concurrency 4 --seed 555000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR" > /data/ymk/ds41/.tier_r6_coldclient.log 2>&1 &
|
||||
COLD_PID=$!
|
||||
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 1541391 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR" > /data/ymk/ds41/.tier_r6_hitclient.log 2>&1 &
|
||||
HIT_PID=$!
|
||||
wait $COLD_PID $HIT_PID
|
||||
echo "===== [$(date +%H:%M:%S)] END: R6_MIX_PARALLEL ====="
|
||||
echo "--- R6 cold client (compute) ---"
|
||||
grep -E 'Input token throughput|Mean TTFT|Request throughput' /data/ymk/ds41/.tier_r6_coldclient.log
|
||||
echo "--- R6 hit client (cache) ---"
|
||||
grep -E 'Input token throughput|Mean TTFT|Request throughput' /data/ymk/ds41/.tier_r6_hitclient.log
|
||||
echo "===== [$(date +%H:%M:%S)] ALL DONE ====="
|
||||
@@ -0,0 +1,27 @@
|
||||
cd /data/ymk/ds41
|
||||
HDR="Authorization=Bearer deepseek-v4-flash-0731-key"
|
||||
PORT=30040
|
||||
MODEL=/data/models/DeepSeek-V4.1-Flash
|
||||
|
||||
echo "===== [$(date +%H:%M:%S)] FLUSH L1+L2(A 只剩 L3) ====="
|
||||
curl -sf -m 10 -X POST http://127.0.0.1:$PORT/flush_cache && echo "flush ok"
|
||||
sleep 5
|
||||
|
||||
echo "===== [$(date +%H:%M:%S)] START: R7_MIX_L3_PLUS_COMPUTE ====="
|
||||
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--num-prompts 4 --max-concurrency 4 --seed 666000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR" > /data/ymk/ds41/.tier_r7_coldclient.log 2>&1 &
|
||||
COLD_PID=$!
|
||||
python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port $PORT --model $MODEL \
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 1541391 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR" > /data/ymk/ds41/.tier_r7_hitclient.log 2>&1 &
|
||||
HIT_PID=$!
|
||||
wait $COLD_PID $HIT_PID
|
||||
echo "===== [$(date +%H:%M:%S)] END: R7 ====="
|
||||
echo "--- R7 cold client (compute, 4x160k mc4) ---"
|
||||
grep -E 'Input token throughput|Mean TTFT|Request throughput' /data/ymk/ds41/.tier_r7_coldclient.log
|
||||
echo "--- R7 hit client (L3, 8x160k mc8) ---"
|
||||
grep -E 'Input token throughput|Mean TTFT|Request throughput' /data/ymk/ds41/.tier_r7_hitclient.log
|
||||
echo "===== [$(date +%H:%M:%S)] ALL DONE ====="
|
||||
@@ -0,0 +1,172 @@
|
||||
# 3x cp2-P + 1x dp2-D PD 分离 + L3 mooncake 互联缓存 + engram host table
|
||||
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-cp2-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-cp2-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-cp2-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-cp2-d
|
||||
environment:
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
@@ -0,0 +1,225 @@
|
||||
# 【L3 dummy client 变体 2026-10-09 深夜,定型=shm+best_effort】sglang→standalone_storage,数据面经本机 mooncake_client(50052 IPC+进程映射),不碰 TCP 传输引擎
|
||||
# 实测 24x160k 热/冷:shm+best_effort +26.4%(唯一正收益);wait_complete 因 all-or-nothing discard 等待白等反而 -17.7%。TCP 传输引擎两组皆负。
|
||||
# 【L3 cache-hit 测试变体 2026-10-09】基于 dockerserve-3p1d-otlp.yml,仅改 P:
|
||||
# --max-total-tokens 800000(L1 缩到 ~0.8M token)+ --hicache-ratio 1.0(L2=L1=0.8M token。⚠️ 教训:ratio 0.05 时 L2 太小,build_storage_backup_spec 拿不到 host 页会静默跳过 L3 写入,热轮无数据可中;V4 hicache 不支持 --hicache-size)
|
||||
# 使 ~2.6M token/P 的语料(48×160k)溢出 L1+L2(1.6M),热轮强制走 L3 mooncake 命中。
|
||||
# 【2026-10-10 L2 放大】hicache-ratio 1.0→4.0(L2=3.2M token/P,宿主可用内存 ~1.9TB 充裕):
|
||||
# #38348 fail-soft 移植后 bench 显示收益被容量掩盖——24x160k 下 host_capacity 拒绝
|
||||
# 0.75M~2.97M token/P(L2 装不下,prefetch 未发起即被拒退回重算)。放大后验证:
|
||||
# ①热轮请求 L3/L2 命中不被容量拒绝;②L3 hit 嫁接进 L2 树后在 L2 长期留存。
|
||||
# 与主模板同容器名/端口,用前必须先 down 主栈。D 与 router 不变。
|
||||
# 原注释: 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram + OTLP trace(2026-09-27)
|
||||
# 基于 dockerserve-3p1d-soak.yml 增加: --enable-trace --trace-modules request,mooncake
|
||||
# --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name ds41-3p1d-<实例名>
|
||||
# 依赖: b300-01 k8s cw jaeger-trace(jaegertracing/all-in-one:1.76.0,badger 6h TTL,hostNetwork 4317)
|
||||
# 安全性: BatchSpanProcessor 导出失败只丢 span;Jaeger 宕机不影响推理服务
|
||||
# 调优旋钮(默认未开): SGLANG_TRACE_ASYNC=1(异步 span 创建), SGLANG_TRACE_LEVEL(默认 3)
|
||||
# router(ds41-router-3p1d,rust)无 OTLP 支持,不产生 span,追踪从各 sglang HTTP 层开始
|
||||
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-baba18bc7d-local-20261010-2310
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python:/sgl-workspace/sglang/python
|
||||
- /data/ymk/ds41/fake-cargo:/root/.cargo/bin/cargo:ro
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
RUSTUP_TOOLCHAIN: "1.92-x86_64-unknown-linux-gnu"
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store-dummy.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 4.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p1
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 4.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p2
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 4.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p3
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
# G9 P0 验证(2026-10-11):D 单独升镜像拿 /query_cache_hit 修复(71c6d8bc5),
|
||||
# 并开 decode radix + hicache L3 客户端(复用 dummy store,只读查询)。
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-6612af9e0-local-20261011-0740
|
||||
container_name: ds41-3p1d-d1
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
RUSTUP_TOOLCHAIN: "1.92-x86_64-unknown-linux-gnu"
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store-dummy.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
--disaggregation-decode-enable-radix-cache
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-d1
|
||||
@@ -0,0 +1,239 @@
|
||||
# 【G9 P0 验证栈:真实 store 3p1d 2026-10-11】基于 dockerserve-3p1d-otlp.yml。
|
||||
# 用途:验证 P0 改动(/v1/loads 排队 token + drain rate 字段、P 侧 /query_cache_hit)。
|
||||
# 与主模板同容器名/端口,用前必须先 down 主栈;router(ds41-router-3p1d)独立不在本文件。
|
||||
# 2026-10-11 改造要点(vs 原 l3test):
|
||||
# - 镜像升 6612af9e0(含 G9 P0 + /query_cache_hit KeyError 修复)+ scheduler.py
|
||||
# 热补丁(L3 查询门限从 enable_decode_hicache 放宽到 enable_hicache_storage,
|
||||
# 否则 P 节点永远查不到 L3)。
|
||||
# - 每实例独立 MOONCAKE_LOCAL_HOSTNAME(a/b/c/d),extra_hosts 互配全部 rank 别名
|
||||
# (真实 store 跨实例副本写要求对端 rank 主机名可解析,见 realstore-x2 注释;
|
||||
# rank 命名规则 {hostname}-d{dp}t{tp}p{pp}c{attn_cp},commit 84fc632088;
|
||||
# D 为 dp2 dp-attention(attn_tp=1),后缀推断为 -d0t0p0c0/-d1t1p0c0,未实证)。
|
||||
# - ⚠️ D 侧 cache 方案实证否决(2026-10-11):SWA 模型下
|
||||
# --disaggregation-decode-enable-radix-cache 先要 SGLANG_ENABLE_UNIFIED_RADIX_TREE=1,
|
||||
# 开了又报 "supports only device-resident cache and is incompatible with
|
||||
# --enable-hierarchical-cache"(kv_cache_builder.py 硬 ValueError)——D 挂不了 L3。
|
||||
# 结论:/query_cache_hit 放 P(L3 全局统一,P 有完整 hicache),D 保持生产配置。
|
||||
# - D mem-fraction 0.80:曾因崩溃 d1 泄漏 ~61GiB 临时降 0.75,全栈 down 掉旧容器后
|
||||
# 驱动即回收(GPU 全零),已恢复 0.80。D max-running 保持 256(512 时 SWA 池挤爆 KV 池)。
|
||||
# P 侧保持 l3test 原状:--max-total-tokens 800000 + ratio 1.0(L1/L2 各 ~0.8M token),
|
||||
# write_through 保证 prefill 后 L3 有数据,供 /query_cache_hit 对拍。
|
||||
# 原注释: 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram + OTLP trace(2026-09-27)
|
||||
# 依赖: mooncake-master 容器(50051,起栈前 restart 清旧 segment 注册);
|
||||
# b300-01 k8s cw jaeger-trace(4317,导出失败只丢 span 不影响服务)
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-e7dc175e9-local-20261011-0940
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-rank-hosts: &rank-hosts
|
||||
# mooncake rank 主机名别名:P(tp2/cp2 interleave)-d0t0p0c0/-d0t1p0c1;
|
||||
# D(dp2 dp-attention)-d0t0p0c0/-d1t1p0c0(推断,起栈后核对 master 注册名)
|
||||
- "localhost-a-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-a-d0t1p0c1:127.0.0.1"
|
||||
- "localhost-b-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-b-d0t1p0c1:127.0.0.1"
|
||||
- "localhost-c-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-c-d0t1p0c1:127.0.0.1"
|
||||
- "localhost-d-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-d-d1t1p0c0:127.0.0.1"
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p1
|
||||
extra_hosts: *rank-hosts
|
||||
environment:
|
||||
<<: *p-environment
|
||||
MOONCAKE_LOCAL_HOSTNAME: localhost-a
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p1
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p2
|
||||
extra_hosts: *rank-hosts
|
||||
environment:
|
||||
<<: *p-environment
|
||||
MOONCAKE_LOCAL_HOSTNAME: localhost-b
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p2
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p3
|
||||
extra_hosts: *rank-hosts
|
||||
environment:
|
||||
<<: *p-environment
|
||||
MOONCAKE_LOCAL_HOSTNAME: localhost-c
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 1.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy best_effort
|
||||
--hicache-size 0
|
||||
--max-total-tokens 800000
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p3
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-d1
|
||||
extra_hosts: *rank-hosts
|
||||
# ⚠️ D 保持生产配置(2026-10-11 实证):SWA 模型下
|
||||
# --disaggregation-decode-enable-radix-cache 与 --enable-hierarchical-cache 互斥
|
||||
# (kv_cache_builder.py 硬 ValueError,"supports only device-resident cache"),
|
||||
# D 无法挂 L3 —— /query_cache_hit 改由 P 节点回答(L3 全局统一,P 有完整 hicache)。
|
||||
# MOONCAKE_LOCAL_HOSTNAME 保留备用(无害)。
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
MOONCAKE_LOCAL_HOSTNAME: localhost-d
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-d1
|
||||
@@ -0,0 +1,196 @@
|
||||
# 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram + OTLP trace(2026-09-27)
|
||||
# 基于 dockerserve-3p1d-soak.yml 增加: --enable-trace --trace-modules request,mooncake
|
||||
# --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name ds41-3p1d-<实例名>
|
||||
# 依赖: b300-01 k8s cw jaeger-trace(jaegertracing/all-in-one:1.76.0,badger 6h TTL,hostNetwork 4317)
|
||||
# 安全性: BatchSpanProcessor 导出失败只丢 span;Jaeger 宕机不影响推理服务
|
||||
# 调优旋钮(默认未开): SGLANG_TRACE_ASYNC=1(异步 span 创建), SGLANG_TRACE_LEVEL(默认 3)
|
||||
# router(ds41-router-3p1d,rust)无 OTLP 支持,不产生 span,追踪从各 sglang HTTP 层开始
|
||||
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave --chunked-prefill-size 32768 --max-prefill-tokens 32768
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p1
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave --chunked-prefill-size 32768 --max-prefill-tokens 32768
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p2
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave --chunked-prefill-size 32768 --max-prefill-tokens 32768
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p3
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-d1
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-d1
|
||||
@@ -0,0 +1,196 @@
|
||||
# 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram + OTLP trace(2026-09-27)
|
||||
# 基于 dockerserve-3p1d-soak.yml 增加: --enable-trace --trace-modules request,mooncake
|
||||
# --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name ds41-3p1d-<实例名>
|
||||
# 依赖: b300-01 k8s cw jaeger-trace(jaegertracing/all-in-one:1.76.0,badger 6h TTL,hostNetwork 4317)
|
||||
# 安全性: BatchSpanProcessor 导出失败只丢 span;Jaeger 宕机不影响推理服务
|
||||
# 调优旋钮(默认未开): SGLANG_TRACE_ASYNC=1(异步 span 创建), SGLANG_TRACE_LEVEL(默认 3)
|
||||
# router(ds41-router-3p1d,rust)无 OTLP 支持,不产生 span,追踪从各 sglang HTTP 层开始
|
||||
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p1
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p2
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p3
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-d1
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-d1
|
||||
@@ -0,0 +1,174 @@
|
||||
# 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram(2026-09-25,mr256,nsys on p1/d1)
|
||||
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-prof-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
nsys launch --session-new=p1prof --cuda-graph-trace=node sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-prof-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-prof-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-prof-d1
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
nsys launch --session-new=d1prof --cuda-graph-trace=node sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
@@ -0,0 +1,174 @@
|
||||
# 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram(2026-09-25,mr256,nsys on p1/d1)
|
||||
# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-d1
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
@@ -0,0 +1,43 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
container_name: ds41-sglang-b4
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30001:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
- PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.60
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--dp-size 4
|
||||
--enable-dp-attention
|
||||
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,43 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
|
||||
container_name: ds41-sglang-b4
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30001:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
- PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.60
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--enable-decoder-swa-bounded-replay
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
|
||||
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,42 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
|
||||
container_name: ds41-sglang-b4
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30001:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--enable-decoder-swa-bounded-replay
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,160 @@
|
||||
# 4×cp2 独立实例(非 PD)+ hicache L3(mooncake) + dspark + engram host table 卸载(2026-09-23 晚)
|
||||
# 用户指定组合:4 cp2 hicache l3 dspark roundrobin loadbalance engram offload
|
||||
# 每实例:tp2 ep2 + interleave CP2 + dspark b5 + hicache L3 write_through(size 0) + engram host table
|
||||
# (tp2 权重必须靠 engram 卸载才放得下,见 goals-track G1.9 反转)
|
||||
# GPU: a=0,1 b=2,3 c=4,5 d=6,7;端口: 30000/30010/30020/30030;rr router=30002
|
||||
# 参考:dockerserve-cpdspark-l3.yml(hicache 参数)、dockerserve-3cp2-pd.yml(cp2/engram 参数)
|
||||
x-common: &common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
# 热补丁(2026-09-23):CP 对称 idle 检查,修 health-check × hicache drain 死锁
|
||||
# 源文件:/data/ymk/sglang(dsv41-pd-visioncp 分支工作区已改);补丁脚本见
|
||||
# D:\B300\experiments\cp2x4\patch_healthcheck_cp_idle.py
|
||||
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
|
||||
environment: &env
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_MS_AUTO_DISC: "0"
|
||||
MOONCAKE_MASTER: 127.0.0.1:50051
|
||||
MOONCAKE_TE_META_DATA_SERVER: P2PHANDSHAKE
|
||||
MOONCAKE_PROTOCOL: tcp
|
||||
MOONCAKE_GLOBAL_SEGMENT_SIZE: 300gb
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
a:
|
||||
<<: *common
|
||||
container_name: ds41-cp2-a
|
||||
environment:
|
||||
<<: *env
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
b:
|
||||
<<: *common
|
||||
container_name: ds41-cp2-b
|
||||
environment:
|
||||
<<: *env
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
c:
|
||||
<<: *common
|
||||
container_name: ds41-cp2-c
|
||||
environment:
|
||||
<<: *env
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
d:
|
||||
<<: *common
|
||||
container_name: ds41-cp2-d2
|
||||
environment:
|
||||
<<: *env
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30030
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
@@ -0,0 +1,44 @@
|
||||
# cp8 单机非 PD 测试(2026-09-23 晚)
|
||||
# 背景:dp8 单机 dspark 健康但长上下文 prefill 结构性慢(单请求只落 1 rank),
|
||||
# 换 cp8(tp8 + interleave prefill CP8,单请求 prefill 切到 8 卡)对照。
|
||||
# 关 engram、无 hicache L3、无 PD。cp+dspark 无 replay 是已知健康组合
|
||||
# (cp4+dspark+replay 三元组必崩,本配置不开 replay)。
|
||||
# GPU 0-7,端口 30000,容器 ds41-cp8,project cp8
|
||||
services:
|
||||
cp8:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-cp8
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
environment:
|
||||
SGLANG_RAGGED_VERIFY_MODE: "static"
|
||||
CUDA_VISIBLE_DEVICES: "0,1,2,3,4,5,6,7"
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 8 --ep-size 8
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
@@ -0,0 +1,50 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536
|
||||
container_name: ds41-cpdspark-l3-b
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
- MC_MS_AUTO_DISC=0
|
||||
- MOONCAKE_MASTER=127.0.0.1:50051
|
||||
- MOONCAKE_TE_META_DATA_SERVER=P2PHANDSHAKE
|
||||
- MOONCAKE_PROTOCOL=tcp
|
||||
- MOONCAKE_GLOBAL_SEGMENT_SIZE=400gb
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4 --ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,51 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536
|
||||
container_name: ds41-cpdspark-l3-b
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
- MC_MS_AUTO_DISC=0
|
||||
- MOONCAKE_MASTER=127.0.0.1:50051
|
||||
- MOONCAKE_TE_META_DATA_SERVER=P2PHANDSHAKE
|
||||
- MOONCAKE_PROTOCOL=tcp
|
||||
- MOONCAKE_GLOBAL_SEGMENT_SIZE=400gb
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4 --ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,50 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536
|
||||
container_name: ds41-cpdspark-l3-a
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
- MC_MS_AUTO_DISC=0
|
||||
- MOONCAKE_MASTER=127.0.0.1:50051
|
||||
- MOONCAKE_TE_META_DATA_SERVER=P2PHANDSHAKE
|
||||
- MOONCAKE_PROTOCOL=tcp
|
||||
- MOONCAKE_GLOBAL_SEGMENT_SIZE=400gb
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4 --ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,51 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536
|
||||
container_name: ds41-cpdspark-l3-a
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
- MC_MS_AUTO_DISC=0
|
||||
- MOONCAKE_MASTER=127.0.0.1:50051
|
||||
- MOONCAKE_TE_META_DATA_SERVER=P2PHANDSHAKE
|
||||
- MOONCAKE_PROTOCOL=tcp
|
||||
- MOONCAKE_GLOBAL_SEGMENT_SIZE=400gb
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4 --ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,44 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-92632a60b-20260922-2100
|
||||
container_name: ds41-cpdspark-nr-b4
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30001:30001"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,44 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
container_name: ds41-cpdspark-nr
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30000:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,44 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
|
||||
container_name: ds41-cpdspark
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30000:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--enable-decoder-swa-bounded-replay
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,49 @@
|
||||
# 单独 D 节点基线(非 PD role):3p1d 的 d1 配置去掉 disaggregation,直接对外服务
|
||||
# 用途:测单 D 节点 decode 基线(阶梯并发 32..80%*cuda-graph-max-bs-decode,
|
||||
# 报 TPOT / 实际并发 / 整实例输出 TPS;prefill 走 L1 radix cache-hit)
|
||||
# GPU 6,7;端口 30001;无 hicache 依赖(L1 即可支撑重复 prompt 命中)
|
||||
# 2026-10-09 建:配合 bench_suite_v2.sh decode
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
d1s:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-d1-standalone
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
@@ -0,0 +1,62 @@
|
||||
# dp2 对照(2026-09-24 上午):验证 dp2(dp-attention)下 dspark 是否正常。
|
||||
# 与 dockerserve-tp2.yml 逐参数对齐,唯一差异 = 加 --dp-size 2 --enable-dp-attention --enable-dp-lm-head。
|
||||
# GPU 6-7,端口 30030 直连。镜像/补丁挂载/L3/engram/tok8 全部相同。
|
||||
# 注:dp+dspark 不能开 --enable-decoder-swa-bounded-replay(本配置本来就没开)。
|
||||
x-common: &common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
|
||||
environment: &env
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_MS_AUTO_DISC: "0"
|
||||
MOONCAKE_MASTER: 127.0.0.1:50051
|
||||
MOONCAKE_TE_META_DATA_SERVER: P2PHANDSHAKE
|
||||
MOONCAKE_PROTOCOL: tcp
|
||||
MOONCAKE_GLOBAL_SEGMENT_SIZE: 300gb
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
dp2:
|
||||
<<: *common
|
||||
container_name: ds41-dp2
|
||||
environment:
|
||||
<<: *env
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--dp-size 2 --enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30030
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
@@ -0,0 +1,42 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
|
||||
container_name: ds41-dp4test
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30000:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--enable-decoder-swa-bounded-replay
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,44 @@
|
||||
# dp8+tp8 单机非 PD 对照测试(2026-09-23 晚)
|
||||
# 背景:3cp2+dp2 PD 部署 dspark accept 回归(32k ctx accept len 3.14→1.31,
|
||||
# 见 experiments/3cp2-pd/bench-20260923.md「未决问题」)。本配置退回单机验证
|
||||
# dspark 健康度:engram host table 关闭、无 hicache L3、无 PD、无 mooncake。
|
||||
# 参数基准:dockerserve-3cp2-pd.yml 的 d 服务(dp 路径),去掉 PD/L3/engram 相关。
|
||||
# GPU 0-7,端口 30000,容器 ds41-dp8,project dp8
|
||||
services:
|
||||
dp8:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-dp8
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
environment:
|
||||
SGLANG_RAGGED_VERIFY_MODE: "static"
|
||||
CUDA_VISIBLE_DEVICES: "0,1,2,3,4,5,6,7"
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 8 --ep-size 8 --dp-size 8
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
@@ -0,0 +1,61 @@
|
||||
# 单独 P 节点基线(非 PD role):3p1d 的 p1 配置去掉 disaggregation,直接对外服务
|
||||
# 用途:测单 P 节点 prefill 基线吞吐(冷)+ hicache cache-hit TPS(热)
|
||||
# GPU 0,1;端口 30000;依赖 mooncake-master(hicache L3,127.0.0.1:50051)
|
||||
# 2026-10-09 建:配合 bench_suite_v2.sh prefill
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
p1s:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-p1-standalone
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: "4"
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
@@ -0,0 +1,47 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
|
||||
container_name: ds41-prefill
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30000:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-decoder-swa-bounded-replay
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2.5
|
||||
--hicache-write-policy write_back
|
||||
--disaggregation-mode prefill
|
||||
--optimistic-prefill-attempts 4
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,51 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-pd-decode
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--dp-size 4
|
||||
--enable-dp-attention
|
||||
--enable-dp-lm-head
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30001
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
@@ -0,0 +1,50 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536
|
||||
container_name: ds41-pd-decode
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5,6,7
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--dp-size 4
|
||||
--enable-dp-attention
|
||||
--enable-dp-lm-head
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30001
|
||||
--enable-cache-report
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,55 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-c74a4037f-20260921-1300
|
||||
container_name: ds41-pd-prefill-dp
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--dp-size 4
|
||||
--enable-dp-attention
|
||||
--enable-dp-lm-head
|
||||
--load-balance-method total_tokens
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30010
|
||||
--enable-cache-report
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2.5
|
||||
--hicache-write-policy write_back
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-bootstrap-port 8918
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,53 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-pd-prefill
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2.5
|
||||
--hicache-write-policy write_back
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
@@ -0,0 +1,52 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536
|
||||
container_name: ds41-pd-prefill
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2.5
|
||||
--hicache-write-policy write_back
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,57 @@
|
||||
# PD2 二分 Round 2 D 侧(2026-09-24):干净 D + 嫌疑项②
|
||||
# SGLANG_DISAGGREGATION_QUEUE_SIZE=16 + SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=32
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-pd2-decode
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=6,7
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
- SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1
|
||||
- SGLANG_DISAGGREGATION_QUEUE_SIZE=16
|
||||
- SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=32
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2
|
||||
--ep-size 2
|
||||
--dp-size 2
|
||||
--enable-dp-attention
|
||||
--enable-dp-lm-head
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30030
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
@@ -0,0 +1,54 @@
|
||||
# PD 干净对照实验 D 节点(2026-09-24 上午):dp2 decode,配 dockerserve-pd2-p.yml 使用。
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-pd2-decode
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=6,7
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
- SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2
|
||||
--ep-size 2
|
||||
--dp-size 2
|
||||
--enable-dp-attention
|
||||
--enable-dp-lm-head
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30030
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
@@ -0,0 +1,68 @@
|
||||
# PD2 二分 Round 1(2026-09-24):干净基线 + 嫌疑项①「P 侧 L3 hicache 块 + mooncake-store.json」
|
||||
# 相对 dockerserve-pd2-p.yml 的差异:
|
||||
# env 加 SGLANG_HICACHE_MOONCAKE_CONFIG_PATH=/data/ymk/ds41/mooncake-store.json
|
||||
# hicache 参数块换成坏部署同款:page_first_direct + direct + write_through + mooncake +
|
||||
# wait_complete + size 0(替代 write_back L2)
|
||||
# D 侧不变(坏部署 D 本无 L3)。探针:bs16 low_entropy decode 看 accept len。
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-pd2-prefill
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
- SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1
|
||||
- SGLANG_HICACHE_MOONCAKE_CONFIG_PATH=/data/ymk/ds41/mooncake-store.json
|
||||
- SGLANG_HICACHE_PREFETCH_IO_THREADS=4
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2
|
||||
--ep-size 2
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30020
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
@@ -0,0 +1,66 @@
|
||||
# PD2 二分 Round 2 P 侧(2026-09-24):R1(L3 块)之上再加嫌疑项②
|
||||
# SGLANG_DISAGGREGATION_QUEUE_SIZE=16 + SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=32
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-pd2-prefill
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
- SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1
|
||||
- SGLANG_HICACHE_MOONCAKE_CONFIG_PATH=/data/ymk/ds41/mooncake-store.json
|
||||
- SGLANG_HICACHE_PREFETCH_IO_THREADS=4
|
||||
- SGLANG_DISAGGREGATION_QUEUE_SIZE=16
|
||||
- SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=32
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2
|
||||
--ep-size 2
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30020
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
@@ -0,0 +1,59 @@
|
||||
# PD 干净对照实验(2026-09-24 上午):验证「cp2-P → dp2-D」传输路径下 dspark 是否正常。
|
||||
# 用户疑问:单机六形态健康不等于 PD 链路健康,可能两边传输没对齐。
|
||||
# 以已验证可用的 dockerserve-pd-{p,d}-vision.yml 为底,缩到 2+2 卡;P 加 engram host table
|
||||
# (2 卡权重放不下,见 G1.9)。P=GPU4-5 端口 30020,D=GPU6-7 端口 30030,mini_lb=30004。
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
container_name: ds41-pd2-prefill
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=4,5
|
||||
- SGLANG_RAGGED_VERIFY_MODE=static
|
||||
- MC_INTRANODE_NVLINK=true
|
||||
- MC_INTRA_NVLINK=true
|
||||
- SGLANG_MOONCAKE_SEND_AUX_TCP=1
|
||||
- SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2
|
||||
--ep-size 2
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0
|
||||
--port 30020
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2.5
|
||||
--hicache-write-policy write_back
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
@@ -0,0 +1,42 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
|
||||
container_name: ds41-replay-off
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30000:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,42 @@
|
||||
services:
|
||||
sglang:
|
||||
image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142
|
||||
container_name: ds41-replay-on
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
ports:
|
||||
- "30000:30000"
|
||||
environment:
|
||||
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--speculative-algorithm DSPARK
|
||||
--speculative-dspark-block-size 5
|
||||
--enable-decoder-swa-bounded-replay
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
@@ -0,0 +1,76 @@
|
||||
# 【跨 P 实例 L3 读写验证 2026-10-10】dockerserve-tier-cp2-realstore.yml 的第二实例:
|
||||
# 与 ds41-tier-t2(A:GPU 0,1,端口 30040)同时跑,共用 mooncake master 50051。
|
||||
# 关键差异:
|
||||
# - GPU 2,3,端口 30041,容器名 ds41-tier-t3
|
||||
# - MOONCAKE_LOCAL_HOSTNAME=localhost-b:mooncake 客户端 hostname 前缀按实例区分
|
||||
# (rank 后缀 -d0t0p0c0/-d0t1p0c1 由 84fc632088 拼在后面的),否则两实例 rank 同名
|
||||
# → master preferred-segment 分不出段归属 → 副本落错进程段 → 跨进程回退 TCP(实例内)
|
||||
# 且跨实例段地址冲突。extra_hosts 别名同步加 -b 前缀(P2P handshake 要求可解析,
|
||||
# ac11e9162d)。
|
||||
# 验证路径:A prime 写 L3 → flush B 的 L1/L2 → B 读同 seed prompts = 跨实例 TCP P2P 命中。
|
||||
x-common: &common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
shm_size: 32gb
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python:/sgl-workspace/sglang/python
|
||||
- /data/ymk/ds41/fake-cargo:/root/.cargo/bin/cargo:ro
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
t3:
|
||||
<<: *common
|
||||
container_name: ds41-tier-t3
|
||||
extra_hosts:
|
||||
- "localhost-b-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-b-d0t1p0c1:127.0.0.1"
|
||||
# 跨实例 P2P:对端实例(realstore compose 的 t2)rank 主机名也要能解析
|
||||
- "localhost-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-d0t1p0c1:127.0.0.1"
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: 1
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_TIMING: 1
|
||||
MOONCAKE_LOCAL_HOSTNAME: localhost-b
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: 4
|
||||
CUDA_VISIBLE_DEVICES: 2,3
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--chunked-prefill-size 32768
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30041
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 4.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--max-total-tokens 1600000
|
||||
@@ -0,0 +1,73 @@
|
||||
# 【L3 真实 store A/B 2026-10-10】同 dockerserve-tier-cp2.yml,唯一差异:
|
||||
# L3 从 dummy standalone(mooncake-store-dummy.json,50052 守护进程)
|
||||
# 切到真实 mooncake store(mooncake-store.json:master 50051 + P2PHANDSHAKE + tcp,
|
||||
# 客户端注册 300GB 自身内存段)——与 k8s 线上 ds41-3p1d 的 L3 模式一致。
|
||||
# 目的:区分 "sglang L3 路径本身慢" 还是 "dummy 后端 per-key 路径慢"(mc 扫描 A/B)。
|
||||
x-common: &common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
shm_size: 32gb
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python:/sgl-workspace/sglang/python
|
||||
- /data/ymk/ds41/fake-cargo:/root/.cargo/bin/cargo:ro
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
t2:
|
||||
<<: *common
|
||||
container_name: ds41-tier-t2
|
||||
extra_hosts:
|
||||
- "localhost-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-d0t1p0c1:127.0.0.1"
|
||||
# 跨实例 P2P:对端实例(x2 compose 的 t3)rank 主机名也要能解析,
|
||||
# 否则 master 把副本分配到对端段后本端 open segment 失败(TRANSFER_FAIL)
|
||||
- "localhost-b-d0t0p0c0:127.0.0.1"
|
||||
- "localhost-b-d0t1p0c1:127.0.0.1"
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: 1
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_HICACHE_TIMING: 1
|
||||
# L3 预取传输并行 lane 数(crc32(rid) 分 lane,每 lane 独立 gloo 组;
|
||||
# 实例内所有 rank 必须一致)。2026-10-10 A/B 见 l3-hicache-deep-dive 文档。
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: 4
|
||||
CUDA_VISIBLE_DEVICES: 0,1
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--chunked-prefill-size 32768
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30040
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 4.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--max-total-tokens 1600000
|
||||
@@ -0,0 +1,65 @@
|
||||
# 【prefill 分层极限测试 2026-10-10】单实例 cp2(tp2+ep2+prefill-cp interleave)+ dspark b5
|
||||
# + hicache L2(ratio 4.0=6.4M token) + L3(shm dummy) + best_effort
|
||||
# 目的:测 纯计算 / L1 / L2 / L3 四档 prefill 极限吞吐 + 混合负载异步聚合验证
|
||||
# L1=1.6M token(装得下 8×160k=1.28M 工作集);L2 6.4M;L3 shm 64GB
|
||||
# GPU 0,1(用前停 p3d1 栈);端口 30040;共用 mooncake-store(50052)/master(50051)
|
||||
x-common: &common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
shm_size: 32gb
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
- /data/ymk/sglang/python:/sgl-workspace/sglang/python
|
||||
- /data/ymk/ds41/fake-cargo:/root/.cargo/bin/cargo:ro
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
t2:
|
||||
<<: *common
|
||||
container_name: ds41-tier-t2
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: 1
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store-dummy.json
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS: 4
|
||||
SGLANG_HICACHE_TIMING: 1
|
||||
SGLANG_HICACHE_STORAGE_BATCH_SIZE: 512
|
||||
CUDA_VISIBLE_DEVICES: 0,1
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--chunked-prefill-size 32768
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30040
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 4.0
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--max-total-tokens 1600000
|
||||
@@ -0,0 +1,92 @@
|
||||
# 纯 tp2 单实例对照(2026-09-24 早):与 dockerserve-cp2x4.yml 逐参数对齐,唯一差异 = 去掉
|
||||
# --enable-prefill-cp --cp-strategy interleave(即无 CP),用于测「tp2 vs cp2」的 prefill 效率差。
|
||||
# GPU 4-5(a)/6-7(b),端口 30020/30030;2×tp2 聚合经 rr router 30003。
|
||||
# 镜像/补丁挂载/L3/engram/tok8 全部与 cp2x4 相同。
|
||||
x-common: &common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
# 与 cp2x4 保持同一份 scheduler.py(含未 commit 的 CP idle 补丁),保证唯一变量是 CP 开关
|
||||
- /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py
|
||||
environment: &env
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_MS_AUTO_DISC: "0"
|
||||
MOONCAKE_MASTER: 127.0.0.1:50051
|
||||
MOONCAKE_TE_META_DATA_SERVER: P2PHANDSHAKE
|
||||
MOONCAKE_PROTOCOL: tcp
|
||||
MOONCAKE_GLOBAL_SEGMENT_SIZE: 300gb
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
tp2:
|
||||
<<: *common
|
||||
container_name: ds41-tp2
|
||||
environment:
|
||||
<<: *env
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
b:
|
||||
<<: *common
|
||||
container_name: ds41-tp2-b
|
||||
environment:
|
||||
<<: *env
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 64
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30030
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
@@ -0,0 +1,43 @@
|
||||
services:
|
||||
sglang:
|
||||
image: uhub.service.ucloud.cn/umirror/sglang:dev-dsv41
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
privileged: true
|
||||
|
||||
ports:
|
||||
- "30000:30000"
|
||||
|
||||
volumes:
|
||||
- /data:/data
|
||||
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
command: >-
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 4
|
||||
--ep-size 4
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--enable-decoder-swa-bounded-replay
|
||||
--cuda-graph-max-bs-decode 64
|
||||
--reasoning-parser auto
|
||||
--tool-call-parser auto
|
||||
--max-running-requests 64
|
||||
--host 0.0.0.0
|
||||
--port 30000
|
||||
--enable-cache-report
|
||||
--enable-metrics
|
||||
--enable-prefill-cp
|
||||
--cp-strategy interleave
|
||||
--json-model-override-args '{"vision_n_layers": 0}'
|
||||
|
||||
@@ -0,0 +1,454 @@
|
||||
# ds41-3p1d: DeepSeek-V4.1-Flash 3P1D PD 分离(cs32k + OTLP trace),RBG 版
|
||||
# 拓扑: 3×prefill(tp2+ep2+cp2, cs32k, dspark, hicache L3) + 1×decode(tp2+ep2+dp2, cg256 mr320, dspark,
|
||||
# load-balance-method=total_requests 按请求数分流) + mooncake-master(hicache L3 store) + router(mini-lb PD)
|
||||
# 全部钉 b300-01(单机 8 卡,hostNetwork + hostPID——mooncake NVLink-intra 跨 Pod CUDA IPC 必需,
|
||||
# 等同 docker compose 的 pid: host;参考 AGENTS.md「部署四要素」)
|
||||
# 镜像已侧载 containerd: docker.io/ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831(IfNotPresent)
|
||||
# 参数来源: deploy/b300-ds41/dockerserve-3p1d-otlp-cs32k.yml(docker compose 版)
|
||||
# 对外: 仅 ClusterIP Service ds41-3p1d:50002(router 用普通 Pod 网络,不占宿主机端口,
|
||||
# 天然不对宿主机/外网暴露);new-api-gw 用集群 DNS http://ds41-3p1d:50002/v1 访问
|
||||
# trace: OTLP → 127.0.0.1:4317(同节点 jaeger-trace cw,badger 6h 轮窗)
|
||||
apiVersion: workloads.x-k8s.io/v1alpha2
|
||||
kind: RoleBasedGroup
|
||||
metadata:
|
||||
name: ds41-3p1d
|
||||
labels:
|
||||
app: ds41-3p1d
|
||||
sglang-model: ds-v41-flash
|
||||
spec:
|
||||
roleTemplates:
|
||||
- name: prefill-base
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
sglang-model: ds-v41-flash
|
||||
sglang-role: prefill
|
||||
spec:
|
||||
hostNetwork: true
|
||||
hostPID: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: b300-01
|
||||
containers:
|
||||
- name: sglang
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec sglang serve \
|
||||
--trust-remote-code \
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash \
|
||||
--tp 2 --ep-size 2 \
|
||||
--enable-prefill-cp --cp-strategy interleave \
|
||||
--chunked-prefill-size 32768 --max-prefill-tokens 32768 \
|
||||
--mem-fraction-static 0.75 \
|
||||
--attention-backend dsv4 \
|
||||
--moe-runner-backend flashinfer_mxfp4 \
|
||||
--cuda-graph-max-bs-decode 32 \
|
||||
--reasoning-parser auto --tool-call-parser auto \
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' \
|
||||
--max-running-requests 256 \
|
||||
--prefill-max-requests 8 \
|
||||
--tokenizer-worker-num 8 \
|
||||
--host 0.0.0.0 --port "$SGLANG_PORT" \
|
||||
--enable-cache-report --enable-metrics \
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5 \
|
||||
--enable-hierarchical-cache \
|
||||
--hicache-ratio 2 \
|
||||
--hicache-mem-layout page_first_direct \
|
||||
--hicache-io-backend direct \
|
||||
--hicache-write-policy write_through \
|
||||
--hicache-storage-backend mooncake \
|
||||
--hicache-storage-prefetch-policy wait_complete \
|
||||
--hicache-size 0 \
|
||||
--disaggregation-mode prefill \
|
||||
--disaggregation-transfer-backend mooncake \
|
||||
--disaggregation-bootstrap-port "$SGLANG_BOOTSTRAP_PORT" \
|
||||
--enable-trace \
|
||||
--trace-modules request,mooncake \
|
||||
--otlp-traces-endpoint 127.0.0.1:4317 \
|
||||
--otlp-service-name "$OTLP_NAME"
|
||||
env:
|
||||
- name: SGLANG_TORCH_PROFILER_DIR
|
||||
value: /data/ymk/prof/torch
|
||||
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||
value: "1"
|
||||
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||
value: static
|
||||
- name: MC_INTRANODE_NVLINK
|
||||
value: "true"
|
||||
- name: MC_INTRA_NVLINK
|
||||
value: "true"
|
||||
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||
value: "1"
|
||||
- name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH
|
||||
value: /data/ymk/ds41/mooncake-store.json
|
||||
- name: SGLANG_HICACHE_PREFETCH_IO_THREADS
|
||||
value: "4"
|
||||
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||
value: "16"
|
||||
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||
value: "32"
|
||||
- name: TZ
|
||||
value: Asia/Shanghai
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: metrics
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 90
|
||||
resources:
|
||||
requests:
|
||||
cpu: "32"
|
||||
memory: 220Gi
|
||||
nvidia.com/gpu: "2"
|
||||
limits:
|
||||
cpu: "96"
|
||||
memory: 950Gi
|
||||
nvidia.com/gpu: "2"
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
- name: sglang-cache
|
||||
mountPath: /root/.cache/sglang
|
||||
- name: dshm
|
||||
mountPath: /dev/shm
|
||||
volumes:
|
||||
- name: data
|
||||
hostPath:
|
||||
path: /data
|
||||
type: Directory
|
||||
- name: sglang-cache
|
||||
hostPath:
|
||||
path: /data/ymk/cache/sglang
|
||||
type: Directory
|
||||
- name: dshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 32Gi
|
||||
- name: decode-base
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
sglang-model: ds-v41-flash
|
||||
sglang-role: decode
|
||||
spec:
|
||||
hostNetwork: true
|
||||
hostPID: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: b300-01
|
||||
containers:
|
||||
- name: sglang
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec sglang serve \
|
||||
--trust-remote-code \
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash \
|
||||
--tp 2 --ep-size 2 --dp-size 2 \
|
||||
--enable-dp-attention --enable-dp-lm-head \
|
||||
--mem-fraction-static 0.83 \
|
||||
--attention-backend dsv4 \
|
||||
--moe-runner-backend flashinfer_mxfp4 \
|
||||
--cuda-graph-max-bs-decode 256 \
|
||||
--load-balance-method total_requests \
|
||||
--reasoning-parser auto --tool-call-parser auto \
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' \
|
||||
--max-running-requests 320 \
|
||||
--tokenizer-worker-num 8 \
|
||||
--host 0.0.0.0 --port 30030 \
|
||||
--enable-cache-report --enable-metrics \
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5 \
|
||||
--disaggregation-mode decode \
|
||||
--disaggregation-transfer-backend mooncake \
|
||||
--disaggregation-bootstrap-port 9001 \
|
||||
--enable-trace \
|
||||
--trace-modules request,mooncake \
|
||||
--otlp-traces-endpoint 127.0.0.1:4317 \
|
||||
--otlp-service-name ds41-3p1d-d1
|
||||
env:
|
||||
- name: SGLANG_TORCH_PROFILER_DIR
|
||||
value: /data/ymk/prof/torch
|
||||
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||
value: "1"
|
||||
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||
value: static
|
||||
- name: MC_INTRANODE_NVLINK
|
||||
value: "true"
|
||||
- name: MC_INTRA_NVLINK
|
||||
value: "true"
|
||||
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||
value: "1"
|
||||
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||
value: "16"
|
||||
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||
value: "32"
|
||||
- name: TZ
|
||||
value: Asia/Shanghai
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 30030
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: metrics
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 90
|
||||
resources:
|
||||
requests:
|
||||
cpu: "32"
|
||||
memory: 100Gi
|
||||
nvidia.com/gpu: "2"
|
||||
limits:
|
||||
cpu: "96"
|
||||
memory: 600Gi
|
||||
nvidia.com/gpu: "2"
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
- name: sglang-cache
|
||||
mountPath: /root/.cache/sglang
|
||||
- name: dshm
|
||||
mountPath: /dev/shm
|
||||
volumes:
|
||||
- name: data
|
||||
hostPath:
|
||||
path: /data
|
||||
type: Directory
|
||||
- name: sglang-cache
|
||||
hostPath:
|
||||
path: /data/ymk/cache/sglang
|
||||
type: Directory
|
||||
- name: dshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 32Gi
|
||||
roles:
|
||||
- name: mooncake-master
|
||||
replicas: 1
|
||||
standalonePattern:
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-mooncake-master
|
||||
spec:
|
||||
hostNetwork: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: b300-01
|
||||
containers:
|
||||
- name: mooncake-master
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec /opt/sglang/bin/mooncake_master \
|
||||
--port 50051 \
|
||||
--enable_http_metadata_server=true \
|
||||
--http_metadata_server_port=18080 \
|
||||
--eviction_high_watermark_ratio=0.90
|
||||
readinessProbe:
|
||||
tcpSocket:
|
||||
port: 50051
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
failureThreshold: 60
|
||||
resources:
|
||||
requests:
|
||||
cpu: "1"
|
||||
memory: 8Gi
|
||||
limits:
|
||||
cpu: "4"
|
||||
memory: 64Gi
|
||||
- name: p1
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- mooncake-master
|
||||
standalonePattern:
|
||||
templateRef:
|
||||
name: prefill-base
|
||||
patch:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-p1
|
||||
spec:
|
||||
containers:
|
||||
- name: sglang
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: "0,1"
|
||||
- name: SGLANG_PORT
|
||||
value: "30000"
|
||||
- name: SGLANG_BOOTSTRAP_PORT
|
||||
value: "8998"
|
||||
- name: OTLP_NAME
|
||||
value: ds41-3p1d-p1
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 30000
|
||||
- name: p2
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- mooncake-master
|
||||
standalonePattern:
|
||||
templateRef:
|
||||
name: prefill-base
|
||||
patch:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-p2
|
||||
spec:
|
||||
containers:
|
||||
- name: sglang
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: "2,3"
|
||||
- name: SGLANG_PORT
|
||||
value: "30010"
|
||||
- name: SGLANG_BOOTSTRAP_PORT
|
||||
value: "8999"
|
||||
- name: OTLP_NAME
|
||||
value: ds41-3p1d-p2
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 30010
|
||||
- name: p3
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- mooncake-master
|
||||
standalonePattern:
|
||||
templateRef:
|
||||
name: prefill-base
|
||||
patch:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-p3
|
||||
spec:
|
||||
containers:
|
||||
- name: sglang
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: "4,5"
|
||||
- name: SGLANG_PORT
|
||||
value: "30020"
|
||||
- name: SGLANG_BOOTSTRAP_PORT
|
||||
value: "9000"
|
||||
- name: OTLP_NAME
|
||||
value: ds41-3p1d-p3
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 30020
|
||||
- name: d1
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- mooncake-master
|
||||
standalonePattern:
|
||||
templateRef:
|
||||
name: decode-base
|
||||
patch:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-d1
|
||||
spec:
|
||||
containers:
|
||||
- name: sglang
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: "6,7"
|
||||
- name: router
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- p1
|
||||
- p2
|
||||
- p3
|
||||
- d1
|
||||
standalonePattern:
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-router
|
||||
spec:
|
||||
# 入口不走 hostNetwork:普通 Pod 网络 + ClusterIP,天然不对宿主机/外网暴露。
|
||||
# P/D 地址用节点内网 IP(hostNetwork 监听在 192.168.1.243 上,Calico 可达)。
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: b300-01
|
||||
containers:
|
||||
- name: router
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec sglang-router launch \
|
||||
--mini-lb --pd-disaggregation \
|
||||
--host 0.0.0.0 --port 50002 \
|
||||
--prefill http://192.168.1.243:30000 8998 \
|
||||
--prefill http://192.168.1.243:30010 8999 \
|
||||
--prefill http://192.168.1.243:30020 9000 \
|
||||
--decode http://192.168.1.243:30030
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 50002
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 30
|
||||
resources:
|
||||
requests:
|
||||
cpu: "1"
|
||||
memory: 4Gi
|
||||
limits:
|
||||
cpu: "8"
|
||||
memory: 16Gi
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: ds41-3p1d
|
||||
labels:
|
||||
app: ds41-3p1d
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
rbg.workloads.x-k8s.io/group-name: ds41-3p1d
|
||||
rbg.workloads.x-k8s.io/role-name: router
|
||||
ports:
|
||||
- name: http
|
||||
port: 50002
|
||||
targetPort: 50002
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PodMonitor
|
||||
metadata:
|
||||
name: ds41-3p1d
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
sglang-model: ds-v41-flash
|
||||
podMetricsEndpoints:
|
||||
- interval: 15s
|
||||
path: /metrics
|
||||
port: metrics
|
||||
@@ -0,0 +1,2 @@
|
||||
#!/bin/sh
|
||||
exit 1
|
||||
@@ -0,0 +1,451 @@
|
||||
# ds41-3p1d: DeepSeek-V4.1-Flash 3P1D PD 分离(cs32k + OTLP trace),RBG 版
|
||||
# 拓扑: 3×prefill(tp2+ep2+cp2, cs32k, dspark, hicache L3) + 1×decode(tp2+ep2+dp2, cg256 mr320, dspark,
|
||||
# load-balance-method=total_requests 按请求数分流) + mooncake-master(hicache L3 store) + router(mini-lb PD)
|
||||
# 全部钉 b300-01(单机 8 卡,hostNetwork + hostPID——mooncake NVLink-intra 跨 Pod CUDA IPC 必需,
|
||||
# 等同 docker compose 的 pid: host;参考 AGENTS.md「部署四要素」)
|
||||
# 镜像已侧载 containerd: docker.io/ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix(IfNotPresent)
|
||||
# 参数来源: deploy/b300-ds41/dockerserve-3p1d-otlp-cs32k.yml(docker compose 版)
|
||||
# 对外: 仅 ClusterIP Service ds41-3p1d:50002(router 用普通 Pod 网络,不占宿主机端口,
|
||||
# 天然不对宿主机/外网暴露);new-api-gw 用集群 DNS http://ds41-3p1d:50002/v1 访问
|
||||
# trace: OTLP → 127.0.0.1:4317(同节点 jaeger-trace cw,badger 6h 轮窗)
|
||||
apiVersion: workloads.x-k8s.io/v1alpha2
|
||||
kind: RoleBasedGroup
|
||||
metadata:
|
||||
name: ds41-3p1d
|
||||
labels:
|
||||
app: ds41-3p1d
|
||||
sglang-model: ds-v41-flash
|
||||
spec:
|
||||
roleTemplates:
|
||||
- name: prefill-base
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
sglang-model: ds-v41-flash
|
||||
sglang-role: prefill
|
||||
spec:
|
||||
hostNetwork: true
|
||||
hostPID: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: b300-01
|
||||
containers:
|
||||
- name: sglang
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec sglang serve \
|
||||
--trust-remote-code \
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash \
|
||||
--tp 2 --ep-size 2 \
|
||||
--enable-prefill-cp --cp-strategy interleave \
|
||||
--chunked-prefill-size 32768 --max-prefill-tokens 32768 \
|
||||
--mem-fraction-static 0.75 \
|
||||
--attention-backend dsv4 \
|
||||
--moe-runner-backend flashinfer_mxfp4 \
|
||||
--cuda-graph-max-bs-decode 32 \
|
||||
--reasoning-parser auto --tool-call-parser auto \
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' \
|
||||
--max-running-requests 256 \
|
||||
--tokenizer-worker-num 8 \
|
||||
--host 0.0.0.0 --port "$SGLANG_PORT" \
|
||||
--enable-cache-report --enable-metrics \
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5 \
|
||||
--enable-hierarchical-cache \
|
||||
--hicache-ratio 2 \
|
||||
--hicache-mem-layout page_first_direct \
|
||||
--hicache-io-backend direct \
|
||||
--hicache-write-policy write_through \
|
||||
--hicache-storage-backend mooncake \
|
||||
--hicache-storage-prefetch-policy wait_complete \
|
||||
--hicache-size 0 \
|
||||
--disaggregation-mode prefill \
|
||||
--disaggregation-transfer-backend mooncake \
|
||||
--disaggregation-bootstrap-port "$SGLANG_BOOTSTRAP_PORT" \
|
||||
--enable-trace \
|
||||
--trace-modules request,mooncake \
|
||||
--otlp-traces-endpoint 127.0.0.1:4317 \
|
||||
--otlp-service-name "$OTLP_NAME"
|
||||
env:
|
||||
- name: SGLANG_TORCH_PROFILER_DIR
|
||||
value: /data/ymk/prof/torch
|
||||
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||
value: "1"
|
||||
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||
value: static
|
||||
- name: MC_INTRANODE_NVLINK
|
||||
value: "true"
|
||||
- name: MC_INTRA_NVLINK
|
||||
value: "true"
|
||||
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||
value: "1"
|
||||
- name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH
|
||||
value: /data/ymk/ds41/mooncake-store.json
|
||||
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||
value: "16"
|
||||
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||
value: "32"
|
||||
- name: TZ
|
||||
value: Asia/Shanghai
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: metrics
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 90
|
||||
resources:
|
||||
requests:
|
||||
cpu: "32"
|
||||
memory: 220Gi
|
||||
nvidia.com/gpu: "2"
|
||||
limits:
|
||||
cpu: "96"
|
||||
memory: 950Gi
|
||||
nvidia.com/gpu: "2"
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
- name: sglang-cache
|
||||
mountPath: /root/.cache/sglang
|
||||
- name: dshm
|
||||
mountPath: /dev/shm
|
||||
volumes:
|
||||
- name: data
|
||||
hostPath:
|
||||
path: /data
|
||||
type: Directory
|
||||
- name: sglang-cache
|
||||
hostPath:
|
||||
path: /data/ymk/cache/sglang
|
||||
type: Directory
|
||||
- name: dshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 32Gi
|
||||
- name: decode-base
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
sglang-model: ds-v41-flash
|
||||
sglang-role: decode
|
||||
spec:
|
||||
hostNetwork: true
|
||||
hostPID: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: b300-01
|
||||
containers:
|
||||
- name: sglang
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec sglang serve \
|
||||
--trust-remote-code \
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash \
|
||||
--tp 2 --ep-size 2 --dp-size 2 \
|
||||
--enable-dp-attention --enable-dp-lm-head \
|
||||
--mem-fraction-static 0.83 \
|
||||
--attention-backend dsv4 \
|
||||
--moe-runner-backend flashinfer_mxfp4 \
|
||||
--cuda-graph-max-bs-decode 256 \
|
||||
--load-balance-method total_requests \
|
||||
--reasoning-parser auto --tool-call-parser auto \
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' \
|
||||
--max-running-requests 320 \
|
||||
--tokenizer-worker-num 8 \
|
||||
--host 0.0.0.0 --port 30030 \
|
||||
--enable-cache-report --enable-metrics \
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5 \
|
||||
--disaggregation-mode decode \
|
||||
--disaggregation-transfer-backend mooncake \
|
||||
--disaggregation-bootstrap-port 9001 \
|
||||
--enable-trace \
|
||||
--trace-modules request,mooncake \
|
||||
--otlp-traces-endpoint 127.0.0.1:4317 \
|
||||
--otlp-service-name ds41-3p1d-d1
|
||||
env:
|
||||
- name: SGLANG_TORCH_PROFILER_DIR
|
||||
value: /data/ymk/prof/torch
|
||||
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||
value: "1"
|
||||
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||
value: static
|
||||
- name: MC_INTRANODE_NVLINK
|
||||
value: "true"
|
||||
- name: MC_INTRA_NVLINK
|
||||
value: "true"
|
||||
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||
value: "1"
|
||||
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||
value: "16"
|
||||
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||
value: "32"
|
||||
- name: TZ
|
||||
value: Asia/Shanghai
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 30030
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: metrics
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 90
|
||||
resources:
|
||||
requests:
|
||||
cpu: "32"
|
||||
memory: 100Gi
|
||||
nvidia.com/gpu: "2"
|
||||
limits:
|
||||
cpu: "96"
|
||||
memory: 600Gi
|
||||
nvidia.com/gpu: "2"
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
- name: sglang-cache
|
||||
mountPath: /root/.cache/sglang
|
||||
- name: dshm
|
||||
mountPath: /dev/shm
|
||||
volumes:
|
||||
- name: data
|
||||
hostPath:
|
||||
path: /data
|
||||
type: Directory
|
||||
- name: sglang-cache
|
||||
hostPath:
|
||||
path: /data/ymk/cache/sglang
|
||||
type: Directory
|
||||
- name: dshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 32Gi
|
||||
roles:
|
||||
- name: mooncake-master
|
||||
replicas: 1
|
||||
standalonePattern:
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-mooncake-master
|
||||
spec:
|
||||
hostNetwork: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: b300-01
|
||||
containers:
|
||||
- name: mooncake-master
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec /opt/sglang/bin/mooncake_master \
|
||||
--port 50051 \
|
||||
--enable_http_metadata_server=true \
|
||||
--http_metadata_server_port=18080 \
|
||||
--eviction_high_watermark_ratio=0.90
|
||||
readinessProbe:
|
||||
tcpSocket:
|
||||
port: 50051
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
failureThreshold: 60
|
||||
resources:
|
||||
requests:
|
||||
cpu: "1"
|
||||
memory: 8Gi
|
||||
limits:
|
||||
cpu: "4"
|
||||
memory: 64Gi
|
||||
- name: p1
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- mooncake-master
|
||||
standalonePattern:
|
||||
templateRef:
|
||||
name: prefill-base
|
||||
patch:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-p1
|
||||
spec:
|
||||
containers:
|
||||
- name: sglang
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: "0,1"
|
||||
- name: SGLANG_PORT
|
||||
value: "30000"
|
||||
- name: SGLANG_BOOTSTRAP_PORT
|
||||
value: "8998"
|
||||
- name: OTLP_NAME
|
||||
value: ds41-3p1d-p1
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 30000
|
||||
- name: p2
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- mooncake-master
|
||||
standalonePattern:
|
||||
templateRef:
|
||||
name: prefill-base
|
||||
patch:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-p2
|
||||
spec:
|
||||
containers:
|
||||
- name: sglang
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: "2,3"
|
||||
- name: SGLANG_PORT
|
||||
value: "30010"
|
||||
- name: SGLANG_BOOTSTRAP_PORT
|
||||
value: "8999"
|
||||
- name: OTLP_NAME
|
||||
value: ds41-3p1d-p2
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 30010
|
||||
- name: p3
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- mooncake-master
|
||||
standalonePattern:
|
||||
templateRef:
|
||||
name: prefill-base
|
||||
patch:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-p3
|
||||
spec:
|
||||
containers:
|
||||
- name: sglang
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: "4,5"
|
||||
- name: SGLANG_PORT
|
||||
value: "30020"
|
||||
- name: SGLANG_BOOTSTRAP_PORT
|
||||
value: "9000"
|
||||
- name: OTLP_NAME
|
||||
value: ds41-3p1d-p3
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 30020
|
||||
- name: d1
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- mooncake-master
|
||||
standalonePattern:
|
||||
templateRef:
|
||||
name: decode-base
|
||||
patch:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-d1
|
||||
spec:
|
||||
containers:
|
||||
- name: sglang
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: "6,7"
|
||||
- name: router
|
||||
replicas: 1
|
||||
dependencies:
|
||||
- p1
|
||||
- p2
|
||||
- p3
|
||||
- d1
|
||||
standalonePattern:
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: ds41-3p1d-router
|
||||
spec:
|
||||
# 入口不走 hostNetwork:普通 Pod 网络 + ClusterIP,天然不对宿主机/外网暴露。
|
||||
# P/D 地址用节点内网 IP(hostNetwork 监听在 192.168.1.243 上,Calico 可达)。
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: b300-01
|
||||
containers:
|
||||
- name: router
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec sglang-router launch \
|
||||
--mini-lb --pd-disaggregation \
|
||||
--host 0.0.0.0 --port 50002 \
|
||||
--prefill http://192.168.1.243:30000 8998 \
|
||||
--prefill http://192.168.1.243:30010 8999 \
|
||||
--prefill http://192.168.1.243:30020 9000 \
|
||||
--decode http://192.168.1.243:30030
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 50002
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 30
|
||||
resources:
|
||||
requests:
|
||||
cpu: "1"
|
||||
memory: 4Gi
|
||||
limits:
|
||||
cpu: "8"
|
||||
memory: 16Gi
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: ds41-3p1d
|
||||
labels:
|
||||
app: ds41-3p1d
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
rbg.workloads.x-k8s.io/group-name: ds41-3p1d
|
||||
rbg.workloads.x-k8s.io/role-name: router
|
||||
ports:
|
||||
- name: http
|
||||
port: 50002
|
||||
targetPort: 50002
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PodMonitor
|
||||
metadata:
|
||||
name: ds41-3p1d
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
sglang-model: ds-v41-flash
|
||||
podMetricsEndpoints:
|
||||
- interval: 15s
|
||||
path: /metrics
|
||||
port: metrics
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"standalone_storage": true,
|
||||
"client_server_address": "127.0.0.1:50052"
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"master_server_address": "127.0.0.1:50051",
|
||||
"metadata_server": "P2PHANDSHAKE",
|
||||
"global_segment_size": "300gb",
|
||||
"protocol": "tcp"
|
||||
}
|
||||
@@ -18,6 +18,8 @@
|
||||
|
||||
namespace sglang {
|
||||
|
||||
using device::math::fast_mod_div_u32_t;
|
||||
|
||||
/// \brief Ratio-2 decode compressor: pair-pool, RMSNorm and the main-KV write.
|
||||
///
|
||||
/// `kv_input` and `kv_state` rows are `2 * kHeadDim` floats, kv then score.
|
||||
@@ -34,8 +36,7 @@ struct Compress2DecodeParams {
|
||||
const int64_t* __restrict__ req; // [num_tokens], req_pool_idx per token
|
||||
const void* __restrict__ raw_out_loc; // [num_tokens] LocT, the FULL slot; 0 marks a padded row
|
||||
uint8_t* __restrict__ kvcache; // [npages, kPageBytes] uint8
|
||||
/// Positions per request slot in the pair-state ring.
|
||||
uint32_t ring_size;
|
||||
fast_mod_div_u32_t ring_size; // Positions per request slot in the pair-state ring.
|
||||
float eps;
|
||||
};
|
||||
|
||||
@@ -43,21 +44,32 @@ struct Compress2DecodeParams {
|
||||
/// At (512, 64) it also keeps the nope/rope split warp-aligned, as the fp8 amax reduction requires.
|
||||
constexpr uint32_t kC2VecSize = 2;
|
||||
|
||||
/// \brief grid = num_tokens, block = kHeadDim / kC2VecSize.
|
||||
/// \brief Where a completing row finds its partner, and who parks a pending row.
|
||||
enum class C2Mode {
|
||||
/// One row per request: the partner comes from the ring, a pending row parks there.
|
||||
Decode,
|
||||
/// `draft_len` consecutive positions per request: `blockIdx.x` is the position inside the
|
||||
/// block, `blockIdx.y` the request, and every row but the first takes its partner from the
|
||||
/// previous `kv_input` row instead of the ring.
|
||||
Verify,
|
||||
/// Arbitrary rows, paired as `_low_ratio_pair_partners` pairs them: the partner is the previous
|
||||
/// row when it is the same request's previous position, else the ring. A pending row does
|
||||
/// nothing here; `c2_prefill_store_kernel` parks the ring rows after this kernel read them.
|
||||
Prefill,
|
||||
};
|
||||
|
||||
/// \brief grid = num_tokens (Decode, Prefill) or `draft_len x batch` (Verify),
|
||||
/// block = kHeadDim / kC2VecSize.
|
||||
///
|
||||
/// An odd position completes a group with its even predecessor; an even one parks in the state.
|
||||
///
|
||||
/// Under target-verify, `draft_len` consecutive positions per request: `blockIdx.x` is the
|
||||
/// position inside the block, `blockIdx.y` the request, and every row but the first takes its
|
||||
/// partner from the previous `kv_input` row instead of the ring.
|
||||
///
|
||||
/// The three reductions have different widths and are not interchangeable: the RMSNorm
|
||||
/// statistic spans the row, an fp8 store scale 64 elements, an fp4 block 16.
|
||||
///
|
||||
/// kLayout is the cache's page format: V4 and V41 store the fake-quantized value; V41_FP4
|
||||
/// stores the e2m1 codes and their e4m3 scales directly, so the fp4 rounding happens once.
|
||||
template <
|
||||
bool kVerify,
|
||||
C2Mode kMode,
|
||||
int64_t kHeadDim,
|
||||
int64_t kRopeDim,
|
||||
int32_t kPageBits,
|
||||
@@ -91,16 +103,21 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
|
||||
|
||||
const auto tx = threadIdx.x;
|
||||
// Verify gives each request a CTA column; decode a flat grid of one row each.
|
||||
const auto row = kVerify ? blockIdx.y * gridDim.x + blockIdx.x : blockIdx.x;
|
||||
const auto row = kMode == C2Mode::Verify ? blockIdx.y * gridDim.x + blockIdx.x : blockIdx.x;
|
||||
PDLWaitPrimary<kUsePDL>();
|
||||
// Slots fit in int32 whatever width the scheduler hands them in.
|
||||
const auto raw_out_loc = static_cast<int32_t>(static_cast<const LocT*>(params.raw_out_loc)[row]);
|
||||
// CUDA graph padding is a completely inert row: do not read schedule, input,
|
||||
// state, or RoPE data, and do not publish output or update the cache.
|
||||
if (raw_out_loc == 0) return PDLTriggerSecondary<kUsePDL>();
|
||||
const auto pos = static_cast<const PosT*>(params.positions)[row];
|
||||
// A completing row reads the slot left by `pos - 1`;
|
||||
// a pending row writes its own slot, so reads and writes stay disjoint.
|
||||
const auto* loc_ptr = static_cast<const LocT*>(params.raw_out_loc);
|
||||
const auto* pos_ptr = static_cast<const PosT*>(params.positions);
|
||||
const auto raw_out_loc = loc_ptr[row];
|
||||
// Prefill rows are many, so a CUDA graph padding row exits before any load.
|
||||
// Decode and verify rows are few: every row issues its loads to keep more in flight.
|
||||
if constexpr (kMode == C2Mode::Prefill) {
|
||||
if (raw_out_loc == 0) return PDLTriggerSecondary<kUsePDL>();
|
||||
}
|
||||
const auto pos = pos_ptr[row];
|
||||
// Prefill leaves a pending row's latent unwritten; its index-K slot is -1, so nothing reads it.
|
||||
if constexpr (kMode == C2Mode::Prefill) {
|
||||
if ((pos & 1) == 0) return PDLTriggerSecondary<kUsePDL>();
|
||||
}
|
||||
const auto rid = params.req[row];
|
||||
|
||||
fp32_vec_t kv_new, score_new;
|
||||
@@ -108,21 +125,32 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
|
||||
score_new.load(params.kv_input + row * kStride, tx + kCTASize);
|
||||
|
||||
const auto ring = static_cast<int64_t>(rid) * params.ring_size;
|
||||
const auto read_row = ring + (pos - 1 + params.ring_size) % params.ring_size;
|
||||
const auto write_row = ring + pos % params.ring_size;
|
||||
const auto write_offset = static_cast<uint32_t>(pos) % params.ring_size;
|
||||
const auto read_offset = write_offset == 0 ? params.ring_size - 1 : write_offset - 1;
|
||||
const auto write_row = ring + write_offset;
|
||||
const auto read_row = ring + read_offset;
|
||||
|
||||
fp32_vec_t kv_old, score_old;
|
||||
const float* partner = params.kv_state + read_row * kStride;
|
||||
if constexpr (kVerify) {
|
||||
if constexpr (kMode == C2Mode::Verify) {
|
||||
if (blockIdx.x != 0) partner = params.kv_input + static_cast<int64_t>(row - 1) * kStride;
|
||||
}
|
||||
if constexpr (kMode == C2Mode::Prefill) {
|
||||
if (row > 0 && params.req[row - 1] == rid && pos_ptr[row - 1] == pos - 1 && loc_ptr[row - 1] != 0) {
|
||||
partner = params.kv_input + static_cast<int64_t>(row - 1) * kStride;
|
||||
}
|
||||
}
|
||||
kv_old.load(partner, tx);
|
||||
score_old.load(partner, tx + kCTASize);
|
||||
|
||||
if ((pos & 1) == 0) {
|
||||
kv_new.store(params.kv_state + write_row * kStride, tx);
|
||||
score_new.store(params.kv_state + write_row * kStride, tx + kCTASize);
|
||||
return PDLTriggerSecondary<kUsePDL>();
|
||||
if constexpr (kMode != C2Mode::Prefill) {
|
||||
// CUDA graph padding case
|
||||
if (raw_out_loc == 0) return PDLTriggerSecondary<kUsePDL>();
|
||||
if ((pos & 1) == 0) {
|
||||
kv_new.store(params.kv_state + write_row * kStride, tx);
|
||||
score_new.store(params.kv_state + write_row * kStride, tx + kCTASize);
|
||||
return PDLTriggerSecondary<kUsePDL>();
|
||||
}
|
||||
}
|
||||
|
||||
constexpr uint32_t kNumWarps = kCTASize / kWarpThreads;
|
||||
@@ -203,9 +231,9 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
|
||||
}
|
||||
}
|
||||
|
||||
const int32_t out_loc = static_cast<uint32_t>(raw_out_loc) >> 1;
|
||||
if constexpr (kLayout == KVLayout::V41_FP4) {
|
||||
// The fp4 cache takes the rotated bf16 value as is: its row quantizer is the fake quant, minus the dequant.
|
||||
const int32_t out_loc = raw_out_loc >> 1;
|
||||
const auto kv_row = Paged::row(params.kvcache, out_loc);
|
||||
return deepseek_v4::v41::store_row<kLayout>(kv_row.data, kv_row.scale, tx, staged);
|
||||
}
|
||||
@@ -228,7 +256,6 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
|
||||
}
|
||||
|
||||
// `raw_out_loc / ratio`; ratio 2 makes it a shift.
|
||||
const int32_t out_loc = raw_out_loc >> 1;
|
||||
const auto kv_row = Paged::row(params.kvcache, out_loc);
|
||||
|
||||
if constexpr (kLayout == KVLayout::V41) {
|
||||
@@ -264,21 +291,63 @@ __global__ __launch_bounds__(kHeadDim / kC2VecSize) void flash_c2_decode_kernel(
|
||||
}
|
||||
}
|
||||
|
||||
/// \brief Prefill ring write-back, launched after `flash_c2_decode_kernel<C2Mode::Prefill>`.
|
||||
///
|
||||
/// Each request's last `ring_size` rows park their `| kv | score |` at `req * ring_size +
|
||||
/// pos % ring_size` for a later step's first partner. It cannot run inside the pooling kernel:
|
||||
/// a chunk whose first row is odd and whose last row is even reads and writes the same ring
|
||||
/// slot. grid = `ring_size x num_reqs`, CTA (k, i) parks the k-th row from the end of request i,
|
||||
/// one float4 per thread; those rows are consecutive positions, so their slots are distinct.
|
||||
template <int64_t kHeadDim, typename PosT, typename LocT, bool kUsePDL>
|
||||
__global__ __launch_bounds__(kHeadDim / 2) void c2_prefill_store_kernel(
|
||||
const Compress2DecodeParams params,
|
||||
const int32_t* __restrict__ extend_start_loc,
|
||||
const int32_t* __restrict__ extend_seq_lens) {
|
||||
using namespace device;
|
||||
constexpr int64_t kStride = kHeadDim * 2;
|
||||
using vec_t = AlignedVector<float, 4>;
|
||||
static_assert(kStride == 4 * (kHeadDim / 2));
|
||||
|
||||
// The pooling kernel reads the ring rows this one overwrites.
|
||||
PDLWaitPrimary<kUsePDL>();
|
||||
const auto bx = static_cast<int32_t>(blockIdx.x);
|
||||
const auto len = extend_seq_lens[blockIdx.y];
|
||||
const auto start_loc = extend_start_loc[blockIdx.y];
|
||||
if (bx >= len) return;
|
||||
const auto row = start_loc + (len - 1 - bx);
|
||||
const auto rid = params.req[row];
|
||||
const auto out = static_cast<const LocT*>(params.raw_out_loc)[row];
|
||||
const auto pos = static_cast<uint32_t>(static_cast<const PosT*>(params.positions)[row]);
|
||||
const auto dst_row = rid * params.ring_size + pos % params.ring_size;
|
||||
if (out == 0) return;
|
||||
vec_t buf;
|
||||
buf.load(params.kv_input + row * kStride, threadIdx.x);
|
||||
PDLTriggerSecondary<kUsePDL>();
|
||||
buf.store(params.kv_state + dst_row * kStride, threadIdx.x);
|
||||
}
|
||||
|
||||
template <int64_t kHeadDim, int64_t kRopeDim, uint32_t kPageSize, deepseek_v4::KVLayout kLayout, bool kUsePDL>
|
||||
struct FlashCompress2Kernel {
|
||||
static constexpr uint32_t kBlockSize = kHeadDim / kC2VecSize;
|
||||
static constexpr int32_t kPageBits = std::bit_width(kPageSize) - 1;
|
||||
static constexpr int64_t kPageBytes = deepseek_v4::kv_page_bytes<kLayout>(kPageSize);
|
||||
static_assert(kLayout != deepseek_v4::KVLayout::V4 || kPageBytes == host::div_ceil(584ll * kPageSize, 576) * 576);
|
||||
template <bool kVerify, typename PosT, typename LocT>
|
||||
template <C2Mode kMode, typename PosT, typename LocT>
|
||||
static constexpr auto kernel =
|
||||
flash_c2_decode_kernel<kVerify, kHeadDim, kRopeDim, kPageBits, PosT, LocT, kLayout, kUsePDL>;
|
||||
flash_c2_decode_kernel<kMode, kHeadDim, kRopeDim, kPageBits, PosT, LocT, kLayout, kUsePDL>;
|
||||
template <typename PosT, typename LocT>
|
||||
static constexpr auto store_kernel = c2_prefill_store_kernel<kHeadDim, PosT, LocT, kUsePDL>;
|
||||
|
||||
/// \brief The (`positions`, `raw_out_loc`) dtype pair, resolved at run time.
|
||||
template <bool kVerify>
|
||||
template <C2Mode kMode>
|
||||
static auto select(const bool pos_i32, const bool loc_i32) {
|
||||
if (pos_i32) return loc_i32 ? kernel<kVerify, int32_t, int32_t> : kernel<kVerify, int32_t, int64_t>;
|
||||
return loc_i32 ? kernel<kVerify, int64_t, int32_t> : kernel<kVerify, int64_t, int64_t>;
|
||||
if (pos_i32) return loc_i32 ? kernel<kMode, int32_t, int32_t> : kernel<kMode, int32_t, int64_t>;
|
||||
return loc_i32 ? kernel<kMode, int64_t, int32_t> : kernel<kMode, int64_t, int64_t>;
|
||||
}
|
||||
|
||||
static auto select_store(const bool pos_i32, const bool loc_i32) {
|
||||
if (pos_i32) return loc_i32 ? store_kernel<int32_t, int32_t> : store_kernel<int32_t, int64_t>;
|
||||
return loc_i32 ? store_kernel<int64_t, int32_t> : store_kernel<int64_t, int64_t>;
|
||||
}
|
||||
|
||||
// The sum of squares reduces through a fixed-size shared array, so the CTA must be whole warps.
|
||||
@@ -302,7 +371,7 @@ struct FlashCompress2Kernel {
|
||||
const tvm::ffi::TensorView kvcache,
|
||||
const int64_t ring_size,
|
||||
const int64_t draft_len) {
|
||||
launch(
|
||||
launch<false>(
|
||||
kv_input,
|
||||
kv_state,
|
||||
kv_output,
|
||||
@@ -317,7 +386,62 @@ struct FlashCompress2Kernel {
|
||||
draft_len);
|
||||
}
|
||||
|
||||
/// \brief The extend-batch pooling kernel, then the ring write-back.
|
||||
///
|
||||
/// Rows may mix requests, prefix offsets and padding; a pending (even) row's `kv_output`
|
||||
/// row is left unwritten. `extend_start_loc` / `extend_seq_lens` are int32 `[num_reqs]`:
|
||||
/// request i owns rows `[start[i], start[i] + len[i])`; zero-length requests are skipped.
|
||||
static void run_prefill_fusion(
|
||||
const tvm::ffi::TensorView kv_input,
|
||||
const tvm::ffi::TensorView kv_state,
|
||||
const tvm::ffi::TensorView kv_output,
|
||||
const tvm::ffi::TensorView norm_weight,
|
||||
const tvm::ffi::TensorView positions,
|
||||
const tvm::ffi::TensorView req,
|
||||
const tvm::ffi::TensorView raw_out_loc,
|
||||
const float eps,
|
||||
const tvm::ffi::TensorView freqs_cis,
|
||||
const tvm::ffi::TensorView kvcache,
|
||||
const int64_t ring_size,
|
||||
const tvm::ffi::TensorView extend_start_loc,
|
||||
const tvm::ffi::TensorView extend_seq_lens) {
|
||||
using namespace host;
|
||||
auto B = SymbolicSize{"num_reqs"};
|
||||
auto device_ = SymbolicDevice{};
|
||||
device_.set_options<kDLGPU>();
|
||||
TensorMatcher({B}).with_dtype<int32_t>().with_device(device_).verify(extend_start_loc);
|
||||
TensorMatcher({B}).with_dtype<int32_t>().with_device(device_).verify(extend_seq_lens);
|
||||
const auto requests = StoreRequests{
|
||||
.extend_start_loc = static_cast<const int32_t*>(extend_start_loc.data_ptr()),
|
||||
.extend_seq_lens = static_cast<const int32_t*>(extend_seq_lens.data_ptr()),
|
||||
.num_reqs = static_cast<uint32_t>(B.unwrap()),
|
||||
};
|
||||
launch<true>(
|
||||
kv_input,
|
||||
kv_state,
|
||||
kv_output,
|
||||
norm_weight,
|
||||
positions,
|
||||
req,
|
||||
raw_out_loc,
|
||||
eps,
|
||||
ring_size,
|
||||
freqs_cis,
|
||||
kvcache,
|
||||
1,
|
||||
&requests);
|
||||
}
|
||||
|
||||
private:
|
||||
struct StoreRequests {
|
||||
const int32_t* extend_start_loc;
|
||||
const int32_t* extend_seq_lens;
|
||||
uint32_t num_reqs;
|
||||
};
|
||||
|
||||
/// \tparam kPrefill `C2Mode::Prefill` plus the ring write-back of `requests`; otherwise
|
||||
/// Decode, or Verify when `draft_len > 1`.
|
||||
template <bool kPrefill>
|
||||
static void launch(
|
||||
const tvm::ffi::TensorView kv_input,
|
||||
const tvm::ffi::TensorView kv_state,
|
||||
@@ -330,7 +454,8 @@ struct FlashCompress2Kernel {
|
||||
const int64_t ring_size,
|
||||
const tvm::ffi::TensorView freqs_cis,
|
||||
const tvm::ffi::TensorView kvcache,
|
||||
const int64_t draft_len) {
|
||||
const int64_t draft_len,
|
||||
const StoreRequests* requests = nullptr) {
|
||||
using namespace host;
|
||||
|
||||
auto N = SymbolicSize{"num_tokens"};
|
||||
@@ -358,7 +483,7 @@ struct FlashCompress2Kernel {
|
||||
|
||||
const auto num_tokens = static_cast<uint32_t>(N.unwrap());
|
||||
if (num_tokens == 0) return;
|
||||
const auto is_verify = draft_len > 1;
|
||||
const auto is_verify = !kPrefill && draft_len > 1;
|
||||
CHECK_HOST(ring_size > 0 && draft_len >= 1);
|
||||
CHECK_HOST(!is_verify || num_tokens % draft_len == 0);
|
||||
CHECK_HOST(!is_verify || ring_size > draft_len)
|
||||
@@ -373,19 +498,28 @@ struct FlashCompress2Kernel {
|
||||
.req = static_cast<const int64_t*>(req.data_ptr()),
|
||||
.raw_out_loc = raw_out_loc.data_ptr(),
|
||||
.kvcache = static_cast<uint8_t*>(kvcache.data_ptr()),
|
||||
.ring_size = static_cast<uint32_t>(ring_size),
|
||||
.ring_size = static_cast<fast_mod_div_u32_t>(ring_size),
|
||||
.eps = eps,
|
||||
};
|
||||
// `LaunchKernel` is move-only, so each arm builds its own.
|
||||
const auto pos_i32 = pos_dtype.is_type<int32_t>();
|
||||
const auto loc_i32 = loc_dtype.is_type<int32_t>();
|
||||
if (is_verify) {
|
||||
if constexpr (kPrefill) {
|
||||
const auto k = select<C2Mode::Prefill>(pos_i32, loc_i32);
|
||||
LaunchKernel(num_tokens, kBlockSize, device_.unwrap()) //
|
||||
.enable_pdl(kUsePDL)(k, params);
|
||||
if (requests->num_reqs == 0) return;
|
||||
const auto store = select_store(pos_i32, loc_i32);
|
||||
const auto grid = dim3{static_cast<uint32_t>(ring_size), requests->num_reqs};
|
||||
LaunchKernel(grid, kHeadDim / 2, device_.unwrap()) //
|
||||
.enable_pdl(kUsePDL)(store, params, requests->extend_start_loc, requests->extend_seq_lens);
|
||||
} else if (is_verify) {
|
||||
const auto block = static_cast<uint32_t>(draft_len);
|
||||
const auto k = select<true>(pos_i32, loc_i32);
|
||||
const auto k = select<C2Mode::Verify>(pos_i32, loc_i32);
|
||||
LaunchKernel(dim3{block, num_tokens / block}, kBlockSize, device_.unwrap()) //
|
||||
.enable_pdl(kUsePDL)(k, params);
|
||||
} else {
|
||||
const auto k = select<false>(pos_i32, loc_i32);
|
||||
const auto k = select<C2Mode::Decode>(pos_i32, loc_i32);
|
||||
LaunchKernel(num_tokens, kBlockSize, device_.unwrap()) //
|
||||
.enable_pdl(kUsePDL)(k, params);
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
#include <sgl_kernel/tensor.h>
|
||||
#include <sgl_kernel/utils.h>
|
||||
|
||||
#include <sgl_kernel/math.cuh>
|
||||
#include <sgl_kernel/runtime.cuh>
|
||||
#include <sgl_kernel/type.cuh>
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
@@ -21,31 +22,7 @@ namespace sglang {
|
||||
|
||||
using device::distributed::PushWorkSpace;
|
||||
using device::distributed::Semaphore;
|
||||
|
||||
// Runtime uint32 division as a multiply-high and a shift (round-up magic,
|
||||
// exact below 2^31); cuda::fast_mod_div needs a newer CCCL than CUDA 13 bundles.
|
||||
struct fast_mod_div_u32_t {
|
||||
uint32_t divisor;
|
||||
uint32_t magic;
|
||||
uint32_t shift;
|
||||
|
||||
__host__ explicit fast_mod_div_u32_t(uint32_t d) : divisor(d), magic(0), shift(0) {
|
||||
if (d > 1) {
|
||||
const uint32_t log2_ceil = 32 - std::countl_zero(d - 1);
|
||||
const uint32_t p = 31 + log2_ceil;
|
||||
magic = static_cast<uint32_t>(((uint64_t{1} << p) + d - 1) / d);
|
||||
shift = p - 32;
|
||||
}
|
||||
}
|
||||
|
||||
__device__ friend uint32_t operator/(uint32_t n, const fast_mod_div_u32_t& fd) {
|
||||
return fd.divisor == 1 ? n : __umulhi(n, fd.magic) >> fd.shift;
|
||||
}
|
||||
|
||||
__device__ friend uint32_t operator%(uint32_t n, const fast_mod_div_u32_t& fd) {
|
||||
return n - (n / fd) * fd.divisor;
|
||||
}
|
||||
};
|
||||
using device::math::fast_mod_div_u32_t;
|
||||
|
||||
template <uint32_t kWorldSize>
|
||||
struct NVLinkCommPushParams {
|
||||
@@ -168,8 +145,7 @@ PUSH_KERNEL void nvlink_push_kernel(const __grid_constant__ NVLinkCommPushParams
|
||||
ptx::st_multimem_16B(vec, dst_ptr_mc, vid);
|
||||
}
|
||||
} else /* reduce-scatter only */ {
|
||||
const auto token_id = vid / params.vecs_per_token_div;
|
||||
const auto offset = vid % params.vecs_per_token_div;
|
||||
const auto [token_id, offset] = math::div(vid, params.vecs_per_token_div);
|
||||
// Both by a compile-time constant, so this is a mask and a shift.
|
||||
const auto dst_rank = token_id % kWorldSize;
|
||||
const auto dst_token_id = token_id / kWorldSize;
|
||||
|
||||
@@ -112,6 +112,53 @@ SGL_DEVICE float fma_f32_bf16(bf16_t a, float b, float acc) {
|
||||
return fmaf(cast<fp32_t>(a), b, acc);
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief Fast uint32 division and modulo by a compile-time constant divisor.
|
||||
*
|
||||
* This struct is only exact when the divisor is less than `2^31`. Larger dividends
|
||||
* will lead to incorrect results. Be careful when using this struct.
|
||||
*/
|
||||
struct fast_mod_div_u32_t {
|
||||
public:
|
||||
explicit constexpr fast_mod_div_u32_t(uint32_t d) : m_divisor(d) {
|
||||
if (d > 1) {
|
||||
const uint32_t log2_ceil = 32 - std::countl_zero(d - 1);
|
||||
const uint32_t p = 31 + log2_ceil;
|
||||
m_magic = static_cast<uint32_t>(((uint64_t{1} << p) + d - 1) / d);
|
||||
m_shift = p - 32;
|
||||
} else {
|
||||
m_magic = 0;
|
||||
m_shift = 0;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
SGL_DEVICE friend uint32_t operator/(T n, const fast_mod_div_u32_t& d) {
|
||||
static_assert(std::is_unsigned_v<T> && sizeof(T) <= sizeof(uint32_t));
|
||||
return d.m_divisor == 1 ? n : __umulhi(n, d.m_magic) >> d.m_shift;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
SGL_DEVICE friend uint32_t operator%(T n, const fast_mod_div_u32_t& d) {
|
||||
static_assert(std::is_unsigned_v<T> && sizeof(T) <= sizeof(uint32_t));
|
||||
return n - (n / d) * d.m_divisor;
|
||||
}
|
||||
|
||||
constexpr operator uint32_t() const {
|
||||
return m_divisor;
|
||||
}
|
||||
|
||||
private:
|
||||
uint32_t m_divisor;
|
||||
uint32_t m_magic;
|
||||
uint32_t m_shift;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
SGL_DEVICE uint2 div(T n, const fast_mod_div_u32_t& d) {
|
||||
return {n / d, n % d};
|
||||
}
|
||||
|
||||
} // namespace device::math
|
||||
|
||||
} // namespace sglang
|
||||
|
||||
@@ -273,6 +273,67 @@ def store_fp4_index_k_cache(
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _gather_fp4_index_k_kernel(
|
||||
cache,
|
||||
cache_stride,
|
||||
slots,
|
||||
payload,
|
||||
scales,
|
||||
n,
|
||||
page_size: tl.constexpr,
|
||||
BLOCK_ROWS: tl.constexpr,
|
||||
):
|
||||
# int32 view of a page: page_size * 16 payload words, then page_size scale words.
|
||||
rows = tl.program_id(0) * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS)
|
||||
valid = rows < n
|
||||
slot = tl.load(slots + rows, valid, 0).to(tl.int64)
|
||||
page = slot // page_size
|
||||
offset = slot - page * page_size
|
||||
page_base = cache + page * cache_stride
|
||||
words = tl.arange(0, 16)
|
||||
payload_words = tl.load(
|
||||
page_base[:, None] + offset[:, None] * 16 + words[None, :], valid[:, None]
|
||||
)
|
||||
tl.store(
|
||||
payload + rows[:, None].to(tl.int64) * 16 + words[None, :],
|
||||
payload_words,
|
||||
valid[:, None],
|
||||
)
|
||||
tl.store(scales + rows, tl.load(page_base + page_size * 16 + offset, valid), valid)
|
||||
|
||||
|
||||
def gather_fp4_index_k(
|
||||
cache: torch.Tensor, slots: torch.Tensor, *, page_size: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Packed fp4 rows at ``slots`` of the index-K ``cache`` (``[pages, page_size *
|
||||
68]`` uint8, ``store_fp4_index_k_cache``'s layout): (payload int8 ``[n, 64]``,
|
||||
ue8m0 scales packed int32 ``[n]``), the layout ``quantize_fp4_indexer_tensor``
|
||||
returns."""
|
||||
assert cache.shape[1] == page_size * INDEX_K_SLOT_BYTES and cache.stride(1) == 1
|
||||
# The kernel addresses slots as `slots + rows`, so a strided view would be read
|
||||
# as if its elements were packed.
|
||||
assert slots.is_contiguous()
|
||||
words = cache.view(torch.int32)
|
||||
n = slots.shape[0]
|
||||
payload = torch.empty((n, 16), device=cache.device, dtype=torch.int32)
|
||||
scales = torch.empty((n,), device=cache.device, dtype=torch.int32)
|
||||
if n > 0:
|
||||
block_rows = 64
|
||||
_gather_fp4_index_k_kernel[(triton.cdiv(n, block_rows),)](
|
||||
words,
|
||||
words.stride(0),
|
||||
slots,
|
||||
payload,
|
||||
scales,
|
||||
n,
|
||||
page_size,
|
||||
BLOCK_ROWS=block_rows,
|
||||
num_warps=4,
|
||||
)
|
||||
return payload.view(torch.int8), scales
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _index_k_rope_pack_kernel(
|
||||
X,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Fused ratio-1 and ratio-2 decode compressors: RMSNorm, RoPE and the FlashMLA
|
||||
cache write in one launch.
|
||||
"""Fused ratio-1 and ratio-2 compressors: RMSNorm, RoPE and the FlashMLA cache
|
||||
write in one launch. The ratio-1 kernel is per token, so decode and extend share
|
||||
it; ratio 2 has a decode/verify entry point and an extend one.
|
||||
|
||||
Ratio 1 takes the bf16 ``wkv`` projection as is: RoPE uses the token's own
|
||||
position and the compressed slot equals the FULL slot. Ratio 2 pair-pools the
|
||||
@@ -17,6 +18,7 @@ import torch
|
||||
from sglang.kernels.jit.utils import (
|
||||
cache_once,
|
||||
is_arch_support_pdl,
|
||||
is_hip_runtime,
|
||||
load_jit,
|
||||
make_cpp_args,
|
||||
)
|
||||
@@ -119,11 +121,12 @@ def _jit_c2_module(
|
||||
is_arch_support_pdl(),
|
||||
)
|
||||
return load_jit(
|
||||
make_name("c2_decode"),
|
||||
make_name("c2"),
|
||||
*args,
|
||||
cuda_files=["deepseek_v4/c2.cuh"],
|
||||
cuda_wrappers=[
|
||||
("decode_fusion", f"FlashCompress2Kernel<{args}>::run_decode_fusion"),
|
||||
("prefill_fusion", f"FlashCompress2Kernel<{args}>::run_prefill_fusion"),
|
||||
],
|
||||
)
|
||||
|
||||
@@ -149,6 +152,12 @@ def c2_decode_norm_rope_store(
|
||||
|
||||
The cache store uses ``raw_out_loc // 2`` as its slot.
|
||||
|
||||
A padded graph row is marked by ``raw_out_loc == 0`` and writes nothing, but
|
||||
decode and verify rows issue their ring load before that check, so a padded
|
||||
row's ``req`` and ``positions`` must still address an in-bounds ring slot.
|
||||
The decode graph buffers zero both tails (``PaddingPolicy.ZERO``); a runner
|
||||
that left stale request indices there would read out of the ring pool.
|
||||
|
||||
:param freqs_cis: ``[max_pos, rope_dim]`` fp32, real/imag interleaved --
|
||||
``torch.view_as_real(freqs).flatten(-2)``. Indexed
|
||||
in-kernel at ``positions - 1``, the position the latent
|
||||
@@ -181,3 +190,60 @@ def c2_decode_norm_rope_store(
|
||||
draft_len,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def c2_prefill_norm_rope_store(
|
||||
kv_input: torch.Tensor,
|
||||
kv_state: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
req: torch.Tensor,
|
||||
raw_out_loc: torch.Tensor,
|
||||
eps: float,
|
||||
freqs_cis: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
extend_start_loc: torch.Tensor,
|
||||
extend_seq_lens: torch.Tensor,
|
||||
*,
|
||||
page_size: int,
|
||||
ring_size: int,
|
||||
layout: Union[KVLayout, str] = KVLayout.V4,
|
||||
out: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
""":func:`c2_decode_norm_rope_store` for an extend batch, then the ring write-back.
|
||||
|
||||
Rows may mix requests, prefix offsets and CUDA graph padding (``raw_out_loc
|
||||
== 0``). An odd position pairs with the previous row when that row is the
|
||||
same request's previous position, else with its ring slot. Afterwards each
|
||||
request's last ``ring_size`` rows are saved to the ring, which a later step
|
||||
reads for its first partner.
|
||||
|
||||
:param extend_start_loc: int32 ``[num_reqs]``, each request's first row.
|
||||
:param extend_seq_lens: int32 ``[num_reqs]``, each request's row count.
|
||||
:return: ``out``, ``[num_tokens, head_dim]`` bf16. Only the rows of odd
|
||||
positions are written; the rest are left as they were.
|
||||
"""
|
||||
num_tokens, fused_dim = kv_input.shape
|
||||
head_dim = fused_dim // 2
|
||||
if out is None:
|
||||
out = kv_input.new_empty((num_tokens, head_dim), dtype=torch.bfloat16)
|
||||
|
||||
layout = KVLayout.parse(layout)
|
||||
module = _jit_c2_module(head_dim, freqs_cis.shape[-1], page_size, layout)
|
||||
module.prefill_fusion(
|
||||
kv_input,
|
||||
kv_state,
|
||||
out,
|
||||
norm_weight,
|
||||
positions,
|
||||
req,
|
||||
raw_out_loc,
|
||||
eps,
|
||||
freqs_cis,
|
||||
# HIP's fp8_e4m3_t is uint8_t, so the kernel matches the fp8 pool as bytes
|
||||
k_cache.view(torch.uint8) if is_hip_runtime() else k_cache,
|
||||
ring_size,
|
||||
extend_start_loc,
|
||||
extend_seq_lens,
|
||||
)
|
||||
return out
|
||||
|
||||
@@ -36,7 +36,7 @@ def _build_swa_token_ids_kernel(
|
||||
tl.store(out_ptr + out_off + i, swa_id)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["top_k"])
|
||||
@triton.jit(do_not_specialize=["num_tokens", "num_reqs", "top_k"])
|
||||
def _combine_topk_swa_indices_kernel(
|
||||
combined_indices_ptr,
|
||||
combined_indices_stride,
|
||||
@@ -49,59 +49,76 @@ def _combine_topk_swa_indices_kernel(
|
||||
gather_lens_ptr,
|
||||
compressed_base_ptr,
|
||||
swa_base_ptr,
|
||||
num_tokens,
|
||||
num_reqs,
|
||||
top_k,
|
||||
COMPRESS_RATIO: tl.constexpr,
|
||||
WINDOW_SIZE: tl.constexpr,
|
||||
PADDED_TOP_K: tl.constexpr,
|
||||
WIDTH: tl.constexpr,
|
||||
PADDED_WIDTH: tl.constexpr,
|
||||
BLOCK_T: tl.constexpr,
|
||||
SEARCH_STEPS: tl.constexpr,
|
||||
):
|
||||
batch_idx = tl.program_id(0)
|
||||
worker_id = tl.program_id(1)
|
||||
num_workers = tl.num_programs(1)
|
||||
# BLOCK_T consecutive tokens per program, so the grid scales with tokens, not requests.
|
||||
token = tl.program_id(0) * BLOCK_T + tl.arange(0, BLOCK_T)
|
||||
valid = token < num_tokens
|
||||
|
||||
# query_start_loc may be a global tensor; rebase to chunk-local offsets
|
||||
# by subtracting the chunk's starting value.
|
||||
base = tl.load(query_start_loc_ptr)
|
||||
query_start = tl.load(query_start_loc_ptr + batch_idx) - base
|
||||
query_end = tl.load(query_start_loc_ptr + batch_idx + 1) - base
|
||||
seq_len = tl.load(seq_lens_ptr + batch_idx)
|
||||
gather_len = tl.load(gather_lens_ptr + batch_idx)
|
||||
compressed_base = tl.load(compressed_base_ptr + batch_idx)
|
||||
swa_base = tl.load(swa_base_ptr + batch_idx)
|
||||
# Each token's request: the last r with query_start_loc[r] - base <= token.
|
||||
lo = tl.zeros([BLOCK_T], dtype=tl.int32)
|
||||
hi = tl.full([BLOCK_T], 0, dtype=tl.int32) + num_reqs
|
||||
for _ in tl.static_range(SEARCH_STEPS):
|
||||
mid = (lo + hi + 1) // 2
|
||||
start = tl.load(query_start_loc_ptr + mid, mask=lo < hi, other=0) - base
|
||||
go_right = (lo < hi) & (start <= token)
|
||||
lo = tl.where(go_right, mid, lo)
|
||||
hi = tl.where((lo < hi) & ~go_right, mid - 1, hi)
|
||||
batch_idx = tl.minimum(lo, num_reqs - 1)
|
||||
# Rows past the last request's queries belong to no request: -1 rows, length 0.
|
||||
owned = valid & (token < tl.load(query_start_loc_ptr + num_reqs) - base)
|
||||
|
||||
pos = tl.load(query_pos_ptr + token, mask=owned, other=0)
|
||||
seq_len = tl.load(seq_lens_ptr + batch_idx, mask=owned, other=0)
|
||||
gather_len = tl.load(gather_lens_ptr + batch_idx, mask=owned, other=0)
|
||||
compressed_base = tl.load(compressed_base_ptr + batch_idx, mask=owned, other=0)
|
||||
swa_base = tl.load(swa_base_ptr + batch_idx, mask=owned, other=0)
|
||||
# SWA portion of the gathered buffer starts from position
|
||||
# (seq_len - gather_len), not 0. The +pos-gather_start formula maps a
|
||||
# query's window back into the workspace's SWA region.
|
||||
gather_start = seq_len - gather_len
|
||||
|
||||
for token_idx in range(query_start + worker_id, query_end, num_workers):
|
||||
pos = tl.load(query_pos_ptr + token_idx)
|
||||
# -1 entries inside the top-k span stay -1 (attention skips them).
|
||||
# top_k=0 disables the compressed portion for SWA-only layers.
|
||||
topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, top_k)
|
||||
swa_len = tl.minimum(pos + 1, WINDOW_SIZE)
|
||||
# -1 entries inside the top-k span stay -1 (attention skips them).
|
||||
# top_k=0 disables the compressed portion for SWA-only layers.
|
||||
topk_len = tl.where(owned, tl.minimum((pos + 1) // COMPRESS_RATIO, top_k), 0)
|
||||
swa_len = tl.where(owned, tl.minimum(pos + 1, WINDOW_SIZE), 0)
|
||||
|
||||
combined_row = token_idx.to(tl.int64) * combined_indices_stride
|
||||
topk_row = token_idx.to(tl.int64) * topk_indices_stride
|
||||
combined_row = token.to(tl.int64)[:, None] * combined_indices_stride
|
||||
topk_row = token.to(tl.int64)[:, None] * topk_indices_stride
|
||||
|
||||
offset = tl.arange(0, PADDED_TOP_K)
|
||||
mask = offset < topk_len
|
||||
topk_vals = tl.load(
|
||||
topk_indices_ptr + topk_row + offset,
|
||||
mask=mask,
|
||||
)
|
||||
tl.store(
|
||||
combined_indices_ptr + combined_row + offset,
|
||||
tl.where(topk_vals >= 0, topk_vals + compressed_base, -1),
|
||||
mask=mask,
|
||||
)
|
||||
# Whole rows, -1 padded, so the output needs no fill pass:
|
||||
# [top-k (-1 holes kept) | window | -1 up to WIDTH].
|
||||
offset = tl.arange(0, PADDED_WIDTH)[None, :]
|
||||
in_topk = offset < topk_len[:, None]
|
||||
topk_vals = tl.load(
|
||||
topk_indices_ptr + topk_row + offset, mask=valid[:, None] & in_topk, other=-1
|
||||
)
|
||||
# Workspace SWA index: swa_base[r] + (gather_offset_in_buffer).
|
||||
# For positions [pos - swa_len + 1, pos], the buffer offsets are
|
||||
# [pos - swa_len + 1 - gather_start, pos - gather_start].
|
||||
swa_start = swa_base + pos - swa_len + 1 - gather_start
|
||||
swa_vals = swa_start[:, None] + (offset - topk_len[:, None])
|
||||
in_swa = offset < (topk_len + swa_len)[:, None]
|
||||
vals = tl.where(
|
||||
in_topk,
|
||||
tl.where(topk_vals >= 0, topk_vals + compressed_base[:, None], -1),
|
||||
tl.where(in_swa, swa_vals, -1),
|
||||
)
|
||||
tl.store(
|
||||
combined_indices_ptr + combined_row + offset,
|
||||
vals,
|
||||
mask=valid[:, None] & (offset < WIDTH),
|
||||
)
|
||||
|
||||
offset = tl.arange(0, WINDOW_SIZE)
|
||||
# Workspace SWA index: swa_base[r] + (gather_offset_in_buffer).
|
||||
# For positions [pos - swa_len + 1, pos], the buffer offsets are
|
||||
# [pos - swa_len + 1 - gather_start, pos - gather_start].
|
||||
tl.store(
|
||||
combined_indices_ptr + combined_row + topk_len + offset,
|
||||
swa_base + offset + pos - swa_len + 1 - gather_start,
|
||||
mask=offset < swa_len,
|
||||
)
|
||||
|
||||
tl.store(combined_lens_ptr + token_idx, topk_len + swa_len)
|
||||
tl.store(combined_lens_ptr + token, topk_len + swa_len, mask=valid)
|
||||
|
||||
@@ -1882,6 +1882,34 @@ def hc_expand(x: torch.Tensor, n: int) -> torch.Tensor:
|
||||
return x.repeat(1, n)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _hc_broadcast_kernel(
|
||||
x_ptr, y_ptr, H, x_stride_m, HC: tl.constexpr, BLOCK_H: tl.constexpr
|
||||
):
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
offs = tl.program_id(1) * BLOCK_H + tl.arange(0, BLOCK_H)
|
||||
mask = offs < H
|
||||
x = tl.load(x_ptr + row * x_stride_m + offs, mask=mask)
|
||||
y_row = y_ptr + row * (HC * H)
|
||||
for k in tl.static_range(HC):
|
||||
tl.store(y_row + k * H + offs, x, mask=mask)
|
||||
|
||||
|
||||
def hc_broadcast(x: torch.Tensor, hc_mult: int) -> torch.Tensor:
|
||||
"""[T, H] -> [T, hc_mult, H], every copy equal to x: the initial mHC residual."""
|
||||
if not x.is_cuda or x.stride(-1) != 1:
|
||||
return x.unsqueeze(1).repeat(1, hc_mult, 1)
|
||||
num_tokens, hidden = x.shape
|
||||
y = x.new_empty(num_tokens, hc_mult, hidden)
|
||||
if num_tokens == 0:
|
||||
return y
|
||||
block_h = 1024
|
||||
_hc_broadcast_kernel[(num_tokens, triton.cdiv(hidden, block_h))](
|
||||
x, y, hidden, x.stride(0), HC=hc_mult, BLOCK_H=block_h, num_warps=4
|
||||
)
|
||||
return y
|
||||
|
||||
|
||||
def hc_contract(x: torch.Tensor, n: int) -> torch.Tensor:
|
||||
return x.unflatten(-1, (n, -1)).mean(dim=-2)
|
||||
|
||||
|
||||
@@ -181,12 +181,15 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
)
|
||||
|
||||
cfg = resolving_view(server_args)
|
||||
if model_config_of(server_args).hf_config.model_type != "deepseek_v41":
|
||||
hf_config = model_config_of(server_args).hf_config
|
||||
if hf_config.model_type != "deepseek_v41":
|
||||
if cfg.enable_encoder_swa_bounded_replay:
|
||||
raise ValueError(
|
||||
"--enable-encoder-swa-bounded-replay requires DeepSeek-V4.1"
|
||||
)
|
||||
return
|
||||
if hf_config.vision_n_layers > 0 and cfg.enable_prefill_cp:
|
||||
_validate_deepseek_v41_vision_prefill_cp(server_args)
|
||||
if cfg.enable_encoder_swa_bounded_replay:
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend
|
||||
|
||||
@@ -197,7 +200,8 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
cfg.cuda_graph_config.prefill.backend != Backend.DISABLED,
|
||||
),
|
||||
("DP attention", cfg.enable_dp_attention),
|
||||
("context parallelism", cfg.attn_cp_size > 1),
|
||||
# Prefill CP declares attn_cp_size and DP attention only later.
|
||||
("context parallelism", cfg.attn_cp_size > 1 or cfg.enable_prefill_cp),
|
||||
("external cache linker", cfg.enable_unified_cache_external_linker),
|
||||
("unified memory", cfg.enable_unified_memory),
|
||||
("PD disaggregation", cfg.disaggregation_mode != "null"),
|
||||
@@ -306,3 +310,40 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
"--enable-decoder-swa-bounded-replay cannot be combined with "
|
||||
f"{feature} yet; disable one of them."
|
||||
)
|
||||
|
||||
|
||||
def _validate_deepseek_v41_vision_prefill_cp(server_args: ServerArgs) -> None:
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
|
||||
|
||||
cfg = resolving_view(server_args)
|
||||
if cfg.cp_strategy != "interleave":
|
||||
raise ValueError(
|
||||
"DeepSeek-V4.1 vision with prefill CP requires --cp-strategy "
|
||||
f"interleave; got {cfg.cp_strategy!r}."
|
||||
)
|
||||
if cfg.cuda_graph_config.prefill.backend != Backend.DISABLED:
|
||||
# The CP runner merges image features eagerly; no capture path replays it.
|
||||
locked = getattr(server_args, "_cuda_graph_config_locked", set())
|
||||
if (Phase.PREFILL, "backend") in locked:
|
||||
raise ValueError(
|
||||
"DeepSeek-V4.1 vision with prefill CP runs eager prefill; remove "
|
||||
"the explicit prefill CUDA graph backend."
|
||||
)
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"validate_deepseek_v41_features",
|
||||
cuda_graph_config=with_phase(
|
||||
cfg.cuda_graph_config, Phase.PREFILL, backend=Backend.DISABLED
|
||||
),
|
||||
)
|
||||
logger.warning(
|
||||
"Disabling the prefill CUDA graph for DeepSeek-V4.1 vision with prefill CP."
|
||||
)
|
||||
if (
|
||||
str(cfg.speculative_algorithm).upper() == "DSPARK"
|
||||
and cfg.enable_decoder_swa_bounded_replay
|
||||
):
|
||||
raise ValueError(
|
||||
"DeepSeek-V4.1 vision with prefill CP does not support DSpark together "
|
||||
"with --enable-decoder-swa-bounded-replay yet."
|
||||
)
|
||||
|
||||
@@ -32,7 +32,7 @@ class TransferKVChunk:
|
||||
# Set when the staging worker first counts this chunk toward the per-room
|
||||
# outstanding count; stays set across re-enqueue on a watermark defer.
|
||||
staging_counted: bool = False
|
||||
# Mori early-send: CUDA event to synchronize before RDMA (optional).
|
||||
# Early-send: GPU event to synchronize before reading KV (optional).
|
||||
wait_event: Optional[object] = None
|
||||
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ from sglang.srt.managers.overlap_utils import RelayPayload
|
||||
from sglang.srt.mem_cache.common import maybe_cache_unfinished_req
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo
|
||||
from sglang.srt.utils.common import is_pin_memory_available
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -140,9 +141,14 @@ class ScheduleBatchDisaggregationDecodeMixin:
|
||||
error_message, HTTPStatus.INTERNAL_SERVER_ERROR
|
||||
)
|
||||
req.grammar.finished = req.finished()
|
||||
# Non-blocking H2D: with overlap this runs on the schedule stream after
|
||||
# its wait on the in-flight forward, so a blocking copy would stall the
|
||||
# host until that forward ends and leave the GPU idle.
|
||||
last_tokens_tensor = torch.tensor(
|
||||
last_tokens, dtype=torch.int64, device=self.device
|
||||
)
|
||||
last_tokens,
|
||||
dtype=torch.int64,
|
||||
pin_memory=is_pin_memory_available(self.device),
|
||||
).to(self.device, non_blocking=True)
|
||||
|
||||
spec_info = self.spec_algorithm.build_disagg_draft_input(
|
||||
self,
|
||||
|
||||
@@ -960,6 +960,8 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
dst_layer_ids: Optional[List[int]] = None,
|
||||
dst_device_data_indices: Optional[npt.NDArray[np.int32]] = None,
|
||||
dst_device_data_ptrs: Optional[set[int]] = None,
|
||||
dst_item_lens: list[int] | None = None,
|
||||
bootstrap_room: Optional[int] = None,
|
||||
) -> int:
|
||||
"""
|
||||
Generic KV cache transfer supporting both MHA and MLA architectures.
|
||||
@@ -1011,6 +1013,21 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
len(dst_data_ptrs),
|
||||
allow_positional_fallback=self.pp_size == 1,
|
||||
)
|
||||
if dst_item_lens is not None:
|
||||
for i, j in pairs:
|
||||
if item_lens[i] != dst_item_lens[j]:
|
||||
assert bootstrap_room is not None
|
||||
failure_reason = (
|
||||
f"{state_type} item length mismatch for paired "
|
||||
f"entries src[{i}]={item_lens[i]} "
|
||||
f"dst[{j}]={dst_item_lens[j]}"
|
||||
)
|
||||
logger.error(failure_reason)
|
||||
self.conclude_failure(
|
||||
bootstrap_room=bootstrap_room,
|
||||
failure_reason=failure_reason,
|
||||
)
|
||||
return -1
|
||||
layers_params = [
|
||||
(src_data_ptrs[i], dst_data_ptrs[j], item_lens[i]) for i, j in pairs
|
||||
]
|
||||
@@ -1020,6 +1037,24 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
src_data_ptrs, dst_data_ptrs, state_type
|
||||
)
|
||||
)
|
||||
if dst_item_lens is not None:
|
||||
_, mapped_dst_lens, _ = self.get_mla_kv_ptrs_with_pp(
|
||||
item_lens, dst_item_lens, state_type
|
||||
)
|
||||
for layer_id in range(layers_current_pp_stage):
|
||||
if item_lens[layer_id] != mapped_dst_lens[layer_id]:
|
||||
assert bootstrap_room is not None
|
||||
failure_reason = (
|
||||
f"{state_type} item length mismatch for positional "
|
||||
f"entry {layer_id}: prefill={item_lens[layer_id]} "
|
||||
f"decode={mapped_dst_lens[layer_id]}"
|
||||
)
|
||||
logger.error(failure_reason)
|
||||
self.conclude_failure(
|
||||
bootstrap_room=bootstrap_room,
|
||||
failure_reason=failure_reason,
|
||||
)
|
||||
return -1
|
||||
layers_params = [
|
||||
(
|
||||
src_kv_ptrs[layer_id],
|
||||
@@ -2110,9 +2145,13 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
),
|
||||
src_layer_ids=src_state_layer_ids,
|
||||
dst_layer_ids=dst_state_layer_ids,
|
||||
dst_item_lens=dst_item_lens,
|
||||
bootstrap_room=req.room,
|
||||
)
|
||||
or rc
|
||||
)
|
||||
if self.request_status.get(req.room, KVPoll.Failed) == KVPoll.Failed:
|
||||
return rc
|
||||
elif st in (StateType.MINIMAX_INDEX_K, StateType.MINIMAX_DENSE_KV):
|
||||
# Compacted layer lists require equal TP and PP=1 on both peers.
|
||||
if self.pp_size is not None and self.pp_size > 1:
|
||||
@@ -2376,6 +2415,19 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
self._staging_outstanding[kv_chunk.room] += 1
|
||||
kv_chunk.staging_counted = True
|
||||
|
||||
# Under overlap scheduling an early cached-prefix send can
|
||||
# precede completion of the previous prefill forward. Wait on
|
||||
# the worker, not the scheduler, before reading source KV.
|
||||
# A room that already failed skips the wait; the status check
|
||||
# below then drops the chunk, including on an abort during it.
|
||||
if kv_chunk.wait_event is not None:
|
||||
if (
|
||||
kv_chunk.room in self.request_status
|
||||
and self.check_status(kv_chunk.room) != KVPoll.Failed
|
||||
):
|
||||
kv_chunk.wait_event.synchronize()
|
||||
kv_chunk.wait_event = None
|
||||
|
||||
if (
|
||||
kv_chunk.room not in self.request_status
|
||||
or self.check_status(kv_chunk.room) == KVPoll.Failed
|
||||
@@ -2588,6 +2640,13 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
executor,
|
||||
target_rank_registration_info,
|
||||
)
|
||||
if (
|
||||
self.request_status.get(
|
||||
kv_chunk.room, KVPoll.Failed
|
||||
)
|
||||
== KVPoll.Failed
|
||||
):
|
||||
break
|
||||
if state_rc != 0:
|
||||
with self.session_lock:
|
||||
self.session_failures[
|
||||
@@ -2919,6 +2978,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
state_indices: Optional[List] = None,
|
||||
num_kv_tokens: Optional[int] = None,
|
||||
trace_ctx: Optional[Union[TraceReqContext, TraceNullContext]] = None,
|
||||
wait_event: Optional[object] = None,
|
||||
):
|
||||
assert self.disaggregation_mode == DisaggregationMode.PREFILL
|
||||
assert not is_last_chunk or (is_last_chunk and aux_index is not None)
|
||||
@@ -2938,12 +2998,22 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
# add further chunks into the transfer queue.
|
||||
return
|
||||
|
||||
# NOTE(shangming): sharding according to the dst_infos to make sure
|
||||
# requests with the same dst_sessions will be added into the same
|
||||
# queue, which enables early abort with failed sessions.
|
||||
# Shard by dst session ports AND bootstrap_room. The port term alone
|
||||
# degenerates to a constant when all requests target a single decode
|
||||
# instance (its session ports are fixed), collapsing every transfer
|
||||
# onto 1/16 of the lanes and throttling P->D intake (measured at
|
||||
# client concurrency 300: #transfer-req piled at 100+ while decode
|
||||
# itself stayed healthy). Mixing in bootstrap_room keeps all chunks
|
||||
# of one request on the same lane (per-request FIFO order preserved)
|
||||
# while spreading distinct rooms across all lanes. Abort/cleanup is
|
||||
# unaffected: chunks are dropped per-room at dequeue via
|
||||
# request_status (see the worker loop) and per-request via
|
||||
# failed_sessions before send — no lane-level purge exists, and the
|
||||
# deferred abort ack is room-scoped (_staging_outstanding). Same
|
||||
# trade-off as mori/conn.py which shards by bootstrap_room alone.
|
||||
dst_infos = self.transfer_infos[bootstrap_room].keys()
|
||||
session_port_sum = sum(int(session.rsplit(":", 1)[1]) for session in dst_infos)
|
||||
shard_idx = session_port_sum % len(self.transfer_queues)
|
||||
shard_idx = (session_port_sum + bootstrap_room) % len(self.transfer_queues)
|
||||
|
||||
if trace_ctx is None:
|
||||
trace_ctx = TraceNullContext()
|
||||
@@ -2958,6 +3028,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
state_indices=state_indices,
|
||||
num_kv_tokens=num_kv_tokens,
|
||||
trace_ctx=trace_ctx,
|
||||
wait_event=wait_event,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -3064,6 +3135,8 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
|
||||
if should_skip:
|
||||
return
|
||||
|
||||
wait_event = getattr(self, "_early_send_wait_event", None)
|
||||
self._early_send_wait_event = None
|
||||
if not is_last_chunk:
|
||||
self.kv_mgr.add_transfer_request(
|
||||
self.bootstrap_room,
|
||||
@@ -3072,6 +3145,7 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
|
||||
False,
|
||||
num_kv_tokens=num_kv_tokens,
|
||||
trace_ctx=self.trace_ctx.copy_for_thread(),
|
||||
wait_event=wait_event,
|
||||
)
|
||||
else:
|
||||
self.kv_mgr.add_transfer_request(
|
||||
@@ -3083,6 +3157,7 @@ class MooncakeKVSender(MooncakeFailureExceptionMixin, CommonKVSender):
|
||||
state_indices=state_indices,
|
||||
num_kv_tokens=num_kv_tokens,
|
||||
trace_ctx=self.trace_ctx.copy_for_thread(),
|
||||
wait_event=wait_event,
|
||||
)
|
||||
self._record_transfer_indices(kv_indices, state_indices)
|
||||
|
||||
|
||||
@@ -21,6 +21,8 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import zlib
|
||||
from array import array
|
||||
from collections import deque
|
||||
from http import HTTPStatus
|
||||
@@ -105,6 +107,64 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
_is_npu = is_npu()
|
||||
|
||||
# DEBUG-ONLY (p2 wedge trap 2026-10-13): per-step batch-decision fingerprint
|
||||
# and event-loop phase logging. Same env as custom_all_reduce_v2 ARDBG.
|
||||
# 1 = sampled/one-line, 2 = + stacks on ONE_SHOT_PUSH. Off by default.
|
||||
_AR_SEQ_DEBUG = int(os.environ.get("SGLANG_AR_SEQ_DEBUG", "0"))
|
||||
_plan_seq = 0
|
||||
_plan_none_seq = 0
|
||||
|
||||
|
||||
def _dbg_plan(self, batch) -> None:
|
||||
global _plan_seq, _plan_none_seq
|
||||
if not _AR_SEQ_DEBUG:
|
||||
return
|
||||
_plan_seq += 1
|
||||
rank = torch.distributed.get_rank()
|
||||
if batch is None:
|
||||
# 空转步采样记录,避免 idle 自旋刷爆日志
|
||||
_plan_none_seq += 1
|
||||
if _plan_none_seq & 511:
|
||||
return
|
||||
logger.info("[ARDBG] plan rank=%s step=%d batch=None", rank, _plan_seq)
|
||||
return
|
||||
reqs = batch.reqs
|
||||
fp = 0
|
||||
ext_total = 0
|
||||
for r in reqs:
|
||||
# crc32 而非 hash():后者按进程随机化种子,跨 rank 不可比
|
||||
fp = (fp * 131 + zlib.crc32(r.rid.encode())) & 0xFFFFFFFFFFFFFFFF
|
||||
er = getattr(r, "extend_range", None)
|
||||
el = int(er.length) if er is not None else -1
|
||||
fp = (fp * 131 + el) & 0xFFFFFFFFFFFFFFFF
|
||||
fp = (fp * 131 + int(getattr(r, "prefix_len", -1))) & 0xFFFFFFFFFFFFFFFF
|
||||
if el > 0:
|
||||
ext_total += el
|
||||
logger.info(
|
||||
"[ARDBG] plan rank=%s step=%d nreq=%d ext=%d fp=%016x",
|
||||
rank,
|
||||
_plan_seq,
|
||||
len(reqs),
|
||||
ext_total,
|
||||
fp,
|
||||
)
|
||||
|
||||
|
||||
def _dbg_phase(label: str, batch=None) -> None:
|
||||
if not _AR_SEQ_DEBUG:
|
||||
return
|
||||
if batch is None and label in ("inflight_enter", "inflight_exit"):
|
||||
# 空转迭代的 inflight 相位价值低,采样记录
|
||||
if _plan_seq & 255:
|
||||
return
|
||||
logger.info(
|
||||
"[ARDBG] phase rank=%s step=%s %s nreq=%s",
|
||||
torch.distributed.get_rank(),
|
||||
_plan_seq,
|
||||
label,
|
||||
len(batch.reqs) if batch is not None else "-",
|
||||
)
|
||||
|
||||
|
||||
def should_force_retry(req: Req) -> bool:
|
||||
"""Test hook to force a request into optimistic prefill retry."""
|
||||
@@ -628,6 +688,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
if batch:
|
||||
set_schedule_time_batch(batch)
|
||||
|
||||
_dbg_plan(self, batch)
|
||||
return NextBatchPlan(batch_to_run=batch, running_batch=running_batch)
|
||||
|
||||
@torch.no_grad()
|
||||
@@ -658,13 +719,17 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
if batch:
|
||||
if self.enable_staging:
|
||||
self.maybe_prefetch_staging_for_batch(batch)
|
||||
_dbg_phase("run_batch_enter", batch)
|
||||
result = self.run_batch(batch)
|
||||
_dbg_phase("run_batch_exit", batch)
|
||||
self.process_batch_result(batch, result)
|
||||
else:
|
||||
self._sched_idled = True
|
||||
self.on_idle()
|
||||
|
||||
_dbg_phase("inflight_enter")
|
||||
self.process_disagg_prefill_inflight_queue()
|
||||
_dbg_phase("inflight_exit")
|
||||
|
||||
# Update last_batch
|
||||
self.last_batch = batch
|
||||
@@ -698,7 +763,9 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
if batch:
|
||||
if self.enable_staging:
|
||||
self.maybe_prefetch_staging_for_batch(batch)
|
||||
_dbg_phase("run_batch_enter", batch)
|
||||
batch_result = self.run_batch(batch)
|
||||
_dbg_phase("run_batch_exit", batch)
|
||||
self._apply_war_barrier()
|
||||
self.result_queue.append((batch.copy(), batch_result))
|
||||
else:
|
||||
@@ -713,7 +780,9 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
# When the server is idle, do self-check and re-init some states
|
||||
self.on_idle()
|
||||
|
||||
_dbg_phase("inflight_enter")
|
||||
self.process_disagg_prefill_inflight_queue()
|
||||
_dbg_phase("inflight_exit")
|
||||
|
||||
# Run sample of the current batch
|
||||
# It depends on the result of the last batch (e.g., grammar), so we run it after the last batch is processed.
|
||||
@@ -851,7 +920,11 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
advance_logprob_pt(i, req)
|
||||
continue
|
||||
|
||||
maybe_cache_unfinished_req(req, self.tree_cache)
|
||||
self.checkpoint_disagg_prefill(req)
|
||||
# G9: the disagg-prefill path bypasses batch_result_processor's
|
||||
# prefill loop, so record the drain event here — final chunk
|
||||
# done, request heading to the inflight (KV transfer) queue.
|
||||
self.record_prefill_drain(req.seqlen)
|
||||
self.disagg_prefill_inflight_queue.append(req)
|
||||
if self.spec_algorithm.is_eagle() and draft_input is not None:
|
||||
req.output_topk_p = draft_input.topk_p[i]
|
||||
@@ -1208,7 +1281,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
chunked_req_to_exclude = set()
|
||||
if (req := self.chunked_req) is not None:
|
||||
chunked_req_to_exclude.add(req)
|
||||
maybe_cache_unfinished_req(req, self.tree_cache, chunked=True)
|
||||
self.checkpoint_disagg_prefill(req, chunked=True)
|
||||
|
||||
if not self.check_bootstrap(req):
|
||||
if is_aborted(req):
|
||||
@@ -1503,10 +1576,21 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
else:
|
||||
self.disagg_prefill_pending_chunk_rids.add(req.rid)
|
||||
|
||||
def checkpoint_disagg_prefill(
|
||||
self: Scheduler, req: Req, chunked: bool = False
|
||||
) -> None:
|
||||
"""Publish the request's computed prefix to the tree cache.
|
||||
|
||||
Name shim for the #42035-era call structure: this branch predates
|
||||
#41520's ``checkpoint_kv_cache`` / ``insert_req`` cache API, so the
|
||||
equivalent here is ``maybe_cache_unfinished_req``.
|
||||
"""
|
||||
maybe_cache_unfinished_req(req, self.tree_cache, chunked=chunked)
|
||||
|
||||
def optimistic_release_and_requeue(self: Scheduler, req: Req) -> None:
|
||||
"""Release KV cache and requeue an optimistic prefill request."""
|
||||
max_attempts = get_disagg().optimistic_prefill_attempts
|
||||
maybe_cache_unfinished_req(req, self.tree_cache)
|
||||
self.checkpoint_disagg_prefill(req)
|
||||
# The cached prefix is evictable once the KV is released. Its length
|
||||
# (capped at what a retry can match) seeds the retry's storage baseline,
|
||||
# so an evicted prefix is looked up in L3 once before it is recomputed.
|
||||
@@ -1519,10 +1603,11 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
)
|
||||
)
|
||||
self._release_aborted_request(req)
|
||||
# Mamba insertion donates the checkpoint and clears its sequence marker.
|
||||
release_kv_cache(
|
||||
req, self.tree_cache, is_insert=not self.tree_cache.supports_mamba()
|
||||
)
|
||||
# The checkpoint above (maybe_cache_unfinished_req) already handed the
|
||||
# prefill KV to the tree; the request is not finished, so the release
|
||||
# only frees the rest. (#42194 semantics; our tree predates #41520's
|
||||
# mamba-donation rework, mamba models are not run on this branch.)
|
||||
release_kv_cache(req, self.tree_cache, is_insert=False)
|
||||
req.reset_for_retract()
|
||||
req.output_ids = array("q")
|
||||
req.start_send_idx = 0
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
from collections import deque
|
||||
from contextlib import nullcontext
|
||||
@@ -43,6 +45,32 @@ if is_npu():
|
||||
DSV4NPUTokenToKVPool,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# DEBUG-ONLY (p2 wedge repro 2026-10-13): env-gated seq logging for host-side
|
||||
# gloo all-reduce sync points. Mirrors SGLANG_AR_SEQ_DEBUG in
|
||||
# custom_all_reduce_v2.py. 1 = one-line, 2 = + python stack. Not for prod.
|
||||
_AR_SEQ_DEBUG = int(os.environ.get("SGLANG_AR_SEQ_DEBUG", "0"))
|
||||
_host_ar_seq = 0
|
||||
|
||||
|
||||
def _dbg_log_host_ar(phase: str, polls) -> None:
|
||||
global _host_ar_seq
|
||||
if not _AR_SEQ_DEBUG:
|
||||
return
|
||||
seq = _host_ar_seq
|
||||
_host_ar_seq += 1
|
||||
# 空转迭代(无 poller)采样 1/256,避免 idle 自旋刷爆日志
|
||||
if (not polls) and seq & 255:
|
||||
return
|
||||
logger.info(
|
||||
"[ARDBG] host rank=%s seq=%d phase=%s polls=%s",
|
||||
dist.get_rank(),
|
||||
seq,
|
||||
phase,
|
||||
polls,
|
||||
)
|
||||
|
||||
#########################
|
||||
# Constants & Enums
|
||||
#########################
|
||||
@@ -213,13 +241,20 @@ def poll_and_all_reduce_attn_cp_tp_group(
|
||||
attn_cp_cpu_group: dist.ProcessGroup,
|
||||
attn_tp_cpu_group: dist.ProcessGroup,
|
||||
):
|
||||
# DEBUG-ONLY
|
||||
_dbg_log_host_ar("enter", len(pollers))
|
||||
# First sync across attn-tp ranks so all TP participants for a given (dp, cp)
|
||||
# shard observe the same status transitions.
|
||||
polls = poll_and_all_reduce(pollers, attn_tp_cpu_group)
|
||||
# DEBUG-ONLY
|
||||
_dbg_log_host_ar("tp-done", polls)
|
||||
|
||||
# Then sync across attn-cp ranks, so all TPxCP participants in one DP shard
|
||||
# converge to the same global status.
|
||||
return _all_reduce_polls(polls, attn_cp_cpu_group)
|
||||
result = _all_reduce_polls(polls, attn_cp_cpu_group)
|
||||
# DEBUG-ONLY
|
||||
_dbg_log_host_ar("cp-done", result)
|
||||
return result
|
||||
|
||||
|
||||
def poll_and_all_reduce_with_staging(
|
||||
|
||||
@@ -25,6 +25,8 @@ the kernel captured in the graph dereferences its row at replay time.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import traceback
|
||||
from contextlib import contextmanager
|
||||
from typing import List, NamedTuple, Optional, Tuple
|
||||
|
||||
@@ -68,6 +70,9 @@ MB = 1024 * 1024
|
||||
_ALIGN_BYTES = 1024
|
||||
_SEMAPHORE_BYTES = 128
|
||||
_MAX_GRAPH_INPUTS = 131072
|
||||
# DEBUG-ONLY (p2 wedge repro 2026-10-13): env-gated seq logging for TP device
|
||||
# all-reduce calls. 1 = one-line per call, 2 = + python stack. Not for prod.
|
||||
_AR_SEQ_DEBUG = int(os.environ.get("SGLANG_AR_SEQ_DEBUG", "0"))
|
||||
# resolved once at import time; explicit constructor sizes take precedence
|
||||
_DEFAULT_MAX_SIZE = envs.SGLANG_CUSTOM_ALL_REDUCE_V2_MAX_SIZE_KB.get() * 1024
|
||||
# forced per-direction workspace sizes; highest priority, override both the
|
||||
@@ -297,6 +302,9 @@ class CustomAllReduceV2:
|
||||
self.pull_enabled,
|
||||
)
|
||||
dist.barrier(group=self.group)
|
||||
# DEBUG-ONLY: per-instance seq counter for SGLANG_AR_SEQ_DEBUG logging
|
||||
self._dbg_seq = 0
|
||||
self._dbg_rank = self.rank
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Algo selection
|
||||
@@ -376,6 +384,27 @@ class CustomAllReduceV2:
|
||||
assert config is not None, f"No config for {nbytes = }"
|
||||
algo, use_graph, use_multicast = config
|
||||
graph_params = self._allocate_graph_row(input, nbytes) if use_graph else None
|
||||
if _AR_SEQ_DEBUG:
|
||||
seq = self._dbg_seq
|
||||
self._dbg_seq += 1
|
||||
# 采样 1/1024:生产高频路径下全量日志会拖垮 P 节点
|
||||
if seq & 1023 == 0:
|
||||
logger.info(
|
||||
"[ARDBG] dev rank=%s seq=%d nbytes=%d algo=%s graph=%s capturing=%s",
|
||||
self._dbg_rank,
|
||||
seq,
|
||||
nbytes,
|
||||
getattr(algo, "name", algo),
|
||||
use_graph,
|
||||
torch.cuda.is_current_stream_capturing(),
|
||||
)
|
||||
if _AR_SEQ_DEBUG >= 2 and getattr(algo, "name", algo) == "ONE_SHOT_PUSH":
|
||||
logger.info(
|
||||
"[ARDBG] stack rank=%s seq=%d\n%s",
|
||||
self._dbg_rank,
|
||||
seq,
|
||||
"".join(traceback.format_stack(limit=15)),
|
||||
)
|
||||
return custom_all_reduce(
|
||||
self.obj,
|
||||
input,
|
||||
|
||||
@@ -1776,6 +1776,18 @@ async def openai_v1_chat_completions(
|
||||
)
|
||||
|
||||
|
||||
@app.post("/query_cache_hit", dependencies=[Depends(validate_json_request)])
|
||||
async def query_cache_hit(request: ChatCompletionRequest, raw_request: Request):
|
||||
"""G9: probe prefix-cache hit (L1/L2 + L3 storage) for a chat request.
|
||||
|
||||
Deployed on decode engines; the PD router calls it before dispatch to
|
||||
decide conditional bypass. Accepts a normal chat body or `input_ids`.
|
||||
"""
|
||||
return await raw_request.app.state.openai_serving_chat.query_cache_hit(
|
||||
request, raw_request
|
||||
)
|
||||
|
||||
|
||||
@app.post(
|
||||
"/v1/embeddings",
|
||||
response_class=ORJSONResponse,
|
||||
|
||||
@@ -1140,6 +1140,73 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
return "text", processed_messages.prompt_ids
|
||||
return "input_ids", processed_messages.prompt_ids
|
||||
|
||||
async def query_cache_hit(
|
||||
self, request: ChatCompletionRequest, raw_request: Request = None
|
||||
) -> ORJSONResponse:
|
||||
"""G9 probe: report how much of this chat request's prompt is already
|
||||
cached (per-rank-best L1/L2 radix + shared L3 storage) without running
|
||||
any prefill. The router uses it for conditional PD-bypass decisions.
|
||||
|
||||
Multimodal requests are rejected (v1): their sentinel-token expansion
|
||||
costs ~10-100ms per image for only ~2% of traffic; the router skips
|
||||
them for bypass anyway.
|
||||
"""
|
||||
try:
|
||||
internal_req, _ = self._convert_to_internal_request(request, raw_request)
|
||||
except Exception as e:
|
||||
return ORJSONResponse(
|
||||
status_code=400,
|
||||
content={"error": f"failed to convert chat request: {e}"},
|
||||
)
|
||||
if internal_req.contains_mm_input():
|
||||
return ORJSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "multimodal requests are not supported by /query_cache_hit"},
|
||||
)
|
||||
try:
|
||||
# 只取 input_ids,不走 _tokenize_one_request——它要求 rid_to_state
|
||||
# 里已有 ReqState(正常生成路径在 tokenize 前注册),纯查询没有。
|
||||
# dsv41 系 chat encoding 直接渲染 prompt_ids;否则对渲染后的
|
||||
# text 走 tokenizer(_tokenize_texts 与真实路径同一入口)。
|
||||
if internal_req.input_ids is not None:
|
||||
input_ids = list(internal_req.input_ids)
|
||||
elif internal_req.text:
|
||||
input_ids, _ = await self.tokenizer_manager._tokenize_texts(
|
||||
internal_req.text, False
|
||||
)
|
||||
else:
|
||||
return ORJSONResponse(
|
||||
status_code=400,
|
||||
content={"error": "request produced neither input_ids nor text"},
|
||||
)
|
||||
from sglang.srt.managers.io_struct import QueryCacheHitReqInput
|
||||
|
||||
result = await self.tokenizer_manager.query_cache_hit(
|
||||
QueryCacheHitReqInput(
|
||||
input_ids=list(input_ids),
|
||||
extra_key=getattr(internal_req, "extra_key", None),
|
||||
cache_salt=getattr(internal_req, "cache_salt", None),
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
return ORJSONResponse(
|
||||
status_code=500, content={"error": f"cache-hit query failed: {e}"}
|
||||
)
|
||||
if result.error is not None:
|
||||
return ORJSONResponse(status_code=500, content={"error": result.error})
|
||||
hit = result.l1_hit_tokens + result.l2_hit_tokens + result.l3_hit_tokens
|
||||
return ORJSONResponse(
|
||||
content={
|
||||
"total_tokens": result.total_tokens,
|
||||
"l1_hit_tokens": result.l1_hit_tokens,
|
||||
"l2_hit_tokens": result.l2_hit_tokens,
|
||||
"l3_hit_tokens": result.l3_hit_tokens,
|
||||
"hit_tokens": hit,
|
||||
"new_tokens": max(0, result.total_tokens - hit),
|
||||
"dp_rank": result.dp_rank,
|
||||
}
|
||||
)
|
||||
|
||||
def _convert_to_internal_request(
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
|
||||
@@ -758,6 +758,16 @@ class Envs:
|
||||
SGLANG_HICACHE_HF3FS_CONFIG_PATH = EnvStr(None)
|
||||
SGLANG_HICACHE_DECODE_OFFLOAD_STRIDE = EnvInt(None)
|
||||
SGLANG_HICACHE_SKIP_HOST_DUPLICATE_RECLAIM = EnvBool(False)
|
||||
# Per-stage timing logs for the L3 storage prefetch path (query / transfer /
|
||||
# ack reduce / drain / graft / load-back). Emits "HC_TIMING ..." lines.
|
||||
SGLANG_HICACHE_TIMING = EnvBool(False)
|
||||
# Pages per batched storage IO call (query / get / put chunking).
|
||||
SGLANG_HICACHE_STORAGE_BATCH_SIZE = EnvInt(128)
|
||||
# Number of parallel prefetch IO lanes (worker threads) per controller.
|
||||
# Operations are sharded by crc32(request_id) so every rank maps a request
|
||||
# to the same lane; each lane owns dedicated completion sync groups.
|
||||
# MUST be identical across all TP/CP/PP ranks of an instance.
|
||||
SGLANG_HICACHE_PREFETCH_IO_THREADS = EnvInt(4)
|
||||
SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR = EnvStr(None)
|
||||
# File-backend LRU eviction (opt-in; sizes accept SI/IEC suffixes, "0" disables).
|
||||
SGLANG_HICACHE_FILE_BACKEND_MAX_SIZE = EnvStr(None)
|
||||
|
||||
@@ -206,6 +206,7 @@ def _create_flashmla_metadata():
|
||||
# only for that shape and go stale silently if FlashMLA retunes it.
|
||||
_FLASHMLA_SCHED_BLOCK_SIZE_N = 64
|
||||
_FLASHMLA_SCHED_FIXED_OVERHEAD = 5
|
||||
_FLASHMLA_SCHED_MAX_SHARED_BYTES = 48 << 10
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=None)
|
||||
@@ -242,6 +243,10 @@ def _maybe_precompute_flashmla_sched_meta(
|
||||
|
||||
b, s_q = q.shape[0], q.shape[1]
|
||||
num_sm_parts = max(_num_sms(q.device.index) // s_q, 1)
|
||||
# Skip when shared memory exceeds the limit.
|
||||
smem_bytes = 4 * (b * 5 + 1 + num_sm_parts * 8)
|
||||
if smem_bytes > _FLASHMLA_SCHED_MAX_SHARED_BYTES:
|
||||
return
|
||||
meta = torch.empty((num_sm_parts, META_INTS), dtype=torch.int32, device=q.device)
|
||||
num_splits = torch.empty((b + 1,), dtype=torch.int32, device=q.device)
|
||||
flashmla_sched_meta(
|
||||
@@ -2543,7 +2548,7 @@ class DeepseekV4AttnBackend(
|
||||
req_to_token=self.req_to_token,
|
||||
full_to_swa=self.token_to_kv_pool.full_to_swa_index_mapping,
|
||||
swa_window_size=SWA_WINDOW,
|
||||
swa_page_size=self.token_to_kv_pool.swa_kv_pool.page_size,
|
||||
swa_page_size=self.token_to_kv_pool.get_swa_key_page_size(),
|
||||
num_qo_tokens=num_qo_tokens,
|
||||
max_seq_len=max(seq_lens_cpu_list),
|
||||
total_swa=total_swa,
|
||||
@@ -2930,6 +2935,22 @@ class DeepseekV4AttnBackend(
|
||||
self._low_ratio_compress_fused(
|
||||
layer, x, req, pos, draft_len=self.speculative_num_draft_tokens
|
||||
)
|
||||
elif (
|
||||
layer.compressor.use_fused_compress
|
||||
and forward_batch.forward_mode.is_extend_without_speculative()
|
||||
and forward_batch.extend_start_loc is not None
|
||||
):
|
||||
# DP padding extends extend_seq_lens but not extend_start_loc, whose
|
||||
# length is the request count; padded rows belong to no request.
|
||||
start_loc = forward_batch.extend_start_loc
|
||||
seq_lens = forward_batch.extend_seq_lens[: start_loc.shape[0]]
|
||||
self._low_ratio_compress_fused(
|
||||
layer,
|
||||
x,
|
||||
req,
|
||||
pos,
|
||||
extend_offsets=(start_loc.to(torch.int32), seq_lens.to(torch.int32)),
|
||||
)
|
||||
else:
|
||||
self._low_ratio_compress_torch(
|
||||
layer,
|
||||
@@ -3001,13 +3022,18 @@ class DeepseekV4AttnBackend(
|
||||
fuse_index_store=_is_sm100_or_newer(),
|
||||
)
|
||||
|
||||
def _low_ratio_compress_fused(self, layer, x, req, pos, *, draft_len=1) -> None:
|
||||
def _low_ratio_compress_fused(
|
||||
self, layer, x, req, pos, *, draft_len=1, extend_offsets=None
|
||||
) -> None:
|
||||
"""Decode or target-verify; an extend batch passes ``extend_offsets``, the int32
|
||||
(first row, row count) of each request."""
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer_rope import (
|
||||
index_k_norm_rope_pack_store,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.low_ratio_compress import (
|
||||
c1_decode_norm_rope_store,
|
||||
c2_decode_norm_rope_store,
|
||||
c2_prefill_norm_rope_store,
|
||||
)
|
||||
|
||||
pool = self.token_to_kv_pool
|
||||
@@ -3040,7 +3066,7 @@ class DeepseekV4AttnBackend(
|
||||
# CompressStatePool stores each request's pending pairs in a position ring.
|
||||
# KVAndScore rows use | kv | score |, addressed as req * ring_size + pos % ring_size.
|
||||
state = pool.get_attention_compress_states(layer_id)
|
||||
latent = c2_decode_norm_rope_store(
|
||||
args = (
|
||||
compressor.project_fused(x),
|
||||
state.kv_score_buffer.kv_score,
|
||||
compressor.norm.weight.data,
|
||||
@@ -3050,11 +3076,23 @@ class DeepseekV4AttnBackend(
|
||||
compressor.norm.eps,
|
||||
freqs_cis,
|
||||
kv_cache,
|
||||
page_size=page_size,
|
||||
ring_size=state.ring_size,
|
||||
draft_len=draft_len,
|
||||
layout=kv_layout,
|
||||
)
|
||||
if extend_offsets is None:
|
||||
latent = c2_decode_norm_rope_store(
|
||||
*args,
|
||||
page_size=page_size,
|
||||
ring_size=state.ring_size,
|
||||
draft_len=draft_len,
|
||||
layout=kv_layout,
|
||||
)
|
||||
else:
|
||||
latent = c2_prefill_norm_rope_store(
|
||||
*args,
|
||||
*extend_offsets,
|
||||
page_size=page_size,
|
||||
ring_size=state.ring_size,
|
||||
layout=kv_layout,
|
||||
)
|
||||
out_loc = core.c2_out_loc
|
||||
|
||||
indexer = layer.indexer
|
||||
@@ -3760,7 +3798,7 @@ class DeepseekV4AttnBackend(
|
||||
compress_ratio
|
||||
)
|
||||
|
||||
swa_kv_page_size = token_to_kv_pool.swa_kv_pool.page_size
|
||||
swa_kv_page_size = token_to_kv_pool.get_swa_key_page_size()
|
||||
assert swa_k_cache.ndim == 2
|
||||
# The kernel detects each cache's format from the last dim of this
|
||||
# view: 584 (V4), 528 (V4.1 fp8) or 288 (V4.1 fp4, extra cache only).
|
||||
|
||||
@@ -1472,9 +1472,10 @@ class IndexerKPool(MultiPlatformOp):
|
||||
if metadata is None:
|
||||
return None
|
||||
|
||||
assert forward_batch.seq_lens_cpu is not None
|
||||
# Spec-v2 decode/verify batches may carry no CPU seq_lens mirror (the
|
||||
# backend opts out of the D2H); test emptiness on the device shape.
|
||||
mode = forward_batch.forward_mode
|
||||
if mode.is_idle() or len(forward_batch.seq_lens_cpu) == 0:
|
||||
if mode.is_idle() or forward_batch.seq_lens.shape[0] == 0:
|
||||
return torch.full(
|
||||
(x.shape[0], self.index_topk + self.index_kpool - 1),
|
||||
-1,
|
||||
|
||||
@@ -306,7 +306,8 @@ class DeepseekSparseAttnBackend(
|
||||
extend_dummy_seqs_capped_by_req_pool: bool = True
|
||||
# Decode/verify/draft graph replay rebuilds metadata from static buffers
|
||||
# (page-table width) and never reads seq_lens_cpu / seq_lens_sum; opt out of
|
||||
# the D2H sync. The eager fallback derives lengths from GPU seq_lens.
|
||||
# the D2H sync. The eager fallback derives lengths from GPU seq_lens, and the
|
||||
# KPool indexer reads the host mirror only for extend, which always has it.
|
||||
needs_cpu_seq_lens: bool = False
|
||||
# init_cuda_graph_state sizes this for every backend, but only the TRT-LLM
|
||||
# branch of __init__ allocates one.
|
||||
@@ -337,7 +338,6 @@ class DeepseekSparseAttnBackend(
|
||||
)
|
||||
self.dsa_index_topk = get_dsa_index_topk(hf_config)
|
||||
self.dsa_index_kpool = get_dsa_index_kpool(hf_config)
|
||||
self.needs_cpu_seq_lens = self.dsa_index_kpool > 1
|
||||
self._init_kpool_metadata_fusion()
|
||||
self.max_context_len = model_runner.model_config.context_len
|
||||
self.num_q_heads = (
|
||||
|
||||
@@ -136,8 +136,8 @@ def combine_topk_swa_indices(
|
||||
|
||||
Returns int32 indices [num_tokens, padded_topk_swa] and per-token scanned-prefix
|
||||
lengths, including -1 entries skipped by attention. Width is padded to 128.
|
||||
Preallocated out_indices must contain -1 outside the written prefix;
|
||||
reuse requires chunk-invariant scanned-prefix lengths. out_lens is overwritten.
|
||||
Rows are written whole and -1 padded, so preallocated out_indices / out_lens
|
||||
are overwritten entirely; rows past the last request are -1 with length 0.
|
||||
"""
|
||||
assert topk_indices.dtype == torch.int32
|
||||
assert query_start_loc.dtype == torch.int32
|
||||
@@ -155,9 +155,9 @@ def combine_topk_swa_indices(
|
||||
num_reqs = seq_lens.shape[0]
|
||||
combined_topk = combined_topk_width(topk, window_size)
|
||||
if out_indices is None:
|
||||
combined_indices = torch.full(
|
||||
# The kernel writes whole -1 padded rows.
|
||||
combined_indices = torch.empty(
|
||||
(num_tokens, combined_topk),
|
||||
-1,
|
||||
dtype=torch.int32,
|
||||
device=topk_indices.device,
|
||||
)
|
||||
@@ -166,7 +166,7 @@ def combine_topk_swa_indices(
|
||||
assert out_indices.dtype == torch.int32
|
||||
combined_indices = out_indices
|
||||
if out_lens is None:
|
||||
combined_lens = torch.zeros(
|
||||
combined_lens = torch.empty(
|
||||
num_tokens, dtype=torch.int32, device=topk_indices.device
|
||||
)
|
||||
else:
|
||||
@@ -174,8 +174,10 @@ def combine_topk_swa_indices(
|
||||
assert out_lens.dtype == torch.int32
|
||||
combined_lens = out_lens
|
||||
|
||||
NUM_WORKERS = 128
|
||||
_combine_topk_swa_indices_kernel[(num_reqs, NUM_WORKERS)](
|
||||
if num_tokens == 0:
|
||||
return combined_indices, combined_lens
|
||||
block_t = 2 # Measured on B200 with num_warps=2, 16384 tokens, bs 1 to 64.
|
||||
_combine_topk_swa_indices_kernel[(triton.cdiv(num_tokens, block_t),)](
|
||||
combined_indices,
|
||||
combined_indices.stride(0),
|
||||
combined_lens,
|
||||
@@ -187,10 +189,16 @@ def combine_topk_swa_indices(
|
||||
gather_lens,
|
||||
compressed_base,
|
||||
swa_base,
|
||||
num_tokens,
|
||||
num_reqs,
|
||||
top_k=topk,
|
||||
COMPRESS_RATIO=compress_ratio,
|
||||
WINDOW_SIZE=window_size,
|
||||
PADDED_TOP_K=triton.next_power_of_2(topk_indices.shape[-1]),
|
||||
WIDTH=combined_topk,
|
||||
PADDED_WIDTH=triton.next_power_of_2(combined_topk),
|
||||
BLOCK_T=block_t,
|
||||
SEARCH_STEPS=max(num_reqs, 1).bit_length(),
|
||||
num_warps=2,
|
||||
)
|
||||
return combined_indices, combined_lens
|
||||
|
||||
|
||||
@@ -239,6 +239,11 @@ class EngramHasher(nn.Module):
|
||||
self.image_token_id: Optional[int] = None
|
||||
self.history: Optional[torch.Tensor] = None
|
||||
self.pad_row = 0
|
||||
# typing only
|
||||
self.token_map: torch.Tensor
|
||||
self.multipliers: torch.Tensor
|
||||
self.primes: torch.Tensor
|
||||
self.offsets: torch.Tensor
|
||||
|
||||
def init_history(self, num_req_slots: int, device) -> None:
|
||||
"""Allocate oldest-first history with a spare row for graph padding."""
|
||||
|
||||
@@ -17,6 +17,7 @@ limitations under the License.
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import zlib
|
||||
from dataclasses import dataclass
|
||||
from queue import Empty, Queue
|
||||
from typing import TYPE_CHECKING, Callable, List, NamedTuple, Optional
|
||||
@@ -32,6 +33,17 @@ from sglang.srt.mem_cache.hicache_storage import (
|
||||
count_pool_hits,
|
||||
)
|
||||
|
||||
from sglang.srt.environ import envs as _envs
|
||||
|
||||
_HC_TIMING = None
|
||||
|
||||
|
||||
def _hc_timing_enabled() -> bool:
|
||||
global _HC_TIMING
|
||||
if _HC_TIMING is None:
|
||||
_HC_TIMING = _envs.SGLANG_HICACHE_TIMING.get()
|
||||
return _HC_TIMING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
||||
from sglang.srt.mem_cache.pool_host import HostKVCache
|
||||
@@ -306,7 +318,18 @@ class HiCacheController:
|
||||
self.attn_tp_group = attn_tp_group
|
||||
self.pp_group = pp_group
|
||||
self.prefetch_hits_sync_groups: List[torch.distributed.ProcessGroup] = []
|
||||
self.prefetch_completion_sync_groups: List[torch.distributed.ProcessGroup] = []
|
||||
# One set of completion sync groups per prefetch IO lane. Each lane's
|
||||
# worker/sync threads form a serial pipeline, so the collective
|
||||
# sequence on a lane's groups is identical across ranks as long as
|
||||
# lane assignment is deterministic (see dispatch_prefetch).
|
||||
self.prefetch_completion_sync_groups_by_lane: List[
|
||||
List[torch.distributed.ProcessGroup]
|
||||
] = []
|
||||
# Read once at construction; attach/reset reuse the stored value so
|
||||
# lane count never changes underneath live sync groups.
|
||||
self.num_prefetch_io_lanes = max(
|
||||
1, _envs.SGLANG_HICACHE_PREFETCH_IO_THREADS.get()
|
||||
)
|
||||
self.mem_pool_device_allocator = token_to_kv_pool_allocator
|
||||
mem_pool_device = token_to_kv_pool_allocator.get_kvcache()
|
||||
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool
|
||||
@@ -335,8 +358,8 @@ class HiCacheController:
|
||||
self.storage_stop_event = threading.Event()
|
||||
|
||||
# Storage control queues, (re)created whenever the storage threads start.
|
||||
self.prefetch_buffer: Optional[Queue[PrefetchOperation]] = None
|
||||
self.prefetch_sync_queue: Optional[Queue[PrefetchAck]] = None
|
||||
self.prefetch_io_queues: List[Queue[PrefetchOperation]] = []
|
||||
self.prefetch_sync_queues: List[Queue[PrefetchAck]] = []
|
||||
self.prefetch_hit_queue: Optional[Queue[StorageOperation]] = None
|
||||
self.ack_prefetch_queue = Queue[PrefetchAck]()
|
||||
self.ack_backup_queue: Optional[Queue[StorageOperation]] = None
|
||||
@@ -444,27 +467,39 @@ class HiCacheController:
|
||||
self.prefetch_thread = threading.Thread(
|
||||
target=self.prefetch_thread_func, daemon=True
|
||||
)
|
||||
self.prefetch_io_aux_thread = threading.Thread(
|
||||
target=self.prefetch_io_aux_func, daemon=True
|
||||
)
|
||||
self.prefetch_sync_thread = threading.Thread(
|
||||
target=self.prefetch_sync_thread_func, daemon=True
|
||||
)
|
||||
self.prefetch_io_aux_threads = [
|
||||
threading.Thread(
|
||||
target=self.prefetch_io_aux_func, args=(lane,), daemon=True
|
||||
)
|
||||
for lane in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
self.prefetch_sync_threads = [
|
||||
threading.Thread(
|
||||
target=self.prefetch_sync_thread_func, args=(lane,), daemon=True
|
||||
)
|
||||
for lane in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
self.backup_thread = threading.Thread(
|
||||
target=self.backup_thread_func, daemon=True
|
||||
)
|
||||
self.prefetch_queue = Queue()
|
||||
self.backup_queue = Queue()
|
||||
self.prefetch_buffer = Queue()
|
||||
self.prefetch_sync_queue = Queue()
|
||||
self.prefetch_io_queues = [
|
||||
Queue() for _ in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
self.prefetch_sync_queues = [
|
||||
Queue() for _ in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
self.prefetch_hit_queue = Queue()
|
||||
self.ack_prefetch_queue = Queue()
|
||||
self.ack_backup_queue = Queue()
|
||||
self.host_mem_release_queue = Queue()
|
||||
|
||||
self.prefetch_thread.start()
|
||||
self.prefetch_io_aux_thread.start()
|
||||
self.prefetch_sync_thread.start()
|
||||
for thread in self.prefetch_io_aux_threads:
|
||||
thread.start()
|
||||
for thread in self.prefetch_sync_threads:
|
||||
thread.start()
|
||||
self.backup_thread.start()
|
||||
|
||||
def _stop_storage_threads(self):
|
||||
@@ -485,10 +520,10 @@ class HiCacheController:
|
||||
self.prefetch_queue.put_nowait(None)
|
||||
if hasattr(self, "backup_queue"):
|
||||
self.backup_queue.put_nowait(None)
|
||||
if hasattr(self, "prefetch_buffer"):
|
||||
self.prefetch_buffer.put_nowait(None)
|
||||
if hasattr(self, "prefetch_sync_queue"):
|
||||
self.prefetch_sync_queue.put_nowait(None)
|
||||
for queue in getattr(self, "prefetch_io_queues", []) or []:
|
||||
queue.put_nowait(None)
|
||||
for queue in getattr(self, "prefetch_sync_queues", []) or []:
|
||||
queue.put_nowait(None)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -498,10 +533,8 @@ class HiCacheController:
|
||||
threads.append(self.prefetch_thread)
|
||||
if hasattr(self, "backup_thread"):
|
||||
threads.append(self.backup_thread)
|
||||
if hasattr(self, "prefetch_io_aux_thread"):
|
||||
threads.append(self.prefetch_io_aux_thread)
|
||||
if hasattr(self, "prefetch_sync_thread"):
|
||||
threads.append(self.prefetch_sync_thread)
|
||||
threads.extend(getattr(self, "prefetch_io_aux_threads", []) or [])
|
||||
threads.extend(getattr(self, "prefetch_sync_threads", []) or [])
|
||||
|
||||
for t in threads:
|
||||
try:
|
||||
@@ -585,7 +618,13 @@ class HiCacheController:
|
||||
# Use dedicated gloo groups so storage prefetch sync is isolated
|
||||
# from other collectives and consistent across CPxTP participants.
|
||||
self.prefetch_hits_sync_groups = self._create_sync_groups()
|
||||
self.prefetch_completion_sync_groups = self._create_sync_groups()
|
||||
# Each IO lane gets its own completion groups: per-lane collectives
|
||||
# stay ordered because a lane is a serial pipeline and dispatch is
|
||||
# deterministic (crc32(request_id) % num_prefetch_io_lanes).
|
||||
self.prefetch_completion_sync_groups_by_lane = [
|
||||
self._create_sync_groups()
|
||||
for _ in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
|
||||
# Select the get and set functions
|
||||
self.page_get_func = self._generic_page_get
|
||||
@@ -620,9 +659,10 @@ class HiCacheController:
|
||||
except Exception:
|
||||
pass
|
||||
self._destroy_sync_groups(self.prefetch_hits_sync_groups)
|
||||
self._destroy_sync_groups(self.prefetch_completion_sync_groups)
|
||||
for lane_groups in self.prefetch_completion_sync_groups_by_lane:
|
||||
self._destroy_sync_groups(lane_groups)
|
||||
self.prefetch_hits_sync_groups = []
|
||||
self.prefetch_completion_sync_groups = []
|
||||
self.prefetch_completion_sync_groups_by_lane = []
|
||||
try:
|
||||
if (
|
||||
hasattr(self, "storage_backend")
|
||||
@@ -660,11 +700,11 @@ class HiCacheController:
|
||||
raise RuntimeError("Stop storage threads failed; detach aborted.") from e
|
||||
|
||||
# Best-effort destroy process groups created for storage ops.
|
||||
self._destroy_sync_groups(
|
||||
self.prefetch_hits_sync_groups + self.prefetch_completion_sync_groups
|
||||
)
|
||||
self._destroy_sync_groups(self.prefetch_hits_sync_groups)
|
||||
for lane_groups in self.prefetch_completion_sync_groups_by_lane:
|
||||
self._destroy_sync_groups(lane_groups)
|
||||
self.prefetch_hits_sync_groups = []
|
||||
self.prefetch_completion_sync_groups = []
|
||||
self.prefetch_completion_sync_groups_by_lane = []
|
||||
|
||||
# Best-effort close (some backends rely on GC/destructor).
|
||||
try:
|
||||
@@ -758,13 +798,17 @@ class HiCacheController:
|
||||
self.ack_load_queue.clear()
|
||||
if self.enable_storage:
|
||||
self.prefetch_thread.join()
|
||||
self.prefetch_io_aux_thread.join()
|
||||
self.prefetch_sync_thread.join()
|
||||
for thread in self.prefetch_io_aux_threads:
|
||||
thread.join()
|
||||
for thread in self.prefetch_sync_threads:
|
||||
thread.join()
|
||||
self.backup_thread.join()
|
||||
self.prefetch_queue.queue.clear()
|
||||
self.backup_queue.queue.clear()
|
||||
self.prefetch_buffer.queue.clear()
|
||||
self.prefetch_sync_queue.queue.clear()
|
||||
for queue in self.prefetch_io_queues:
|
||||
queue.queue.clear()
|
||||
for queue in self.prefetch_sync_queues:
|
||||
queue.queue.clear()
|
||||
self.prefetch_hit_queue.queue.clear()
|
||||
self.ack_prefetch_queue.queue.clear()
|
||||
self.ack_backup_queue.queue.clear()
|
||||
@@ -777,18 +821,26 @@ class HiCacheController:
|
||||
self.prefetch_thread = threading.Thread(
|
||||
target=self.prefetch_thread_func, daemon=True
|
||||
)
|
||||
self.prefetch_io_aux_thread = threading.Thread(
|
||||
target=self.prefetch_io_aux_func, daemon=True
|
||||
)
|
||||
self.prefetch_sync_thread = threading.Thread(
|
||||
target=self.prefetch_sync_thread_func, daemon=True
|
||||
)
|
||||
self.prefetch_io_aux_threads = [
|
||||
threading.Thread(
|
||||
target=self.prefetch_io_aux_func, args=(lane,), daemon=True
|
||||
)
|
||||
for lane in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
self.prefetch_sync_threads = [
|
||||
threading.Thread(
|
||||
target=self.prefetch_sync_thread_func, args=(lane,), daemon=True
|
||||
)
|
||||
for lane in range(self.num_prefetch_io_lanes)
|
||||
]
|
||||
self.backup_thread = threading.Thread(
|
||||
target=self.backup_thread_func, daemon=True
|
||||
)
|
||||
self.prefetch_thread.start()
|
||||
self.prefetch_io_aux_thread.start()
|
||||
self.prefetch_sync_thread.start()
|
||||
for thread in self.prefetch_io_aux_threads:
|
||||
thread.start()
|
||||
for thread in self.prefetch_sync_threads:
|
||||
thread.start()
|
||||
self.backup_thread.start()
|
||||
|
||||
def has_inflight_device_transfers(self) -> bool:
|
||||
@@ -1075,7 +1127,23 @@ class HiCacheController:
|
||||
count += 1
|
||||
return count
|
||||
|
||||
def _page_transfer(self, operation: PrefetchOperation) -> int:
|
||||
def dispatch_prefetch(self, operation: PrefetchOperation) -> None:
|
||||
"""Route a prefetch operation to its IO lane.
|
||||
|
||||
Lane assignment must be deterministic and identical on every rank:
|
||||
each lane owns dedicated completion sync groups, and the collective
|
||||
sequence on a lane is the filtered dispatch order. crc32 (not the
|
||||
per-process salted builtin hash) keeps the mapping stable across
|
||||
processes. All chunks of one operation stay in one lane, preserving
|
||||
the per-operation ack order the sync thread reduces over.
|
||||
"""
|
||||
lane = (
|
||||
zlib.crc32(str(operation.request_id).encode())
|
||||
% self.num_prefetch_io_lanes
|
||||
)
|
||||
self.prefetch_io_queues[lane].put(operation)
|
||||
|
||||
def _page_transfer(self, operation: PrefetchOperation, sync_queue: Queue) -> int:
|
||||
# Transfer batch by batch
|
||||
prefix_keys = operation.prefix_keys
|
||||
kv_derived_transfers = [
|
||||
@@ -1087,8 +1155,8 @@ class HiCacheController:
|
||||
completed_pages = 0
|
||||
for i in range(0, len(operation.hash_value), STORAGE_BATCH_SIZE):
|
||||
# When an error is occurred, we should keep looping and produce the same number of
|
||||
# PrefetchAck as other ranks do, because prefetch_sync_thread (i.e. consumer of
|
||||
# prefetch_sync_queue) perform reduce on the results. This is so tricky.
|
||||
# PrefetchAck as other ranks do, because the prefetch sync thread (i.e. consumer of
|
||||
# the lane sync queue) perform reduce on the results. This is so tricky.
|
||||
if all_success and operation.is_terminated():
|
||||
all_success = False
|
||||
if all_success:
|
||||
@@ -1100,6 +1168,7 @@ class HiCacheController:
|
||||
# Get one batch token, and update the completed_tokens if succeed
|
||||
extra_info = HiCacheStorageExtraInfo(prefix_keys=prefix_keys)
|
||||
|
||||
_hc_tb = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
hit_pages = self._page_transfer_kv_batch(
|
||||
operation,
|
||||
batch_hashes,
|
||||
@@ -1107,6 +1176,14 @@ class HiCacheController:
|
||||
extra_info,
|
||||
kv_derived_transfers,
|
||||
)
|
||||
if _hc_timing_enabled():
|
||||
logger.info(
|
||||
"HC_TIMING stage=xfer_batch req=%s keys=%d ms=%.1f hit=%d",
|
||||
operation.request_id,
|
||||
len(batch_hashes),
|
||||
(time.monotonic() - _hc_tb) * 1e3,
|
||||
hit_pages,
|
||||
)
|
||||
# Check termination
|
||||
if hit_pages != len(batch_hashes):
|
||||
all_success = False
|
||||
@@ -1118,7 +1195,7 @@ class HiCacheController:
|
||||
completed_tokens=completed_pages * self.page_size,
|
||||
operation=operation,
|
||||
)
|
||||
self.prefetch_sync_queue.put(ack)
|
||||
sync_queue.put(ack)
|
||||
return completed_pages
|
||||
|
||||
def _page_transfer_kv_batch(
|
||||
@@ -1160,18 +1237,33 @@ class HiCacheController:
|
||||
# Clamp to minimal number of hits.
|
||||
return min([kv_hits, *sidecar_hits.values()])
|
||||
|
||||
def prefetch_io_aux_func(self):
|
||||
def prefetch_io_aux_func(self, lane: int = 0):
|
||||
"""
|
||||
Auxiliary function conducting IO operations for prefetching.
|
||||
|
||||
Each lane has its own worker thread and queues; an operation's whole
|
||||
chunk sequence stays on one lane so the sync thread sees it in order.
|
||||
"""
|
||||
io_queue = self.prefetch_io_queues[lane]
|
||||
sync_queue = self.prefetch_sync_queues[lane]
|
||||
while not self.storage_stop_event.is_set():
|
||||
try:
|
||||
operation = self.prefetch_buffer.get(block=True, timeout=1)
|
||||
operation = io_queue.get(block=True, timeout=1)
|
||||
if operation is None:
|
||||
continue
|
||||
self._page_transfer(operation)
|
||||
_hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
self._page_transfer(operation, sync_queue)
|
||||
if _hc_timing_enabled():
|
||||
logger.info(
|
||||
"HC_TIMING stage=xfer req=%s lane=%d wait_ms=%.1f xfer_ms=%.1f",
|
||||
operation.request_id,
|
||||
lane,
|
||||
(_hc_t0 - getattr(operation, "query_done_ts", operation.start_time))
|
||||
* 1e3,
|
||||
(time.monotonic() - _hc_t0) * 1e3,
|
||||
)
|
||||
|
||||
self.prefetch_sync_queue.put(
|
||||
sync_queue.put(
|
||||
PrefetchAck(
|
||||
rid=operation.request_id,
|
||||
completed_req=True,
|
||||
@@ -1216,7 +1308,16 @@ class HiCacheController:
|
||||
for start in range(0, len(page_hashes), STORAGE_BATCH_SIZE):
|
||||
batch_hashes = page_hashes[start : start + STORAGE_BATCH_SIZE]
|
||||
extra_info = HiCacheStorageExtraInfo(prefix_keys=prefix_keys)
|
||||
_hc_tb = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
hit_page_num = self.storage_backend.batch_exists(batch_hashes, extra_info)
|
||||
if _hc_timing_enabled():
|
||||
logger.info(
|
||||
"HC_TIMING stage=query_batch req=%s keys=%d ms=%.1f hit=%d",
|
||||
operation.request_id,
|
||||
len(batch_hashes),
|
||||
(time.monotonic() - _hc_tb) * 1e3,
|
||||
hit_page_num,
|
||||
)
|
||||
hash_value.extend(batch_hashes[:hit_page_num])
|
||||
storage_query_count += hit_page_num * self.page_size
|
||||
if hit_page_num < len(batch_hashes):
|
||||
@@ -1235,10 +1336,12 @@ class HiCacheController:
|
||||
operation = self.prefetch_queue.get(block=True, timeout=1)
|
||||
if operation is None:
|
||||
continue
|
||||
_hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
if operation.is_terminated():
|
||||
hash_value, storage_hit_count = [], 0
|
||||
else:
|
||||
hash_value, storage_hit_count = self._storage_hit_query(operation)
|
||||
_hc_t1 = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
storage_hit_count_tensor = torch.tensor(
|
||||
storage_hit_count, dtype=torch.int
|
||||
)
|
||||
@@ -1248,6 +1351,18 @@ class HiCacheController:
|
||||
self.prefetch_hits_sync_groups,
|
||||
)
|
||||
storage_hit_count = storage_hit_count_tensor.item()
|
||||
if _hc_timing_enabled():
|
||||
_hc_t2 = time.monotonic()
|
||||
operation.query_done_ts = _hc_t2
|
||||
logger.info(
|
||||
"HC_TIMING stage=query req=%s wait_ms=%.1f query_ms=%.1f "
|
||||
"hit_reduce_ms=%.1f hit_tokens=%d",
|
||||
operation.request_id,
|
||||
(_hc_t0 - operation.start_time) * 1e3,
|
||||
(_hc_t1 - _hc_t0) * 1e3,
|
||||
(_hc_t2 - _hc_t1) * 1e3,
|
||||
storage_hit_count,
|
||||
)
|
||||
|
||||
# Record the TP-synced hit count; the scheduler thread decides
|
||||
# at drain time whether to revoke (below threshold) or allocate.
|
||||
@@ -1329,28 +1444,41 @@ class HiCacheController:
|
||||
except Empty:
|
||||
continue
|
||||
|
||||
def prefetch_sync_thread_func(self):
|
||||
def prefetch_sync_thread_func(self, lane: int = 0):
|
||||
"""Synchronize prefetch results across all PP and TP ranks."""
|
||||
sync_queue = self.prefetch_sync_queues[lane]
|
||||
groups = self.prefetch_completion_sync_groups_by_lane[lane]
|
||||
while not self.storage_stop_event.is_set():
|
||||
try:
|
||||
ack = self.prefetch_sync_queue.get(block=True, timeout=1)
|
||||
ack = sync_queue.get(block=True, timeout=1)
|
||||
if ack is None:
|
||||
continue
|
||||
self._reduce_prefetch_ack(ack)
|
||||
self._reduce_prefetch_ack(ack, groups)
|
||||
if _hc_timing_enabled() and ack.completed_req:
|
||||
ack.operation.final_ack_enqueue_ts = time.monotonic()
|
||||
self.ack_prefetch_queue.put(ack)
|
||||
except Empty:
|
||||
continue
|
||||
|
||||
def _reduce_prefetch_ack(self, ack: PrefetchAck) -> None:
|
||||
def _reduce_prefetch_ack(
|
||||
self, ack: PrefetchAck, groups: List[torch.distributed.ProcessGroup]
|
||||
) -> None:
|
||||
"""Synchronize all ranks to agree on a PrefetchAck."""
|
||||
if ack.completed_tokens is not None:
|
||||
# Determine the minimal successful prefix of tokens.
|
||||
_hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
completed_tokens_tensor = torch.tensor(
|
||||
ack.completed_tokens, dtype=torch.int
|
||||
)
|
||||
self._all_reduce(
|
||||
completed_tokens_tensor,
|
||||
torch.distributed.ReduceOp.MIN,
|
||||
self.prefetch_completion_sync_groups,
|
||||
groups,
|
||||
)
|
||||
ack.completed_tokens = completed_tokens_tensor.item()
|
||||
if _hc_timing_enabled():
|
||||
logger.info(
|
||||
"HC_TIMING stage=ack_reduce req=%s ms=%.1f",
|
||||
ack.operation.request_id,
|
||||
(time.monotonic() - _hc_t0) * 1e3,
|
||||
)
|
||||
|
||||
@@ -2141,6 +2141,27 @@ class SetInternalStateReqOutput(BaseReq, kw_only=True):
|
||||
updated: bool
|
||||
|
||||
|
||||
class QueryCacheHitReqInput(BaseReq, kw_only=True):
|
||||
"""G9: read-only probe of prefix-cache hit for a token sequence.
|
||||
|
||||
Sent to every DP rank of a (decode) engine; each rank answers with its own
|
||||
L1/L2 radix match plus the shared L3 storage hit continuing from it.
|
||||
"""
|
||||
|
||||
input_ids: List[int]
|
||||
extra_key: Optional[str] = None
|
||||
cache_salt: Optional[str] = None
|
||||
|
||||
|
||||
class QueryCacheHitReqOutput(BaseReq, kw_only=True):
|
||||
total_tokens: int = 0
|
||||
l1_hit_tokens: int = 0
|
||||
l2_hit_tokens: int = 0
|
||||
l3_hit_tokens: int = 0
|
||||
dp_rank: int = 0
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
class ProfileReqType(Enum):
|
||||
START_PROFILE = 1
|
||||
STOP_PROFILE = 2
|
||||
|
||||
@@ -197,6 +197,12 @@ class LoadSnapshot(msgspec.Struct, omit_defaults=True):
|
||||
num_running_reqs: int = 0
|
||||
num_waiting_reqs: int = 0
|
||||
num_waiting_uncached_tokens: int = 0
|
||||
# G9 queue-time estimation (总输入长度口径, self-consistent numerator/denominator):
|
||||
# sum of total input lengths (req.seqlen) over all pending queues, and the
|
||||
# self-learned drain rate in the same unit — completed prefills' total
|
||||
# input tokens / wall second over a sliding window.
|
||||
queued_total_input_tokens: int = 0
|
||||
drain_rate_tokens_per_s: float = 0.0
|
||||
num_used_tokens: int = 0
|
||||
num_total_tokens: int = 0
|
||||
# num_total_tokens minus tokens still awaiting a KV transfer (equal to it
|
||||
|
||||
@@ -0,0 +1,486 @@
|
||||
"""One owner rank encodes each image span and broadcasts it to the ranks that
|
||||
run the same prefill chunk; every agreement precedes the payload it guards."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from contextlib import contextmanager
|
||||
from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence, Tuple
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
disable_symmetric_memory_context,
|
||||
restore_symmetric_memory_context,
|
||||
)
|
||||
from sglang.srt.mem_cache.multimodal_cache import EmbeddingResult, MultiModalStaticCache
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SpanKey = Tuple[Optional[int], int]
|
||||
SpanEncoder = Callable[[List[Any]], torch.Tensor | List[torch.Tensor]]
|
||||
SpanSignature = Callable[[Any, int], Tuple[Any, ...]]
|
||||
|
||||
LOCAL_HIT = 0
|
||||
OWNER_CACHE_BROADCAST = 1
|
||||
OWNER_ENCODE_BROADCAST = 2
|
||||
|
||||
PHASE_PREPARE = "prepare"
|
||||
PHASE_FEATURES = "features"
|
||||
PHASE_FINALIZE = "finalize"
|
||||
|
||||
|
||||
class MmOwnerProtocolError(RuntimeError):
|
||||
"""Raised with identical text on every group member after a group-agreed failure."""
|
||||
|
||||
|
||||
class ImageSpanRequest(msgspec.Struct, frozen=True):
|
||||
hash: Optional[int]
|
||||
span_len: int
|
||||
item: Any
|
||||
inside_chunk: bool
|
||||
duplicates: List[Any] = []
|
||||
|
||||
|
||||
class ImageSpanKey(msgspec.Struct, frozen=True):
|
||||
hash: Optional[int]
|
||||
span_len: int
|
||||
geometry: Optional[Tuple[Any, ...]]
|
||||
|
||||
|
||||
class RankManifest(msgspec.Struct, frozen=True):
|
||||
rank: int
|
||||
keys: List[ImageSpanKey]
|
||||
cached: List[bool]
|
||||
dtype: str
|
||||
width: int
|
||||
rids: List[str]
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
class OwnerPlan(msgspec.Struct, frozen=True):
|
||||
actions: List[int]
|
||||
owners: List[int]
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
class RankStatus(msgspec.Struct, frozen=True):
|
||||
rank: int
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
def select_owner_group(parallel) -> Optional[Any]:
|
||||
"""The group whose members all execute the same requests, or None when a
|
||||
single rank already encodes every image it sees."""
|
||||
replication = parallel.tp_size // parallel.attn_dp_size
|
||||
if replication <= 1:
|
||||
return None
|
||||
if parallel.attn_cp_size == 1:
|
||||
group = parallel.attn_tp_group
|
||||
elif parallel.attn_dp_size == 1 and parallel.attn_cp_size == parallel.tp_size:
|
||||
group = parallel.attn_cp_group
|
||||
else:
|
||||
return None
|
||||
return group if group.world_size == replication else None
|
||||
|
||||
|
||||
def has_owner_span_work(
|
||||
mm_inputs: Sequence[Any],
|
||||
extend_prefix_lens: Sequence[int],
|
||||
extend_seq_lens: Sequence[int],
|
||||
) -> bool:
|
||||
"""Host-side mirror of the per-image scheduling path: does any raw
|
||||
single-span image overlap the chunk on every rank of the group."""
|
||||
for mm_input, prefix_len, extend_len in zip(
|
||||
mm_inputs, extend_prefix_lens, extend_seq_lens
|
||||
):
|
||||
if mm_input is None or extend_len <= 0:
|
||||
continue
|
||||
items = [item for item in mm_input.mm_items if item is not None]
|
||||
if not items or any(
|
||||
item.precomputed_embeddings is not None or len(item.offsets) != 1
|
||||
for item in items
|
||||
):
|
||||
continue
|
||||
for item in items:
|
||||
start, end = item.offsets[0]
|
||||
if end >= prefix_len and start < prefix_len + extend_len:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class MmOwnerSession(msgspec.Struct):
|
||||
group: Any
|
||||
device: Any
|
||||
dtype: Any
|
||||
width: int
|
||||
rids: List[str]
|
||||
signature: Any
|
||||
engaged: bool
|
||||
phase: str = PHASE_PREPARE
|
||||
in_collective: bool = False
|
||||
|
||||
def resolve(
|
||||
self,
|
||||
requests: Sequence[ImageSpanRequest],
|
||||
cache: MultiModalStaticCache,
|
||||
encode: SpanEncoder,
|
||||
) -> Dict[SpanKey, torch.Tensor]:
|
||||
if not self.engaged:
|
||||
raise RuntimeError(
|
||||
"owner protocol reached for a chunk whose host metadata has no image span"
|
||||
)
|
||||
# Owners allocate different amounts than receivers, so none of these
|
||||
# buffers may come out of a symmetric pool.
|
||||
saved_context = disable_symmetric_memory_context()
|
||||
try:
|
||||
return _resolve_owner_features(self, requests, cache, encode)
|
||||
finally:
|
||||
restore_symmetric_memory_context(saved_context)
|
||||
|
||||
def features_ready(self) -> None:
|
||||
self._complete()
|
||||
self.phase = PHASE_FINALIZE
|
||||
|
||||
@contextmanager
|
||||
def uncaptured(self) -> Iterator[None]:
|
||||
# A failure inside a collective leaves the group in an unknown state;
|
||||
# no later exchange may try to agree on it.
|
||||
self.in_collective = True
|
||||
yield
|
||||
self.in_collective = False
|
||||
|
||||
@contextmanager
|
||||
def fence(self) -> Iterator[None]:
|
||||
try:
|
||||
yield
|
||||
except Exception as exc:
|
||||
self._fail(exc)
|
||||
raise
|
||||
self._complete()
|
||||
|
||||
def _fail(self, exc: BaseException) -> None:
|
||||
if (
|
||||
not self.engaged
|
||||
or self.in_collective
|
||||
or isinstance(exc, MmOwnerProtocolError)
|
||||
):
|
||||
raise exc
|
||||
text = _describe(self, self.phase, exc)
|
||||
if self.phase == PHASE_PREPARE:
|
||||
try:
|
||||
_exchange_manifest(self, _manifest(self, [], [], error=text))
|
||||
except MmOwnerProtocolError as agreed:
|
||||
raise agreed from exc
|
||||
_exchange_status(self, text, exc)
|
||||
|
||||
def _complete(self) -> None:
|
||||
if not self.engaged:
|
||||
return
|
||||
if self.phase == PHASE_PREPARE:
|
||||
raise RuntimeError(
|
||||
f"owner protocol {self.phase} completed without a manifest exchange"
|
||||
)
|
||||
error = None
|
||||
cause = None
|
||||
try:
|
||||
_synchronize(self.device)
|
||||
except Exception as exc:
|
||||
cause = exc
|
||||
error = _describe(self, self.phase, exc)
|
||||
_exchange_status(self, error, cause)
|
||||
|
||||
|
||||
def _manifest(
|
||||
session: MmOwnerSession,
|
||||
keys: List[ImageSpanKey],
|
||||
cached: List[bool],
|
||||
error: Optional[str] = None,
|
||||
) -> RankManifest:
|
||||
return RankManifest(
|
||||
rank=session.group.rank_in_group,
|
||||
keys=keys,
|
||||
cached=cached,
|
||||
dtype=str(session.dtype),
|
||||
width=session.width,
|
||||
rids=list(session.rids),
|
||||
error=error,
|
||||
)
|
||||
|
||||
|
||||
def _exchange_manifest(session: MmOwnerSession, manifest: RankManifest) -> OwnerPlan:
|
||||
group = session.group
|
||||
with session.uncaptured():
|
||||
manifests = group.all_gather_object(manifest)
|
||||
plan = _plan_or_error(session, manifests) if group.rank_in_group == 0 else None
|
||||
plan = group.broadcast_object(plan, src=0)
|
||||
session.phase = PHASE_FEATURES
|
||||
if plan.error is not None:
|
||||
raise MmOwnerProtocolError(plan.error)
|
||||
return plan
|
||||
|
||||
|
||||
def _plan_or_error(session: MmOwnerSession, manifests: List[RankManifest]) -> OwnerPlan:
|
||||
try:
|
||||
return _make_plan(manifests)
|
||||
except Exception as exc:
|
||||
return OwnerPlan(actions=[], owners=[], error=_describe(session, "plan", exc))
|
||||
|
||||
|
||||
def _exchange_status(
|
||||
session: MmOwnerSession, error: Optional[str], cause: Optional[BaseException]
|
||||
) -> None:
|
||||
with session.uncaptured():
|
||||
statuses = session.group.all_gather_object(
|
||||
RankStatus(rank=session.group.rank_in_group, error=error)
|
||||
)
|
||||
_raise_first_error(statuses, cause)
|
||||
|
||||
|
||||
def _resolve_owner_features(
|
||||
session: MmOwnerSession,
|
||||
requests: Sequence[ImageSpanRequest],
|
||||
cache: MultiModalStaticCache,
|
||||
encode: SpanEncoder,
|
||||
) -> Dict[SpanKey, torch.Tensor]:
|
||||
group = session.group
|
||||
features: Dict[SpanKey, torch.Tensor] = {}
|
||||
keys: List[ImageSpanKey] = []
|
||||
cached: List[bool] = []
|
||||
error = None
|
||||
try:
|
||||
keys, cached = _pin_local_cache(session, requests, cache, features)
|
||||
except Exception as exc:
|
||||
error = _describe(session, "manifest", exc)
|
||||
plan = _exchange_manifest(session, _manifest(session, keys, cached, error))
|
||||
|
||||
if all(action == LOCAL_HIT for action in plan.actions):
|
||||
return features
|
||||
|
||||
buffers: Dict[int, torch.Tensor] = {}
|
||||
error = None
|
||||
try:
|
||||
buffers = _prepare_transfers(session, requests, keys, plan, features, encode)
|
||||
_synchronize(session.device)
|
||||
except Exception as exc:
|
||||
error = _describe(session, "encode", exc)
|
||||
_exchange_status(session, error, None)
|
||||
|
||||
with session.uncaptured():
|
||||
for index, (action, owner) in enumerate(zip(plan.actions, plan.owners)):
|
||||
if action != LOCAL_HIT:
|
||||
group.broadcast(buffers[index], src=owner)
|
||||
|
||||
for index, key in enumerate(keys):
|
||||
if plan.actions[index] == LOCAL_HIT:
|
||||
continue
|
||||
span = buffers[index]
|
||||
features[(key.hash, key.span_len)] = span
|
||||
cache.set(key.hash, EmbeddingResult(embedding=span))
|
||||
return features
|
||||
|
||||
|
||||
def _pin_local_cache(
|
||||
session: MmOwnerSession,
|
||||
requests: Sequence[ImageSpanRequest],
|
||||
cache: MultiModalStaticCache,
|
||||
features: Dict[SpanKey, torch.Tensor],
|
||||
) -> Tuple[List[ImageSpanKey], List[bool]]:
|
||||
keys: List[ImageSpanKey] = []
|
||||
cached: List[bool] = []
|
||||
for request in requests:
|
||||
if request.hash is None:
|
||||
raise ValueError(
|
||||
f"image span of {request.span_len} tokens has no content hash"
|
||||
)
|
||||
geometry = session.signature(request.item, request.span_len)
|
||||
for duplicate in request.duplicates:
|
||||
other = session.signature(duplicate, request.span_len)
|
||||
if other != geometry:
|
||||
raise ValueError(
|
||||
f"image hash {request.hash} ({request.span_len} tokens) occurs "
|
||||
f"with different geometry: {geometry} vs {other}"
|
||||
)
|
||||
keys.append(
|
||||
ImageSpanKey(
|
||||
hash=request.hash, span_len=request.span_len, geometry=geometry
|
||||
)
|
||||
)
|
||||
span = _valid_cached_span(session, cache, request)
|
||||
if span is not None:
|
||||
features[(request.hash, request.span_len)] = span
|
||||
cached.append(span is not None)
|
||||
return keys, cached
|
||||
|
||||
|
||||
def _valid_cached_span(
|
||||
session: MmOwnerSession,
|
||||
cache: MultiModalStaticCache,
|
||||
request: ImageSpanRequest,
|
||||
) -> Optional[torch.Tensor]:
|
||||
entry = cache.get_single(request.hash)
|
||||
if entry is None:
|
||||
return None
|
||||
span = entry.embedding
|
||||
if (
|
||||
span.dim() == 2
|
||||
and span.shape[0] == request.span_len
|
||||
and span.shape[1] == session.width
|
||||
and span.dtype == session.dtype
|
||||
and span.device == session.device
|
||||
):
|
||||
return span
|
||||
logger.warning(
|
||||
"Discarding cached multimodal embedding that cannot serve the current "
|
||||
"image span: cache_key=%s expected=(%d, %d, %s) cached=(%s, %s).",
|
||||
request.hash,
|
||||
request.span_len,
|
||||
session.width,
|
||||
session.dtype,
|
||||
tuple(span.shape),
|
||||
span.dtype,
|
||||
)
|
||||
cache.free(request.hash, None)
|
||||
return None
|
||||
|
||||
|
||||
def _make_plan(manifests: List[RankManifest]) -> OwnerPlan:
|
||||
for manifest in manifests:
|
||||
if manifest.error is not None:
|
||||
return OwnerPlan(actions=[], owners=[], error=manifest.error)
|
||||
lead = manifests[0]
|
||||
for manifest in manifests[1:]:
|
||||
if (manifest.keys, manifest.dtype, manifest.width, manifest.rids) != (
|
||||
lead.keys,
|
||||
lead.dtype,
|
||||
lead.width,
|
||||
lead.rids,
|
||||
):
|
||||
return OwnerPlan(
|
||||
actions=[],
|
||||
owners=[],
|
||||
error=(
|
||||
"image manifest mismatch between group ranks 0 and "
|
||||
f"{manifest.rank}: rids={lead.rids} vs {manifest.rids}, "
|
||||
f"keys={lead.keys} vs {manifest.keys}, "
|
||||
f"dtype={lead.dtype} vs {manifest.dtype}, "
|
||||
f"width={lead.width} vs {manifest.width}"
|
||||
),
|
||||
)
|
||||
replication = len(manifests)
|
||||
actions: List[int] = []
|
||||
owners: List[int] = []
|
||||
for index, key in enumerate(lead.keys):
|
||||
owner = key.hash % replication
|
||||
if all(manifest.cached[index] for manifest in manifests):
|
||||
action = LOCAL_HIT
|
||||
elif manifests[owner].cached[index]:
|
||||
action = OWNER_CACHE_BROADCAST
|
||||
else:
|
||||
action = OWNER_ENCODE_BROADCAST
|
||||
actions.append(action)
|
||||
owners.append(owner)
|
||||
return OwnerPlan(actions=actions, owners=owners)
|
||||
|
||||
|
||||
def _prepare_transfers(
|
||||
session: MmOwnerSession,
|
||||
requests: Sequence[ImageSpanRequest],
|
||||
keys: List[ImageSpanKey],
|
||||
plan: OwnerPlan,
|
||||
features: Dict[SpanKey, torch.Tensor],
|
||||
encode: SpanEncoder,
|
||||
) -> Dict[int, torch.Tensor]:
|
||||
rank = session.group.rank_in_group
|
||||
buffers: Dict[int, torch.Tensor] = {}
|
||||
owned: List[int] = []
|
||||
for index, (action, owner) in enumerate(zip(plan.actions, plan.owners)):
|
||||
if action == LOCAL_HIT:
|
||||
continue
|
||||
if owner != rank:
|
||||
try:
|
||||
buffers[index] = _new_span_buffer(session, keys[index])
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
f"receive buffer for image hash {keys[index].hash} shape "
|
||||
f"{(keys[index].span_len, session.width)} {session.dtype} "
|
||||
f"failed: {type(exc).__name__}: {exc}"
|
||||
) from exc
|
||||
elif action == OWNER_CACHE_BROADCAST:
|
||||
key = (keys[index].hash, keys[index].span_len)
|
||||
buffers[index] = features[key].contiguous()
|
||||
else:
|
||||
owned.append(index)
|
||||
if owned:
|
||||
owned_hashes = [keys[index].hash for index in owned]
|
||||
try:
|
||||
encoded = encode([requests[index].item for index in owned])
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
f"owner encode of image hashes {owned_hashes} failed: "
|
||||
f"{type(exc).__name__}: {exc}"
|
||||
) from exc
|
||||
spans = _split_spans(encoded, [keys[index].span_len for index in owned])
|
||||
for index, span in zip(owned, spans):
|
||||
buffers[index] = _validated_span(session, keys[index], span)
|
||||
return buffers
|
||||
|
||||
|
||||
def _new_span_buffer(session: MmOwnerSession, key: ImageSpanKey) -> torch.Tensor:
|
||||
return torch.empty(
|
||||
(key.span_len, session.width), device=session.device, dtype=session.dtype
|
||||
)
|
||||
|
||||
|
||||
def _split_spans(
|
||||
encoded: torch.Tensor | List[torch.Tensor], span_lens: List[int]
|
||||
) -> List[torch.Tensor]:
|
||||
if isinstance(encoded, list):
|
||||
if len(encoded) != len(span_lens):
|
||||
raise ValueError(
|
||||
f"encoder returned {len(encoded)} spans for {len(span_lens)} images"
|
||||
)
|
||||
return [span.reshape(-1, span.shape[-1]) for span in encoded]
|
||||
encoded = encoded.reshape(-1, encoded.shape[-1])
|
||||
if encoded.shape[0] != sum(span_lens):
|
||||
raise ValueError(
|
||||
f"encoder returned {encoded.shape[0]} rows for spans of {span_lens}"
|
||||
)
|
||||
return list(torch.split(encoded, span_lens, dim=0))
|
||||
|
||||
|
||||
def _validated_span(
|
||||
session: MmOwnerSession, key: ImageSpanKey, span: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
expected = (key.span_len, session.width)
|
||||
if tuple(span.shape) != expected or span.dtype != session.dtype:
|
||||
raise ValueError(
|
||||
f"encoded span for hash={key.hash} has shape {tuple(span.shape)} "
|
||||
f"dtype {span.dtype}; expected {expected} {session.dtype}"
|
||||
)
|
||||
if span.device != session.device:
|
||||
span = span.to(session.device)
|
||||
return span.contiguous()
|
||||
|
||||
|
||||
def _synchronize(device) -> None:
|
||||
if device.type == "cuda":
|
||||
torch.cuda.current_stream(device).synchronize()
|
||||
|
||||
|
||||
def _describe(session: MmOwnerSession, stage: str, exc: BaseException) -> str:
|
||||
return (
|
||||
f"multimodal owner protocol failed during {stage} on group rank "
|
||||
f"{session.group.rank_in_group} (global rank "
|
||||
f"{session.group.ranks[session.group.rank_in_group]}, rids={list(session.rids)}): "
|
||||
f"{type(exc).__name__}: {exc}"
|
||||
)
|
||||
|
||||
|
||||
def _raise_first_error(
|
||||
statuses: List[RankStatus], cause: Optional[BaseException]
|
||||
) -> None:
|
||||
for status in statuses:
|
||||
if status.error is not None:
|
||||
raise MmOwnerProtocolError(status.error) from cause
|
||||
@@ -5,6 +5,7 @@ from typing import Callable, Dict, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.managers.mm_owner_embedding import ImageSpanRequest, MmOwnerSession
|
||||
from sglang.srt.managers.schedule_batch import MultimodalDataItem
|
||||
from sglang.srt.mem_cache.multimodal_cache import EmbeddingResult, MultiModalStaticCache
|
||||
from sglang.srt.multimodal.evs import EVSEmbeddingResult
|
||||
@@ -339,43 +340,24 @@ def _batch_encode_per_image_misses(
|
||||
unique_misses: Dict[Tuple[Optional[int], int], Tuple[MultimodalDataItem, int]] = {}
|
||||
hash_to_embedding: Dict[Tuple[Optional[int], int], torch.Tensor] = {}
|
||||
|
||||
# Phase 1a: find overlapping items per request and collect cache misses
|
||||
for req_info in per_image_requests:
|
||||
chunk_start = req_info.extend_prefix_len
|
||||
chunk_end = chunk_start + req_info.extend_seq_len # exclusive
|
||||
overlapping = []
|
||||
if req_info.extend_seq_len > 0:
|
||||
for idx, (item, (start, end)) in enumerate(
|
||||
zip(req_info.items, req_info.items_offset)
|
||||
):
|
||||
if end >= chunk_start and start < chunk_end:
|
||||
overlapping.append((idx, item, start, end))
|
||||
req_info.overlapping = overlapping
|
||||
|
||||
for _idx, item, start, end in overlapping:
|
||||
expected_token_count = end - start + 1
|
||||
cache_key = (item.hash, expected_token_count)
|
||||
if cache_key in hash_to_embedding:
|
||||
# Phase 1a: collect cache misses over the unique overlapping spans
|
||||
for span in _collect_image_span_requests(per_image_requests):
|
||||
cache_key = (span.hash, span.span_len)
|
||||
cached = embedding_cache.get_single(span.hash)
|
||||
if cached is not None:
|
||||
cached_embedding = cached.embedding
|
||||
cached_token_count = _embedding_token_count(cached_embedding)
|
||||
if cached_token_count == span.span_len:
|
||||
hash_to_embedding[cache_key] = cached_embedding
|
||||
continue
|
||||
cached = embedding_cache.get_single(item.hash)
|
||||
if cached is not None:
|
||||
cached_embedding = cached.embedding
|
||||
cached_token_count = _embedding_token_count(cached_embedding)
|
||||
if cached_token_count == expected_token_count:
|
||||
hash_to_embedding[cache_key] = cached_embedding
|
||||
else:
|
||||
_discard_mismatched_cached_embedding(
|
||||
item.hash, expected_token_count, cached_token_count
|
||||
)
|
||||
unique_misses[cache_key] = (item, expected_token_count)
|
||||
elif cache_key not in unique_misses:
|
||||
if (
|
||||
start >= chunk_start
|
||||
and end < chunk_end
|
||||
and item.can_defer_cuda_ipc_feature_reconstruction()
|
||||
):
|
||||
item.model_specific_data[BORROW_CUDA_IPC_FEATURE_KEY] = True
|
||||
unique_misses[cache_key] = (item, expected_token_count)
|
||||
_discard_mismatched_cached_embedding(
|
||||
span.hash, span.span_len, cached_token_count
|
||||
)
|
||||
elif (
|
||||
span.inside_chunk and span.item.can_defer_cuda_ipc_feature_reconstruction()
|
||||
):
|
||||
span.item.model_specific_data[BORROW_CUDA_IPC_FEATURE_KEY] = True
|
||||
unique_misses[cache_key] = (span.item, span.span_len)
|
||||
|
||||
# Phase 1b: single ViT call for all unique cache misses
|
||||
if unique_misses:
|
||||
@@ -412,6 +394,52 @@ def _batch_encode_per_image_misses(
|
||||
return hash_to_embedding
|
||||
|
||||
|
||||
def _collect_image_span_requests(
|
||||
per_image_requests: List[PerImageRequestInfo],
|
||||
) -> List[ImageSpanRequest]:
|
||||
spans: Dict[
|
||||
Tuple[Optional[int], int],
|
||||
Tuple[MultimodalDataItem, bool, List[MultimodalDataItem]],
|
||||
] = {}
|
||||
for req_info in per_image_requests:
|
||||
chunk_start = req_info.extend_prefix_len
|
||||
chunk_end = chunk_start + req_info.extend_seq_len # exclusive
|
||||
overlapping = []
|
||||
if req_info.extend_seq_len > 0:
|
||||
for idx, (item, (start, end)) in enumerate(
|
||||
zip(req_info.items, req_info.items_offset)
|
||||
):
|
||||
if end >= chunk_start and start < chunk_end:
|
||||
overlapping.append((idx, item, start, end))
|
||||
req_info.overlapping = overlapping
|
||||
|
||||
for _idx, item, start, end in overlapping:
|
||||
cache_key = (item.hash, end - start + 1)
|
||||
if cache_key in spans:
|
||||
spans[cache_key][2].append(item)
|
||||
continue
|
||||
spans[cache_key] = (item, start >= chunk_start and end < chunk_end, [])
|
||||
return [
|
||||
ImageSpanRequest(
|
||||
hash=item_hash,
|
||||
span_len=span_len,
|
||||
item=item,
|
||||
inside_chunk=inside_chunk,
|
||||
duplicates=duplicates,
|
||||
)
|
||||
for (item_hash, span_len), (item, inside_chunk, duplicates) in spans.items()
|
||||
]
|
||||
|
||||
|
||||
def _owner_span_encoder(data_embedding_func: DataEmbeddingFunc, device: torch.device):
|
||||
def encode(items: List[MultimodalDataItem]):
|
||||
if not _can_skip_pre_embed_feature_move(data_embedding_func):
|
||||
_move_items_to_device(items, device)
|
||||
return data_embedding_func(items)
|
||||
|
||||
return encode
|
||||
|
||||
|
||||
def _get_chunked_embedding_by_item(
|
||||
data_embedding_func: DataEmbeddingFunc,
|
||||
embedding_items_per_req: List[MultimodalDataItem],
|
||||
@@ -537,6 +565,7 @@ def _get_chunked_prefill_embedding(
|
||||
extend_length: List[int],
|
||||
items_offset_list: List[List[Tuple[int, int]]],
|
||||
input_ids: torch.Tensor,
|
||||
mm_owner: Optional[MmOwnerSession] = None,
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor]:
|
||||
"""
|
||||
Chunked prefill embedding: encode items across all requests and extract
|
||||
@@ -598,7 +627,22 @@ def _get_chunked_prefill_embedding(
|
||||
|
||||
# Phase 1: batch encode all per-image cache misses in ONE ViT call
|
||||
hash_to_embedding: Dict[Tuple[Optional[int], int], torch.Tensor] = {}
|
||||
if per_image_requests:
|
||||
if per_image_requests and mm_owner is not None:
|
||||
# The owner protocol must see every overlapping span before any local
|
||||
# cache filtering: a rank-local hit can never skip a group collective.
|
||||
span_requests = _collect_image_span_requests(per_image_requests)
|
||||
if mm_owner.engaged:
|
||||
hash_to_embedding = mm_owner.resolve(
|
||||
span_requests,
|
||||
cache=embedding_cache,
|
||||
encode=_owner_span_encoder(data_embedding_func, device),
|
||||
)
|
||||
elif span_requests:
|
||||
raise RuntimeError(
|
||||
"owner eligibility saw no image span in this chunk, but "
|
||||
f"scheduling found {len(span_requests)}"
|
||||
)
|
||||
elif per_image_requests:
|
||||
hash_to_embedding = _batch_encode_per_image_misses(
|
||||
data_embedding_func, per_image_requests, device
|
||||
)
|
||||
@@ -701,6 +745,7 @@ def get_embedding_and_mask(
|
||||
prefix_length: List[int],
|
||||
extend_length: List[int],
|
||||
items_offset_list: List[List[Tuple[int, int]]],
|
||||
mm_owner: Optional[MmOwnerSession] = None,
|
||||
) -> Tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor]:
|
||||
"""
|
||||
Generate multimodal embeddings and create a mask for identifying their positions in the input sequence.
|
||||
@@ -741,6 +786,7 @@ def get_embedding_and_mask(
|
||||
extend_length,
|
||||
items_offset_list,
|
||||
input_ids,
|
||||
mm_owner=mm_owner,
|
||||
)
|
||||
if embedding is None:
|
||||
return None, None, input_ids
|
||||
|
||||
@@ -10,6 +10,7 @@ import pickle
|
||||
import sys
|
||||
from abc import abstractmethod
|
||||
from collections import defaultdict
|
||||
from contextlib import nullcontext
|
||||
from multiprocessing import shared_memory
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
@@ -24,6 +25,7 @@ from sglang.srt.managers.io_struct import (
|
||||
TokenizedEmbeddingReqInput,
|
||||
TokenizedGenerateReqInput,
|
||||
)
|
||||
from sglang.srt.managers.mm_owner_embedding import MmOwnerSession
|
||||
|
||||
# Preserve the existing initialization import for downstream callers.
|
||||
from sglang.srt.managers.mm_schedule import (
|
||||
@@ -397,6 +399,7 @@ def embed_mm_inputs(
|
||||
data_embedding_func_mapping: Dict[Modality, DataEmbeddingFunc] = None,
|
||||
placeholder_tokens: dict[Modality, List[int]] = None,
|
||||
use_deepstack: Dict[Modality, bool] = {},
|
||||
mm_owner: Optional[MmOwnerSession] = None,
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""
|
||||
Embed multimodal inputs and integrate them with text token embeddings.
|
||||
@@ -478,6 +481,7 @@ def embed_mm_inputs(
|
||||
prefix_length=extend_prefix_lens,
|
||||
extend_length=extend_seq_lens,
|
||||
items_offset_list=items_offsets,
|
||||
mm_owner=mm_owner,
|
||||
)
|
||||
|
||||
if use_deepstack.get(modality, None) and embedding is not None:
|
||||
@@ -498,7 +502,12 @@ def embed_mm_inputs(
|
||||
# filled with the hash values of the multimodal for the prefix matching in the radix attention.
|
||||
# There values are useless because their embeddings will be replaced by vision embeddings anyway.
|
||||
input_ids.clamp_(min=0, max=vocab_size - 1)
|
||||
input_embeds = input_embedding(input_ids)
|
||||
if mm_owner is not None:
|
||||
# The text embedding may all-reduce across TP; a rank-local failure in
|
||||
# feature preparation has to be agreed on before any rank enters it.
|
||||
mm_owner.features_ready()
|
||||
with mm_owner.uncaptured() if mm_owner is not None else nullcontext():
|
||||
input_embeds = input_embedding(input_ids)
|
||||
|
||||
# deepstack embedding
|
||||
if use_deepstack:
|
||||
@@ -525,7 +534,9 @@ def embed_mm_inputs(
|
||||
_scatter_mm_embedding(dest=input_embeds, mask=mask, src=embedding)
|
||||
if use_deepstack.get(modality, None):
|
||||
_scatter_mm_embedding(
|
||||
dest=input_deepstack_embeds, mask=mask, src=deepstack_embeddings[i]
|
||||
dest=input_deepstack_embeds,
|
||||
mask=mask,
|
||||
src=deepstack_embeddings[i],
|
||||
)
|
||||
|
||||
return input_embeds, other_info
|
||||
|
||||
@@ -1034,12 +1034,12 @@ class Req(ReqDllmMixin):
|
||||
else self.origin_input_ids
|
||||
) # Before image padding
|
||||
# Each decode stage's output ids. Append-only by contract:
|
||||
# _refresh_fill_ids infers how many output tokens are already in
|
||||
# refresh_fill_ids infers how many output tokens are already in
|
||||
# full_untruncated_fill_ids from lengths alone, so in-place rewrites
|
||||
# that preserve length would silently corrupt fill_ids.
|
||||
self.output_ids = array("q")
|
||||
# Full untruncated sequence: origin + output (+ DLLM mask block).
|
||||
# Kept in sync by _refresh_fill_ids; admission only updates
|
||||
# Kept in sync by refresh_fill_ids; admission only updates
|
||||
# extend_range, never mutates this array's length.
|
||||
self.full_untruncated_fill_ids = array("q")
|
||||
self.extend_range: Optional[Range] = None
|
||||
@@ -1533,7 +1533,7 @@ class Req(ReqDllmMixin):
|
||||
def get_fill_ids(self) -> array:
|
||||
return self.full_untruncated_fill_ids[: self.extend_range.end]
|
||||
|
||||
def _refresh_fill_ids(self) -> None:
|
||||
def refresh_fill_ids(self) -> None:
|
||||
"""Keep full_untruncated_fill_ids == origin_input_ids + output_ids by
|
||||
appending only the new output tokens.
|
||||
|
||||
@@ -1562,7 +1562,7 @@ class Req(ReqDllmMixin):
|
||||
self._init_fill_ids_for_dllm()
|
||||
self.determine_dllm_phase()
|
||||
else:
|
||||
self._refresh_fill_ids()
|
||||
self.refresh_fill_ids()
|
||||
|
||||
input_len = len(self.full_untruncated_fill_ids)
|
||||
|
||||
@@ -3107,7 +3107,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
for req, prefix_len in zip(
|
||||
running_batch.reqs, running_prefix_lens, strict=True
|
||||
):
|
||||
req._refresh_fill_ids()
|
||||
req.refresh_fill_ids()
|
||||
req.set_extend_range(prefix_len, prefix_len + 1)
|
||||
|
||||
# Decode tokens of the running portion live in future_map.output_tokens_buf.
|
||||
@@ -3185,7 +3185,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
# cannot stand in for it. Rows past bs are a beam tail, not requests.
|
||||
seq_lens = self.seq_lens_cpu[:bs].tolist()
|
||||
for req, seq_len in zip(self.reqs, seq_lens, strict=True):
|
||||
req._refresh_fill_ids()
|
||||
req.refresh_fill_ids()
|
||||
# end runs one past full_untruncated_fill_ids while output_ids
|
||||
# trails; safe only while decoding_reqs suppresses cache_unfinished_req.
|
||||
req.set_extend_range(seq_len - 1, seq_len)
|
||||
|
||||
@@ -1019,6 +1019,15 @@ class PrefillAdder:
|
||||
else AddReqResult.CONTINUE
|
||||
)
|
||||
|
||||
def can_share_extend_batch(self, req: Req) -> bool:
|
||||
# Token embedding overrides embed the batch's raw input_ids before the
|
||||
# model runs, and that lookup cannot index multimodal placeholder hash IDs.
|
||||
if req.positional_embed_overrides is not None:
|
||||
return all(r.multimodal_inputs is None for r in self.can_run_list)
|
||||
if req.multimodal_inputs is not None:
|
||||
return all(r.positional_embed_overrides is None for r in self.can_run_list)
|
||||
return True
|
||||
|
||||
def add_chunked_req(self, req: Req):
|
||||
if self.dllm_config is not None:
|
||||
_rem_tokens = self._get_dllm_remain_tokens()
|
||||
|
||||
@@ -161,6 +161,8 @@ from sglang.srt.managers.io_struct import (
|
||||
PauseGenerationReqInput,
|
||||
PdRoleSwitchReqInput,
|
||||
ProfileReq,
|
||||
QueryCacheHitReqInput,
|
||||
QueryCacheHitReqOutput,
|
||||
ReleaseMemoryOccupationReqInput,
|
||||
RemoveExternalCorpusReqInput,
|
||||
RemoveExternalCorpusReqOutput,
|
||||
@@ -1806,6 +1808,7 @@ class Scheduler(
|
||||
(ShutdownReq, self.handle_shutdown),
|
||||
(GetInternalStateReq, self.get_internal_state),
|
||||
(SetInternalStateReq, self.set_internal_state),
|
||||
(QueryCacheHitReqInput, self.handle_query_cache_hit),
|
||||
(RpcReqInput, self.handle_rpc_request),
|
||||
(ExpertDistributionReq, self.expert_distribution_handle),
|
||||
(LoadLoRAAdapterReqInput, self.load_lora_adapter),
|
||||
@@ -2097,9 +2100,13 @@ class Scheduler(
|
||||
vmm_errors = self._materialize_cuda_vmm_inputs(recv_req)
|
||||
|
||||
# Skip health check when server is busy — ongoing requests already carry health info.
|
||||
if is_health_check_generate_req(recv_req) and not self.is_fully_idle(
|
||||
for_health_check=True
|
||||
):
|
||||
# NOTE: the admit/skip decision must be identical on every CP/TP rank.
|
||||
# is_fully_idle() includes rank-local hicache drain queues, which diverge
|
||||
# across ranks right after activity; a divergent decision lets one rank
|
||||
# dispatch the health-check generate while others piggyback-skip, breaking
|
||||
# collective ordering (deadlock: one rank blocks in the hicache drain
|
||||
# all_reduce while another waits in the CP request broadcast).
|
||||
if is_health_check_generate_req(recv_req) and not self.is_sched_idle_cp_symmetric():
|
||||
self.return_health_check_ipcs.append(
|
||||
getattr(recv_req, "http_worker_ipc", None)
|
||||
)
|
||||
@@ -2427,6 +2434,11 @@ class Scheduler(
|
||||
self._prev_step: Optional[Tuple[int, float, bool]] = None
|
||||
self._prev_prefill_end_ts: Optional[float] = None
|
||||
self._sched_idled = False
|
||||
# G9: sliding-window drain events (monotonic_ts, total_input_tokens) for
|
||||
# the queue-time estimator. Recorded once per request when its final
|
||||
# prefill chunk completes (batch_result_processor), in the same
|
||||
# total-input-length unit as LoadSnapshot.queued_total_input_tokens.
|
||||
self.prefill_drain_window: deque = deque()
|
||||
self.load_inquirer = SchedulerLoadInquirer(
|
||||
disaggregation_mode=self.disaggregation_mode,
|
||||
server_args=self.server_args,
|
||||
@@ -2465,8 +2477,35 @@ class Scheduler(
|
||||
),
|
||||
get_total_prefill_busy_us=lambda: self.total_prefill_busy_us,
|
||||
get_decode_moment_totals=lambda: self.decode_moment_totals,
|
||||
get_prefill_drain_rate=self.get_prefill_drain_rate,
|
||||
)
|
||||
|
||||
PREFILL_DRAIN_WINDOW_S = 60.0
|
||||
|
||||
def record_prefill_drain(self, total_input_tokens: int) -> None:
|
||||
"""G9: one drain event per request whose final prefill chunk completed.
|
||||
|
||||
Tokens are the request's total input length (seqlen), matching the
|
||||
queued_total_input_tokens numerator so the router's
|
||||
wait = queued_tokens / drain_rate is unit-consistent (决策②).
|
||||
"""
|
||||
now = time.monotonic()
|
||||
self.prefill_drain_window.append((now, total_input_tokens))
|
||||
cutoff = now - self.PREFILL_DRAIN_WINDOW_S
|
||||
while self.prefill_drain_window and self.prefill_drain_window[0][0] < cutoff:
|
||||
self.prefill_drain_window.popleft()
|
||||
|
||||
def get_prefill_drain_rate(self) -> float:
|
||||
"""Completed prefill input tokens per wall second over the window."""
|
||||
now = time.monotonic()
|
||||
cutoff = now - self.PREFILL_DRAIN_WINDOW_S
|
||||
while self.prefill_drain_window and self.prefill_drain_window[0][0] < cutoff:
|
||||
self.prefill_drain_window.popleft()
|
||||
if not self.prefill_drain_window:
|
||||
return 0.0
|
||||
span = max(now - self.prefill_drain_window[0][0], 1.0)
|
||||
return sum(tokens for _, tokens in self.prefill_drain_window) / span
|
||||
|
||||
def init_output_streamer(self) -> None:
|
||||
self.output_streamer = self.get_output_streamer_class()(
|
||||
send_to_detokenizer=self.ipc_channels.send_to_detokenizer,
|
||||
@@ -2517,6 +2556,7 @@ class Scheduler(
|
||||
output_streamer=self.output_streamer,
|
||||
beam_coordinator=self.beam_coordinator,
|
||||
abort_request=self.abort_request,
|
||||
record_prefill_drain=self.record_prefill_drain,
|
||||
)
|
||||
|
||||
def init_req_max_new_tokens(self, req):
|
||||
@@ -3940,6 +3980,8 @@ class Scheduler(
|
||||
for req in self.waiting_queue:
|
||||
if self.enable_lora and not self.can_schedule_lora_req(req, running_loras):
|
||||
continue
|
||||
if not adder.can_share_extend_batch(req):
|
||||
break
|
||||
|
||||
running_bs = len(running_batch.reqs)
|
||||
candidate_beam_width = (
|
||||
@@ -4980,6 +5022,31 @@ class Scheduler(
|
||||
else:
|
||||
self.metrics_reporter.record_scheduler_active()
|
||||
|
||||
def is_sched_idle_cp_symmetric(self) -> bool:
|
||||
"""Idle check using only state that is identical across CP/TP ranks.
|
||||
|
||||
Request/batch/queue state is collectively maintained (requests arrive
|
||||
via broadcast, batches are collectively scheduled), so every rank
|
||||
computes the same result. Rank-local hicache drain and disagg transfer
|
||||
queues are deliberately excluded: those are exactly the terms that
|
||||
diverge across ranks and caused the CP health-check deadlock
|
||||
(hicache drain all_reduce vs CP request broadcast cross-collective
|
||||
wait, seen on cp2/cp4 + hicache L3 right after router health checks).
|
||||
|
||||
Used only for health-check admission; all other idle logic keeps using
|
||||
is_fully_idle().
|
||||
"""
|
||||
return (
|
||||
self.running_batch.is_empty()
|
||||
and self.chunked_req is None
|
||||
and not self.dllm_manager.any_staging_reqs()
|
||||
and (self.last_batch is None or self.last_batch.is_empty())
|
||||
and (not self.enable_overlap or len(self.result_queue) == 0)
|
||||
and self._pp_microbatches_drained()
|
||||
and len(self.waiting_queue) == 0
|
||||
and len(self.grammar_manager.grammar_queue) == 0
|
||||
)
|
||||
|
||||
def is_fully_idle(self, for_health_check=False) -> bool:
|
||||
# Health check piggybacks on running requests in process_output.
|
||||
# Only running_batch + waiting_queue guarantee active GPU processing;
|
||||
@@ -5184,6 +5251,86 @@ class Scheduler(
|
||||
success = False
|
||||
return success
|
||||
|
||||
def handle_query_cache_hit(self, recv_req: QueryCacheHitReqInput):
|
||||
"""G9: read-only prefix-hit probe (L1/L2 radix + L3 storage).
|
||||
|
||||
The router calls this (via /query_cache_hit on the decode engine) to
|
||||
decide conditional PD-bypass. Runs inside the scheduler event loop so
|
||||
radix/storage state is consistent, and reuses the exact admission-time
|
||||
match + L3 query logic (mirrors DecodeHiCachePreallocMixin
|
||||
._build_decode_prefix_match) without creating a Req or mutating
|
||||
request state. Node access metadata (LRU) may be touched; a hit found
|
||||
here can still be evicted before the real request arrives (inherent
|
||||
TOCTOU, acceptable for an estimate).
|
||||
"""
|
||||
from sglang.srt.mem_cache.base_prefix_cache import MatchPrefixParams
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
|
||||
total = len(recv_req.input_ids)
|
||||
# 树节点键是 array("q"),RadixKey.match_at 断言两侧类型一致——
|
||||
# 查询侧必须同样用 array,否则树非空时匹配直接炸(2026-10-11 实测)。
|
||||
from array import array
|
||||
|
||||
query_ids = array("q", recv_req.input_ids)
|
||||
try:
|
||||
match_result = self.tree_cache.match_prefix(
|
||||
MatchPrefixParams(
|
||||
key=RadixKey(
|
||||
token_ids=query_ids,
|
||||
extra_key=recv_req.extra_key,
|
||||
cache_salt=recv_req.cache_salt,
|
||||
)
|
||||
)
|
||||
)
|
||||
l1_hit = len(match_result.device_indices)
|
||||
l2_hit = getattr(match_result, "host_hit_length", 0) or 0
|
||||
l3_hit = 0
|
||||
last_host_node = getattr(match_result, "last_host_node", None)
|
||||
# L3 查询条件:本调度器挂了 hicache 存储后端即可(P 节点 HiCache
|
||||
# write_through/write_back 场景),不必是 decode-hicache 模式。
|
||||
# SWA 模型的 decode radix 与 hicache 互斥(kv_cache_builder 硬限制),
|
||||
# 若只认 enable_decode_hicache,P 节点永远查不到 L3。
|
||||
can_query_l3 = getattr(self, "enable_decode_hicache", False) or getattr(
|
||||
self, "enable_hicache_storage", False
|
||||
)
|
||||
if (
|
||||
can_query_l3
|
||||
and last_host_node is not None
|
||||
and hasattr(self.tree_cache, "query_storage_hit_length")
|
||||
and (
|
||||
self.tree_cache.is_backuped(last_host_node)
|
||||
or self.tree_cache.is_root(last_host_node)
|
||||
)
|
||||
):
|
||||
matched_len = l1_hit + l2_hit
|
||||
suffix_tokens = query_ids[matched_len:]
|
||||
last_hash = self.tree_cache.get_last_hash_value(last_host_node)
|
||||
prefix_keys = (
|
||||
self.tree_cache.get_prefix_hash_values(last_host_node)
|
||||
if getattr(
|
||||
self.tree_cache, "hicache_storage_pass_prefix_keys", False
|
||||
)
|
||||
else None
|
||||
)
|
||||
l3_hit = self.tree_cache.query_storage_hit_length(
|
||||
last_host_node,
|
||||
suffix_tokens,
|
||||
last_hash,
|
||||
prefix_keys,
|
||||
extra_key=recv_req.extra_key,
|
||||
cache_salt=recv_req.cache_salt,
|
||||
)
|
||||
return QueryCacheHitReqOutput(
|
||||
total_tokens=total,
|
||||
l1_hit_tokens=l1_hit,
|
||||
l2_hit_tokens=l2_hit,
|
||||
l3_hit_tokens=l3_hit,
|
||||
dp_rank=int(get_parallel().dp_rank or 0),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"handle_query_cache_hit failed: {e}")
|
||||
return QueryCacheHitReqOutput(total_tokens=total, error=str(e))
|
||||
|
||||
def get_internal_state(self, recv_req: GetInternalStateReq):
|
||||
# Resolved config (pristine server_args + post-publish overrides) so a
|
||||
# readback reflects values changed via /set_internal_state, not startup.
|
||||
|
||||
@@ -112,6 +112,8 @@ class SchedulerBatchResultProcessor:
|
||||
output_streamer: SchedulerOutputStreamer
|
||||
beam_coordinator: BeamCoordinator
|
||||
abort_request: Callable
|
||||
# G9: scheduler-side drain-event recorder for the queue-time estimator.
|
||||
record_prefill_drain: Callable
|
||||
|
||||
def process_batch_result_prebuilt(self, batch: ScheduleBatch):
|
||||
assert self.disaggregation_mode == DisaggregationMode.DECODE
|
||||
@@ -343,6 +345,9 @@ class SchedulerBatchResultProcessor:
|
||||
|
||||
if req.inflight_middle_chunks <= 0:
|
||||
req.time_stats.set_prefill_finished_time()
|
||||
# G9: exactly once per request (final chunk only) — feed the
|
||||
# queue-time estimator with the drained total input length.
|
||||
self.record_prefill_drain(req.seqlen)
|
||||
|
||||
if sampling_mask_finish_reason is not None:
|
||||
req.to_finish = sampling_mask_finish_reason
|
||||
|
||||
@@ -54,6 +54,7 @@ class SchedulerLoadInquirer:
|
||||
get_total_prefill_uncached_tokens: Callable
|
||||
get_total_prefill_busy_us: Callable
|
||||
get_decode_moment_totals: Callable
|
||||
get_prefill_drain_rate: Callable
|
||||
|
||||
def _get_num_pending_tokens(self, chunk_deduct: int = 0) -> int:
|
||||
"""Get the total number of tokens pending prefill.
|
||||
@@ -128,9 +129,10 @@ class SchedulerLoadInquirer:
|
||||
num_used_tokens, kv_token_usage = (
|
||||
self.pool_stats_observer.get_pool_stats().get_kv_token_stats()
|
||||
)
|
||||
num_total_tokens = num_used_tokens + sum(
|
||||
queued_total_input_tokens = sum(
|
||||
req.seqlen for queue in pending_token_queues for req in queue
|
||||
)
|
||||
num_total_tokens = num_used_tokens + queued_total_input_tokens
|
||||
num_active_tokens = max(0, num_total_tokens - awaiting_kv_tokens)
|
||||
|
||||
memory = None
|
||||
@@ -220,6 +222,8 @@ class SchedulerLoadInquirer:
|
||||
num_running_reqs=num_running_reqs,
|
||||
num_waiting_reqs=num_waiting_reqs,
|
||||
num_waiting_uncached_tokens=self.get_num_waiting_uncached_tokens(),
|
||||
queued_total_input_tokens=queued_total_input_tokens,
|
||||
drain_rate_tokens_per_s=round(self.get_prefill_drain_rate(), 2),
|
||||
num_used_tokens=num_used_tokens,
|
||||
num_total_tokens=num_total_tokens,
|
||||
num_active_tokens=num_active_tokens,
|
||||
|
||||
@@ -54,6 +54,8 @@ from sglang.srt.managers.io_struct import (
|
||||
ProfileReq,
|
||||
ProfileReqOutput,
|
||||
ProfileReqType,
|
||||
QueryCacheHitReqInput,
|
||||
QueryCacheHitReqOutput,
|
||||
ReleaseMemoryOccupationReqInput,
|
||||
ReleaseMemoryOccupationReqOutput,
|
||||
RemoveExternalCorpusReqInput,
|
||||
@@ -129,6 +131,7 @@ _COMMUNICATOR_SPECS = [
|
||||
("detach_hicache_storage", DetachHiCacheStorageReqOutput),
|
||||
("profile", ProfileReqOutput),
|
||||
("get_internal_state", GetInternalStateReqOutput),
|
||||
("query_cache_hit", QueryCacheHitReqOutput),
|
||||
("set_internal_state", SetInternalStateReqOutput),
|
||||
("expert_distribution", ExpertDistributionReqOutput),
|
||||
("update_lora_adapter", LoRAUpdateOutput),
|
||||
@@ -905,6 +908,37 @@ class TokenizerControlMixin:
|
||||
self.auto_create_handle_loop()
|
||||
return await self.dumper_control_communicator(obj)
|
||||
|
||||
async def query_cache_hit(
|
||||
self: TokenizerManager, obj: QueryCacheHitReqInput
|
||||
) -> QueryCacheHitReqOutput:
|
||||
"""G9: fan the probe out to all DP ranks and keep the best hit.
|
||||
|
||||
The decode node's DP controller routes new requests to any rank, so
|
||||
the bypass decision uses the best per-rank L1/L2 (L3 is shared); the
|
||||
real request still goes through normal dispatch afterwards.
|
||||
"""
|
||||
self.auto_create_handle_loop()
|
||||
results: List[QueryCacheHitReqOutput] = await self.query_cache_hit_communicator(
|
||||
obj
|
||||
)
|
||||
ok = [r for r in results if r.error is None]
|
||||
if not ok:
|
||||
msg = (
|
||||
"; ".join(r.error or "empty response" for r in results)
|
||||
or "no scheduler responded"
|
||||
)
|
||||
return QueryCacheHitReqOutput(total_tokens=len(obj.input_ids), error=msg)
|
||||
best = max(
|
||||
ok, key=lambda r: r.l1_hit_tokens + r.l2_hit_tokens + r.l3_hit_tokens
|
||||
)
|
||||
return QueryCacheHitReqOutput(
|
||||
total_tokens=best.total_tokens,
|
||||
l1_hit_tokens=best.l1_hit_tokens,
|
||||
l2_hit_tokens=best.l2_hit_tokens,
|
||||
l3_hit_tokens=best.l3_hit_tokens,
|
||||
dp_rank=best.dp_rank,
|
||||
)
|
||||
|
||||
async def get_loads(
|
||||
self: TokenizerManager,
|
||||
include: Optional[List[str]] = None,
|
||||
|
||||
@@ -1279,6 +1279,16 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
raise ValueError(
|
||||
"encoder SWA replay cannot return cached prompt logprobs"
|
||||
)
|
||||
requests_embed_overrides = obj.positional_embed_overrides is not None or (
|
||||
isinstance(obj, EmbeddingReqInput)
|
||||
and obj.embed_overrides is not None
|
||||
and obj.embed_override_token_id is not None
|
||||
)
|
||||
if requests_embed_overrides and obj.contains_mm_input():
|
||||
raise ValueError(
|
||||
"embedding overrides cannot be combined with image, video, or audio "
|
||||
"inputs"
|
||||
)
|
||||
_max_req_len = self.context_len
|
||||
input_token_num = len(input_ids) if input_ids is not None else 0
|
||||
input_token_num += self.num_reserved_tokens
|
||||
|
||||
@@ -866,3 +866,15 @@ class PureSWATokenToKVPoolAllocator(SWATokenToKVPoolAllocator):
|
||||
|
||||
def is_swa_req_ring(allocator) -> bool:
|
||||
return isinstance(allocator, SWATokenToKVPoolAllocator) and allocator.swa_req_ring
|
||||
|
||||
|
||||
def swa_kept_out_of_tree(allocator) -> bool:
|
||||
"""Whether radix-tree nodes do not own the SWA state: a per-request SWA ring,
|
||||
or the DeepSeek-V4.1 encoder-replay request window, which replay rebuilds on a
|
||||
prefix hit. SWA bookkeeping must not gate prefix matching for either."""
|
||||
if not isinstance(allocator, SWATokenToKVPoolAllocator):
|
||||
return False
|
||||
if allocator.swa_req_ring:
|
||||
return True
|
||||
kvcache = getattr(allocator, "_kvcache", None)
|
||||
return getattr(kvcache, "request_window", None) is not None
|
||||
|
||||
@@ -659,6 +659,12 @@ class DeepSeekV4IndexerPool(KVCache):
|
||||
from the page layout [page_size * 64 payload | page_size * 4 scale]."""
|
||||
assert self.use_fp4_indexer, "packed readback only applies to the fp4 layout"
|
||||
buf = self.index_k_with_scale_buffer[layer_id - self.start_layer]
|
||||
if buf.is_cuda:
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer import (
|
||||
gather_fp4_index_k,
|
||||
)
|
||||
|
||||
return gather_fp4_index_k(buf, slots, page_size=self.page_size)
|
||||
slots = slots.to(torch.int64)
|
||||
p = self.page_size
|
||||
page, off = (slots // p).unsqueeze(-1), slots % p
|
||||
@@ -1855,6 +1861,11 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
# swa_kv_pool is None under the request window and unified_kv.
|
||||
return self.kv_layout
|
||||
|
||||
def get_swa_key_page_size(self) -> int:
|
||||
if self.request_window is not None:
|
||||
return self.request_window.page_size
|
||||
return self.swa_kv_pool.page_size
|
||||
|
||||
def get_swa_key_bytes_per_token(self) -> int:
|
||||
"""Last dim of the ``(pages, page_size, 1, bytes)`` view the attention
|
||||
kernel detects the SWA cache's format from."""
|
||||
|
||||
@@ -91,7 +91,7 @@ def auto_size_hicache(
|
||||
return
|
||||
requested = get_memory().hicache_ratio
|
||||
device_bytes = _estimate_hicache_bytes(params, draft_plan)
|
||||
budget = int(host_memory_budget_bytes() * fraction)
|
||||
budget = int(host_memory_budget_bytes(auto_size=True) * fraction)
|
||||
ratio = min(requested, budget * (1 - _ALLOCATION_SLACK_FRACTION) / device_bytes)
|
||||
# One collective before any pool is built: PP stages own different pool
|
||||
# counts, so a per-pool collective could deadlock.
|
||||
|
||||
@@ -20,7 +20,7 @@ if TYPE_CHECKING:
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Max pages per batched storage IO call.
|
||||
STORAGE_BATCH_SIZE = 128
|
||||
STORAGE_BATCH_SIZE = envs.SGLANG_HICACHE_STORAGE_BATCH_SIZE.get()
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -653,7 +653,7 @@ class HiRadixCache(RadixCache):
|
||||
: alloc_len // self.page_size
|
||||
]
|
||||
operation.host_indices = host_indices
|
||||
cc.prefetch_buffer.put(operation)
|
||||
cc.dispatch_prefetch(operation)
|
||||
|
||||
def _drain_ack_prefetch():
|
||||
for ack in _drain_queue(cc.ack_prefetch_queue, n_ack_prefetch):
|
||||
|
||||
@@ -15,6 +15,11 @@ def _unescape_mount_path(value: str) -> str:
|
||||
return re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), value)
|
||||
|
||||
|
||||
def _memory_stat(directory: Path) -> dict[str, int]:
|
||||
lines = (directory / "memory.stat").read_text().splitlines()
|
||||
return {key: int(value) for key, value in map(str.split, lines)}
|
||||
|
||||
|
||||
def _cgroup_memory_headroom(proc_root: Path = Path("/proc")) -> int | None:
|
||||
memberships = {}
|
||||
try:
|
||||
@@ -65,6 +70,13 @@ def _cgroup_memory_headroom(proc_root: Path = Path("/proc")) -> int | None:
|
||||
usage_name = (
|
||||
"memory.current" if filesystem == "cgroup2" else "memory.usage_in_bytes"
|
||||
)
|
||||
# File LRU pages, as MemAvailable counts; shmem sits on the anon LRU.
|
||||
# v1's unprefixed counters omit descendants, unlike its usage.
|
||||
page_cache_keys = (
|
||||
("active_file", "inactive_file")
|
||||
if filesystem == "cgroup2"
|
||||
else ("total_active_file", "total_inactive_file")
|
||||
)
|
||||
while True:
|
||||
for name in limits:
|
||||
try:
|
||||
@@ -78,7 +90,10 @@ def _cgroup_memory_headroom(proc_root: Path = Path("/proc")) -> int | None:
|
||||
# Do not silently ignore an unreadable usage file for a known
|
||||
# limit: falling back to host RAM could overrun the container.
|
||||
usage = int((directory / usage_name).read_text())
|
||||
remaining = max(0, limit - usage)
|
||||
stat = _memory_stat(directory)
|
||||
page_cache = sum(stat[key] for key in page_cache_keys)
|
||||
# Charged page cache is reclaimed under the limit, not used.
|
||||
remaining = max(0, limit - max(0, usage - page_cache))
|
||||
headroom = remaining if headroom is None else min(headroom, remaining)
|
||||
if directory == mount:
|
||||
break
|
||||
@@ -90,10 +105,25 @@ def _cgroup_memory_headroom(proc_root: Path = Path("/proc")) -> int | None:
|
||||
return headroom
|
||||
|
||||
|
||||
def available_host_memory_bytes() -> int:
|
||||
"""Conservative allocatable RAM; charged file cache is not assumed reclaimable."""
|
||||
def available_host_memory_bytes(*, allow_cgroup_fallback: bool = False) -> int:
|
||||
"""Allocatable RAM, counting reclaimable page cache as free."""
|
||||
available = psutil.virtual_memory().available
|
||||
cgroup_headroom = _cgroup_memory_headroom()
|
||||
try:
|
||||
cgroup_headroom = _cgroup_memory_headroom()
|
||||
except (OSError, RuntimeError) as exc:
|
||||
if not allow_cgroup_fallback:
|
||||
raise RuntimeError(
|
||||
"Cannot automatically size HiCache host memory; "
|
||||
"set --hicache-size explicitly (GB per rank), leaving room "
|
||||
"for other memory use within the container limit."
|
||||
) from exc
|
||||
logger.warning(
|
||||
"Cannot determine cgroup memory headroom: %s. Checking explicitly "
|
||||
"sized HiCache pools against host available memory; ensure all "
|
||||
"local ranks' pools fit within the container memory limit.",
|
||||
exc,
|
||||
)
|
||||
return available
|
||||
if cgroup_headroom is not None:
|
||||
logger.info(
|
||||
"HiCache memory headroom: host %.1f GiB, cgroup %.1f GiB",
|
||||
|
||||
@@ -15,6 +15,7 @@ import torch
|
||||
|
||||
from sglang.srt.managers.cache_controller import (
|
||||
CacheOperation,
|
||||
_hc_timing_enabled,
|
||||
)
|
||||
from sglang.srt.managers.cache_controller import (
|
||||
HiCacheController as BaseHiCacheController,
|
||||
@@ -1005,7 +1006,7 @@ class HybridCacheController(BaseHiCacheController):
|
||||
if not self._allocate_pp_prefetch_buffers(ticket, operation):
|
||||
operation.host_indices = torch.empty(0, dtype=torch.int64)
|
||||
operation.mark_terminate()
|
||||
self.prefetch_buffer.put(operation)
|
||||
self.dispatch_prefetch(operation)
|
||||
except Exception:
|
||||
logger.exception("PP HiCache ticket processing failed.")
|
||||
if operation is None or not operation.hash_value:
|
||||
@@ -1015,7 +1016,7 @@ class HybridCacheController(BaseHiCacheController):
|
||||
operation.host_indices = torch.empty(0, dtype=torch.int64)
|
||||
operation.mark_terminate()
|
||||
# Preserve every KV/sidecar ACK even when local allocation fails.
|
||||
self.prefetch_buffer.put(operation)
|
||||
self.dispatch_prefetch(operation)
|
||||
finally:
|
||||
if is_source and objects:
|
||||
self.pp_prefetch_command_queue.task_done()
|
||||
@@ -1103,15 +1104,15 @@ class HybridCacheController(BaseHiCacheController):
|
||||
)
|
||||
return host_indices, device_indices, resolved_pool_transfers
|
||||
|
||||
def _page_transfer(self, operation: PrefetchOperation) -> bool:
|
||||
def _page_transfer(self, operation: PrefetchOperation, sync_queue: Queue) -> bool:
|
||||
# KV pools and KV-derived pools first — determines actual completed page count
|
||||
kv_completed_pages = super()._page_transfer(operation)
|
||||
kv_completed_pages = super()._page_transfer(operation, sync_queue)
|
||||
|
||||
# Read non-KV derived sidecar pool, e.g. SWA, Mamba.
|
||||
self._page_transfer_sidecar(operation, kv_completed_pages)
|
||||
self._page_transfer_sidecar(operation, kv_completed_pages, sync_queue)
|
||||
|
||||
def _page_transfer_sidecar(
|
||||
self, operation: PrefetchOperation, kv_completed_pages: int
|
||||
self, operation: PrefetchOperation, kv_completed_pages: int, sync_queue: Queue
|
||||
) -> None:
|
||||
if operation.pool_transfers is None:
|
||||
return
|
||||
@@ -1143,10 +1144,10 @@ class HybridCacheController(BaseHiCacheController):
|
||||
transfers_nonkv, extra_info=extra_info
|
||||
)
|
||||
pool_hits = count_pool_hits(results)
|
||||
# Emit PrefetchAck to prefetch_sync_queue, even the operation has been canceled by the
|
||||
# Emit PrefetchAck to the lane sync queue, even the operation has been canceled by the
|
||||
# scheduler thread. The prefetch sync thread expects the same number of PrefetchAck objects
|
||||
# to perform all_reduce.
|
||||
self.prefetch_sync_queue.put(
|
||||
sync_queue.put(
|
||||
PrefetchAck(
|
||||
rid=operation.request_id,
|
||||
operation=operation,
|
||||
@@ -1420,9 +1421,11 @@ class HybridCacheController(BaseHiCacheController):
|
||||
pool.device_indices = source.device_indices
|
||||
return extra_pools
|
||||
|
||||
def _reduce_prefetch_ack(self, ack: PrefetchAck) -> None:
|
||||
def _reduce_prefetch_ack(
|
||||
self, ack: PrefetchAck, groups: List[torch.distributed.ProcessGroup]
|
||||
) -> None:
|
||||
# Handle KV-derived pool.
|
||||
super()._reduce_prefetch_ack(ack)
|
||||
super()._reduce_prefetch_ack(ack, groups)
|
||||
|
||||
# Handle other sidecar pools, e.g. SWA, Mamba.
|
||||
if ack.pool_hits is not None:
|
||||
@@ -1435,20 +1438,24 @@ class HybridCacheController(BaseHiCacheController):
|
||||
self._all_reduce(
|
||||
packed,
|
||||
torch.distributed.ReduceOp.MIN,
|
||||
self.prefetch_completion_sync_groups,
|
||||
groups,
|
||||
)
|
||||
for i, pool in enumerate(PoolName):
|
||||
ack.pool_hits[pool.value] = packed[i].item()
|
||||
|
||||
def prefetch_sync_thread_func(self) -> None:
|
||||
def prefetch_sync_thread_func(self, lane: int = 0) -> None:
|
||||
"""Synchronize progressive ACKs and publish PP ticket completion."""
|
||||
sync_queue = self.prefetch_sync_queues[lane]
|
||||
groups = self.prefetch_completion_sync_groups_by_lane[lane]
|
||||
while not self.storage_stop_event.is_set():
|
||||
try:
|
||||
ack = self.prefetch_sync_queue.get(block=True, timeout=1)
|
||||
ack = sync_queue.get(block=True, timeout=1)
|
||||
if ack is None:
|
||||
continue
|
||||
self._reduce_prefetch_ack(ack)
|
||||
self._reduce_prefetch_ack(ack, groups)
|
||||
if not getattr(ack.operation, "is_pp_broadcast", False):
|
||||
if _hc_timing_enabled() and ack.completed_req:
|
||||
ack.operation.final_ack_enqueue_ts = time.monotonic()
|
||||
self.ack_prefetch_queue.put(ack)
|
||||
continue
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user