diff --git a/deploy/b300-ds41/dockerserve-3p1d-l3dummy.yml b/deploy/b300-ds41/dockerserve-3p1d-l3dummy.yml index e422cf4265..639282cac8 100644 --- a/deploy/b300-ds41/dockerserve-3p1d-l3dummy.yml +++ b/deploy/b300-ds41/dockerserve-3p1d-l3dummy.yml @@ -175,6 +175,9 @@ services: d: <<: *sglang-common + # G9 P0 验证(2026-10-11):D 单独升镜像拿 /query_cache_hit 修复(71c6d8bc5), + # 并开 decode radix + hicache L3 客户端(复用 dummy store,只读查询)。 + image: ymkymx/sglang:dsv41-pd-visioncp-6612af9e0-local-20261011-0740 container_name: ds41-3p1d-d1 environment: SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch @@ -186,6 +189,8 @@ services: SGLANG_MOONCAKE_SEND_AUX_TCP: "1" SGLANG_DISAGGREGATION_QUEUE_SIZE: "16" SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32" + SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store-dummy.json + SGLANG_HICACHE_PREFETCH_IO_THREADS: "4" CUDA_VISIBLE_DEVICES: "6,7" command: > sglang serve @@ -199,7 +204,7 @@ services: --cuda-graph-max-bs-decode 256 --reasoning-parser auto --tool-call-parser auto --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' - --max-running-requests 512 + --max-running-requests 256 --tokenizer-worker-num 8 --host 0.0.0.0 --port 30001 --enable-cache-report --enable-metrics @@ -207,6 +212,13 @@ services: --disaggregation-mode decode --disaggregation-transfer-backend mooncake --disaggregation-bootstrap-port 9001 + --disaggregation-decode-enable-radix-cache + --enable-hierarchical-cache + --hicache-ratio 1.0 + --hicache-mem-layout page_first_direct + --hicache-io-backend direct + --hicache-storage-backend mooncake + --hicache-storage-prefetch-policy best_effort --enable-trace --trace-modules request,mooncake --otlp-traces-endpoint 127.0.0.1:4317 diff --git a/deploy/b300-ds41/dockerserve-3p1d-l3test.yml b/deploy/b300-ds41/dockerserve-3p1d-l3test.yml index fb187561f4..a2b1ccf630 100644 --- a/deploy/b300-ds41/dockerserve-3p1d-l3test.yml +++ b/deploy/b300-ds41/dockerserve-3p1d-l3test.yml @@ -1,20 +1,31 @@ -# 【L3 cache-hit 测试变体 2026-10-09】基于 dockerserve-3p1d-otlp.yml,仅改 P: -# --max-total-tokens 800000(L1 缩到 ~0.8M token)+ --hicache-ratio 1.0(L2=L1=0.8M token。⚠️ 教训:ratio 0.05 时 L2 太小,build_storage_backup_spec 拿不到 host 页会静默跳过 L3 写入,热轮无数据可中;V4 hicache 不支持 --hicache-size) -# 使 ~2.6M token/P 的语料(48×160k)溢出 L1+L2(1.6M),热轮强制走 L3 mooncake 命中。 -# 与主模板同容器名/端口,用前必须先 down 主栈。D 与 router 不变。 +# 【G9 P0 验证栈:真实 store 3p1d 2026-10-11】基于 dockerserve-3p1d-otlp.yml。 +# 用途:验证 P0 改动(/v1/loads 排队 token + drain rate 字段、P 侧 /query_cache_hit)。 +# 与主模板同容器名/端口,用前必须先 down 主栈;router(ds41-router-3p1d)独立不在本文件。 +# 2026-10-11 改造要点(vs 原 l3test): +# - 镜像升 6612af9e0(含 G9 P0 + /query_cache_hit KeyError 修复)+ scheduler.py +# 热补丁(L3 查询门限从 enable_decode_hicache 放宽到 enable_hicache_storage, +# 否则 P 节点永远查不到 L3)。 +# - 每实例独立 MOONCAKE_LOCAL_HOSTNAME(a/b/c/d),extra_hosts 互配全部 rank 别名 +# (真实 store 跨实例副本写要求对端 rank 主机名可解析,见 realstore-x2 注释; +# rank 命名规则 {hostname}-d{dp}t{tp}p{pp}c{attn_cp},commit 84fc632088; +# D 为 dp2 dp-attention(attn_tp=1),后缀推断为 -d0t0p0c0/-d1t1p0c0,未实证)。 +# - ⚠️ D 侧 cache 方案实证否决(2026-10-11):SWA 模型下 +# --disaggregation-decode-enable-radix-cache 先要 SGLANG_ENABLE_UNIFIED_RADIX_TREE=1, +# 开了又报 "supports only device-resident cache and is incompatible with +# --enable-hierarchical-cache"(kv_cache_builder.py 硬 ValueError)——D 挂不了 L3。 +# 结论:/query_cache_hit 放 P(L3 全局统一,P 有完整 hicache),D 保持生产配置。 +# - D mem-fraction 0.80:曾因崩溃 d1 泄漏 ~61GiB 临时降 0.75,全栈 down 掉旧容器后 +# 驱动即回收(GPU 全零),已恢复 0.80。D max-running 保持 256(512 时 SWA 池挤爆 KV 池)。 +# P 侧保持 l3test 原状:--max-total-tokens 800000 + ratio 1.0(L1/L2 各 ~0.8M token), +# write_through 保证 prefill 后 L3 有数据,供 /query_cache_hit 对拍。 # 原注释: 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram + OTLP trace(2026-09-27) -# 基于 dockerserve-3p1d-soak.yml 增加: --enable-trace --trace-modules request,mooncake -# --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name ds41-3p1d-<实例名> -# 依赖: b300-01 k8s cw jaeger-trace(jaegertracing/all-in-one:1.76.0,badger 6h TTL,hostNetwork 4317) -# 安全性: BatchSpanProcessor 导出失败只丢 span;Jaeger 宕机不影响推理服务 -# 调优旋钮(默认未开): SGLANG_TRACE_ASYNC=1(异步 span 创建), SGLANG_TRACE_LEVEL(默认 3) -# router(ds41-router-3p1d,rust)无 OTLP 支持,不产生 span,追踪从各 sglang HTTP 层开始 -# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md +# 依赖: mooncake-master 容器(50051,起栈前 restart 清旧 segment 注册); +# b300-01 k8s cw jaeger-trace(4317,导出失败只丢 span 不影响服务) # GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7 # 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002 # bootstrap: P1=8998 P2=8999 P3=9000 D=9001 x-sglang-common: &sglang-common - image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831 + image: ymkymx/sglang:dsv41-pd-visioncp-6612af9e0-local-20261011-0740 shm_size: "32gb" ipc: host pid: host @@ -31,6 +42,18 @@ x-sglang-common: &sglang-common count: all capabilities: [gpu] +x-rank-hosts: &rank-hosts + # mooncake rank 主机名别名:P(tp2/cp2 interleave)-d0t0p0c0/-d0t1p0c1; + # D(dp2 dp-attention)-d0t0p0c0/-d1t1p0c0(推断,起栈后核对 master 注册名) + - "localhost-a-d0t0p0c0:127.0.0.1" + - "localhost-a-d0t1p0c1:127.0.0.1" + - "localhost-b-d0t0p0c0:127.0.0.1" + - "localhost-b-d0t1p0c1:127.0.0.1" + - "localhost-c-d0t0p0c0:127.0.0.1" + - "localhost-c-d0t1p0c1:127.0.0.1" + - "localhost-d-d0t0p0c0:127.0.0.1" + - "localhost-d-d1t1p0c0:127.0.0.1" + x-p-environment: &p-environment SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" @@ -47,8 +70,10 @@ services: p1: <<: *sglang-common container_name: ds41-3p1d-p1 + extra_hosts: *rank-hosts environment: <<: *p-environment + MOONCAKE_LOCAL_HOSTNAME: localhost-a CUDA_VISIBLE_DEVICES: "0,1" command: > sglang serve @@ -87,8 +112,10 @@ services: p2: <<: *sglang-common container_name: ds41-3p1d-p2 + extra_hosts: *rank-hosts environment: <<: *p-environment + MOONCAKE_LOCAL_HOSTNAME: localhost-b CUDA_VISIBLE_DEVICES: "2,3" command: > sglang serve @@ -127,8 +154,10 @@ services: p3: <<: *sglang-common container_name: ds41-3p1d-p3 + extra_hosts: *rank-hosts environment: <<: *p-environment + MOONCAKE_LOCAL_HOSTNAME: localhost-c CUDA_VISIBLE_DEVICES: "4,5" command: > sglang serve @@ -167,6 +196,12 @@ services: d: <<: *sglang-common container_name: ds41-3p1d-d1 + extra_hosts: *rank-hosts + # ⚠️ D 保持生产配置(2026-10-11 实证):SWA 模型下 + # --disaggregation-decode-enable-radix-cache 与 --enable-hierarchical-cache 互斥 + # (kv_cache_builder.py 硬 ValueError,"supports only device-resident cache"), + # D 无法挂 L3 —— /query_cache_hit 改由 P 节点回答(L3 全局统一,P 有完整 hicache)。 + # MOONCAKE_LOCAL_HOSTNAME 保留备用(无害)。 environment: SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" @@ -176,6 +211,7 @@ services: SGLANG_MOONCAKE_SEND_AUX_TCP: "1" SGLANG_DISAGGREGATION_QUEUE_SIZE: "16" SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32" + MOONCAKE_LOCAL_HOSTNAME: localhost-d CUDA_VISIBLE_DEVICES: "6,7" command: > sglang serve diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index c794b0ff89..ea128956b1 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -852,6 +852,10 @@ class SchedulerDisaggregationPrefillMixin: continue self.checkpoint_disagg_prefill(req) + # G9: the disagg-prefill path bypasses batch_result_processor's + # prefill loop, so record the drain event here — final chunk + # done, request heading to the inflight (KV transfer) queue. + self.record_prefill_drain(req.seqlen) self.disagg_prefill_inflight_queue.append(req) if self.spec_algorithm.is_eagle() and draft_input is not None: req.output_topk_p = draft_input.topk_p[i] diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 00663bdc33..93ed7c6f67 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -5267,11 +5267,16 @@ class Scheduler( from sglang.srt.mem_cache.radix_cache import RadixKey total = len(recv_req.input_ids) + # 树节点键是 array("q"),RadixKey.match_at 断言两侧类型一致—— + # 查询侧必须同样用 array,否则树非空时匹配直接炸(2026-10-11 实测)。 + from array import array + + query_ids = array("q", recv_req.input_ids) try: match_result = self.tree_cache.match_prefix( MatchPrefixParams( key=RadixKey( - token_ids=recv_req.input_ids, + token_ids=query_ids, extra_key=recv_req.extra_key, cache_salt=recv_req.cache_salt, ) @@ -5281,8 +5286,15 @@ class Scheduler( l2_hit = getattr(match_result, "host_hit_length", 0) or 0 l3_hit = 0 last_host_node = getattr(match_result, "last_host_node", None) + # L3 查询条件:本调度器挂了 hicache 存储后端即可(P 节点 HiCache + # write_through/write_back 场景),不必是 decode-hicache 模式。 + # SWA 模型的 decode radix 与 hicache 互斥(kv_cache_builder 硬限制), + # 若只认 enable_decode_hicache,P 节点永远查不到 L3。 + can_query_l3 = getattr(self, "enable_decode_hicache", False) or getattr( + self, "enable_hicache_storage", False + ) if ( - getattr(self, "enable_decode_hicache", False) + can_query_l3 and last_host_node is not None and hasattr(self.tree_cache, "query_storage_hit_length") and ( @@ -5291,7 +5303,7 @@ class Scheduler( ) ): matched_len = l1_hit + l2_hit - suffix_tokens = recv_req.input_ids[matched_len:] + suffix_tokens = query_ids[matched_len:] last_hash = self.tree_cache.get_last_hash_value(last_host_node) prefix_keys = ( self.tree_cache.get_prefix_hash_values(last_host_node) diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index d62147fb26..dd658df956 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -1920,11 +1920,46 @@ class UnifiedRadixCache(BasePrefixCache): if len(prefetch_key) < self.prefetch_threshold: return 0 + # DeepSeek V4 keeps no physical KV objects in storage (logical anchor), + # so a plain batch_exists on KV keys always misses; the usable prefix is + # decided by the side pools. Mirror prefetch_from_storage's transfer + # construction so the query goes through batch_exists_v2. Both hooks are + # pure sizing/descriptor builders: prepare_prefetch only computes + # staging_tokens, and PREFETCH-phase transfers carry placeholder keys + # whose length sizes the TRAILING_PAGES window for batch_exists_v2. + comp_xfers: dict[ComponentType, list[PoolTransfer]] = {} + for ct in self.tree_components: + if ct == BASE_COMPONENT_TYPE: + continue + prep = self.components[ct].prepare_prefetch( + last_host_node_id, prefetch_tokens=len(prefetch_key) + ) + if prep.staging_tokens == 0: + continue + transfers = self.tree_core.build_hicache_transfers( + ct, + last_host_node_id, + CacheTransferPhase.PREFETCH, + token_ids=prefetch_key.token_ids, + prefetch_tokens=len(prefetch_key), + staging_tokens=prep.staging_tokens, + last_hash=last_hash, + ) + if transfers: + comp_xfers[ct] = transfers + kv_xfer = PoolTransfer(name=PoolName.KV, host_indices=None) + sidecar_xfers = self._build_sidecar_transfers( + CacheTransferPhase.PREFETCH, kv_xfer, comp_xfers + ) + aux_xfers = [x for xfers in comp_xfers.values() for x in xfers] + aux_xfers.extend(sidecar_xfers) + operation = PrefetchOperation( CacheRequestHandle("__storage_hit_query__", 0), prefetch_key, last_hash, prefix_keys, + pool_transfers=aux_xfers or None, ) _, storage_hit_count = self.cache_controller._storage_hit_query(operation) storage_hit_count_tensor = torch.tensor(storage_hit_count, dtype=torch.int)