From 7d5759b2d92ee0e98d7c08d7acbdabb2c92fdb4d Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Sat, 10 Oct 2026 17:55:14 +0800 Subject: [PATCH] =?UTF-8?q?hicache:=20STORAGE=5FBATCH=5FSIZE=20=E5=8F=AFen?= =?UTF-8?q?v=E5=8C=96=EF=BC=88SGLANG=5FHICACHE=5FSTORAGE=5FBATCH=5FSIZE?= =?UTF-8?q?=EF=BC=8C=E9=BB=98=E8=AE=A4128=E4=B8=8D=E5=8F=98=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 打点显示每批存储 IO 有 ~30-70ms 固定开销(与传输介质无关), 调大批次可摊薄。env 化便于不动代码做 128/512/1024 对照。 --- python/sglang/srt/environ.py | 2 ++ python/sglang/srt/mem_cache/hicache_storage.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 746bce4da1..06a691994e 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -761,6 +761,8 @@ class Envs: # Per-stage timing logs for the L3 storage prefetch path (query / transfer / # ack reduce / drain / graft / load-back). Emits "HC_TIMING ..." lines. SGLANG_HICACHE_TIMING = EnvBool(False) + # Pages per batched storage IO call (query / get / put chunking). + SGLANG_HICACHE_STORAGE_BATCH_SIZE = EnvInt(128) SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR = EnvStr(None) # File-backend LRU eviction (opt-in; sizes accept SI/IEC suffixes, "0" disables). SGLANG_HICACHE_FILE_BACKEND_MAX_SIZE = EnvStr(None) diff --git a/python/sglang/srt/mem_cache/hicache_storage.py b/python/sglang/srt/mem_cache/hicache_storage.py index 04e5ab8c84..0e35b769b4 100644 --- a/python/sglang/srt/mem_cache/hicache_storage.py +++ b/python/sglang/srt/mem_cache/hicache_storage.py @@ -20,7 +20,7 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) # Max pages per batched storage IO call. -STORAGE_BATCH_SIZE = 128 +STORAGE_BATCH_SIZE = envs.SGLANG_HICACHE_STORAGE_BATCH_SIZE.get() @dataclass