[Feat] [AMD] disagg: Support DSV4 decode radix cache on ROCm unified KV (bf16/fp8, DSpark) (#30929)

Co-authored-by: HAI <hixiao@gmail.com>
Co-authored-by: Duyi-Wang <duyi.wang@amd.com>
This commit is contained in:
AMD-yanfeiwang
2026-10-16 02:39:27 -07:00
committed by GitHub
co-authored by HAI Duyi-Wang
parent 42eb556e66
commit f05da43d44
3 changed files with 90 additions and 23 deletions
@@ -3057,7 +3057,7 @@ This option does not disable required attention or MoE collectives, including ex
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--disaggregation-decode-enable-radix-cache`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Enable radix cache on decode server (PD mode). Caches KV prefixes to avoid redundant transfers. Incompatible with --enable-hisparse, speculative decoding, and --disaggregation-transfer-backend fake.</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Enable radix cache on decode server (PD mode). Caches KV prefixes to avoid redundant transfers. Incompatible with --enable-hisparse, --disaggregation-transfer-backend fake, and speculative algorithms other than EAGLE, EAGLE3, and DSPARK.</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>`False`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>bool flag (set to enable)</td>
</tr>
+47 -22
View File
@@ -245,6 +245,47 @@ def resolve_decode_retraction_backup(*, tp_worker: BaseTpWorker) -> str:
return backend
def validate_decode_radix_cache(
*,
model_config: ModelConfig,
token_to_kv_pool: object,
is_hybrid_swa: bool,
enable_hierarchical_cache: bool,
) -> None:
"""Reject decode-side radix cache on KV layouts its prefix reuse cannot serve.
SWA models go through the unified tree, whose component pools preserve the
full-attention prefix while the SWA window is transferred fresh. Hybrid
SSM/KDA uses UnifiedRadixCache's Mamba component (match + lock + CoW), the
same path as colocated serving.
"""
if not is_hybrid_swa:
return
if enable_hierarchical_cache:
raise ValueError(
"--disaggregation-decode-enable-radix-cache with sliding "
"window attention (SWA) models currently supports only "
"device-resident cache and is incompatible with "
"--enable-hierarchical-cache."
)
# DeepSeek-V4 qualifies only on the unified KV layout (bf16 or fp8 rows): its
# SWA ring and c128 state are request-scoped and always transferred whole, so
# the tree shares only the compressed full-attention pages.
if getattr(model_config, "is_deepseek_v4_arch", False) and not getattr(
token_to_kv_pool, "_unified_kv", False
):
raise ValueError(
"--disaggregation-decode-enable-radix-cache with DeepSeek-V4 "
"requires the unified KV layout "
"(SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton)."
)
if getattr(model_config, "is_hybrid_swa_compress", False):
raise ValueError(
"--disaggregation-decode-enable-radix-cache does not support "
"SWA-compress models (e.g. Gemma4 / MiMo-V2) yet."
)
def build_kv_cache(
*,
server_args: ServerArgs,
@@ -297,32 +338,16 @@ def build_kv_cache(
"Transformers backend to avoid multimodal prefix-cache mismatches."
)
# Decode-side radix cache supports SWA only through the unified tree, whose
# component pools preserve the full-attention prefix while transferring the
# SWA window fresh. Hybrid SSM/KDA uses UnifiedRadixCache's Mamba
# component (match + lock + CoW), the same path as colocated serving.
if (
get_disagg().disaggregation_decode_enable_radix_cache
and get_disagg().disaggregation_mode == "decode"
):
if is_hybrid_swa:
if enable_hierarchical_cache:
raise ValueError(
"--disaggregation-decode-enable-radix-cache with sliding "
"window attention (SWA) models currently supports only "
"device-resident cache and is incompatible with "
"--enable-hierarchical-cache."
)
if getattr(model_config, "is_deepseek_v4_arch", False):
raise ValueError(
"--disaggregation-decode-enable-radix-cache does not support "
"DeepSeek-V4 (DSA) compressed KV (c4/c128/indexer) yet."
)
if getattr(model_config, "is_hybrid_swa_compress", False):
raise ValueError(
"--disaggregation-decode-enable-radix-cache does not support "
"SWA-compress models (e.g. Gemma4 / MiMo-V2) yet."
)
validate_decode_radix_cache(
model_config=model_config,
token_to_kv_pool=token_to_kv_pool,
is_hybrid_swa=is_hybrid_swa,
enable_hierarchical_cache=enable_hierarchical_cache,
)
effective_chunked_prefill_size = get_schedule().chunked_prefill_size
if model_config.is_multimodal and uses_transformers_backend:
@@ -0,0 +1,42 @@
"""Decode-side radix cache admits DeepSeek-V4 only on the unified KV layout.
Both the bf16 and fp8 unified rows are admitted; the paged layout stays rejected.
python -m pytest test/registered/unit/mem_cache/test_decode_radix_cache_validation.py -v
"""
import unittest
from types import SimpleNamespace
from sglang.srt.mem_cache.kv_cache_builder import validate_decode_radix_cache
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=1, suite="base-a-test-cpu")
_DSV4 = SimpleNamespace(is_deepseek_v4_arch=True, is_hybrid_swa_compress=False)
def _validate(token_to_kv_pool):
validate_decode_radix_cache(
model_config=_DSV4,
token_to_kv_pool=token_to_kv_pool,
is_hybrid_swa=True,
enable_hierarchical_cache=False,
)
class TestValidateDecodeRadixCacheDSV4(CustomTestCase):
def test_unified_kv_bf16_is_admitted(self):
_validate(SimpleNamespace(_unified_kv=True, _unified_kv_fp8=False))
def test_unified_kv_fp8_is_admitted(self):
_validate(SimpleNamespace(_unified_kv=True, _unified_kv_fp8=True))
def test_paged_layout_is_rejected(self):
with self.assertRaisesRegex(ValueError, "unified KV layout"):
_validate(SimpleNamespace(_unified_kv=False, _unified_kv_fp8=False))
if __name__ == "__main__":
unittest.main()