[Feat] [AMD] disagg: Support DSV4 decode radix cache on ROCm unified KV (bf16/fp8, DSpark) (#30929)
Co-authored-by: HAI <hixiao@gmail.com> Co-authored-by: Duyi-Wang <duyi.wang@amd.com>
This commit is contained in:
co-authored by
HAI
Duyi-Wang
parent
42eb556e66
commit
f05da43d44
@@ -3057,7 +3057,7 @@ This option does not disable required attention or MoE collectives, including ex
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--disaggregation-decode-enable-radix-cache`</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Enable radix cache on decode server (PD mode). Caches KV prefixes to avoid redundant transfers. Incompatible with --enable-hisparse, speculative decoding, and --disaggregation-transfer-backend fake.</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Enable radix cache on decode server (PD mode). Caches KV prefixes to avoid redundant transfers. Incompatible with --enable-hisparse, --disaggregation-transfer-backend fake, and speculative algorithms other than EAGLE, EAGLE3, and DSPARK.</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>`False`</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>bool flag (set to enable)</td>
|
||||
</tr>
|
||||
|
||||
@@ -245,6 +245,47 @@ def resolve_decode_retraction_backup(*, tp_worker: BaseTpWorker) -> str:
|
||||
return backend
|
||||
|
||||
|
||||
def validate_decode_radix_cache(
|
||||
*,
|
||||
model_config: ModelConfig,
|
||||
token_to_kv_pool: object,
|
||||
is_hybrid_swa: bool,
|
||||
enable_hierarchical_cache: bool,
|
||||
) -> None:
|
||||
"""Reject decode-side radix cache on KV layouts its prefix reuse cannot serve.
|
||||
|
||||
SWA models go through the unified tree, whose component pools preserve the
|
||||
full-attention prefix while the SWA window is transferred fresh. Hybrid
|
||||
SSM/KDA uses UnifiedRadixCache's Mamba component (match + lock + CoW), the
|
||||
same path as colocated serving.
|
||||
"""
|
||||
if not is_hybrid_swa:
|
||||
return
|
||||
if enable_hierarchical_cache:
|
||||
raise ValueError(
|
||||
"--disaggregation-decode-enable-radix-cache with sliding "
|
||||
"window attention (SWA) models currently supports only "
|
||||
"device-resident cache and is incompatible with "
|
||||
"--enable-hierarchical-cache."
|
||||
)
|
||||
# DeepSeek-V4 qualifies only on the unified KV layout (bf16 or fp8 rows): its
|
||||
# SWA ring and c128 state are request-scoped and always transferred whole, so
|
||||
# the tree shares only the compressed full-attention pages.
|
||||
if getattr(model_config, "is_deepseek_v4_arch", False) and not getattr(
|
||||
token_to_kv_pool, "_unified_kv", False
|
||||
):
|
||||
raise ValueError(
|
||||
"--disaggregation-decode-enable-radix-cache with DeepSeek-V4 "
|
||||
"requires the unified KV layout "
|
||||
"(SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton)."
|
||||
)
|
||||
if getattr(model_config, "is_hybrid_swa_compress", False):
|
||||
raise ValueError(
|
||||
"--disaggregation-decode-enable-radix-cache does not support "
|
||||
"SWA-compress models (e.g. Gemma4 / MiMo-V2) yet."
|
||||
)
|
||||
|
||||
|
||||
def build_kv_cache(
|
||||
*,
|
||||
server_args: ServerArgs,
|
||||
@@ -297,32 +338,16 @@ def build_kv_cache(
|
||||
"Transformers backend to avoid multimodal prefix-cache mismatches."
|
||||
)
|
||||
|
||||
# Decode-side radix cache supports SWA only through the unified tree, whose
|
||||
# component pools preserve the full-attention prefix while transferring the
|
||||
# SWA window fresh. Hybrid SSM/KDA uses UnifiedRadixCache's Mamba
|
||||
# component (match + lock + CoW), the same path as colocated serving.
|
||||
if (
|
||||
get_disagg().disaggregation_decode_enable_radix_cache
|
||||
and get_disagg().disaggregation_mode == "decode"
|
||||
):
|
||||
if is_hybrid_swa:
|
||||
if enable_hierarchical_cache:
|
||||
raise ValueError(
|
||||
"--disaggregation-decode-enable-radix-cache with sliding "
|
||||
"window attention (SWA) models currently supports only "
|
||||
"device-resident cache and is incompatible with "
|
||||
"--enable-hierarchical-cache."
|
||||
)
|
||||
if getattr(model_config, "is_deepseek_v4_arch", False):
|
||||
raise ValueError(
|
||||
"--disaggregation-decode-enable-radix-cache does not support "
|
||||
"DeepSeek-V4 (DSA) compressed KV (c4/c128/indexer) yet."
|
||||
)
|
||||
if getattr(model_config, "is_hybrid_swa_compress", False):
|
||||
raise ValueError(
|
||||
"--disaggregation-decode-enable-radix-cache does not support "
|
||||
"SWA-compress models (e.g. Gemma4 / MiMo-V2) yet."
|
||||
)
|
||||
validate_decode_radix_cache(
|
||||
model_config=model_config,
|
||||
token_to_kv_pool=token_to_kv_pool,
|
||||
is_hybrid_swa=is_hybrid_swa,
|
||||
enable_hierarchical_cache=enable_hierarchical_cache,
|
||||
)
|
||||
|
||||
effective_chunked_prefill_size = get_schedule().chunked_prefill_size
|
||||
if model_config.is_multimodal and uses_transformers_backend:
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Decode-side radix cache admits DeepSeek-V4 only on the unified KV layout.
|
||||
|
||||
Both the bf16 and fp8 unified rows are admitted; the paged layout stays rejected.
|
||||
|
||||
python -m pytest test/registered/unit/mem_cache/test_decode_radix_cache_validation.py -v
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.mem_cache.kv_cache_builder import validate_decode_radix_cache
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=1, suite="base-a-test-cpu")
|
||||
|
||||
_DSV4 = SimpleNamespace(is_deepseek_v4_arch=True, is_hybrid_swa_compress=False)
|
||||
|
||||
|
||||
def _validate(token_to_kv_pool):
|
||||
validate_decode_radix_cache(
|
||||
model_config=_DSV4,
|
||||
token_to_kv_pool=token_to_kv_pool,
|
||||
is_hybrid_swa=True,
|
||||
enable_hierarchical_cache=False,
|
||||
)
|
||||
|
||||
|
||||
class TestValidateDecodeRadixCacheDSV4(CustomTestCase):
|
||||
def test_unified_kv_bf16_is_admitted(self):
|
||||
_validate(SimpleNamespace(_unified_kv=True, _unified_kv_fp8=False))
|
||||
|
||||
def test_unified_kv_fp8_is_admitted(self):
|
||||
_validate(SimpleNamespace(_unified_kv=True, _unified_kv_fp8=True))
|
||||
|
||||
def test_paged_layout_is_rejected(self):
|
||||
with self.assertRaisesRegex(ValueError, "unified KV layout"):
|
||||
_validate(SimpleNamespace(_unified_kv=False, _unified_kv_fp8=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user