[Fix] muse_spark: support unified KV memory with TRT-LLM (#45356)

This commit is contained in:
Baizhou Zhang
2026-10-20 12:54:53 -07:00
committed by GitHub
parent 29274b41c6
commit cfec1b91bb
5 changed files with 123 additions and 5 deletions
@@ -1156,9 +1156,13 @@ class KVCacheConfigurator:
# A fused draft stores its rows in the host's KV dtype. An explicit
# draft dtype resolves here as the draft runner will; `auto` follows
# the draft's quant config, which only the draft runner knows, so it
# is checked when the draft binds.
# is checked when the draft binds. DFLASH-family FA4 is already known:
# it always uses the draft's compute dtype, including with default/auto
# KV settings. Decline fusion now if that differs from the host.
draft_kv_cache_dtype = get_spec().speculative_draft_kv_cache_dtype
if draft_kv_cache_dtype not in (None, "auto"):
if draft_kv_cache_dtype not in (None, "auto") or (
self.spec_algorithm.is_dflash_family() and draft_backend == "fa4"
):
from sglang.srt.mem_cache.kv_cache_dtype import configure_kv_cache_dtype
_, draft_kv_dtype = configure_kv_cache_dtype(
+5 -1
View File
@@ -91,7 +91,7 @@ from sglang.srt.model_executor.forward_context import (
get_token_to_kv_pool,
)
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.runtime_context import get_forward, get_parallel
from sglang.srt.runtime_context import get_forward, get_memory, get_parallel
from sglang.srt.state_capturer.routed_experts import get_global_experts_capturer
from sglang.srt.utils import add_prefix, is_cpu, use_intel_amx_backend
from sglang.srt.utils.common import get_device_capability, is_cuda
@@ -353,6 +353,10 @@ class MuseSpark1xAttention(nn.Module):
def _fused_kv_store(
self, forward_batch: ForwardBatch
) -> Optional[Tuple[torch.Tensor, torch.Tensor, torch.Tensor]]:
# Unified pools interleave layers in each token row. Reuse the prefill
# path, whose attention-backend cache writer supports these strides.
if get_memory().enable_unified_memory:
return None
backend = get_attn_backend()
pool = get_token_to_kv_pool()
if not (