[Fix] muse_spark: support unified KV memory with TRT-LLM (#45356)
This commit is contained in:
@@ -1156,9 +1156,13 @@ class KVCacheConfigurator:
|
||||
# A fused draft stores its rows in the host's KV dtype. An explicit
|
||||
# draft dtype resolves here as the draft runner will; `auto` follows
|
||||
# the draft's quant config, which only the draft runner knows, so it
|
||||
# is checked when the draft binds.
|
||||
# is checked when the draft binds. DFLASH-family FA4 is already known:
|
||||
# it always uses the draft's compute dtype, including with default/auto
|
||||
# KV settings. Decline fusion now if that differs from the host.
|
||||
draft_kv_cache_dtype = get_spec().speculative_draft_kv_cache_dtype
|
||||
if draft_kv_cache_dtype not in (None, "auto"):
|
||||
if draft_kv_cache_dtype not in (None, "auto") or (
|
||||
self.spec_algorithm.is_dflash_family() and draft_backend == "fa4"
|
||||
):
|
||||
from sglang.srt.mem_cache.kv_cache_dtype import configure_kv_cache_dtype
|
||||
|
||||
_, draft_kv_dtype = configure_kv_cache_dtype(
|
||||
|
||||
@@ -91,7 +91,7 @@ from sglang.srt.model_executor.forward_context import (
|
||||
get_token_to_kv_pool,
|
||||
)
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.runtime_context import get_forward, get_parallel
|
||||
from sglang.srt.runtime_context import get_forward, get_memory, get_parallel
|
||||
from sglang.srt.state_capturer.routed_experts import get_global_experts_capturer
|
||||
from sglang.srt.utils import add_prefix, is_cpu, use_intel_amx_backend
|
||||
from sglang.srt.utils.common import get_device_capability, is_cuda
|
||||
@@ -353,6 +353,10 @@ class MuseSpark1xAttention(nn.Module):
|
||||
def _fused_kv_store(
|
||||
self, forward_batch: ForwardBatch
|
||||
) -> Optional[Tuple[torch.Tensor, torch.Tensor, torch.Tensor]]:
|
||||
# Unified pools interleave layers in each token row. Reuse the prefill
|
||||
# path, whose attention-backend cache writer supports these strides.
|
||||
if get_memory().enable_unified_memory:
|
||||
return None
|
||||
backend = get_attn_backend()
|
||||
pool = get_token_to_kv_pool()
|
||||
if not (
|
||||
|
||||
Reference in New Issue
Block a user