feat(unified-memory): read unified pool from attention backends fa3/flashinfer/trtllm_mha/flashmla (#34613)
Co-authored-by: Caihua Li <caihua.li@bytedance.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
This commit is contained in:
co-authored by
Caihua Li
Claude Fable 5
Cheng Wan
parent
29578d5578
commit
8bb776dc48
@@ -12,6 +12,7 @@ from sglang.kernels.ops.kvcache.kv_indices import (
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.dcp.layout import filter_dcp_local_chunk_kv_indices
|
||||
from sglang.srt.model_executor.forward_context import (
|
||||
get_attn_backend,
|
||||
get_req_to_token_pool,
|
||||
get_token_to_kv_pool,
|
||||
)
|
||||
@@ -90,6 +91,10 @@ class ForwardBatchDeepSeekMHAMixin:
|
||||
self.prefix_chunk_starts_cpu[idx],
|
||||
self.prefix_chunk_seq_lens_cpu[idx],
|
||||
)
|
||||
# None on a backend that never bound a translator.
|
||||
src = get_attn_backend().kv_index_translator
|
||||
if src is not None:
|
||||
chunk_kv_indices = src.translate_full_attn_ids(chunk_kv_indices)
|
||||
self.prefix_chunk_kv_indices.append(chunk_kv_indices)
|
||||
|
||||
# Here we suppose the length of each chunk is equal
|
||||
@@ -235,5 +240,9 @@ class ForwardBatchDeepSeekMHAMixin:
|
||||
kv_indices,
|
||||
req_to_token.shape[1],
|
||||
)
|
||||
# None on a backend that never bound a translator.
|
||||
src = get_attn_backend().kv_index_translator
|
||||
if src is not None:
|
||||
kv_indices = src.translate_full_attn_ids(kv_indices)
|
||||
self.mha_one_shot_kv_indices = kv_indices
|
||||
return kv_indices
|
||||
|
||||
Reference in New Issue
Block a user