fix(unified-memory): forward the KV-index translator through every wrapper backend (#37307)
Co-authored-by: Caihua Li <caihua.li@bytedance.com> Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
This commit is contained in:
co-authored by
Caihua Li
Cheng Wan
parent
e6f21cdadc
commit
22337e9c56
@@ -1011,6 +1011,9 @@ class ModelRunner:
|
||||
self.attn_backend = backends.attn_backend
|
||||
self.decode_attn_backend = backends.decode_attn_backend
|
||||
self.decode_attn_backend_group = backends.decode_attn_backend_group
|
||||
self.kv_index_translator.assert_backends_carry_translator(
|
||||
[self.attn_backend, self.decode_attn_backend]
|
||||
)
|
||||
|
||||
if get_parallel().dcp_enabled and get_parallel().dcp_replicate_q_proj:
|
||||
self._prepare_replicated_q_proj()
|
||||
|
||||
Reference in New Issue
Block a user