refactor(unified-memory): translate the KV write location once, at ForwardBatch construction (#35245)
Co-authored-by: Caihua Li <caihua.li@bytedance.com> Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
This commit is contained in:
co-authored by
Caihua Li
Cheng Wan
parent
4f997a432a
commit
29578d5578
@@ -411,6 +411,7 @@ class MockModelRunner(ModelRunner):
|
||||
page_size=case.page_size,
|
||||
get_kvcache=lambda: self.token_to_kv_pool,
|
||||
)
|
||||
self.init_kv_index_translator()
|
||||
self.attn_cp_size = 1
|
||||
self.attention_chunk_size = None
|
||||
self.hisparse_coordinator = None
|
||||
|
||||
@@ -404,6 +404,7 @@ class DSAMockModelRunner(ModelRunner):
|
||||
kv_cache_dim=pool_kv_cache_dim,
|
||||
)
|
||||
self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size)
|
||||
self.init_kv_index_translator()
|
||||
self.attn_cp_size = 1
|
||||
self.attention_chunk_size = None
|
||||
self.hisparse_coordinator = None
|
||||
|
||||
@@ -384,6 +384,7 @@ class DualChunkMockModelRunner(ModelRunner):
|
||||
enable_alt_stream=False,
|
||||
)
|
||||
self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size)
|
||||
self.init_kv_index_translator()
|
||||
self.attn_cp_size = 1
|
||||
self.attention_chunk_size = None
|
||||
self.hisparse_coordinator = None
|
||||
|
||||
@@ -318,6 +318,7 @@ class MockGDNModelRunner(ModelRunner):
|
||||
page_size=case.page_size,
|
||||
get_kvcache=lambda: self.token_to_kv_pool,
|
||||
)
|
||||
self.init_kv_index_translator()
|
||||
self.attn_cp_size = 1
|
||||
self.attention_chunk_size = None
|
||||
self.hisparse_coordinator = None
|
||||
|
||||
@@ -317,6 +317,7 @@ class MockKDAModelRunner(ModelRunner):
|
||||
enable_alt_stream=False,
|
||||
)
|
||||
self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size)
|
||||
self.init_kv_index_translator()
|
||||
self.attn_cp_size = 1
|
||||
self.attention_chunk_size = None
|
||||
self.hisparse_coordinator = None
|
||||
|
||||
@@ -326,6 +326,7 @@ class MockLightningModelRunner(ModelRunner):
|
||||
enable_alt_stream=False,
|
||||
)
|
||||
self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size)
|
||||
self.init_kv_index_translator()
|
||||
self.attn_cp_size = 1
|
||||
self.attention_chunk_size = None
|
||||
self.hisparse_coordinator = None
|
||||
|
||||
@@ -453,6 +453,7 @@ class MockMamba2ModelRunner(ModelRunner):
|
||||
enable_alt_stream=False,
|
||||
)
|
||||
self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size)
|
||||
self.init_kv_index_translator()
|
||||
self.attn_cp_size = 1
|
||||
self.attention_chunk_size = None
|
||||
self.hisparse_coordinator = None
|
||||
|
||||
@@ -310,6 +310,7 @@ class MockMLAModelRunner(ModelRunner):
|
||||
enable_memory_saver=False,
|
||||
)
|
||||
self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size)
|
||||
self.init_kv_index_translator()
|
||||
self.attn_cp_size = 1
|
||||
self.attention_chunk_size = None
|
||||
self.hisparse_coordinator = None
|
||||
|
||||
Reference in New Issue
Block a user