[AMD] DSv4: bring HIP compress-state pool into the memory_saver KV_CACHE region (#31747)
Co-authored-by: Zhiyao Jiang <jessicajiang324@gmail.com>
This commit is contained in:
co-authored by
Zhiyao Jiang
parent
c32c4ef79c
commit
f5bcd00e16
@@ -153,11 +153,6 @@ class CompressStatePool:
|
||||
allocation boilerplate. Requires ``self._size`` and ``self.last_dim``
|
||||
to be set already.
|
||||
"""
|
||||
if _is_hip:
|
||||
self.kv_score_buffer = KVAndScore(
|
||||
torch.empty((self._size, self.last_dim), dtype=dtype, device=device)
|
||||
)
|
||||
else:
|
||||
self.memory_saver_adapter = TorchMemorySaverAdapter.create(
|
||||
enable=enable_memory_saver
|
||||
)
|
||||
|
||||
@@ -646,9 +646,6 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
|
||||
self._init_compressed_layer_mapping()
|
||||
|
||||
if _is_hip:
|
||||
self._init_paged_compress_states(False)
|
||||
else:
|
||||
self._init_paged_compress_states(enable_memory_saver)
|
||||
|
||||
def get_unified_kv(self, layer_id: int) -> torch.Tensor:
|
||||
|
||||
Reference in New Issue
Block a user