minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[AMD] DSv4: bring HIP compress-state pool into the memory_saver KV_CACHE region (#31747)

Browse Source 
Co-authored-by: Zhiyao Jiang <jessicajiang324@gmail.com>
This commit is contained in:
Xinyu Jiang
2026-07-29 02:41:08 -07:00
committed by GitHub
co-authored by Zhiyao Jiang
parent c32c4ef79c
commit f5bcd00e16
2 changed files with 19 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/mem_cache/deepseek_v4_compress_state.py
-5
View File
@@ -153,11 +153,6 @@ class CompressStatePool:
allocation boilerplate. Requires ``self._size`` and ``self.last_dim`` allocation boilerplate. Requires ``self._size`` and ``self.last_dim``
to be set already. to be set already.
""" """
if _is_hip:
self.kv_score_buffer = KVAndScore(
torch.empty((self._size, self.last_dim), dtype=dtype, device=device)
)
else:
self.memory_saver_adapter = TorchMemorySaverAdapter.create( self.memory_saver_adapter = TorchMemorySaverAdapter.create(
enable=enable_memory_saver enable=enable_memory_saver
) )
python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py
-3
View File
@@ -646,9 +646,6 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
self._init_compressed_layer_mapping() self._init_compressed_layer_mapping()
if _is_hip:
self._init_paged_compress_states(False)
else:
self._init_paged_compress_states(enable_memory_saver) self._init_paged_compress_states(enable_memory_saver)
def get_unified_kv(self, layer_id: int) -> torch.Tensor: def get_unified_kv(self, layer_id: int) -> torch.Tensor: