Fix --hicache-size allocating ~2x host memory on hybrid Mamba (#32915)

This commit is contained in:
luchangli
2026-08-01 02:37:57 +08:00
committed by GitHub
parent 89f4a80c1f
commit 26486a957d
3 changed files with 26 additions and 2 deletions
@@ -1111,6 +1111,9 @@ class MambaPool:
subdims_per_tensor += [subdims] * self.num_mamba_layers
return subdims_per_tensor
def get_kv_size_bytes(self):
return self.mamba_cache.mem_usage_bytes()
class HybridReqToTokenPool(ReqToTokenPool):
"""A memory pool that maps a request to its token locations."""