Fix --hicache-size allocating ~2x host memory on hybrid SWA (#32373)
Co-authored-by: cctry <cctry@fb.com> Co-authored-by: Zhiqiang Xie <xiezhq@stanford.edu>
This commit is contained in:
co-authored by
cctry
Zhiqiang Xie
parent
9989077f24
commit
2c63a2f12b
@@ -0,0 +1,31 @@
|
||||
"""Unit test for hybrid HiCache fixed-size budget splitting."""
|
||||
|
||||
import unittest
|
||||
|
||||
from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import _split_hicache_size
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=1, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class _Pool:
|
||||
def __init__(self, kv_bytes):
|
||||
self._kv_bytes = kv_bytes
|
||||
|
||||
def get_kv_size_bytes(self):
|
||||
return self._kv_bytes
|
||||
|
||||
|
||||
class TestSplitHicacheSize(CustomTestCase):
|
||||
def test_splits_total_budget_by_device_bytes(self):
|
||||
# scalar and (k, v) tuple return shapes both supported
|
||||
shares = _split_hicache_size(
|
||||
100, (_Pool(75 * 10**9), _Pool((15 * 10**9, 10 * 10**9)))
|
||||
)
|
||||
self.assertEqual(shares, (75.0, 25.0)) # proportional to device KV bytes
|
||||
self.assertEqual(sum(shares), 100) # total budget preserved, not doubled
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user