[BUG FIX]Fix DSA CPU offload mamba indices signature (#27645)
This commit is contained in:
@@ -2360,13 +2360,13 @@ class DSATokenToKVPool(MLATokenToKVPool):
|
|||||||
pool=self, buf=buf, loc=loc, index_k=index_k, index_k_scale=index_k_scale
|
pool=self, buf=buf, loc=loc, index_k=index_k, index_k_scale=index_k_scale
|
||||||
)
|
)
|
||||||
|
|
||||||
def get_cpu_copy(self, indices):
|
def get_cpu_copy(self, indices, mamba_indices=None):
|
||||||
# DSA keeps a page-indexed index_k_with_scale_buffer alongside kv_buffer.
|
# DSA keeps a page-indexed index_k_with_scale_buffer alongside kv_buffer.
|
||||||
# Retract frees the slots/pages and they get reused by other reqs'
|
# Retract frees the slots/pages and they get reused by other reqs'
|
||||||
# set_index_k_scale_buffer, so we must offload it here too -- otherwise
|
# set_index_k_scale_buffer, so we must offload it here too -- otherwise
|
||||||
# resume restores kv_buffer but leaves foreign index/scale in place and
|
# resume restores kv_buffer but leaves foreign index/scale in place and
|
||||||
# DSA attention reads garbage at those token positions.
|
# DSA attention reads garbage at those token positions.
|
||||||
kv_cache_cpu = super().get_cpu_copy(indices)
|
kv_cache_cpu = super().get_cpu_copy(indices, mamba_indices=mamba_indices)
|
||||||
|
|
||||||
page_indices = indices[:: self.page_size] // self.page_size
|
page_indices = indices[:: self.page_size] // self.page_size
|
||||||
torch.cuda.synchronize()
|
torch.cuda.synchronize()
|
||||||
@@ -2385,8 +2385,10 @@ class DSATokenToKVPool(MLATokenToKVPool):
|
|||||||
|
|
||||||
return {"kv": kv_cache_cpu, "index_k": index_k_cpu}
|
return {"kv": kv_cache_cpu, "index_k": index_k_cpu}
|
||||||
|
|
||||||
def load_cpu_copy(self, kv_cache_cpu_dict, indices):
|
def load_cpu_copy(self, kv_cache_cpu_dict, indices, mamba_indices=None):
|
||||||
super().load_cpu_copy(kv_cache_cpu_dict["kv"], indices)
|
super().load_cpu_copy(
|
||||||
|
kv_cache_cpu_dict["kv"], indices, mamba_indices=mamba_indices
|
||||||
|
)
|
||||||
|
|
||||||
page_indices = indices[:: self.page_size] // self.page_size
|
page_indices = indices[:: self.page_size] // self.page_size
|
||||||
index_k_cpu = kv_cache_cpu_dict["index_k"]
|
index_k_cpu = kv_cache_cpu_dict["index_k"]
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import inspect
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
@@ -15,6 +16,14 @@ from sglang.test.ci.ci_register import register_cuda_ci
|
|||||||
register_cuda_ci(est_time=9, stage="base-b", runner_config="1-gpu-small")
|
register_cuda_ci(est_time=9, stage="base-b", runner_config="1-gpu-small")
|
||||||
|
|
||||||
|
|
||||||
|
class TestDSAOffloadSignatures(unittest.TestCase):
|
||||||
|
def test_cpu_copy_methods_accept_mamba_indices(self):
|
||||||
|
for method_name in ("get_cpu_copy", "load_cpu_copy"):
|
||||||
|
with self.subTest(method_name=method_name):
|
||||||
|
signature = inspect.signature(getattr(DSATokenToKVPool, method_name))
|
||||||
|
self.assertIn("mamba_indices", signature.parameters)
|
||||||
|
|
||||||
|
|
||||||
class TestDSAHiCacheTransfer(unittest.TestCase):
|
class TestDSAHiCacheTransfer(unittest.TestCase):
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
if not torch.cuda.is_available():
|
if not torch.cuda.is_available():
|
||||||
|
|||||||
Reference in New Issue
Block a user