Support mxfp8 KV cache in PD transfer (#35718)
This commit is contained in:
@@ -3403,6 +3403,9 @@ class MHATokenToKVPoolMXFP8(MHATokenToKVPool):
|
||||
],
|
||||
device=self.device,
|
||||
)
|
||||
# This override replaces the base allocation, so the PD-transfer
|
||||
# descriptors for the packed data buffers are built here too.
|
||||
self._kv_buffer_descs = self._build_kv_buffer_descs()
|
||||
|
||||
def _clear_buffers(self):
|
||||
del self.k_buffer
|
||||
@@ -3552,11 +3555,21 @@ class MHATokenToKVPoolMXFP8(MHATokenToKVPool):
|
||||
def load_cpu_copy(self, kv_cache_cpu, indices, mamba_indices=None):
|
||||
raise NotImplementedError("CPU offloading is unsupported for MXFP8 KV cache.")
|
||||
|
||||
def get_contiguous_buf_infos(self):
|
||||
raise NotImplementedError(
|
||||
"KV transfer / disaggregation is unsupported for MXFP8 KV cache "
|
||||
"(scale buffers are not exposed)."
|
||||
)
|
||||
def get_kv_scale_buf_infos(self):
|
||||
"""(ptrs, lens, item_lens) for the UE8M0 scale buffers, k then v.
|
||||
|
||||
The interleaved layout puts pages on the leading axis, so a page's
|
||||
scales are one contiguous row; the flat layout is per slot.
|
||||
"""
|
||||
tensors = self.k_scale_buffer + self.v_scale_buffer
|
||||
ptrs = [t.data_ptr() for t in tensors]
|
||||
lens = [t.nbytes for t in tensors]
|
||||
row_bytes = [t[0].nbytes for t in tensors]
|
||||
if self.mxfp8_sf_interleaved:
|
||||
item_lens = row_bytes
|
||||
else:
|
||||
item_lens = [rb * self.page_size for rb in row_bytes]
|
||||
return ptrs, lens, item_lens
|
||||
|
||||
def set_kv_buffer_prefix_valid(self, *args, **kwargs):
|
||||
raise NotImplementedError(
|
||||
|
||||
Reference in New Issue
Block a user