Support mxfp8 KV cache in PD transfer (#35718)

This commit is contained in:
Ke Bao
2026-08-21 13:05:06 +08:00
committed by GitHub
parent f64080fbaf
commit 73a2c117c6
8 changed files with 121 additions and 15 deletions
+18 -5
View File
@@ -3403,6 +3403,9 @@ class MHATokenToKVPoolMXFP8(MHATokenToKVPool):
],
device=self.device,
)
# This override replaces the base allocation, so the PD-transfer
# descriptors for the packed data buffers are built here too.
self._kv_buffer_descs = self._build_kv_buffer_descs()
def _clear_buffers(self):
del self.k_buffer
@@ -3552,11 +3555,21 @@ class MHATokenToKVPoolMXFP8(MHATokenToKVPool):
def load_cpu_copy(self, kv_cache_cpu, indices, mamba_indices=None):
raise NotImplementedError("CPU offloading is unsupported for MXFP8 KV cache.")
def get_contiguous_buf_infos(self):
raise NotImplementedError(
"KV transfer / disaggregation is unsupported for MXFP8 KV cache "
"(scale buffers are not exposed)."
)
def get_kv_scale_buf_infos(self):
"""(ptrs, lens, item_lens) for the UE8M0 scale buffers, k then v.
The interleaved layout puts pages on the leading axis, so a page's
scales are one contiguous row; the flat layout is per slot.
"""
tensors = self.k_scale_buffer + self.v_scale_buffer
ptrs = [t.data_ptr() for t in tensors]
lens = [t.nbytes for t in tensors]
row_bytes = [t[0].nbytes for t in tensors]
if self.mxfp8_sf_interleaved:
item_lens = row_bytes
else:
item_lens = [rb * self.page_size for rb in row_bytes]
return ptrs, lens, item_lens
def set_kv_buffer_prefix_valid(self, *args, **kwargs):
raise NotImplementedError(