[AMD][DSV4] feat: enable PD-disagg with fp8 unified_kv on gfx950 (#39166)
This commit is contained in:
@@ -1349,6 +1349,26 @@ class CommonKVManager(BaseKVManager):
|
||||
c128_full = sum(1 for r in mla_ratios if r == 128)
|
||||
kv_layout_len = 2 * c4_full + c128_full
|
||||
|
||||
# A DSV4 two-pool peer registers a rope group beside each KV group and
|
||||
# doubles the ring, and the switch that turns that on is read per
|
||||
# process. Mixed peers would fall into the cuts below, match whichever
|
||||
# side is shorter, and then index past the other one once per request.
|
||||
if state_type in (StateType.SWA, StateType.DSV4_REQUEST_STATE):
|
||||
single_pool_len = two_pool_len = None
|
||||
elif state_type == StateType.SWA_RING:
|
||||
single_pool_len, two_pool_len = len(mla_ratios), 2 * len(mla_ratios)
|
||||
else:
|
||||
single_pool_len, two_pool_len = kv_layout_len, 3 * c4_full + 2 * c128_full
|
||||
peer_lens = {len(src_kv_ptrs), len(dst_kv_ptrs)}
|
||||
if single_pool_len is not None and peer_lens == {single_pool_len, two_pool_len}:
|
||||
raise ValueError(
|
||||
"PD peers disagree on the compressed-MLA KV layout: prefill "
|
||||
f"registered {len(src_kv_ptrs)} regions, decode "
|
||||
f"{len(dst_kv_ptrs)} ({single_pool_len} is one pool per layer, "
|
||||
f"{two_pool_len} is the fp8 two-pool). "
|
||||
"SGLANG_DSV4_UNIFIED_KV_FP8 must be set the same on both sides."
|
||||
)
|
||||
|
||||
c4_off_s = sum(1 for r in mla_ratios[:start_layer] if r == 4)
|
||||
c4_off_e = sum(1 for r in mla_ratios[:end_layer] if r == 4)
|
||||
c128_off_s = sum(1 for r in mla_ratios[:start_layer] if r == 128)
|
||||
|
||||
@@ -891,8 +891,7 @@ class DeepSeekV4UnifiedKVPool:
|
||||
|
||||
def get_buf_infos(self) -> Tuple[List[int], List[int], List[int]]:
|
||||
if self.fp8:
|
||||
# One pointer and row size per layer would describe only the nope pool.
|
||||
# TODO(danli103): report both pools once a consumer needs them.
|
||||
# one ptr/layer; PD uses get_contiguous_buf_infos / SWA_RING
|
||||
raise NotImplementedError(
|
||||
"get_buf_infos describes one pool per layer; the fp8 rope pool "
|
||||
"would be dropped (SGLANG_DSV4_UNIFIED_KV_FP8=1)."
|
||||
@@ -1229,41 +1228,44 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
data_lens: List[int] = []
|
||||
item_lens: List[int] = []
|
||||
|
||||
if self._unified_kv_fp8:
|
||||
# One pointer and one buf[0]-sized row per layer covers only the fp8 nope
|
||||
# pool; the remote side would decode rows against stale rope.
|
||||
# TODO(danli103): ship the rope pool as a second per-layer entry.
|
||||
raise NotImplementedError(
|
||||
"PD disaggregation is not supported with "
|
||||
"SGLANG_DSV4_UNIFIED_KV_FP8=1 (the transfer assumes a single "
|
||||
"unified pool; the rope pool would be silently dropped)."
|
||||
)
|
||||
|
||||
def append_page_buffer(buf: torch.Tensor) -> None:
|
||||
assert buf.ndim == 2, f"expected 2D buffer, got {buf.ndim}D"
|
||||
data_ptrs.append(buf.data_ptr())
|
||||
data_lens.append(buf.nbytes)
|
||||
item_lens.append(buf[0].nbytes)
|
||||
|
||||
def append_unified_compress(buf: torch.Tensor, ratio: int) -> None:
|
||||
# Compressed pages sit after the SWA ring; PD indices are page ids
|
||||
# into this region. SWA itself ships as StateType.SWA_RING.
|
||||
assert buf is not None, "unified kv buffer not allocated"
|
||||
assert buf.ndim == 2, f"expected 2D buffer, got {buf.ndim}D"
|
||||
swa_pages = self.unified_kv_pool.swa_pages
|
||||
row_bytes = buf[0].nbytes
|
||||
rows_per_page = self.page_size // ratio
|
||||
compress_rows = buf.shape[0] - swa_pages
|
||||
data_ptrs.append(buf.data_ptr() + swa_pages * row_bytes)
|
||||
data_lens.append(compress_rows * row_bytes)
|
||||
item_lens.append(rows_per_page * row_bytes)
|
||||
|
||||
stage_ratios = self.compression_ratios[self._stage_start : self._stage_end]
|
||||
# Registration order defines the PD wire layout: C4 KV, C4 indexer, C128 KV.
|
||||
# Keep each indexer immediately after the KV buffers of the same ratio.
|
||||
# [C4 KV, C4 indexer, C128 KV], and under fp8 a rope group (128 B/row)
|
||||
# after each KV group. Rope keeps the KV page indices -- shift them and
|
||||
# decode reads a page whose rope half came from some other page.
|
||||
for ratio, kv_pool in self.kv_pools.items():
|
||||
if self._unified_kv:
|
||||
# Unified buffers store token rows after the SWA ring. Transfer
|
||||
# compressed pages from the offset; SWA ships as StateType.SWA_RING.
|
||||
swa_pages = self.unified_kv_pool.swa_pages
|
||||
for local_layer_id, layer_ratio in enumerate(stage_ratios):
|
||||
if layer_ratio != ratio:
|
||||
continue
|
||||
buf = self.unified_kv_pool.kv_buffer[local_layer_id]
|
||||
assert buf.ndim == 2, f"expected 2D buffer, got {buf.ndim}D"
|
||||
row_bytes = buf[0].nbytes
|
||||
rows_per_page = self.page_size // ratio
|
||||
compress_rows = buf.shape[0] - swa_pages
|
||||
data_ptrs.append(buf.data_ptr() + swa_pages * row_bytes)
|
||||
data_lens.append(compress_rows * row_bytes)
|
||||
item_lens.append(rows_per_page * row_bytes)
|
||||
append_unified_compress(
|
||||
self.unified_kv_pool.kv_buffer[local_layer_id], ratio
|
||||
)
|
||||
if self._unified_kv_fp8:
|
||||
for local_layer_id, layer_ratio in enumerate(stage_ratios):
|
||||
if layer_ratio != ratio:
|
||||
continue
|
||||
append_unified_compress(
|
||||
self.unified_kv_pool.kv_buffer_rope[local_layer_id], ratio
|
||||
)
|
||||
elif kv_pool is not None:
|
||||
for buf in kv_pool.kv_buffer:
|
||||
append_page_buffer(buf)
|
||||
@@ -1289,6 +1291,12 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
data_lens.append(buf.nbytes)
|
||||
item_lens.append(buf[0].nbytes * index_pages_per_full_page)
|
||||
|
||||
if self._unified_kv_fp8 and data_ptrs:
|
||||
logger.info(
|
||||
f"DSV4 fp8 two-pool registered {len(data_ptrs)} KV regions for "
|
||||
"PD; the peer must run with SGLANG_DSV4_UNIFIED_KV_FP8 too"
|
||||
)
|
||||
|
||||
return data_ptrs, data_lens, item_lens
|
||||
|
||||
def get_unified_swa_ring_buf_infos(self) -> Tuple[List[int], List[int], List[int]]:
|
||||
@@ -1300,20 +1308,21 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
item_lens: List[int] = []
|
||||
if not self._unified_kv:
|
||||
return data_ptrs, data_lens, item_lens
|
||||
|
||||
def append_ring(bufs) -> None:
|
||||
swa_pages = self.unified_kv_pool.swa_pages
|
||||
for buf in bufs:
|
||||
assert buf is not None, "unified kv buffer not allocated"
|
||||
assert buf.ndim == 2, f"expected 2D buffer, got {buf.ndim}D"
|
||||
row_bytes = buf[0].nbytes
|
||||
data_ptrs.append(buf.data_ptr())
|
||||
data_lens.append(swa_pages * row_bytes)
|
||||
item_lens.append(row_bytes)
|
||||
|
||||
append_ring(self.unified_kv_pool.kv_buffer)
|
||||
# all-nope then all-rope, same grouping as kv_data. don't interleave.
|
||||
if self._unified_kv_fp8:
|
||||
# Ring half of the fp8 gap in get_contiguous_buf_infos; fix both together.
|
||||
raise NotImplementedError(
|
||||
"PD disaggregation is not supported with "
|
||||
"SGLANG_DSV4_UNIFIED_KV_FP8=1 (the SWA_RING component assumes a "
|
||||
"single unified pool; the rope pool would be silently dropped)."
|
||||
)
|
||||
swa_pages = self.unified_kv_pool.swa_pages
|
||||
for buf in self.unified_kv_pool.kv_buffer:
|
||||
assert buf.ndim == 2, f"expected 2D buffer, got {buf.ndim}D"
|
||||
row_bytes = buf[0].nbytes
|
||||
data_ptrs.append(buf.data_ptr())
|
||||
data_lens.append(swa_pages * row_bytes)
|
||||
item_lens.append(row_bytes)
|
||||
append_ring(self.unified_kv_pool.kv_buffer_rope)
|
||||
return data_ptrs, data_lens, item_lens
|
||||
|
||||
def _unified_page_views(
|
||||
|
||||
@@ -140,6 +140,30 @@ def _get_dsv4_compress_state_dtype_sizes() -> tuple[int, int]:
|
||||
)
|
||||
|
||||
|
||||
def check_dsv4_unified_fp8_pd_supported(
|
||||
*, unified_fp8: bool, disaggregation_mode: str, pp_size: int, enable_hisparse: bool
|
||||
) -> None:
|
||||
"""PP and HiSparse still index kv_data as one region per layer; fp8 PD adds rope groups."""
|
||||
if not unified_fp8 or disaggregation_mode == "null":
|
||||
return
|
||||
if pp_size > 1:
|
||||
raise ValueError(
|
||||
"SGLANG_DSV4_UNIFIED_KV_FP8=1 does not support PD disaggregation "
|
||||
f"with pp_size={pp_size}: the PP re-slicing of the per-stage KV "
|
||||
"regions has no coverage for the extra rope regions. Run PD with "
|
||||
"pp_size=1 or unset the fp8 switch."
|
||||
)
|
||||
if enable_hisparse:
|
||||
# HiSparse appends its device tail to kv_data and locates it as
|
||||
# dst_kv_ptrs[c4_layer_num:] (mooncake/conn.py, decode.py), i.e. the
|
||||
# slice that fp8 fills with the C4 rope regions.
|
||||
raise ValueError(
|
||||
"SGLANG_DSV4_UNIFIED_KV_FP8=1 does not support PD disaggregation "
|
||||
"with --enable-hisparse: the host/device split locates its device "
|
||||
"regions by layer count, which the fp8 rope regions shift."
|
||||
)
|
||||
|
||||
|
||||
class MemoryPoolConfigurator:
|
||||
"""Base class for memory pool configurators.
|
||||
|
||||
@@ -1084,15 +1108,12 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
|
||||
"env switch to get a bf16 unified pool."
|
||||
)
|
||||
|
||||
# get_contiguous_buf_infos prices a row as buf[0].nbytes, which under fp8
|
||||
# covers the nope pool only; fail at startup rather than at the first transfer.
|
||||
# TODO(danli103): drop this once the transfer ships the rope pool.
|
||||
if self._unified_fp8 and self.disaggregation_mode != "null":
|
||||
raise ValueError(
|
||||
"SGLANG_DSV4_UNIFIED_KV_FP8=1 does not support PD disaggregation "
|
||||
f"(disaggregation_mode={self.disaggregation_mode!r}). Unset the fp8 "
|
||||
"switch or run without disaggregation."
|
||||
)
|
||||
check_dsv4_unified_fp8_pd_supported(
|
||||
unified_fp8=self._unified_fp8,
|
||||
disaggregation_mode=self.disaggregation_mode,
|
||||
pp_size=kvc.pp_size,
|
||||
enable_hisparse=get_memory().enable_hisparse,
|
||||
)
|
||||
|
||||
if self.is_speculative:
|
||||
# Ring is sized once here, so it must serve the largest adaptive tier.
|
||||
|
||||
Reference in New Issue
Block a user