From 9c3262d855072e5e273a681d22b24f7e61339da5 Mon Sep 17 00:00:00 2001 From: amd-danli103 Date: Thu, 1 Oct 2026 14:42:31 +0800 Subject: [PATCH] [AMD][DSV4] feat: enable PD-disagg with fp8 unified_kv on gfx950 (#39166) --- .../sglang/srt/disaggregation/common/conn.py | 20 + .../srt/mem_cache/deepseek_v4_memory_pool.py | 85 +++-- .../srt/model_executor/pool_configurator.py | 39 +- .../dsv4/test_dsv4_pd_disagg_fp8_nixl.py | 359 ++++++++++++++++++ .../test_disaggregation_wire.py | 27 ++ .../test_dsv4_unified_fp8_pd.py | 287 ++++++++++++++ .../mem_cache/test_dsv4_compressed_pools.py | 38 +- .../model_executor/test_pool_configurator.py | 36 ++ 8 files changed, 839 insertions(+), 52 deletions(-) create mode 100644 test/manual/dsv4/test_dsv4_pd_disagg_fp8_nixl.py create mode 100644 test/registered/unit/disaggregation/test_dsv4_unified_fp8_pd.py diff --git a/python/sglang/srt/disaggregation/common/conn.py b/python/sglang/srt/disaggregation/common/conn.py index 7ff8b3997a..6816e7bf9d 100644 --- a/python/sglang/srt/disaggregation/common/conn.py +++ b/python/sglang/srt/disaggregation/common/conn.py @@ -1349,6 +1349,26 @@ class CommonKVManager(BaseKVManager): c128_full = sum(1 for r in mla_ratios if r == 128) kv_layout_len = 2 * c4_full + c128_full + # A DSV4 two-pool peer registers a rope group beside each KV group and + # doubles the ring, and the switch that turns that on is read per + # process. Mixed peers would fall into the cuts below, match whichever + # side is shorter, and then index past the other one once per request. + if state_type in (StateType.SWA, StateType.DSV4_REQUEST_STATE): + single_pool_len = two_pool_len = None + elif state_type == StateType.SWA_RING: + single_pool_len, two_pool_len = len(mla_ratios), 2 * len(mla_ratios) + else: + single_pool_len, two_pool_len = kv_layout_len, 3 * c4_full + 2 * c128_full + peer_lens = {len(src_kv_ptrs), len(dst_kv_ptrs)} + if single_pool_len is not None and peer_lens == {single_pool_len, two_pool_len}: + raise ValueError( + "PD peers disagree on the compressed-MLA KV layout: prefill " + f"registered {len(src_kv_ptrs)} regions, decode " + f"{len(dst_kv_ptrs)} ({single_pool_len} is one pool per layer, " + f"{two_pool_len} is the fp8 two-pool). " + "SGLANG_DSV4_UNIFIED_KV_FP8 must be set the same on both sides." + ) + c4_off_s = sum(1 for r in mla_ratios[:start_layer] if r == 4) c4_off_e = sum(1 for r in mla_ratios[:end_layer] if r == 4) c128_off_s = sum(1 for r in mla_ratios[:start_layer] if r == 128) diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index c271f777e3..f364e89d06 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -891,8 +891,7 @@ class DeepSeekV4UnifiedKVPool: def get_buf_infos(self) -> Tuple[List[int], List[int], List[int]]: if self.fp8: - # One pointer and row size per layer would describe only the nope pool. - # TODO(danli103): report both pools once a consumer needs them. + # one ptr/layer; PD uses get_contiguous_buf_infos / SWA_RING raise NotImplementedError( "get_buf_infos describes one pool per layer; the fp8 rope pool " "would be dropped (SGLANG_DSV4_UNIFIED_KV_FP8=1)." @@ -1229,41 +1228,44 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): data_lens: List[int] = [] item_lens: List[int] = [] - if self._unified_kv_fp8: - # One pointer and one buf[0]-sized row per layer covers only the fp8 nope - # pool; the remote side would decode rows against stale rope. - # TODO(danli103): ship the rope pool as a second per-layer entry. - raise NotImplementedError( - "PD disaggregation is not supported with " - "SGLANG_DSV4_UNIFIED_KV_FP8=1 (the transfer assumes a single " - "unified pool; the rope pool would be silently dropped)." - ) - def append_page_buffer(buf: torch.Tensor) -> None: assert buf.ndim == 2, f"expected 2D buffer, got {buf.ndim}D" data_ptrs.append(buf.data_ptr()) data_lens.append(buf.nbytes) item_lens.append(buf[0].nbytes) + def append_unified_compress(buf: torch.Tensor, ratio: int) -> None: + # Compressed pages sit after the SWA ring; PD indices are page ids + # into this region. SWA itself ships as StateType.SWA_RING. + assert buf is not None, "unified kv buffer not allocated" + assert buf.ndim == 2, f"expected 2D buffer, got {buf.ndim}D" + swa_pages = self.unified_kv_pool.swa_pages + row_bytes = buf[0].nbytes + rows_per_page = self.page_size // ratio + compress_rows = buf.shape[0] - swa_pages + data_ptrs.append(buf.data_ptr() + swa_pages * row_bytes) + data_lens.append(compress_rows * row_bytes) + item_lens.append(rows_per_page * row_bytes) + stage_ratios = self.compression_ratios[self._stage_start : self._stage_end] - # Registration order defines the PD wire layout: C4 KV, C4 indexer, C128 KV. - # Keep each indexer immediately after the KV buffers of the same ratio. + # [C4 KV, C4 indexer, C128 KV], and under fp8 a rope group (128 B/row) + # after each KV group. Rope keeps the KV page indices -- shift them and + # decode reads a page whose rope half came from some other page. for ratio, kv_pool in self.kv_pools.items(): if self._unified_kv: - # Unified buffers store token rows after the SWA ring. Transfer - # compressed pages from the offset; SWA ships as StateType.SWA_RING. - swa_pages = self.unified_kv_pool.swa_pages for local_layer_id, layer_ratio in enumerate(stage_ratios): if layer_ratio != ratio: continue - buf = self.unified_kv_pool.kv_buffer[local_layer_id] - assert buf.ndim == 2, f"expected 2D buffer, got {buf.ndim}D" - row_bytes = buf[0].nbytes - rows_per_page = self.page_size // ratio - compress_rows = buf.shape[0] - swa_pages - data_ptrs.append(buf.data_ptr() + swa_pages * row_bytes) - data_lens.append(compress_rows * row_bytes) - item_lens.append(rows_per_page * row_bytes) + append_unified_compress( + self.unified_kv_pool.kv_buffer[local_layer_id], ratio + ) + if self._unified_kv_fp8: + for local_layer_id, layer_ratio in enumerate(stage_ratios): + if layer_ratio != ratio: + continue + append_unified_compress( + self.unified_kv_pool.kv_buffer_rope[local_layer_id], ratio + ) elif kv_pool is not None: for buf in kv_pool.kv_buffer: append_page_buffer(buf) @@ -1289,6 +1291,12 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): data_lens.append(buf.nbytes) item_lens.append(buf[0].nbytes * index_pages_per_full_page) + if self._unified_kv_fp8 and data_ptrs: + logger.info( + f"DSV4 fp8 two-pool registered {len(data_ptrs)} KV regions for " + "PD; the peer must run with SGLANG_DSV4_UNIFIED_KV_FP8 too" + ) + return data_ptrs, data_lens, item_lens def get_unified_swa_ring_buf_infos(self) -> Tuple[List[int], List[int], List[int]]: @@ -1300,20 +1308,21 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): item_lens: List[int] = [] if not self._unified_kv: return data_ptrs, data_lens, item_lens + + def append_ring(bufs) -> None: + swa_pages = self.unified_kv_pool.swa_pages + for buf in bufs: + assert buf is not None, "unified kv buffer not allocated" + assert buf.ndim == 2, f"expected 2D buffer, got {buf.ndim}D" + row_bytes = buf[0].nbytes + data_ptrs.append(buf.data_ptr()) + data_lens.append(swa_pages * row_bytes) + item_lens.append(row_bytes) + + append_ring(self.unified_kv_pool.kv_buffer) + # all-nope then all-rope, same grouping as kv_data. don't interleave. if self._unified_kv_fp8: - # Ring half of the fp8 gap in get_contiguous_buf_infos; fix both together. - raise NotImplementedError( - "PD disaggregation is not supported with " - "SGLANG_DSV4_UNIFIED_KV_FP8=1 (the SWA_RING component assumes a " - "single unified pool; the rope pool would be silently dropped)." - ) - swa_pages = self.unified_kv_pool.swa_pages - for buf in self.unified_kv_pool.kv_buffer: - assert buf.ndim == 2, f"expected 2D buffer, got {buf.ndim}D" - row_bytes = buf[0].nbytes - data_ptrs.append(buf.data_ptr()) - data_lens.append(swa_pages * row_bytes) - item_lens.append(row_bytes) + append_ring(self.unified_kv_pool.kv_buffer_rope) return data_ptrs, data_lens, item_lens def _unified_page_views( diff --git a/python/sglang/srt/model_executor/pool_configurator.py b/python/sglang/srt/model_executor/pool_configurator.py index 74163e6830..b6a54aad5a 100644 --- a/python/sglang/srt/model_executor/pool_configurator.py +++ b/python/sglang/srt/model_executor/pool_configurator.py @@ -140,6 +140,30 @@ def _get_dsv4_compress_state_dtype_sizes() -> tuple[int, int]: ) +def check_dsv4_unified_fp8_pd_supported( + *, unified_fp8: bool, disaggregation_mode: str, pp_size: int, enable_hisparse: bool +) -> None: + """PP and HiSparse still index kv_data as one region per layer; fp8 PD adds rope groups.""" + if not unified_fp8 or disaggregation_mode == "null": + return + if pp_size > 1: + raise ValueError( + "SGLANG_DSV4_UNIFIED_KV_FP8=1 does not support PD disaggregation " + f"with pp_size={pp_size}: the PP re-slicing of the per-stage KV " + "regions has no coverage for the extra rope regions. Run PD with " + "pp_size=1 or unset the fp8 switch." + ) + if enable_hisparse: + # HiSparse appends its device tail to kv_data and locates it as + # dst_kv_ptrs[c4_layer_num:] (mooncake/conn.py, decode.py), i.e. the + # slice that fp8 fills with the C4 rope regions. + raise ValueError( + "SGLANG_DSV4_UNIFIED_KV_FP8=1 does not support PD disaggregation " + "with --enable-hisparse: the host/device split locates its device " + "regions by layer count, which the fp8 rope regions shift." + ) + + class MemoryPoolConfigurator: """Base class for memory pool configurators. @@ -1084,15 +1108,12 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator): "env switch to get a bf16 unified pool." ) - # get_contiguous_buf_infos prices a row as buf[0].nbytes, which under fp8 - # covers the nope pool only; fail at startup rather than at the first transfer. - # TODO(danli103): drop this once the transfer ships the rope pool. - if self._unified_fp8 and self.disaggregation_mode != "null": - raise ValueError( - "SGLANG_DSV4_UNIFIED_KV_FP8=1 does not support PD disaggregation " - f"(disaggregation_mode={self.disaggregation_mode!r}). Unset the fp8 " - "switch or run without disaggregation." - ) + check_dsv4_unified_fp8_pd_supported( + unified_fp8=self._unified_fp8, + disaggregation_mode=self.disaggregation_mode, + pp_size=kvc.pp_size, + enable_hisparse=get_memory().enable_hisparse, + ) if self.is_speculative: # Ring is sized once here, so it must serve the largest adaptive tier. diff --git a/test/manual/dsv4/test_dsv4_pd_disagg_fp8_nixl.py b/test/manual/dsv4/test_dsv4_pd_disagg_fp8_nixl.py new file mode 100644 index 0000000000..f7c4985729 --- /dev/null +++ b/test/manual/dsv4/test_dsv4_pd_disagg_fp8_nixl.py @@ -0,0 +1,359 @@ +"""DSV4 Flash PD-disagg: fp8 two-pool vs bf16, same CLI besides the env. + +Default transfer probe order is mooncake_tcp, mooncake, mori, nixl. +Force with SGLANG_TEST_PD_TRANSFER_BACKEND. MoE a2a stays none unless +SGLANG_TEST_PD_MOE_A2A=mori. GSM8K n/bar: SGLANG_TEST_PD_GSM8K_N (200), +SGLANG_TEST_PD_GSM8K_BAR (0.95). +""" + +import os +import unittest +from types import SimpleNamespace + +import requests + +from sglang.test.few_shot_gsm8k import run_eval as run_gsm8k_eval +from sglang.test.server_fixtures.disaggregation_fixture import ( + PDDisaggregationServerBase, +) +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + popen_launch_pd_server, +) + +DSV4_FLASH_MODEL_PATH = os.environ.get( + "SGLANG_TEST_DSV4_FLASH_MODEL", + "sgl-project/DeepSeek-V4-Flash-FP8", +) + +DSV4_FLASH_ENV = { + # local DeepSeek-V4-Flash snapshot has expert_dtype=fp4 in config.json; + # handbook Flash-FP8 sets this to 0. 0 allocates an fp8 w13 (4096) and + # then copy_ from packed fp4 (2048) blows up in fused_moe_triton. + "SGLANG_DSV4_FP4_EXPERTS": os.environ.get("SGLANG_DSV4_FP4_EXPERTS", "1"), + "SGLANG_HACK_FLASHMLA_BACKEND": "unified_kv_triton", + "SGLANG_OPT_USE_AITER_BATCHED_GEMM": "1", + "SGLANG_USE_AITER": "1", + # unset: aiter CK 2stage on small-M warmup has no kernel + "AITER_BF16_FP8_MOE_BOUND": "0", + "SGLANG_USE_ROCM700A": "0", +} + +_PD_ARGS = [ + "--trust-remote-code", + "--tp", + "4", + "--dp", + "4", + "--enable-dp-attention", + "--disable-shared-experts-fusion", + "--cuda-graph-max-bs-decode", + "128", + "--max-running-requests", + "256", + "--mem-fraction-static", + "0.7", + "--attention-backend", + "dsv4", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--watchdog-timeout", + "900", +] + + +def _probe_transfer(name: str) -> None: + if name in ("mooncake", "mooncake_tcp"): + import mooncake # noqa: F401 + elif name == "mori": + import mori.io # noqa: F401 + elif name == "nixl": + from nixl._api import nixl_agent # noqa: F401 + else: + raise ImportError(f"unknown PD transfer backend {name}") + + +def _transfer_backend_args(): + # mooncake_tcp first: single-node P+D over RDMA has failed ibv_reg_mr on the + # NICs we have. Cross-node still wants mooncake; nixl hits the same thing. + forced = os.environ.get("SGLANG_TEST_PD_TRANSFER_BACKEND") + order = [] + if forced: + order.append(forced) + for name in ("mooncake_tcp", "mooncake", "mori", "nixl"): + if name not in order: + order.append(name) + errors = [] + for name in order: + try: + _probe_transfer(name) + print(f"PD transfer backend={name}") + return ["--disaggregation-transfer-backend", name] + except Exception as e: + errors.append(f"{name}: {type(e).__name__}: {e}") + raise RuntimeError("no PD transfer backend importable: " + "; ".join(errors)) + + +def _moe_a2a_args(): + # default none = fused TP-MoE. mori EP is an overlay, not DeepEP. + forced = os.environ.get("SGLANG_TEST_PD_MOE_A2A", "none") + if forced in ("", "none"): + print("PD moe-a2a-backend=none") + return [] + if forced != "mori": + print(f"PD moe-a2a-backend={forced} (passthrough)") + return ["--moe-a2a-backend", forced] + try: + import mori # noqa: F401 + except Exception as e: + raise RuntimeError(f"SGLANG_TEST_PD_MOE_A2A=mori but import failed: {e}") from e + print("PD moe-a2a-backend=mori ep-size=4") + return [ + "--moe-a2a-backend", + "mori", + "--ep-size", + "4", + "--deepep-mode", + "normal", + ] + + +class _DSV4FlashPDBase(PDDisaggregationServerBase): + # pytest collects unittest.TestCase subclasses even when the name is _Foo. + __test__ = False + moe_a2a_args = [] + transfer_backend = [] + capture_per_side_logs = False + + @classmethod + def _arm_env(cls): + env = dict(DSV4_FLASH_ENV) + env["SGLANG_DSV4_UNIFIED_KV_FP8"] = "1" if cls.fp8 else "0" + backend = "" + if getattr(cls, "transfer_backend", None): + backend = cls.transfer_backend[-1] + if backend == "nixl": + env.setdefault("UCX_TLS", "tcp,sm,self,rocm") + if backend == "mooncake": + env.setdefault("MC_GID_INDEX", "1") + if "--moe-a2a-backend" in getattr(cls, "moe_a2a_args", []): + env.setdefault("SGLANG_MORI_RECV_BOUND", "1") + # FP4 experts on this snapshot; cookbook: both env vars for FP4. + env.setdefault("SGLANG_MORI_DISPATCH_DTYPE", "mxfp8") + return env + + @classmethod + def setUpClass(cls): + try: + import torch + + n = torch.cuda.device_count() + except Exception as e: + raise unittest.SkipTest(f"cuda unavailable: {e}") from e + if n < 8: + raise unittest.SkipTest(f"Flash PD E2E needs 8 GPUs, got {n}") + super().setUpClass() + cls.transfer_backend = _transfer_backend_args() + cls.moe_a2a_args = _moe_a2a_args() + rdma_env = os.environ.get("SGLANG_TEST_RDMA_DEVICE") + cls.rdma_devices = ["--disaggregation-ib-device", rdma_env] if rdma_env else [] + cls.model = DSV4_FLASH_MODEL_PATH + if cls.capture_per_side_logs: + cls._open_side_log_files() + # P+D together has SIGKILL'd around 7 min while /health was still 503. + ready_timeout = int(os.environ.get("SGLANG_TEST_PD_READY_TIMEOUT", "3600")) + cls.start_prefill() + try: + cls.wait_server_ready( + cls.prefill_url + "/health", + timeout=ready_timeout, + process=cls.process_prefill, + ) + cls.start_decode() + cls.wait_server_ready( + cls.decode_url + "/health", + timeout=ready_timeout, + process=cls.process_decode, + ) + except Exception: + cls._dump_side_logs() + raise + cls.launch_lb() + + @classmethod + def start_prefill(cls): + prefill_args = [ + "--disaggregation-mode", + "prefill", + "--disaggregation-bootstrap-port", + cls.bootstrap_port, + "--nccl-port", + cls.prefill_nccl_port, + "--base-gpu-id", + "0", + *_PD_ARGS, + *cls.moe_a2a_args, + *cls.transfer_backend, + *cls.rdma_devices, + ] + cls.process_prefill = popen_launch_pd_server( + cls.model, + cls.prefill_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=prefill_args, + env=cls._arm_env(), + return_stdout_stderr=( + (cls._prefill_stdout_buf, cls._prefill_stderr_buf) + if cls.capture_per_side_logs + else None + ), + ) + + @classmethod + def start_decode(cls): + decode_args = [ + "--disaggregation-mode", + "decode", + "--disaggregation-bootstrap-port", + cls.bootstrap_port, + "--nccl-port", + cls.decode_nccl_port, + "--base-gpu-id", + "4", + *_PD_ARGS, + *cls.moe_a2a_args, + *cls.transfer_backend, + *cls.rdma_devices, + ] + cls.process_decode = popen_launch_pd_server( + cls.model, + cls.decode_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=decode_args, + env=cls._arm_env(), + return_stdout_stderr=( + (cls._decode_stdout_buf, cls._decode_stderr_buf) + if cls.capture_per_side_logs + else None + ), + ) + + @classmethod + def _open_side_log_files(cls): + # StringIO dies with SIGKILL; a real file keeps the last lines. + logdir = os.environ.get("SGLANG_TEST_PD_LOGDIR") + cls._log_files = [] + if not logdir: + return + os.makedirs(logdir, exist_ok=True) + arm = "fp8" if getattr(cls, "fp8", False) else "bf16" + for side in ("prefill", "decode"): + for stream in ("stdout", "stderr"): + path = os.path.join(logdir, f"pd_e2e_{arm}_{side}.{stream}.log") + fh = open(path, "w", buffering=1) + setattr(cls, f"_{side}_{stream}_buf", fh) + cls._log_files.append(fh) + + @classmethod + def _dump_side_logs(cls): + logdir = os.environ.get("SGLANG_TEST_PD_LOGDIR") + if not logdir: + return + arm = "fp8" if getattr(cls, "fp8", False) else "bf16" + for fh in getattr(cls, "_log_files", []): + try: + fh.flush() + except Exception: + pass + for side in ("prefill", "decode"): + for stream in ("stdout", "stderr"): + path = os.path.join(logdir, f"pd_e2e_{arm}_{side}.{stream}.log") + try: + with open(path) as f: + text = f.read() + except OSError: + text = "" + tail = text[-4000:] + if tail: + print(f"===== {side} {stream} (tail) =====\n{tail}") + + @classmethod + def tearDownClass(cls): + try: + super().tearDownClass() + finally: + for fh in getattr(cls, "_log_files", []): + try: + fh.close() + except Exception: + pass + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=int(os.environ.get("SGLANG_TEST_PD_GSM8K_N", "200")), + max_new_tokens=512, + parallel=64, + host=f"http://{self.base_host}", + port=int(self.lb_port), + ) + metrics = run_gsm8k_eval(args) + print(f"{metrics=}") + bar = float(os.environ.get("SGLANG_TEST_PD_GSM8K_BAR", "0.95")) + self.assertGreater(metrics["accuracy"], bar) + + def _recall_marker(self, marker, filler_tokens, marker_first): + filler = " ".join(f"tok{i}" for i in range(filler_tokens)) + stated = f"The secret marker is {marker}." + body = f"{stated}\n{filler}" if marker_first else f"{filler}\n{stated}" + r = requests.post( + f"{self.lb_url}/v1/completions", + json={ + "model": self.model, + "prompt": f"{body}\nRepeat the secret marker and nothing else:", + "max_tokens": 16, + "temperature": 0, + }, + timeout=180, + ) + r.raise_for_status() + return r.json()["choices"][0]["text"] + + def test_rope_probe_recalls_marker_in_swa_window(self): + # Recent marker: decode answers out of the SWA ring, which ships as + # StateType.SWA_RING. GSM8K can pass with stale rope, this can't. + text = self._recall_marker("74931", 64, marker_first=False) + print(f"rope_probe swa text={text!r}") + self.assertIn("74931", text) + + def test_rope_probe_recalls_marker_beyond_swa_window(self): + # Same probe with the marker pushed out of the window, so the answer has + # to come from the compressed pages -- the groups fp8 splits in two. Bump + # SGLANG_TEST_PD_ROPE_PROBE_TOKENS if the model's window grows; if the + # window probe above passes and this one fails, suspect rope, and only + # then the model losing the needle. + filler = int(os.environ.get("SGLANG_TEST_PD_ROPE_PROBE_TOKENS", "2048")) + text = self._recall_marker("58207", filler, marker_first=True) + print(f"rope_probe compressed text={text!r}") + self.assertIn("58207", text) + + +class TestDSV4FlashPDDisaggFp8(_DSV4FlashPDBase): + __test__ = True + fp8 = True + + +class TestDSV4FlashPDDisaggBf16(_DSV4FlashPDBase): + __test__ = True + fp8 = False + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/disaggregation/test_disaggregation_wire.py b/test/registered/unit/disaggregation/test_disaggregation_wire.py index fb81e7ee6f..ac09fac0c1 100644 --- a/test/registered/unit/disaggregation/test_disaggregation_wire.py +++ b/test/registered/unit/disaggregation/test_disaggregation_wire.py @@ -1449,6 +1449,33 @@ class TestDSV4DraftStateRegistration(unittest.TestCase): self.assertEqual(kv_args.state_data_lens[-1], expected_infos[1]) self.assertEqual(kv_args.state_item_lens[-1], expected_infos[2]) + def test_fp8_draft_ring_registers_rope(self): + target = _make_dsv4_target(unified=True) + draft = _make_dsv4_draft(unified=True) + draft._unified_kv_fp8 = True + draft.unified_kv_pool.kv_buffer_rope = [ + torch.empty((524, 64), dtype=torch.bfloat16) + ] + expected_infos = draft.get_unified_swa_ring_buf_infos() + self.assertEqual(len(expected_infos[0]), 2) + self.assertEqual(expected_infos[2], [16, 128]) + + kv_args = KVArgs() + setup_state_kv_args(kv_args, target, draft) + self.assertEqual(kv_args.state_types[-1], StateType.SWA_RING) + self.assertEqual(kv_args.state_data_ptrs[-1], expected_infos[0]) + self.assertEqual(kv_args.state_item_lens[-1], expected_infos[2]) + + def test_dspark_draft_ignores_stray_rope_buffer(self): + draft = _make_dsv4_draft(unified=True) + draft._unified_kv_fp8 = False + draft.unified_kv_pool.kv_buffer_rope = [ + torch.empty((524, 64), dtype=torch.bfloat16) + ] + ptrs, _, items = draft.get_unified_swa_ring_buf_infos() + self.assertEqual(len(ptrs), 1) + self.assertEqual(items, [16]) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/disaggregation/test_dsv4_unified_fp8_pd.py b/test/registered/unit/disaggregation/test_dsv4_unified_fp8_pd.py new file mode 100644 index 0000000000..f28911cb76 --- /dev/null +++ b/test/registered/unit/disaggregation/test_dsv4_unified_fp8_pd.py @@ -0,0 +1,287 @@ +import contextlib +import unittest +from types import SimpleNamespace + +import torch + +from sglang.srt.disaggregation.base.conn import KVArgs, StateType +from sglang.srt.disaggregation.common.conn import CommonKVManager +from sglang.srt.disaggregation.utils import setup_state_kv_args +from sglang.srt.mem_cache.deepseek_v4_memory_pool import ( + DSV4_FP8_NOPE_ROW_BYTES, + DeepSeekV4TokenToKVPool, + DeepSeekV4UnifiedKVPool, +) +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=3, suite="base-a-test-cpu") + +NOPE_DIM = 448 +ROPE_DIM = 64 +ROPE_ROW_BYTES = ROPE_DIM * 2 +PAGE_SIZE = 256 +SWA_RING = 8 +NUM_SLOTS = 2 +NUM_BLOCKS = 4 + + +class _StubMemorySaver: + def region(self, _tag): + return contextlib.nullcontext() + + +def _unified_pool(stage_ratios, *, fp8): + return DeepSeekV4UnifiedKVPool( + stage_ratios=stage_ratios, + num_slots=NUM_SLOTS, + num_blocks=NUM_BLOCKS, + page_size=PAGE_SIZE, + qk_nope_head_dim=NOPE_DIM, + qk_rope_head_dim=ROPE_DIM, + device="cpu", + memory_saver_adapter=_StubMemorySaver(), + custom_mem_pool=None, + swa_ring_size=SWA_RING, + fp8=fp8, + ) + + +def _token_pool(stage_ratios, *, fp8, indexer=None): + uni = _unified_pool(stage_ratios, fp8=fp8) + pool = DeepSeekV4TokenToKVPool.__new__(DeepSeekV4TokenToKVPool) + pool._unified_kv = True + pool._unified_kv_fp8 = fp8 + pool.page_size = PAGE_SIZE + pool.compression_ratios = list(stage_ratios) + pool._stage_start = 0 + pool._stage_end = len(stage_ratios) + pool.unified_kv_pool = uni + pool.unified_swa_window = 128 + pool.unified_swa_ring_size = SWA_RING + pool.unified_swa_pages = uni.swa_pages + # same insertion order as _init_compressed_pools + pool.kv_pools = {4: None, 128: None} + pool.index_pools = {} if indexer is None else {4: indexer} + pool.compress_state_pools = [None] * len(stage_ratios) + pool.indexer_compress_state_pools = [None] * len(stage_ratios) + pool.get_state_buf_infos = lambda: ([], [], []) + pool.get_request_state_buf_infos = lambda: ([], [], []) + return pool + + +def _paint_rows(buf, start, end, salt): + u8 = buf.view(torch.uint8).reshape(buf.shape[0], -1) + for i in range(start, end): + u8[i, 0] = (i + salt) % 256 + if u8.shape[1] > 1: + u8[i, 1] = salt % 256 + + +def _pp_mgr(start, end, ratios): + mgr = object.__new__(CommonKVManager) + mgr.kv_args = SimpleNamespace( + prefill_start_layer=start, + prefill_end_layer=end, + mla_compression_ratios=ratios, + ) + return mgr + + +class TestDSV4UnifiedFp8PdLayout(CustomTestCase): + STAGE = [4, 128] + + def test_bf16_contiguous_and_ring_stay_one_pool(self): + pool = _token_pool(self.STAGE, fp8=False) + ptrs, _, items = pool.get_contiguous_buf_infos() + self.assertEqual(len(ptrs), 2) + self.assertEqual(items, [PAGE_SIZE // 4 * 1024, PAGE_SIZE // 128 * 1024]) + ring_ptrs, _, ring_items = pool.get_unified_swa_ring_buf_infos() + self.assertEqual(len(ring_ptrs), 2) + self.assertEqual(ring_items, [1024, 1024]) + + def test_fp8_adds_rope_groups_with_128_vs_512_item_len(self): + indexer_buf = torch.empty((2, 8), dtype=torch.uint8) + indexer = SimpleNamespace(contiguous_page_row_buffers=lambda: [indexer_buf]) + pool = _token_pool(self.STAGE, fp8=True, indexer=indexer) + ptrs, _, items = pool.get_contiguous_buf_infos() + # C4_nope, C4_rope, C4_indexer, C128_nope, C128_rope + self.assertEqual(len(ptrs), 5) + c4_rows = PAGE_SIZE // 4 + c128_rows = PAGE_SIZE // 128 + self.assertEqual( + items, + [ + c4_rows * DSV4_FP8_NOPE_ROW_BYTES, + c4_rows * ROPE_ROW_BYTES, + indexer_buf[0].nbytes, + c128_rows * DSV4_FP8_NOPE_ROW_BYTES, + c128_rows * ROPE_ROW_BYTES, + ], + ) + self.assertEqual(items[1] / items[0], ROPE_ROW_BYTES / DSV4_FP8_NOPE_ROW_BYTES) + ring_ptrs, _, ring_items = pool.get_unified_swa_ring_buf_infos() + self.assertEqual(len(ring_ptrs), 4) + self.assertEqual( + ring_items, + [ + DSV4_FP8_NOPE_ROW_BYTES, + DSV4_FP8_NOPE_ROW_BYTES, + ROPE_ROW_BYTES, + ROPE_ROW_BYTES, + ], + ) + + def test_fake_copy_moves_nope_and_rope(self): + src = _token_pool(self.STAGE, fp8=True) + dst = _token_pool(self.STAGE, fp8=True) + swa = src.unified_kv_pool.swa_pages + page_ids = [0, 1] + ring_rows = [0, SWA_RING, SWA_RING + 3] + + for layer, salt in enumerate((11, 22)): + nope = src.unified_kv_pool.kv_buffer[layer] + rope = src.unified_kv_pool.kv_buffer_rope[layer] + _paint_rows(nope, swa, nope.shape[0], salt) + _paint_rows(rope, swa, rope.shape[0], salt + 50) + _paint_rows(nope, 0, swa, salt + 7) + _paint_rows(rope, 0, swa, salt + 57) + + src_kv = src.get_contiguous_buf_infos() + self.assertEqual(len(src_kv[0]), 4) + kv_groups = ( + (4, "nope"), + (4, "rope"), + (128, "nope"), + (128, "rope"), + ) + for i, (ratio, kind) in enumerate(kv_groups): + local = self.STAGE.index(ratio) + buf_s = ( + src.unified_kv_pool.kv_buffer[local] + if kind == "nope" + else src.unified_kv_pool.kv_buffer_rope[local] + ) + buf_d = ( + dst.unified_kv_pool.kv_buffer[local] + if kind == "nope" + else dst.unified_kv_pool.kv_buffer_rope[local] + ) + row_bytes = buf_s[0].nbytes + rows_per_page = PAGE_SIZE // ratio + self.assertEqual(src_kv[0][i], buf_s.data_ptr() + swa * row_bytes) + self.assertEqual(src_kv[2][i], rows_per_page * row_bytes) + s_u8 = buf_s.view(torch.uint8).reshape(buf_s.shape[0], -1) + d_u8 = buf_d.view(torch.uint8).reshape(buf_d.shape[0], -1) + for p in page_ids: + a = swa + p * rows_per_page + d_u8[a : a + rows_per_page].copy_(s_u8[a : a + rows_per_page]) + self.assertTrue( + torch.equal( + s_u8[a : a + rows_per_page], d_u8[a : a + rows_per_page] + ) + ) + + src_ring = src.get_unified_swa_ring_buf_infos() + self.assertEqual(len(src_ring[0]), 4) + ring_src = list(src.unified_kv_pool.kv_buffer) + list( + src.unified_kv_pool.kv_buffer_rope + ) + ring_dst = list(dst.unified_kv_pool.kv_buffer) + list( + dst.unified_kv_pool.kv_buffer_rope + ) + for i, (buf_s, buf_d) in enumerate(zip(ring_src, ring_dst)): + self.assertEqual(src_ring[0][i], buf_s.data_ptr()) + self.assertEqual(src_ring[2][i], buf_s[0].nbytes) + s_u8 = buf_s.view(torch.uint8).reshape(buf_s.shape[0], -1) + d_u8 = buf_d.view(torch.uint8).reshape(buf_d.shape[0], -1) + for r in ring_rows: + d_u8[r].copy_(s_u8[r]) + self.assertTrue(torch.equal(s_u8[r], d_u8[r])) + + def test_dspark_draft_ring_stays_single_pool(self): + draft = _token_pool([0], fp8=False) + draft.compression_ratios = [0] + ptrs, _, items = draft.get_unified_swa_ring_buf_infos() + self.assertEqual(len(ptrs), 1) + self.assertEqual(items, [1024]) + self.assertIsNone(draft.unified_kv_pool.kv_buffer_rope[0]) + + def test_fp8_draft_ring_includes_rope(self): + draft = _token_pool([0], fp8=True) + draft.compression_ratios = [0] + ptrs, _, items = draft.get_unified_swa_ring_buf_infos() + self.assertEqual(len(ptrs), 2) + self.assertEqual(items, [DSV4_FP8_NOPE_ROW_BYTES, ROPE_ROW_BYTES]) + kv_args = KVArgs() + target = _token_pool(self.STAGE, fp8=True) + setup_state_kv_args(kv_args, target, draft) + self.assertEqual(kv_args.state_types[-1], StateType.SWA_RING) + self.assertEqual(kv_args.state_item_lens[-1], items) + + def test_get_buf_infos_still_refuses(self): + uni = _unified_pool(self.STAGE, fp8=True) + with self.assertRaises(NotImplementedError): + uni.get_buf_infos() + # #37778 already ships the rope host pool; PD must not re-refuse it. + pool = _token_pool(self.STAGE, fp8=True) + pool.unified_region_buffers(4) + self.assertIsNotNone(pool.unified_rope_region_buffers(4)) + + +class TestDSV4UnifiedFp8PpSlice(CustomTestCase): + RATIOS = [4, 4, 128, 4] + # kv_data groups: 2*c4 + c128 bf16, 3*c4 + 2*c128 under the fp8 two-pool + BF16_KV_LEN = 7 + FP8_KV_LEN = 11 + + def _slice(self, dst, start, end, state_type=None, src=None): + mgr = _pp_mgr(start, end, self.RATIOS) + if src is None: + src = list(range(100, 100 + 8)) + return mgr._mla_slice_ptrs_for_pp(src, dst, self.RATIOS, state_type) + + def test_bf16_kv_layout_unchanged(self): + # [C4 x3 | indexer x3 | C128 x1] + dst = list(range(self.BF16_KV_LEN)) + _, sliced = self._slice(dst, 0, 2) + self.assertEqual(sliced, [0, 1, 3, 4]) + _, sliced = self._slice(dst, 2, 3) + self.assertEqual(sliced, [6]) + + def test_same_layout_peers_never_reach_the_slicer(self): + # PD with pp_size=1 always lands here, whichever pool layout is in use; + # the fp8 group counts are only ever produced on both sides at once. + mgr = _pp_mgr(0, len(self.RATIOS), self.RATIOS) + for n in (self.BF16_KV_LEN, self.FP8_KV_LEN): + ptrs = list(range(n)) + src, dst, count = mgr.get_mla_kv_ptrs_with_pp(ptrs, list(ptrs)) + self.assertEqual((src, dst, count), (ptrs, ptrs, n)) + + def test_mixed_fp8_and_bf16_peers_are_rejected(self): + cases = ( + (None, self.BF16_KV_LEN, self.FP8_KV_LEN), + (None, self.FP8_KV_LEN, self.BF16_KV_LEN), + (StateType.SWA_RING, len(self.RATIOS), 2 * len(self.RATIOS)), + (StateType.SWA_RING, 2 * len(self.RATIOS), len(self.RATIOS)), + ) + for state_type, src_len, dst_len in cases: + with self.subTest(state_type=state_type, src_len=src_len): + with self.assertRaisesRegex(ValueError, "SGLANG_DSV4_UNIFIED_KV_FP8"): + self._slice( + list(range(dst_len)), + 1, + 3, + state_type, + src=list(range(100, 100 + src_len)), + ) + + def test_swa_ring_bf16_slice_unchanged(self): + # stage [1, 3) registers 2 of the 4 bf16 rings + dst = list(range(4)) + _, sliced = self._slice(dst, 1, 3, StateType.SWA_RING, src=[100, 101]) + self.assertEqual(sliced, [1, 2]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/mem_cache/test_dsv4_compressed_pools.py b/test/registered/unit/mem_cache/test_dsv4_compressed_pools.py index 4447164ac6..91e47e5083 100644 --- a/test/registered/unit/mem_cache/test_dsv4_compressed_pools.py +++ b/test/registered/unit/mem_cache/test_dsv4_compressed_pools.py @@ -59,12 +59,15 @@ class TestDSV4CompressedPools(CustomTestCase): self.assertEqual(pool.get_state_buf_infos(), ([], [], [])) def test_pp_mapping_and_pd_buffer_order(self): - for unified, stage_ratios in product( - (False, True), ([4, 0, 128, 4], [128], [0]) + for unified, fp8, stage_ratios in product( + (False, True), (False, True), ([4, 0, 128, 4], [128], [0]) ): - with self.subTest(unified=unified, stage_ratios=stage_ratios): + if fp8 and not unified: + continue + with self.subTest(unified=unified, fp8=fp8, stage_ratios=stage_ratios): pool = DeepSeekV4TokenToKVPool.__new__(DeepSeekV4TokenToKVPool) pool._unified_kv = unified + pool._unified_kv_fp8 = fp8 pool.uniform_fp8 = False pool.kv_layout = KVLayout.V4 pool.compressed_kv_layout_option = None @@ -127,8 +130,13 @@ class TestDSV4CompressedPools(CustomTestCase): buffers = [ torch.empty((9, 8), dtype=torch.uint8) for _ in stage_ratios ] + rope_buffers = [ + torch.empty((9, 16), dtype=torch.uint8) for _ in stage_ratios + ] pool.unified_kv_pool = SimpleNamespace( - swa_pages=2, kv_buffer=buffers + swa_pages=2, + kv_buffer=buffers, + kv_buffer_rope=rope_buffers, ) def kv_entries(ratio): @@ -137,6 +145,14 @@ class TestDSV4CompressedPools(CustomTestCase): for buf, r in zip(buffers, stage_ratios) if r == ratio ] + + def rope_entries(ratio): + return [ + (buf.data_ptr() + 32, 112, 256 // ratio * 16) + for buf, r in zip(rope_buffers, stage_ratios) + if r == ratio + ] + else: def kv_entries(ratio): @@ -145,10 +161,22 @@ class TestDSV4CompressedPools(CustomTestCase): for b in pool.kv_pools[ratio].kv_buffer ] + def rope_entries(ratio): + return [] + indexer_entries = [ (b.data_ptr(), b.nbytes, b[0].nbytes) for b in indexer_buffers ] - expected = kv_entries(4) + indexer_entries + kv_entries(128) + if fp8: + expected = ( + kv_entries(4) + + rope_entries(4) + + indexer_entries + + kv_entries(128) + + rope_entries(128) + ) + else: + expected = kv_entries(4) + indexer_entries + kv_entries(128) actual = list(zip(*pool.get_contiguous_buf_infos())) self.assertEqual(actual, expected) diff --git a/test/registered/unit/model_executor/test_pool_configurator.py b/test/registered/unit/model_executor/test_pool_configurator.py index ff62f8b7dc..3fa29e5ce0 100644 --- a/test/registered/unit/model_executor/test_pool_configurator.py +++ b/test/registered/unit/model_executor/test_pool_configurator.py @@ -1393,6 +1393,42 @@ class TestSWAPoolFloor(CustomTestCase): target = slots * cfg._swa_ring_size * 640 * cfg.num_layers_total self.assertEqual(cfg._fixed_swa_bytes(mrr), int(target * cfg._spec_infl)) + def test_dsv4_fp8_pd_refuses_pp_and_hisparse(self): + from sglang.srt.model_executor.pool_configurator import ( + check_dsv4_unified_fp8_pd_supported, + ) + + base = dict( + unified_fp8=True, + disaggregation_mode="prefill", + pp_size=1, + enable_hisparse=False, + ) + check_dsv4_unified_fp8_pd_supported(**base) + # bf16 PD keeps both + check_dsv4_unified_fp8_pd_supported( + **{**base, "unified_fp8": False, "pp_size": 2, "enable_hisparse": True} + ) + # fp8 without PD keeps both + check_dsv4_unified_fp8_pd_supported( + **{ + **base, + "disaggregation_mode": "null", + "pp_size": 2, + "enable_hisparse": True, + } + ) + for mode in ("prefill", "decode"): + for key, value, message in ( + ("pp_size", 2, "pp_size=2"), + ("enable_hisparse", True, "enable-hisparse"), + ): + with self.subTest(disaggregation_mode=mode, refused=key): + with self.assertRaisesRegex(ValueError, message): + check_dsv4_unified_fp8_pd_supported( + **{**base, "disaggregation_mode": mode, key: value} + ) + if __name__ == "__main__": unittest.main()