diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index b54e876aeb..0f0edd888b 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -1359,10 +1359,12 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): each page to ensure correctness for any page_size and head-slicing configuration. This may introduce performance overhead (increased TTFT) for long sequences. """ + from sglang.srt.disaggregation.common.staging_buffer import ( + compute_head_slice_params, + ) + # Extract configuration - local_tp_rank_in_group = self.kv_args.engine_rank % self.attn_tp_size src_kv_item_len = self.kv_args.kv_item_lens[0] - dst_tp_rank_in_group = dst_tp_rank % dst_attn_tp_size page_size = self.kv_args.page_size # Use total KV head count (not per-rank) for correct head distribution. @@ -1371,36 +1373,23 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): if total_kv_heads <= 0: total_kv_heads = self.kv_args.kv_head_num * self.attn_tp_size - src_heads_per_rank = max(1, total_kv_heads // self.attn_tp_size) dst_heads_per_rank = max(1, total_kv_heads // dst_attn_tp_size) bytes_per_head_slice_to_send = ( dst_kv_item_len // page_size // dst_heads_per_rank ) - # GQA replication: how many prefill ranks share the same KV head - src_replication = max(1, self.attn_tp_size // total_kv_heads) - - # Determine slicing parameters based on TP configuration - if self.attn_tp_size > dst_attn_tp_size: - # Send KVCache from multiple prefill instances to 1 decode instance - src_head_start_offset = 0 - num_heads_to_send = src_heads_per_rank - unique_head_idx = local_tp_rank_in_group // src_replication - dst_head_start_offset = ( - unique_head_idx * src_heads_per_rank - ) % dst_heads_per_rank - else: - # Send KVCache from 1 prefill instance to multiple decode instances - # GQA replication (total_kv_heads < dst_attn_tp_size): consecutive decode - # ranks share one KV head (QKVParallelLinear: tp_rank // num_kv_head_replicas), - # so map by integer division NOT modulo or ranks 1..r-1 fetch the wrong head. - dst_replication = max(1, dst_attn_tp_size // total_kv_heads) - unique_dst_head_idx = dst_tp_rank_in_group // dst_replication - src_head_start_offset = ( - unique_dst_head_idx * dst_heads_per_rank - ) % src_heads_per_rank - num_heads_to_send = dst_heads_per_rank - dst_head_start_offset = 0 + ( + src_head_start_offset, + num_heads_to_send, + dst_head_start_offset, + _, + ) = compute_head_slice_params( + self.attn_tp_size, + dst_attn_tp_size, + self.kv_args.engine_rank, + dst_tp_rank, + total_kv_heads, + ) src_data_ptrs = self.kv_args.kv_data_ptrs src_layer_ids = self.kv_args.kv_layer_ids diff --git a/python/sglang/srt/disaggregation/mori/conn.py b/python/sglang/srt/disaggregation/mori/conn.py index bafb0ce5df..475f38bcb6 100644 --- a/python/sglang/srt/disaggregation/mori/conn.py +++ b/python/sglang/srt/disaggregation/mori/conn.py @@ -931,6 +931,10 @@ class MoriKVManager(CommonKVManager): return grouped_plan.materialize(item_len) def _build_tp_slice_config(self, peer_info: KVArgsRegisterInfo) -> TPSliceConfig: + from sglang.srt.disaggregation.common.staging_buffer import ( + compute_head_slice_params, + ) + page_size = self.kv_args.page_size src_item_len = self.kv_args.kv_item_lens[0] @@ -946,27 +950,21 @@ class MoriKVManager(CommonKVManager): if total_kv_heads <= 0: total_kv_heads = self.kv_args.kv_head_num * prefill_tp_size - src_heads_per_rank = max(1, total_kv_heads // prefill_tp_size) dst_heads_per_rank = max(1, total_kv_heads // decode_tp_size) bytes_per_head_slice = bytes_per_token_dst // dst_heads_per_rank if bytes_per_head_slice == 0: raise ValueError("Head slice size evaluates to zero") - src_replication = max(1, prefill_tp_size // total_kv_heads) - - local_tp_rank = self.kv_args.engine_rank % prefill_tp_size - dst_tp_rank = peer_info.decode_tp_rank % decode_tp_size - - if prefill_tp_size > decode_tp_size: - src_head_start = 0 - num_heads_to_send = src_heads_per_rank - unique_head_idx = local_tp_rank // src_replication - dst_head_start = (unique_head_idx * src_heads_per_rank) % dst_heads_per_rank - else: - src_head_start = (dst_tp_rank * dst_heads_per_rank) % src_heads_per_rank - num_heads_to_send = dst_heads_per_rank - dst_head_start = 0 + src_head_start, num_heads_to_send, dst_head_start, _ = ( + compute_head_slice_params( + prefill_tp_size, + decode_tp_size, + self.kv_args.engine_rank, + peer_info.decode_tp_rank, + total_kv_heads, + ) + ) src_head_slice_offset = src_head_start * bytes_per_head_slice dst_head_slice_offset = dst_head_start * bytes_per_head_slice diff --git a/python/sglang/srt/disaggregation/nixl/conn.py b/python/sglang/srt/disaggregation/nixl/conn.py index cc70acf1d3..3af974d3b3 100644 --- a/python/sglang/srt/disaggregation/nixl/conn.py +++ b/python/sglang/srt/disaggregation/nixl/conn.py @@ -834,6 +834,10 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): interleave num_groups per token, peers select via head_group_idx. prefill_tp > decode_tp: num_groups=1. Dst dlist is per-peer. """ + from sglang.srt.disaggregation.common.staging_buffer import ( + compute_head_slice_params, + ) + decode_tp_size = decode_kv_args.decode_tp_size dst_kv_item_len = decode_kv_args.dst_kv_item_len prefill_tp_size = self.attn_tp_size @@ -844,39 +848,25 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): if total_kv_heads <= 0: total_kv_heads = self.kv_args.kv_head_num * prefill_tp_size - src_heads_per_rank = max(1, total_kv_heads // prefill_tp_size) dst_heads_per_rank = max(1, total_kv_heads // decode_tp_size) bytes_per_head_slice = dst_kv_item_len // page_size // dst_heads_per_rank if prefill_tp_size > decode_tp_size: # Multiple prefill ranks feed one decode rank: each prefill rank sends # all its src heads to a specific head-range in the decode rank. - src_replication = max(1, prefill_tp_size // total_kv_heads) - local_tp_rank_in_group = self.kv_args.engine_rank % prefill_tp_size + _, num_heads_to_send, dst_head_start, _ = compute_head_slice_params( + prefill_tp_size, + decode_tp_size, + self.kv_args.engine_rank, + decode_kv_args.decode_tp_rank, + total_kv_heads, + ) num_groups = 1 - num_heads_to_send = src_heads_per_rank head_group_idx = 0 - unique_head_idx = local_tp_rank_in_group // src_replication - dst_head_start = (unique_head_idx * src_heads_per_rank) % dst_heads_per_rank dst_head_offset = dst_head_start * bytes_per_head_slice else: # One prefill rank feeds multiple decode ranks: interleave num_groups # head-groups in the src dlist so each decode rank picks its slice. - # - # Under GQA the decode side can have MORE attn-TP ranks than there are - # KV heads (decode_tp_size > total_kv_heads). In that case consecutive - # decode ranks replicate a shared KV head, so the src dlist must - # interleave one group per UNIQUE source head-slice, not one per decode - # rank -- otherwise it addresses past the registered KV region and - # prep_xfer_dlist raises NIXL_ERR_NOT_FOUND. - # - # Reuse the shared replicated-KV head map (integer division under - # replication, not modulo) that the mooncake backend already relies - # on, so the two backends stay in sync. - from sglang.srt.disaggregation.common.staging_buffer import ( - compute_head_slice_params, - ) - src_head_start, num_heads_to_send, _, _ = compute_head_slice_params( prefill_tp_size, decode_tp_size, @@ -884,9 +874,9 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): decode_kv_args.decode_tp_rank, total_kv_heads, ) - # num_groups (distinct head-groups packed in one prefill rank's src - # region) and head_group_idx (this peer's group) are NIXL-specific and - # not returned by the shared helper, so derive them here. + # One group per UNIQUE head-slice, not per decode rank: replicating + # ranks share one, and over-counting addresses past the registered + # KV region, where prep_xfer_dlist raises NIXL_ERR_NOT_FOUND. dst_replication = max(1, decode_tp_size // total_kv_heads) num_groups = decode_tp_size // prefill_tp_size // dst_replication head_group_idx = src_head_start // dst_heads_per_rank diff --git a/test/registered/disaggregation/test_disaggregation_different_tp.py b/test/registered/disaggregation/test_disaggregation_different_tp.py index 67145d5956..9862a6f442 100644 --- a/test/registered/disaggregation/test_disaggregation_different_tp.py +++ b/test/registered/disaggregation/test_disaggregation_different_tp.py @@ -257,85 +257,6 @@ class TestDisaggregationMooncakeMHAPrefillLargerTP(PDDisaggregationServerBase): self.assertGreater(metrics["score"], 0.60) -class TestDisaggregationMooncakeMHADecodeLargerTP(PDDisaggregationServerBase): - @classmethod - def setUpClass(cls): - super().setUpClass() - # Temporarily disable JIT DeepGEMM - envs.SGLANG_ENABLE_JIT_DEEPGEMM.set(False) - - cls.model = try_cached_model(DEFAULT_MODEL_NAME_FOR_TEST) - - # Non blocking start servers - cls.start_prefill() - cls.start_decode() - - # Block until both - cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) - cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) - - cls.launch_lb() - - @classmethod - def start_prefill(cls): - prefill_args = [ - "--trust-remote-code", - "--disaggregation-mode", - "prefill", - "--disaggregation-bootstrap-port", - cls.bootstrap_port, - "--tp", - "2", - "--enable-metrics", - "--enable-request-time-stats-logging", - ] - prefill_args += cls.transfer_backend + cls.rdma_devices_for(range(2)) - cls.process_prefill = popen_launch_pd_server( - cls.model, - cls.prefill_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=prefill_args, - ) - - @classmethod - def start_decode(cls): - decode_args = [ - "--trust-remote-code", - "--disaggregation-mode", - "decode", - "--disaggregation-bootstrap-port", - cls.bootstrap_port, - "--tp", - "4", - "--base-gpu-id", - "4", - "--enable-metrics", - "--enable-request-time-stats-logging", - ] - decode_args += cls.transfer_backend + cls.rdma_devices_for(range(4, 8)) - cls.process_decode = popen_launch_pd_server( - cls.model, - cls.decode_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=decode_args, - ) - - def test_gsm8k(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="gsm8k", - api="completion", - max_tokens=512, - num_examples=200, - num_threads=128, - ) - metrics = run_eval(args) - print(f"Evaluation metrics: {metrics}") - - self.assertGreater(metrics["score"], 0.60) - - # The prefill staging buffer is auto-sized to one chunk (chunked_prefill_size); # the decode ring is sized manually. STAGING_ENV = { @@ -344,87 +265,6 @@ STAGING_ENV = { } -class TestDisaggregationStagingPrefillLargerTP(PDDisaggregationServerBase): - """Prefill TP=4 -> Decode TP=2 with staging buffer enabled (MHA model).""" - - @classmethod - def setUpClass(cls): - super().setUpClass() - envs.SGLANG_ENABLE_JIT_DEEPGEMM.set(False) - - cls.model = try_cached_model(DEFAULT_MODEL_NAME_FOR_TEST) - - cls.start_prefill() - cls.start_decode() - - cls.wait_server_ready(cls.prefill_url + "/health", process=cls.process_prefill) - cls.wait_server_ready(cls.decode_url + "/health", process=cls.process_decode) - - cls.launch_lb() - - @classmethod - def start_prefill(cls): - prefill_args = [ - "--trust-remote-code", - "--disaggregation-mode", - "prefill", - "--disaggregation-bootstrap-port", - cls.bootstrap_port, - "--tp", - "4", - "--enable-metrics", - "--enable-request-time-stats-logging", - ] - prefill_args += cls.transfer_backend + cls.rdma_devices_for(range(4)) - env = {**os.environ, **STAGING_ENV} - cls.process_prefill = popen_launch_pd_server( - cls.model, - cls.prefill_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=prefill_args, - env=env, - ) - - @classmethod - def start_decode(cls): - decode_args = [ - "--trust-remote-code", - "--disaggregation-mode", - "decode", - "--disaggregation-bootstrap-port", - cls.bootstrap_port, - "--tp", - "2", - "--base-gpu-id", - "4", - "--enable-metrics", - "--enable-request-time-stats-logging", - ] - decode_args += cls.transfer_backend + cls.rdma_devices_for(range(4, 6)) - env = {**os.environ, **STAGING_ENV} - cls.process_decode = popen_launch_pd_server( - cls.model, - cls.decode_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=decode_args, - env=env, - ) - - def test_gsm8k(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="gsm8k", - api="completion", - max_tokens=512, - num_examples=200, - num_threads=128, - ) - metrics = run_eval(args) - print(f"[Staging PrefillLargerTP] Evaluation metrics: {metrics}") - self.assertGreater(metrics["score"], 0.60) - - class TestDisaggregationStagingDecodeLargerTP(PDDisaggregationServerBase): """Prefill TP=2 -> Decode TP=4 with staging buffer enabled (MHA model).""" diff --git a/test/registered/unit/disaggregation/test_head_slice_params.py b/test/registered/unit/disaggregation/test_head_slice_params.py new file mode 100644 index 0000000000..e9bacfffa8 --- /dev/null +++ b/test/registered/unit/disaggregation/test_head_slice_params.py @@ -0,0 +1,104 @@ +"""Unit tests for srt/disaggregation/common/staging_buffer -- compute_head_slice_params. + +Wrong head indices do not fail loudly: the transfer delivers the wrong channels +and the only symptom is a garbled end-to-end accuracy score. The replication +cases are the ones no end-to-end test can reach -- every heterogeneous-TP suite +runs tp <= total_kv_heads on both sides, where a modulo map and the correct +divide-by-replication map agree. + +Expected values are derived by hand from the head-distribution rules, never by +calling the implementation, so a bug in it cannot make both sides agree. +""" + +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") + + +import unittest + +from sglang.srt.disaggregation.common.staging_buffer import compute_head_slice_params +from sglang.test.test_utils import CustomTestCase + + +class TestHeadSliceParamsGather(CustomTestCase): + SRC_TP, DST_TP, KV_HEADS = 4, 2, 8 + + def _call(self, src_rank, dst_rank=0): + return compute_head_slice_params( + self.SRC_TP, self.DST_TP, src_rank, dst_rank, self.KV_HEADS + ) + + def test_prefill_ranks_tile_the_decode_head_range(self): + self.assertEqual([self._call(r)[2] for r in range(4)], [0, 2, 0, 2]) + + def test_rank_is_taken_modulo_the_tp_group(self): + for src_rank in range(self.SRC_TP): + self.assertEqual(self._call(src_rank), self._call(src_rank + self.SRC_TP)) + + +class TestHeadSliceParamsScatter(CustomTestCase): + SRC_TP, DST_TP, KV_HEADS = 2, 4, 8 + + def _call(self, dst_rank, src_rank=0): + return compute_head_slice_params( + self.SRC_TP, self.DST_TP, src_rank, dst_rank, self.KV_HEADS + ) + + def test_decode_ranks_walk_the_prefill_head_range(self): + self.assertEqual([self._call(d)[0] for d in range(4)], [0, 2, 0, 2]) + + +class TestHeadSliceParamsScatterReplication(CustomTestCase): + """More decode ranks than KV heads: consecutive decode ranks share a head.""" + + SRC_TP, DST_TP, KV_HEADS = 1, 4, 2 + + def _call(self, dst_rank): + return compute_head_slice_params( + self.SRC_TP, self.DST_TP, 0, dst_rank, self.KV_HEADS + ) + + def test_replicating_decode_ranks_read_the_same_head(self): + starts = [self._call(d)[0] for d in range(4)] + self.assertEqual(starts, [0, 0, 1, 1]) + self.assertNotEqual(starts, [0, 1, 0, 1], "a modulo map would give this") + for dst_rank in range(self.DST_TP): + self.assertEqual( + self._call(dst_rank)[1], 1, "max(1, 2 // 4) clamps to one head" + ) + + +class TestHeadSliceParamsGatherReplication(CustomTestCase): + """More prefill ranks than KV heads -- the src_replication branch.""" + + SRC_TP, DST_TP, KV_HEADS = 8, 2, 4 + + def _call(self, src_rank): + return compute_head_slice_params( + self.SRC_TP, self.DST_TP, src_rank, 0, self.KV_HEADS + ) + + def test_replicating_prefill_ranks_write_the_same_head(self): + starts = [self._call(r)[2] for r in range(8)] + self.assertEqual(starts, [0, 0, 1, 1, 0, 0, 1, 1]) + self.assertNotEqual( + starts, [0, 1, 0, 1, 0, 1, 0, 1], "a modulo map would give this" + ) + for src_rank in range(self.SRC_TP): + self.assertEqual( + self._call(src_rank)[1], 1, "max(1, 4 // 8) clamps to one head" + ) + + +class TestHeadSliceParamsEqualTp(CustomTestCase): + def test_equal_tp_copies_the_whole_rank_slice(self): + for rank in range(4): + src_start, num_heads, dst_start, _ = compute_head_slice_params( + 4, 4, rank, rank, 8 + ) + self.assertEqual((src_start, num_heads, dst_start), (0, 2, 0)) + + +if __name__ == "__main__": + unittest.main()