diff --git a/python/sglang/srt/managers/prefill_delayer.py b/python/sglang/srt/managers/prefill_delayer.py index 88e3d5fcb5..da93dbfaa0 100644 --- a/python/sglang/srt/managers/prefill_delayer.py +++ b/python/sglang/srt/managers/prefill_delayer.py @@ -129,9 +129,10 @@ class PrefillDelayer: # Fields packed per rank into the all-gather tensor: prefillable, # token_watermark_force_allow, running_batch, max_prefill_bs, - # waiting_queue_len. + # waiting_queue_len. The gather spans the TP group, whose ranks are + # laid out DP-major over attn_cp_size * attn_tp_size. self._global_info_buffer = torch.empty( - (dp_size_dim, parallel.attn_tp_size, 5), + (dp_size_dim, parallel.attn_cp_size * parallel.attn_tp_size, 5), dtype=torch.int64, device=self._gather_device, ) diff --git a/test/registered/unit/managers/test_prefill_delayer_gather.py b/test/registered/unit/managers/test_prefill_delayer_gather.py new file mode 100644 index 0000000000..8cadc7334b --- /dev/null +++ b/test/registered/unit/managers/test_prefill_delayer_gather.py @@ -0,0 +1,68 @@ +import unittest +from unittest.mock import patch + +import torch + +from sglang.srt.managers.prefill_delayer import PrefillDelayer +from sglang.srt.runtime_context import get_parallel +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase, published_topology + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +TOPOLOGIES = { + "tp": dict(tp_size=4), + "attn_cp": dict(tp_size=4, attn_cp_size=2), + "dp_attention": dict(tp_size=4, dp_size=2, enable_dp_attention=True), + "dp_attention_attn_cp": dict( + tp_size=8, dp_size=2, attn_cp_size=2, enable_dp_attention=True + ), +} + + +def _all_gather_over_tp(output, local, group=None): + """All-gather over the TP group, as ``all_gather_into_tensor`` checks it.""" + world_size = get_parallel().tp_size + if output.numel() != world_size * local.numel(): + raise RuntimeError( + f"output holds {output.numel()} elements, not {world_size} x " + f"{local.numel()}" + ) + rows = output.view(world_size, -1) + for rank in range(world_size): + rows[rank] = local + 100 * rank + + +class TestPrefillDelayerGather(CustomTestCase): + def test_gather_returns_the_first_rank_of_each_dp_group(self): + for name, topology in TOPOLOGIES.items(): + with ( + self.subTest(name), + published_topology(**topology), + patch( + "sglang.srt.managers.prefill_delayer.all_gather_single", + side_effect=_all_gather_over_tp, + ), + ): + delayer = PrefillDelayer( + cpu_group=None, max_delay_passes=4, token_usage_low_watermark=None + ) + info = delayer._gather_info( + local_prefillable=True, + local_token_watermark_force_allow=False, + running_batch=3, + max_prefill_bs=4, + waiting_queue_len=5, + ) + parallel = get_parallel() + dp_groups = parallel.dp_size if parallel.enable_dp_attention else 1 + ranks_per_dp_group = parallel.tp_size // dp_groups + local = torch.tensor([1, 0, 3, 4, 5]) + expected = torch.stack( + [local + 100 * g * ranks_per_dp_group for g in range(dp_groups)] + ) + self.assertTrue(torch.equal(info, expected), f"{info} != {expected}") + + +if __name__ == "__main__": + unittest.main()