diff --git a/python/sglang/srt/batch_overlap/two_batch_overlap.py b/python/sglang/srt/batch_overlap/two_batch_overlap.py index e82a169b9a..220e1b1b29 100644 --- a/python/sglang/srt/batch_overlap/two_batch_overlap.py +++ b/python/sglang/srt/batch_overlap/two_batch_overlap.py @@ -823,6 +823,9 @@ class TboForwardBatchPreparer: _original_num_tokens=None, global_num_tokens_gpu=None, global_num_tokens_cpu=None, + # Children publish no per-rank list of their own; the parent + # published the gather sizes before it was split. + global_num_tokens_padded_cpu=None, global_dp_buffer_len=global_dp_buffer_len, global_num_tokens_for_logprob_gpu=None, global_num_tokens_for_logprob_cpu=None, diff --git a/python/sglang/srt/layers/dp_attention.py b/python/sglang/srt/layers/dp_attention.py index 39bbaad4fb..38a0c3409c 100644 --- a/python/sglang/srt/layers/dp_attention.py +++ b/python/sglang/srt/layers/dp_attention.py @@ -337,6 +337,24 @@ def set_local_dp_buffer_len(local_dp_buffer_len: int) -> None: _DpGatheredBufferWrapper.set_local_dp_buffer_len(local_dp_buffer_len) +def set_dp_buffer_len_from_batch(forward_batch: ForwardBatch) -> None: + """Publish the DP gather sizes ``forward_batch`` carries: the buffer + length, the per-rank token counts as padded for the gather, this rank's + entry, and the padding mode. Capture batches carry no separately padded + list, so the raw counts stand in for it.""" + global_num_tokens = forward_batch.global_num_tokens_padded_cpu + if global_num_tokens is None: + global_num_tokens = forward_batch.global_num_tokens_cpu + dp_rank = get_parallel().attn_dp_rank if len(global_num_tokens) > 1 else 0 + set_dp_buffer_len( + forward_batch.global_dp_buffer_len, + global_num_tokens[dp_rank], + forward_batch.dp_padding_mode.is_max_len(), + global_num_tokens, + forward_batch.global_num_tokens_gpu, + ) + + def get_dp_global_num_tokens() -> List[int]: return _DpGatheredBufferWrapper.get_dp_global_num_tokens() diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index 1bd0830e24..45d6c19f47 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -46,7 +46,7 @@ from sglang.srt.kv_canary.req_to_expected_token_ids_manager import ( from sglang.srt.layers.dp_attention import ( DpPaddingMode, dp_gather_slot, - set_dp_buffer_len, + set_dp_buffer_len_from_batch, set_is_extend_in_batch, world_dp_gather_enabled, ) @@ -688,6 +688,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): dp_local_start_pos: Optional[torch.Tensor] = None # cached info at runtime dp_local_num_tokens: Optional[torch.Tensor] = None # cached info at runtime global_dp_buffer_len: Optional[int] = None + # global_num_tokens_cpu as published for the DP gather: attn-TP aligned and, + # under MAX_LEN, padded to the max. None when the raw list already is. + global_num_tokens_padded_cpu: Optional[List[int]] = None # For Qwen2-VL mrope_positions: torch.Tensor = None @@ -1584,13 +1587,8 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): ) self.global_dp_buffer_len = buffer_len - set_dp_buffer_len( - buffer_len, - num_tokens, - dp_padding_mode.is_max_len(), - global_num_tokens, - self.global_num_tokens_gpu, - ) + self.global_num_tokens_padded_cpu = global_num_tokens + set_dp_buffer_len_from_batch(self) set_is_extend_in_batch(self.is_extend_in_batch) bs = self.batch_size diff --git a/test/registered/unit/batch_overlap/test_tbo_filter_batch_marker.py b/test/registered/unit/batch_overlap/test_tbo_filter_batch_marker.py index cba5a44efd..5d1b63805c 100644 --- a/test/registered/unit/batch_overlap/test_tbo_filter_batch_marker.py +++ b/test/registered/unit/batch_overlap/test_tbo_filter_batch_marker.py @@ -61,9 +61,11 @@ class TestTboFilterBatchMarker(CustomTestCase): parent = _make_target_verify_batch(8) parent._original_batch_size = 8 parent._original_num_tokens = 8 + parent.global_num_tokens_padded_cpu = [8, 0, 0, 0] child = _filter(parent, lo=0, hi=4) self.assertIsNone(child._original_batch_size) self.assertIsNone(child._original_num_tokens) + self.assertIsNone(child.global_num_tokens_padded_cpu) def test_filter_batch_resets_plan_marker_on_children(self): child = _filter(_make_target_verify_batch(8), lo=0, hi=4) diff --git a/test/registered/unit/layers/test_dp_attention.py b/test/registered/unit/layers/test_dp_attention.py new file mode 100644 index 0000000000..ac51d0de5a --- /dev/null +++ b/test/registered/unit/layers/test_dp_attention.py @@ -0,0 +1,79 @@ +"""CPU unit tests for publishing DP-attention buffer sizes from a ForwardBatch.""" + +import unittest +from types import SimpleNamespace + +import torch + +from sglang.srt.layers.dp_attention import ( + DpPaddingMode, + _DpGatheredBufferWrapper, + get_dp_global_num_tokens, + get_global_dp_buffer_len, + get_local_dp_buffer_len, + is_dp_max_padding, + set_dp_buffer_len, + set_dp_buffer_len_from_batch, +) +from sglang.srt.runtime_context import get_parallel +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +def _batch(**overrides): + fields = dict( + global_dp_buffer_len=8, + global_num_tokens_cpu=[3, 1], + global_num_tokens_padded_cpu=[4, 4], + global_num_tokens_gpu=torch.tensor([3, 1]), + dp_padding_mode=DpPaddingMode.MAX_LEN, + ) + fields.update(overrides) + return SimpleNamespace(**fields) + + +class TestSetDpBufferLenFromBatch(unittest.TestCase): + def setUp(self): + self.addCleanup(set_dp_buffer_len, 0, 0, False) + + def test_publishes_the_padded_list_and_this_ranks_entry(self): + batch = _batch() + with get_parallel().override(attn_dp_rank=1): + set_dp_buffer_len_from_batch(batch) + self.assertEqual(get_global_dp_buffer_len(), 8) + self.assertEqual(get_local_dp_buffer_len(), 4) + self.assertTrue(is_dp_max_padding()) + self.assertEqual(get_dp_global_num_tokens(), [4, 4]) + self.assertIs( + _DpGatheredBufferWrapper.get_dp_global_num_tokens_gpu(), + batch.global_num_tokens_gpu, + ) + + def test_falls_back_to_the_raw_list_when_no_padded_list_is_carried(self): + batch = _batch( + global_dp_buffer_len=4, + global_num_tokens_padded_cpu=None, + dp_padding_mode=DpPaddingMode.SUM_LEN, + ) + with get_parallel().override(attn_dp_rank=0): + set_dp_buffer_len_from_batch(batch) + self.assertEqual(get_local_dp_buffer_len(), 3) + self.assertFalse(is_dp_max_padding()) + self.assertEqual(get_dp_global_num_tokens(), [3, 1]) + + def test_a_single_entry_list_ignores_the_rank(self): + batch = _batch( + global_dp_buffer_len=3, + global_num_tokens_cpu=[3], + global_num_tokens_padded_cpu=None, + global_num_tokens_gpu=torch.tensor([3]), + dp_padding_mode=DpPaddingMode.SUM_LEN, + ) + with get_parallel().override(attn_dp_rank=1): + set_dp_buffer_len_from_batch(batch) + self.assertEqual(get_local_dp_buffer_len(), 3) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/model_executor/test_mlp_sync_hybrid_target_verify.py b/test/registered/unit/model_executor/test_mlp_sync_hybrid_target_verify.py index c978ed2f1a..a85292d316 100644 --- a/test/registered/unit/model_executor/test_mlp_sync_hybrid_target_verify.py +++ b/test/registered/unit/model_executor/test_mlp_sync_hybrid_target_verify.py @@ -53,7 +53,7 @@ def _sync(forward_mode, num_reqs, global_num_tokens): get_exec=lambda: no_prefill_graph, _elastic_should_preserve_local_token_counts=lambda **kwargs: False, dp_gather_slot=lambda: 0, - set_dp_buffer_len=lambda *args: None, + set_dp_buffer_len_from_batch=lambda *args: None, set_is_extend_in_batch=lambda *args: None, _is_cpu=True, ),