diff --git a/python/sglang/srt/layers/deep_gemm_wrapper/compile_utils.py b/python/sglang/srt/layers/deep_gemm_wrapper/compile_utils.py index 2db04ae05d..5caf3ea83c 100644 --- a/python/sglang/srt/layers/deep_gemm_wrapper/compile_utils.py +++ b/python/sglang/srt/layers/deep_gemm_wrapper/compile_utils.py @@ -16,7 +16,11 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import ( restore_symmetric_memory_context, ) from sglang.srt.environ import envs -from sglang.srt.layers.deep_gemm_wrapper.configurer import ENABLE_JIT_DEEPGEMM +from sglang.srt.layers.deep_gemm_wrapper.configurer import ( + DEEPGEMM_NEED_TMA_ALIGNED_SCALES, + DEEPGEMM_SCALE_UE8M0, + ENABLE_JIT_DEEPGEMM, +) from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.runtime_context import ( get_device, @@ -275,13 +279,43 @@ class _BaseWarmupExecutor: @staticmethod def get_memory_requirement( kernel_type: DeepGemmKernelType, max_m: int, n: int, k: int, num_groups: int - ) -> int: - # Return the required memory space in GB for warmup executor + ) -> float: + # FP32 input scales remain live while DeepGEMM converts their layout. _GB = 1 << 30 + lhs_scales = max_m * ceil_div(k, _BLOCK_SIZE) * 4 + rhs_scales = ceil_div(n, _BLOCK_SIZE) * ceil_div(k, _BLOCK_SIZE) * 4 + scale_workspace = _get_fp8_scale_workspace( + max_m, + n, + k, + lhs_groups=( + num_groups + if kernel_type == DeepGemmKernelType.GROUPED_GEMM_NT_F8F8BF16_MASKED + else 1 + ), + rhs_groups=( + 1 if kernel_type == DeepGemmKernelType.GEMM_NT_F8F8BF16 else num_groups + ), + ) if kernel_type == DeepGemmKernelType.GEMM_NT_F8F8BF16: - return (max_m * k + n * k + max_m * n * 2) / _GB + return ( + max_m * k + + n * k + + max_m * n * 2 + + lhs_scales + + rhs_scales + + scale_workspace + ) / _GB elif kernel_type == DeepGemmKernelType.GROUPED_GEMM_NT_F8F8BF16_CONTIG: - return (max_m * k + num_groups * n * k + max_m * 4 + max_m * n * 2) / _GB + return ( + max_m * k + + num_groups * n * k + + max_m * 4 + + max_m * n * 2 + + lhs_scales + + scale_workspace + + num_groups * rhs_scales + ) / _GB elif kernel_type == DeepGemmKernelType.GROUPED_GEMM_NT_BF16_CONTIG: return ( max_m * k * 2 + num_groups * n * k * 2 + max_m * 4 + max_m * n * 2 @@ -292,6 +326,8 @@ class _BaseWarmupExecutor: + num_groups * n * k + num_groups * 4 + num_groups * max_m * n * 2 + + num_groups * (lhs_scales + rhs_scales) + + scale_workspace ) / _GB elif kernel_type == DeepGemmKernelType.GEMM_NT_BF16BF16F32: # bf16 lhs + bf16 rhs + fp32 out @@ -310,6 +346,33 @@ class _BaseWarmupExecutor: raise NotImplementedError +def _packed_scale_workspace(rows: int, cols: int, groups: int) -> tuple[int, int]: + packed_bytes = groups * ceil_align(rows, 4) * ceil_div(cols, 4) * 4 + peak_bytes = packed_bytes + if groups > 1 and rows * cols % 4: + # DeepGEMM's PyTorch fallback keeps the initial packed allocation while + # shifting FP32 bits to uint8, padding, and transposing a second copy. + token_scales = groups * rows * cols + peak_bytes += max(5 * token_scales, token_scales + 2 * packed_bytes) + return packed_bytes, peak_bytes + + +def _get_fp8_scale_workspace( + max_m: int, n: int, k: int, lhs_groups: int, rhs_groups: int +) -> int: + scale_k = ceil_div(k, _BLOCK_SIZE) + if DEEPGEMM_NEED_TMA_ALIGNED_SCALES: + return lhs_groups * ceil_align(max_m, 4) * scale_k * 4 + if DEEPGEMM_SCALE_UE8M0: + lhs_packed, lhs_peak = _packed_scale_workspace(max_m, scale_k, lhs_groups) + _, rhs_peak = _packed_scale_workspace(n, scale_k, rhs_groups) + # RHS block scales are broadcast to token scales before packing. The + # int64 index tensor dies before packing; the packed LHS stays live. + rhs_broadcast = rhs_groups * n * scale_k * 4 + return max(lhs_peak, lhs_packed + rhs_broadcast + max(n * 8, rhs_peak)) + return 0 + + def _empty_token_fp8(size): *dims, k = size return ( diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index 916ecc2eb1..e55ec24227 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -139,6 +139,7 @@ from sglang.srt.runtime_context import ( from sglang.srt.speculative.eagle_utils import get_draft_input_from_target_hidden_dim from sglang.srt.utils import ( get_available_gpu_memory, + get_cuda_graph_max_batch_size, is_cuda, is_npu, require_attn_tp_gather, @@ -162,6 +163,15 @@ _MAX_PREFILL_CUDA_GRAPH_PADDING_FACTOR = 2 _CHUNKED_PREFIX_VARIANTS = (1, 2, 4, 8, 16) +def get_prefill_num_tokens_to_capture(capture_tokens: Sequence[int]) -> list[int]: + """Align capture and replay buckets to the same TP/CP token-shard widths. + + Dummy capture batches bypass eager padding. Unequal shards can mismatch + collectives when a dense FFN runs with moe_dense_tp_size=1. + """ + return sorted({get_cuda_graph_max_batch_size(n) for n in capture_tokens}) + + def _chunked_prefix_variant(num_chunks: int) -> str: return f"chunked_prefix:{num_chunks}" @@ -335,7 +345,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): # tc_piecewise) — one shape knob per phase. capture_tokens = prefill_config.bs assert capture_tokens is not None, "cuda_graph_config[prefill].bs is not set" - self.capture_num_tokens = sorted(capture_tokens) + self.capture_num_tokens = get_prefill_num_tokens_to_capture(capture_tokens) assert self.capture_num_tokens, "cuda_graph_config[prefill].bs is empty" # --- runner bounds -------------------------------------------- diff --git a/test/registered/unit/layers/deep_gemm_wrapper/test_warmup_memory.py b/test/registered/unit/layers/deep_gemm_wrapper/test_warmup_memory.py new file mode 100644 index 0000000000..480a6381b1 --- /dev/null +++ b/test/registered/unit/layers/deep_gemm_wrapper/test_warmup_memory.py @@ -0,0 +1,156 @@ +"""CPU checks for DeepGEMM warmup buffers and scale conversion workspace.""" + +import unittest +from contextlib import ExitStack +from unittest.mock import patch + +import torch + +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel + +maybe_stub_sgl_kernel() + +from sglang.srt.layers.deep_gemm_wrapper import compile_utils +from sglang.srt.layers.deep_gemm_wrapper.compile_utils import ( + DeepGemmKernelType, + _BaseWarmupExecutor, +) + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +def _allocate_on_meta(factory): + def allocate(*args, **kwargs): + kwargs["device"] = "meta" + return factory(*args, **kwargs) + + return allocate + + +def _storage_bytes(*tensors): + return sum(tensor.untyped_storage().nbytes() for tensor in tensors) + + +def _pack_scales_on_meta(scales): + scales = scales.unsqueeze(0) if scales.ndim == 2 else scales + groups, rows, cols = scales.shape + aligned_rows = (rows + 3) // 4 * 4 + packed_cols = (cols + 3) // 4 + initial = torch.empty_strided( + (groups, rows, packed_cols), + (aligned_rows * packed_cols, 1, aligned_rows), + dtype=torch.int32, + device="meta", + ) + peak = _storage_bytes(initial) + if groups > 1 and rows * cols % 4: + shifted = scales.view(torch.int32) >> 23 + unpacked = shifted.to(torch.uint8) + peak = max(peak, _storage_bytes(initial, shifted, unpacked)) + padded = torch.zeros( + (groups, aligned_rows, packed_cols * 4), dtype=torch.uint8, device="meta" + ) + packed = torch.empty_strided( + (groups, aligned_rows, packed_cols), + (aligned_rows * packed_cols, 1, aligned_rows), + dtype=torch.int32, + device="meta", + ) + peak = max(peak, _storage_bytes(initial, unpacked, padded, packed)) + return packed, peak + return initial, peak + + +def _scale_workspace_on_meta(executor, architecture): + if not hasattr(executor, "lhs_s") or architecture == "musa": + return 0 + scales = executor.lhs_s + if architecture == "hopper": + scales = scales.unsqueeze(0) if scales.ndim == 2 else scales + _, rows, cols = scales.shape + aligned_rows = (rows + 3) // 4 * 4 + workspace = torch.empty_strided( + scales.shape, + (aligned_rows * cols, 1, aligned_rows), + dtype=scales.dtype, + device="meta", + ) + return _storage_bytes(workspace) + lhs_packed, lhs_peak = _pack_scales_on_meta(scales) + rows = executor.rhs_q.shape[-2] + indices = torch.arange(rows, device="meta") // 128 + broadcast = executor.rhs_s.index_select(-2, indices) + _, rhs_peak = _pack_scales_on_meta(broadcast) + return max( + lhs_peak, + _storage_bytes(lhs_packed, broadcast) + max(_storage_bytes(indices), rhs_peak), + ) + + +class TestWarmupMemory(CustomTestCase): + def test_budget_covers_executor_buffers(self): + for architecture in ("hopper", "blackwell", "musa"): + for max_m, n, k, num_groups in ( + (9, 131, 259, 3), # Ceil division and grouped packing fallback. + (16384, 4096, 7168, 32), # V3 DP8/EP8 startup OOM shape. + ): + for kernel_type in DeepGemmKernelType: + with self.subTest( + architecture=architecture, kernel_type=kernel_type, max_m=max_m + ): + with ExitStack() as stack: + stack.enter_context( + patch.object( + compile_utils, + "DEEPGEMM_NEED_TMA_ALIGNED_SCALES", + architecture == "hopper", + ) + ) + stack.enter_context( + patch.object( + compile_utils, + "DEEPGEMM_SCALE_UE8M0", + architecture == "blackwell", + ) + ) + for name in ("empty", "ones", "zeros"): + factory = getattr(torch, name) + stack.enter_context( + patch.object( + torch, + name, + side_effect=_allocate_on_meta(factory), + ) + ) + executor = _BaseWarmupExecutor.create( + kernel_type, + max_m=max_m, + n=n, + k=k, + num_groups=num_groups, + ) + budget_gib = _BaseWarmupExecutor.get_memory_requirement( + kernel_type, + max_m=max_m, + n=n, + k=k, + num_groups=num_groups, + ) + allocated_bytes = _storage_bytes( + *[ + tensor + for tensor in vars(executor).values() + if isinstance(tensor, torch.Tensor) + ] + ) + peak_bytes = allocated_bytes + _scale_workspace_on_meta( + executor, architecture + ) + self.assertGreaterEqual(budget_gib * (1 << 30), peak_bytes) + # Upper bounds include the last row's TMA padding. + self.assertLess(budget_gib * (1 << 30) - peak_bytes, 64) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/model_executor/test_prefill_cuda_graph_runner_helpers.py b/test/registered/unit/model_executor/test_prefill_cuda_graph_runner_helpers.py index de0d7273cf..7f554e313f 100644 --- a/test/registered/unit/model_executor/test_prefill_cuda_graph_runner_helpers.py +++ b/test/registered/unit/model_executor/test_prefill_cuda_graph_runner_helpers.py @@ -8,6 +8,7 @@ from unittest.mock import Mock, patch import torch +from sglang.srt.layers.moe.utils import MoeA2ABackend from sglang.srt.model_executor.cuda_graph_buffer_registry import ( build_prefill_registry, ) @@ -19,9 +20,11 @@ from sglang.srt.model_executor.runner.prefill_cuda_graph_runner import ( PrefillCudaGraphRunner, _build_layer_model_forward_kwargs, _resolve_transformer_layer_model, + get_prefill_num_tokens_to_capture, ) from sglang.srt.model_executor.runner_utils.buffers import PrefillInputBuffers from sglang.srt.model_loader.utils import resolve_language_model +from sglang.srt.runtime_context import get_context, get_flags, get_parallel from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -56,6 +59,67 @@ def _make_pp_buffers_and_registry(): class TestPrefillCudaGraphRunnerHelpers(CustomTestCase): + def test_capture_buckets_keep_tp_shards_equal(self): + override = get_context().override_server_args( + enable_two_batch_overlap=False, + ) + override.install() + self.addCleanup(override.restore) + for attn_tp_size, dense_tp_size, expected in ( + (8, 1, [8, 16, 24, 32]), + (4, 1, [4, 8, 12, 16, 20, 24, 28, 32]), + (1, 1, [4, 8, 12, 16, 20, 24, 28, 32]), + (8, None, [4, 8, 12, 16, 20, 24, 28, 32]), + ): + with ( + self.subTest(attn_tp_size=attn_tp_size, dense_tp_size=dense_tp_size), + get_parallel().override( + tp_size=8, + moe_ep_size=1, + moe_dp_size=1, + moe_tp_size=8, + attn_tp_size=attn_tp_size, + attn_dp_size=8 // attn_tp_size, + attn_cp_size=1, + num_dp_ranks=8 // attn_tp_size, + attn_dp_enabled=attn_tp_size < 8, + moe_dense_tp_size=dense_tp_size, + disable_attn_tp_gather=False, + enable_dp_lm_head=True, + ), + get_flags().moe.override(a2a_backend=MoeA2ABackend.NONE), + ): + buckets = get_prefill_num_tokens_to_capture(list(range(4, 33, 4))) + self.assertEqual(buckets, expected) + if dense_tp_size == 1: + replay_tokens = PrefillCudaGraphRunner._pad_to_bucket(28, buckets) + shards = torch.arange(replay_tokens).tensor_split(attn_tp_size) + self.assertEqual(len({shard.numel() for shard in shards}), 1) + + def test_capture_ceiling_rounds_up_for_sequence_sharding(self): + override = get_context().override_server_args(enable_two_batch_overlap=False) + override.install() + self.addCleanup(override.restore) + with ( + get_parallel().override( + tp_size=8, + moe_ep_size=1, + moe_dp_size=1, + moe_tp_size=8, + attn_tp_size=8, + attn_dp_size=1, + attn_cp_size=1, + num_dp_ranks=1, + attn_dp_enabled=False, + moe_dense_tp_size=1, + disable_attn_tp_gather=False, + ), + get_flags().moe.override(a2a_backend=MoeA2ABackend.NONE), + ): + buckets = get_prefill_num_tokens_to_capture([4, 12, 28]) + self.assertEqual(buckets, [8, 16, 32]) + self.assertEqual(PrefillCudaGraphRunner._pad_to_bucket(28, buckets), 32) + def test_qwen_hc_restore_round_trip(self): runner = PrefillCudaGraphRunner.__new__(PrefillCudaGraphRunner) runner._qwen_bcg_hc_sidechannel = True