From 79e5c4bb9b438db8f06333f0bf9759221cee1ef3 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Fri, 23 Oct 2026 17:24:06 -0700 Subject: [PATCH] [Perf] dsv41: Trim per-step DSpark decode overhead (#46176) --- python/sglang/kernels/ops/layernorm/mhc.py | 57 ++++++++++++++++ python/sglang/srt/layers/aux_hidden_states.py | 25 ++++--- python/sglang/srt/models/deepseek_v4.py | 67 +++++++++++++------ .../sglang/srt/models/deepseek_v4_dspark.py | 18 +++-- .../srt/models/deepseek_v4_replay_tail.py | 4 +- .../ops/layernorm/test_mhc_stream_mean.py | 62 +++++++++++++++++ .../unit/models/test_deepseek_v4_boundary.py | 5 +- 7 files changed, 201 insertions(+), 37 deletions(-) create mode 100644 test/registered/kernels/ops/layernorm/test_mhc_stream_mean.py diff --git a/python/sglang/kernels/ops/layernorm/mhc.py b/python/sglang/kernels/ops/layernorm/mhc.py index e1f29ab61b..398eef6da7 100644 --- a/python/sglang/kernels/ops/layernorm/mhc.py +++ b/python/sglang/kernels/ops/layernorm/mhc.py @@ -2185,6 +2185,32 @@ def _mhc_combine_kernel( tl.store(y_ptr + pid_m * y_stride_m + offs_h, acc, mask=mask) +@triton.jit +def _mhc_stream_mean_kernel( + x_ptr, + y_ptr, + H, + x_stride_m, + x_stride_k, + y_stride_m, + HC: tl.constexpr, + BLOCK_H: tl.constexpr, +): + pid_m = tl.program_id(0) + pid_h = tl.program_id(1) + offs_h = pid_h * BLOCK_H + tl.arange(0, BLOCK_H) + mask = offs_h < H + x_row = x_ptr + pid_m.to(tl.int64) * x_stride_m + offs_h + # Streams are added in order onto a +0 fp32 accumulator, as torch.mean's + # reduction does (so an all -0 row averages to +0). + acc = tl.zeros([BLOCK_H], dtype=tl.float32) + for k in tl.static_range(HC): + acc += tl.load(x_row + k * x_stride_k, mask=mask, other=0.0).to(tl.float32) + tl.store( + y_ptr + pid_m.to(tl.int64) * y_stride_m + offs_h, acc * (1.0 / HC), mask=mask + ) + + @triton.jit def _hc_mix_stats_partial_kernel( x_ptr, @@ -2736,3 +2762,34 @@ def mhc_combine( BLOCK_H=block_h, ) return y + + +def mhc_stream_mean(x: torch.Tensor, out: torch.Tensor | None = None) -> torch.Tensor: + """y[m, h] = x[m, :, h].mean(), bitwise equal to ``x.mean(dim=1)``, written + into ``out`` when given.""" + m, hc, h = x.shape + # Bitwise equality with torch's CUDA mean holds for 4 streams (the mHC width) + # over whole 1024-element blocks; torch orders some other widths differently. + if ( + _is_hip + or x.device.type != "cuda" + or hc != 4 + or h % 1024 + or x.stride(2) != 1 + or (out is not None and (out.dtype != x.dtype or out.stride(1) != 1)) + ): + y = x.mean(dim=1) + return y if out is None else out.copy_(y) + y = torch.empty((m, h), dtype=x.dtype, device=x.device) if out is None else out + block_h = 1024 + _mhc_stream_mean_kernel[(m, triton.cdiv(h, block_h))]( + x, + y, + h, + x.stride(0), + x.stride(1), + y.stride(0), + HC=hc, + BLOCK_H=block_h, + ) + return y diff --git a/python/sglang/srt/layers/aux_hidden_states.py b/python/sglang/srt/layers/aux_hidden_states.py index f429f100fb..dae02e26fa 100644 --- a/python/sglang/srt/layers/aux_hidden_states.py +++ b/python/sglang/srt/layers/aux_hidden_states.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, List, Optional, Union +from typing import TYPE_CHECKING, Iterable, List, Optional, Union import torch @@ -49,21 +49,30 @@ class AuxHiddenStatePacker: ) -> AuxHiddenStatePacker: return cls(num_captures, out=forward_batch.aux_hidden_states_buffer) - def append(self, hidden: torch.Tensor) -> None: - feature_size = int(hidden.shape[-1]) + def slot(self, like: torch.Tensor) -> torch.Tensor: + """Take the next capture's view of the packed buffer, shaped like ``like``, + for a producer that writes it in place.""" + feature_size = int(like.shape[-1]) if self._feature_size is None: self._feature_size = feature_size - shape = (*hidden.shape[:-1], feature_size * self._num_captures) + shape = (*like.shape[:-1], feature_size * self._num_captures) if self._buffer is None: - self._buffer = hidden.new_empty(shape) - elif self._buffer.shape != shape or self._buffer.dtype != hidden.dtype: + self._buffer = like.new_empty(shape) + elif self._buffer.shape != shape or self._buffer.dtype != like.dtype: raise ValueError( f"aux hidden output buffer is {tuple(self._buffer.shape)} " - f"{self._buffer.dtype}, captures need {shape} {hidden.dtype}" + f"{self._buffer.dtype}, captures need {shape} {like.dtype}" ) start = self._idx * self._feature_size - self._buffer[..., start : start + self._feature_size].copy_(hidden) self._idx += 1 + return self._buffer[..., start : start + self._feature_size] + + def append(self, hidden: torch.Tensor) -> None: + self.slot(hidden).copy_(hidden) + + def extend(self, hiddens: Iterable[torch.Tensor]) -> None: + for hidden in hiddens: + self.append(hidden) def capture(self, hidden: torch.Tensor, *, owned: bool = False) -> None: """Write directly to the final packed buffer, without an intermediate copy.""" diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index bf94b44b91..41ceec4485 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -72,6 +72,7 @@ from sglang.srt.layers.attention.dsv4.dsv41_sparse import ( DeepseekV41Indexer, ) from sglang.srt.layers.attention.dsv4.indexer import C4Indexer +from sglang.srt.layers.aux_hidden_states import AuxHiddenStatePacker from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx from sglang.srt.layers.cp.utils import ( cp_gather_full_sequence_states, @@ -461,6 +462,17 @@ def _fused_wo_a_arch_supported() -> bool: return _is_cuda and torch.cuda.get_device_capability()[0] == 10 +def _wo_a_bf16_bmm_token_major(o: torch.Tensor, wo_a: torch.Tensor) -> torch.Tensor: + result = torch.empty( + (o.shape[0], wo_a.shape[0], wo_a.shape[1]), dtype=o.dtype, device=o.device + ) + # The strided destination keeps the einsum reduction while producing the + # layout wo_b consumes; draft warmup/capture can enter with grad enabled. + with torch.no_grad(): + torch.bmm(o.transpose(0, 1), wo_a.transpose(1, 2), out=result.transpose(0, 1)) + return result + + def _apply_wo_a_bf16_matmul( o: torch.Tensor, wo_a: torch.Tensor, @@ -525,16 +537,20 @@ def _apply_wo_a_bf16_matmul( if fuse_mxfp8_quant: return Mxfp8SwizzledInput(*wo_a_bf16_small_batch_mxfp8(o, wo_a)) return wo_a_bf16_small_batch(o, wo_a) - result = torch.empty( - (o.shape[0], wo_a.shape[0], wo_a.shape[1]), dtype=o.dtype, device=o.device - ) - # The strided destination keeps the einsum reduction while producing the - # layout wo_b consumes; draft warmup/capture can enter with grad enabled. - with torch.no_grad(): - torch.bmm( - o.transpose(0, 1), wo_a.transpose(1, 2), out=result.transpose(0, 1) - ) - return result + return _wo_a_bf16_bmm_token_major(o, wo_a) + # The DSpark draft runs its block as target verify past 384 rows and with any + # group count (8 without attention TP); the strided bmm still matches einsum. + if ( + fast_path + and _is_cuda + and is_target_verify + and o.shape[0] > 0 + and get_platform().is_blackwell + and o.dtype == wo_a.dtype == torch.bfloat16 + and o.stride(2) == 1 + and wo_a.is_contiguous() + ): + return _wo_a_bf16_bmm_token_major(o, wo_a) # the 16-row decode kernels also serve V4.1's target-verify rows if (is_decode or is_target_verify) and hip_fast_path: y = _hip.wo_a_fp8_grid_matmul(o, wo_a, fp8_grid, emit_fp8=emit_fp8) @@ -3887,7 +3903,7 @@ class DeepseekV4Model(nn.Module): input_ids: torch.Tensor, input_ids_global: torch.Tensor, capture_dspark: bool, - dspark_aux_hidden_states: List[torch.Tensor], + dspark_aux_hidden_states: Optional[AuxHiddenStatePacker], ) -> Tuple[torch.Tensor, torch.Tensor, Optional[LateLayerTail]]: assert self.pp_group.world_size == 1, "pre-mix hand-off across PP is not wired" hash_ids = None @@ -4035,7 +4051,8 @@ class DeepseekV4Model(nn.Module): ), ) ) - dspark_aux_hidden_states.extend(aux) + if capture_dspark: + dspark_aux_hidden_states.extend(aux) residual_batch.clear(forward_batch) bcg_deepseek_v4_exit_late_layer_tail(tail_sentinel) return residual, pre, tail @@ -4062,7 +4079,8 @@ class DeepseekV4Model(nn.Module): capture_dspark=capture_dspark, dp_rows=tail_graph_dp_rows, ) - dspark_aux_hidden_states.extend(aux) + if capture_dspark: + dspark_aux_hidden_states.extend(aux) residual_batch.clear(forward_batch) attn_backend.exit_late_layer_tail(saved_full, forward_batch) return residual, pre, tail @@ -4076,7 +4094,8 @@ class DeepseekV4Model(nn.Module): capture_dspark=capture_dspark, dp_rows=tail_graph_dp_rows, ) - dspark_aux_hidden_states.extend(aux) + if capture_dspark: + dspark_aux_hidden_states.extend(aux) residual_batch.clear(forward_batch) return residual, pre, None if late_dp_counts is not None and i == self.late_layer_start: @@ -4103,11 +4122,13 @@ class DeepseekV4Model(nn.Module): if _is_cuda: hidden_states = residual_batch.set_written(state, forward_batch) if capture_dspark and i in self.dspark_layers_to_capture: + from sglang.kernels.ops.layernorm.mhc import mhc_stream_mean + # The draft head reads the attention input of its target layers. aux = state.residual if tail is not None and i < self.late_layer_start: aux = tail.rows(aux) - dspark_aux_hidden_states.append(aux.mean(dim=1)) + mhc_stream_mean(aux, out=dspark_aux_hidden_states.slot(aux[:, 0])) del aux ctx = ( nullcontext() @@ -4346,7 +4367,11 @@ class DeepseekV4Model(nn.Module): input_ids_global = getattr(forward_batch, "input_ids_global", input_ids) capture_dspark = self.dspark_layers_to_capture is not None - dspark_aux_hidden_states: List[torch.Tensor] = [] + dspark_aux_hidden_states = ( + AuxHiddenStatePacker(len(self.dspark_layers_to_capture)) + if capture_dspark + else None + ) attn_backend = get_attn_backend() if _is_npu and forward_batch.attn_cp_metadata is not None: @@ -4438,6 +4463,8 @@ class DeepseekV4Model(nn.Module): prev_comb=prev_comb, ) if capture_dspark and i in self.dspark_layers_to_capture: + from sglang.kernels.ops.layernorm.mhc import mhc_stream_mean + if _is_cuda: completed = hc_boundary.materialize( hidden_states, forward_batch, consume=False @@ -4448,7 +4475,9 @@ class DeepseekV4Model(nn.Module): ) else: completed = hidden_states - dspark_aux_hidden_states.append(completed.mean(dim=1)) + mhc_stream_mean( + completed, out=dspark_aux_hidden_states.slot(completed[:, 0]) + ) if _is_cuda: hidden_states = residual_batch.read_output( hidden_states, forward_batch, hc_boundary.FinalRead(), None @@ -4488,7 +4517,7 @@ class DeepseekV4Model(nn.Module): ) if capture_dspark: - return (hidden_states, pre_hc_head), dspark_aux_hidden_states + return (hidden_states, pre_hc_head), dspark_aux_hidden_states.finalize() return hidden_states, pre_hc_head @@ -4843,7 +4872,7 @@ class DeepseekV4ForCausalLM(nn.Module): hidden_states = hidden_states[:n] pre_hc_head = pre_hc_head[:n] if aux_hidden_states is not None: - aux_hidden_states = [a[:n] for a in aux_hidden_states] + aux_hidden_states = aux_hidden_states[:n] logits_metadata = LogitsMetadata.from_forward_batch(forward_batch) logits_metadata.extend_seq_lens = tail.extend_seq_lens logits_metadata.extend_seq_lens_cpu = tail.extend_seq_lens_cpu diff --git a/python/sglang/srt/models/deepseek_v4_dspark.py b/python/sglang/srt/models/deepseek_v4_dspark.py index 18a376dc71..9703af5897 100644 --- a/python/sglang/srt/models/deepseek_v4_dspark.py +++ b/python/sglang/srt/models/deepseek_v4_dspark.py @@ -468,6 +468,17 @@ class DSparkV4MarkovHead(BaseDSparkMarkovHead): "Disable SGLANG_DSPARK_OPT_MARKOV_W2_TP_SHARD." ) self._shard_group = shard_group + self._tp_shard = MarkovW2ShardGeometry( + tp_size=tp_size, + org_vocab_start=int(lm_head.shard_indices.org_vocab_start_index), + org_vocab_end=int(lm_head.shard_indices.org_vocab_end_index), + num_embeddings_per_partition=per_partition, + num_embeddings_padded=num_padded, + ) + # A rank holding the whole unpadded vocab has nothing to gather, so the + # draft step adds the bias, keeps the logits and selects in one pass. + if per_partition == self.vocab_size: + return vocab_gather = make_vocab_gather( shard_group, local_width=per_partition, @@ -477,13 +488,6 @@ class DSparkV4MarkovHead(BaseDSparkMarkovHead): if shard_group.rank == 0: cls_name = type(vocab_gather).__name__ logger.info("DSpark markov_w2 vocab gather: %s", cls_name) - self._tp_shard = MarkovW2ShardGeometry( - tp_size=tp_size, - org_vocab_start=int(lm_head.shard_indices.org_vocab_start_index), - org_vocab_end=int(lm_head.shard_indices.org_vocab_end_index), - num_embeddings_per_partition=per_partition, - num_embeddings_padded=num_padded, - ) self.vocab_shard = VocabShard( start=self._tp_shard.org_vocab_start, gather=vocab_gather ) diff --git a/python/sglang/srt/models/deepseek_v4_replay_tail.py b/python/sglang/srt/models/deepseek_v4_replay_tail.py index df29976a36..bfb2dc8d2e 100644 --- a/python/sglang/srt/models/deepseek_v4_replay_tail.py +++ b/python/sglang/srt/models/deepseek_v4_replay_tail.py @@ -587,8 +587,10 @@ class DecoderReplayTail: hidden_states = state = hc_boundary.materialize( hidden_states, forward_batch, next=next_boundary ) + from sglang.kernels.ops.layernorm.mhc import mhc_stream_mean + # The draft head reads the attention input of its target layers. - aux.append(state.residual.mean(dim=1)) + aux.append(mhc_stream_mean(state.residual)) ctx = ( nullcontext() if check_cuda_graph_backend(Phase.PREFILL, Backend.TC_PIECEWISE) diff --git a/test/registered/kernels/ops/layernorm/test_mhc_stream_mean.py b/test/registered/kernels/ops/layernorm/test_mhc_stream_mean.py new file mode 100644 index 0000000000..1d53b1e4a0 --- /dev/null +++ b/test/registered/kernels/ops/layernorm/test_mhc_stream_mean.py @@ -0,0 +1,62 @@ +import sys + +import pytest +import torch + +from sglang.kernels.ops.layernorm.mhc import mhc_stream_mean +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large") + +pytestmark = pytest.mark.skipif( + not torch.cuda.is_available() or torch.version.cuda is None, + reason="mhc_stream_mean requires CUDA", +) + + +def _assert_bitwise(got: torch.Tensor, ref: torch.Tensor): + assert got.shape == ref.shape and got.dtype == ref.dtype + assert torch.equal(got.view(torch.uint8), ref.view(torch.uint8)) + + +@pytest.mark.parametrize("m", [0, 7, 4097]) +@pytest.mark.parametrize("h", [1024, 4096]) +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float32]) +def test_matches_torch_mean_bitwise(m: int, h: int, dtype: torch.dtype): + # The kernel path relies on torch's CUDA mean adding the 4 streams in order in + # fp32; a change in torch's reduction order turns this red. + torch.manual_seed(0) + x = torch.randn(m, 4, h, device="cuda", dtype=dtype) * 1e3 + _assert_bitwise(mhc_stream_mean(x), x.mean(dim=1)) + + +def test_extreme_values_bitwise(): + torch.manual_seed(0) + x = torch.randn(64, 4, 4096, device="cuda", dtype=torch.bfloat16) + x[0, :, :4] = torch.tensor([float("inf"), float("-inf"), float("nan"), -0.0]) + x[1] = 3e38 + x[2] = 2.0**-60 + _assert_bitwise(mhc_stream_mean(x), x.mean(dim=1)) + + +def test_strided_rows_and_out_slot(): + torch.manual_seed(0) + base = torch.randn(256, 4, 4096 + 128, device="cuda", dtype=torch.bfloat16) + x = base[..., :4096] + packed = torch.empty(256, 3 * 4096, device="cuda", dtype=torch.bfloat16) + slot = packed[:, 4096 : 2 * 4096] + got = mhc_stream_mean(x, out=slot) + assert got.data_ptr() == slot.data_ptr() + _assert_bitwise(slot.clone(), x.mean(dim=1)) + + +def test_out_with_other_dtype_takes_the_fallback(): + torch.manual_seed(0) + x = torch.randn(32, 4, 4096, device="cuda", dtype=torch.bfloat16) + out = torch.empty(32, 4096, device="cuda", dtype=torch.float32) + mhc_stream_mean(x, out=out) + _assert_bitwise(out, x.mean(dim=1).float()) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__])) diff --git a/test/registered/unit/models/test_deepseek_v4_boundary.py b/test/registered/unit/models/test_deepseek_v4_boundary.py index 94fe68b27e..3f277b26eb 100644 --- a/test/registered/unit/models/test_deepseek_v4_boundary.py +++ b/test/registered/unit/models/test_deepseek_v4_boundary.py @@ -20,6 +20,7 @@ register_cpu_ci(est_time=5, suite="base-a-test-cpu") class TestHcConsumer(unittest.TestCase): def test_dspark_tap_hands_materialized_stream_to_next_layer(self): + from sglang.srt.layers.aux_hidden_states import AuxHiddenStatePacker from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.models import deepseek_v4 @@ -58,7 +59,7 @@ class TestHcConsumer(unittest.TestCase): residual_stream=None, ) source = torch.arange(24.0).reshape(3, 2, 4) - aux = [] + aux = AuxHiddenStatePacker(1) recorder = SimpleNamespace(with_current_layer=lambda _: nullcontext()) with ( patch.object(deepseek_v4, "_is_cuda", True), @@ -75,7 +76,7 @@ class TestHcConsumer(unittest.TestCase): positions, source, batch, positions, positions, True, aux ) torch.testing.assert_close(output, source + 3) - torch.testing.assert_close(aux[0], (source + 1).mean(1)) + torch.testing.assert_close(aux.finalize(), (source + 1).mean(1)) torch.testing.assert_close(pre, torch.ones(3, 2)) self.assertIsNone(tail) self.assertIsNone(batch.residual_stream)