[Perf] dsv41: Trim per-step DSpark decode overhead (#46176)
This commit is contained in:
@@ -2185,6 +2185,32 @@ def _mhc_combine_kernel(
|
||||
tl.store(y_ptr + pid_m * y_stride_m + offs_h, acc, mask=mask)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _mhc_stream_mean_kernel(
|
||||
x_ptr,
|
||||
y_ptr,
|
||||
H,
|
||||
x_stride_m,
|
||||
x_stride_k,
|
||||
y_stride_m,
|
||||
HC: tl.constexpr,
|
||||
BLOCK_H: tl.constexpr,
|
||||
):
|
||||
pid_m = tl.program_id(0)
|
||||
pid_h = tl.program_id(1)
|
||||
offs_h = pid_h * BLOCK_H + tl.arange(0, BLOCK_H)
|
||||
mask = offs_h < H
|
||||
x_row = x_ptr + pid_m.to(tl.int64) * x_stride_m + offs_h
|
||||
# Streams are added in order onto a +0 fp32 accumulator, as torch.mean's
|
||||
# reduction does (so an all -0 row averages to +0).
|
||||
acc = tl.zeros([BLOCK_H], dtype=tl.float32)
|
||||
for k in tl.static_range(HC):
|
||||
acc += tl.load(x_row + k * x_stride_k, mask=mask, other=0.0).to(tl.float32)
|
||||
tl.store(
|
||||
y_ptr + pid_m.to(tl.int64) * y_stride_m + offs_h, acc * (1.0 / HC), mask=mask
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _hc_mix_stats_partial_kernel(
|
||||
x_ptr,
|
||||
@@ -2736,3 +2762,34 @@ def mhc_combine(
|
||||
BLOCK_H=block_h,
|
||||
)
|
||||
return y
|
||||
|
||||
|
||||
def mhc_stream_mean(x: torch.Tensor, out: torch.Tensor | None = None) -> torch.Tensor:
|
||||
"""y[m, h] = x[m, :, h].mean(), bitwise equal to ``x.mean(dim=1)``, written
|
||||
into ``out`` when given."""
|
||||
m, hc, h = x.shape
|
||||
# Bitwise equality with torch's CUDA mean holds for 4 streams (the mHC width)
|
||||
# over whole 1024-element blocks; torch orders some other widths differently.
|
||||
if (
|
||||
_is_hip
|
||||
or x.device.type != "cuda"
|
||||
or hc != 4
|
||||
or h % 1024
|
||||
or x.stride(2) != 1
|
||||
or (out is not None and (out.dtype != x.dtype or out.stride(1) != 1))
|
||||
):
|
||||
y = x.mean(dim=1)
|
||||
return y if out is None else out.copy_(y)
|
||||
y = torch.empty((m, h), dtype=x.dtype, device=x.device) if out is None else out
|
||||
block_h = 1024
|
||||
_mhc_stream_mean_kernel[(m, triton.cdiv(h, block_h))](
|
||||
x,
|
||||
y,
|
||||
h,
|
||||
x.stride(0),
|
||||
x.stride(1),
|
||||
y.stride(0),
|
||||
HC=hc,
|
||||
BLOCK_H=block_h,
|
||||
)
|
||||
return y
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, List, Optional, Union
|
||||
from typing import TYPE_CHECKING, Iterable, List, Optional, Union
|
||||
|
||||
import torch
|
||||
|
||||
@@ -49,21 +49,30 @@ class AuxHiddenStatePacker:
|
||||
) -> AuxHiddenStatePacker:
|
||||
return cls(num_captures, out=forward_batch.aux_hidden_states_buffer)
|
||||
|
||||
def append(self, hidden: torch.Tensor) -> None:
|
||||
feature_size = int(hidden.shape[-1])
|
||||
def slot(self, like: torch.Tensor) -> torch.Tensor:
|
||||
"""Take the next capture's view of the packed buffer, shaped like ``like``,
|
||||
for a producer that writes it in place."""
|
||||
feature_size = int(like.shape[-1])
|
||||
if self._feature_size is None:
|
||||
self._feature_size = feature_size
|
||||
shape = (*hidden.shape[:-1], feature_size * self._num_captures)
|
||||
shape = (*like.shape[:-1], feature_size * self._num_captures)
|
||||
if self._buffer is None:
|
||||
self._buffer = hidden.new_empty(shape)
|
||||
elif self._buffer.shape != shape or self._buffer.dtype != hidden.dtype:
|
||||
self._buffer = like.new_empty(shape)
|
||||
elif self._buffer.shape != shape or self._buffer.dtype != like.dtype:
|
||||
raise ValueError(
|
||||
f"aux hidden output buffer is {tuple(self._buffer.shape)} "
|
||||
f"{self._buffer.dtype}, captures need {shape} {hidden.dtype}"
|
||||
f"{self._buffer.dtype}, captures need {shape} {like.dtype}"
|
||||
)
|
||||
start = self._idx * self._feature_size
|
||||
self._buffer[..., start : start + self._feature_size].copy_(hidden)
|
||||
self._idx += 1
|
||||
return self._buffer[..., start : start + self._feature_size]
|
||||
|
||||
def append(self, hidden: torch.Tensor) -> None:
|
||||
self.slot(hidden).copy_(hidden)
|
||||
|
||||
def extend(self, hiddens: Iterable[torch.Tensor]) -> None:
|
||||
for hidden in hiddens:
|
||||
self.append(hidden)
|
||||
|
||||
def capture(self, hidden: torch.Tensor, *, owned: bool = False) -> None:
|
||||
"""Write directly to the final packed buffer, without an intermediate copy."""
|
||||
|
||||
@@ -72,6 +72,7 @@ from sglang.srt.layers.attention.dsv4.dsv41_sparse import (
|
||||
DeepseekV41Indexer,
|
||||
)
|
||||
from sglang.srt.layers.attention.dsv4.indexer import C4Indexer
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStatePacker
|
||||
from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx
|
||||
from sglang.srt.layers.cp.utils import (
|
||||
cp_gather_full_sequence_states,
|
||||
@@ -461,6 +462,17 @@ def _fused_wo_a_arch_supported() -> bool:
|
||||
return _is_cuda and torch.cuda.get_device_capability()[0] == 10
|
||||
|
||||
|
||||
def _wo_a_bf16_bmm_token_major(o: torch.Tensor, wo_a: torch.Tensor) -> torch.Tensor:
|
||||
result = torch.empty(
|
||||
(o.shape[0], wo_a.shape[0], wo_a.shape[1]), dtype=o.dtype, device=o.device
|
||||
)
|
||||
# The strided destination keeps the einsum reduction while producing the
|
||||
# layout wo_b consumes; draft warmup/capture can enter with grad enabled.
|
||||
with torch.no_grad():
|
||||
torch.bmm(o.transpose(0, 1), wo_a.transpose(1, 2), out=result.transpose(0, 1))
|
||||
return result
|
||||
|
||||
|
||||
def _apply_wo_a_bf16_matmul(
|
||||
o: torch.Tensor,
|
||||
wo_a: torch.Tensor,
|
||||
@@ -525,16 +537,20 @@ def _apply_wo_a_bf16_matmul(
|
||||
if fuse_mxfp8_quant:
|
||||
return Mxfp8SwizzledInput(*wo_a_bf16_small_batch_mxfp8(o, wo_a))
|
||||
return wo_a_bf16_small_batch(o, wo_a)
|
||||
result = torch.empty(
|
||||
(o.shape[0], wo_a.shape[0], wo_a.shape[1]), dtype=o.dtype, device=o.device
|
||||
)
|
||||
# The strided destination keeps the einsum reduction while producing the
|
||||
# layout wo_b consumes; draft warmup/capture can enter with grad enabled.
|
||||
with torch.no_grad():
|
||||
torch.bmm(
|
||||
o.transpose(0, 1), wo_a.transpose(1, 2), out=result.transpose(0, 1)
|
||||
)
|
||||
return result
|
||||
return _wo_a_bf16_bmm_token_major(o, wo_a)
|
||||
# The DSpark draft runs its block as target verify past 384 rows and with any
|
||||
# group count (8 without attention TP); the strided bmm still matches einsum.
|
||||
if (
|
||||
fast_path
|
||||
and _is_cuda
|
||||
and is_target_verify
|
||||
and o.shape[0] > 0
|
||||
and get_platform().is_blackwell
|
||||
and o.dtype == wo_a.dtype == torch.bfloat16
|
||||
and o.stride(2) == 1
|
||||
and wo_a.is_contiguous()
|
||||
):
|
||||
return _wo_a_bf16_bmm_token_major(o, wo_a)
|
||||
# the 16-row decode kernels also serve V4.1's target-verify rows
|
||||
if (is_decode or is_target_verify) and hip_fast_path:
|
||||
y = _hip.wo_a_fp8_grid_matmul(o, wo_a, fp8_grid, emit_fp8=emit_fp8)
|
||||
@@ -3887,7 +3903,7 @@ class DeepseekV4Model(nn.Module):
|
||||
input_ids: torch.Tensor,
|
||||
input_ids_global: torch.Tensor,
|
||||
capture_dspark: bool,
|
||||
dspark_aux_hidden_states: List[torch.Tensor],
|
||||
dspark_aux_hidden_states: Optional[AuxHiddenStatePacker],
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, Optional[LateLayerTail]]:
|
||||
assert self.pp_group.world_size == 1, "pre-mix hand-off across PP is not wired"
|
||||
hash_ids = None
|
||||
@@ -4035,7 +4051,8 @@ class DeepseekV4Model(nn.Module):
|
||||
),
|
||||
)
|
||||
)
|
||||
dspark_aux_hidden_states.extend(aux)
|
||||
if capture_dspark:
|
||||
dspark_aux_hidden_states.extend(aux)
|
||||
residual_batch.clear(forward_batch)
|
||||
bcg_deepseek_v4_exit_late_layer_tail(tail_sentinel)
|
||||
return residual, pre, tail
|
||||
@@ -4062,7 +4079,8 @@ class DeepseekV4Model(nn.Module):
|
||||
capture_dspark=capture_dspark,
|
||||
dp_rows=tail_graph_dp_rows,
|
||||
)
|
||||
dspark_aux_hidden_states.extend(aux)
|
||||
if capture_dspark:
|
||||
dspark_aux_hidden_states.extend(aux)
|
||||
residual_batch.clear(forward_batch)
|
||||
attn_backend.exit_late_layer_tail(saved_full, forward_batch)
|
||||
return residual, pre, tail
|
||||
@@ -4076,7 +4094,8 @@ class DeepseekV4Model(nn.Module):
|
||||
capture_dspark=capture_dspark,
|
||||
dp_rows=tail_graph_dp_rows,
|
||||
)
|
||||
dspark_aux_hidden_states.extend(aux)
|
||||
if capture_dspark:
|
||||
dspark_aux_hidden_states.extend(aux)
|
||||
residual_batch.clear(forward_batch)
|
||||
return residual, pre, None
|
||||
if late_dp_counts is not None and i == self.late_layer_start:
|
||||
@@ -4103,11 +4122,13 @@ class DeepseekV4Model(nn.Module):
|
||||
if _is_cuda:
|
||||
hidden_states = residual_batch.set_written(state, forward_batch)
|
||||
if capture_dspark and i in self.dspark_layers_to_capture:
|
||||
from sglang.kernels.ops.layernorm.mhc import mhc_stream_mean
|
||||
|
||||
# The draft head reads the attention input of its target layers.
|
||||
aux = state.residual
|
||||
if tail is not None and i < self.late_layer_start:
|
||||
aux = tail.rows(aux)
|
||||
dspark_aux_hidden_states.append(aux.mean(dim=1))
|
||||
mhc_stream_mean(aux, out=dspark_aux_hidden_states.slot(aux[:, 0]))
|
||||
del aux
|
||||
ctx = (
|
||||
nullcontext()
|
||||
@@ -4346,7 +4367,11 @@ class DeepseekV4Model(nn.Module):
|
||||
input_ids_global = getattr(forward_batch, "input_ids_global", input_ids)
|
||||
|
||||
capture_dspark = self.dspark_layers_to_capture is not None
|
||||
dspark_aux_hidden_states: List[torch.Tensor] = []
|
||||
dspark_aux_hidden_states = (
|
||||
AuxHiddenStatePacker(len(self.dspark_layers_to_capture))
|
||||
if capture_dspark
|
||||
else None
|
||||
)
|
||||
|
||||
attn_backend = get_attn_backend()
|
||||
if _is_npu and forward_batch.attn_cp_metadata is not None:
|
||||
@@ -4438,6 +4463,8 @@ class DeepseekV4Model(nn.Module):
|
||||
prev_comb=prev_comb,
|
||||
)
|
||||
if capture_dspark and i in self.dspark_layers_to_capture:
|
||||
from sglang.kernels.ops.layernorm.mhc import mhc_stream_mean
|
||||
|
||||
if _is_cuda:
|
||||
completed = hc_boundary.materialize(
|
||||
hidden_states, forward_batch, consume=False
|
||||
@@ -4448,7 +4475,9 @@ class DeepseekV4Model(nn.Module):
|
||||
)
|
||||
else:
|
||||
completed = hidden_states
|
||||
dspark_aux_hidden_states.append(completed.mean(dim=1))
|
||||
mhc_stream_mean(
|
||||
completed, out=dspark_aux_hidden_states.slot(completed[:, 0])
|
||||
)
|
||||
if _is_cuda:
|
||||
hidden_states = residual_batch.read_output(
|
||||
hidden_states, forward_batch, hc_boundary.FinalRead(), None
|
||||
@@ -4488,7 +4517,7 @@ class DeepseekV4Model(nn.Module):
|
||||
)
|
||||
|
||||
if capture_dspark:
|
||||
return (hidden_states, pre_hc_head), dspark_aux_hidden_states
|
||||
return (hidden_states, pre_hc_head), dspark_aux_hidden_states.finalize()
|
||||
|
||||
return hidden_states, pre_hc_head
|
||||
|
||||
@@ -4843,7 +4872,7 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
hidden_states = hidden_states[:n]
|
||||
pre_hc_head = pre_hc_head[:n]
|
||||
if aux_hidden_states is not None:
|
||||
aux_hidden_states = [a[:n] for a in aux_hidden_states]
|
||||
aux_hidden_states = aux_hidden_states[:n]
|
||||
logits_metadata = LogitsMetadata.from_forward_batch(forward_batch)
|
||||
logits_metadata.extend_seq_lens = tail.extend_seq_lens
|
||||
logits_metadata.extend_seq_lens_cpu = tail.extend_seq_lens_cpu
|
||||
|
||||
@@ -468,6 +468,17 @@ class DSparkV4MarkovHead(BaseDSparkMarkovHead):
|
||||
"Disable SGLANG_DSPARK_OPT_MARKOV_W2_TP_SHARD."
|
||||
)
|
||||
self._shard_group = shard_group
|
||||
self._tp_shard = MarkovW2ShardGeometry(
|
||||
tp_size=tp_size,
|
||||
org_vocab_start=int(lm_head.shard_indices.org_vocab_start_index),
|
||||
org_vocab_end=int(lm_head.shard_indices.org_vocab_end_index),
|
||||
num_embeddings_per_partition=per_partition,
|
||||
num_embeddings_padded=num_padded,
|
||||
)
|
||||
# A rank holding the whole unpadded vocab has nothing to gather, so the
|
||||
# draft step adds the bias, keeps the logits and selects in one pass.
|
||||
if per_partition == self.vocab_size:
|
||||
return
|
||||
vocab_gather = make_vocab_gather(
|
||||
shard_group,
|
||||
local_width=per_partition,
|
||||
@@ -477,13 +488,6 @@ class DSparkV4MarkovHead(BaseDSparkMarkovHead):
|
||||
if shard_group.rank == 0:
|
||||
cls_name = type(vocab_gather).__name__
|
||||
logger.info("DSpark markov_w2 vocab gather: %s", cls_name)
|
||||
self._tp_shard = MarkovW2ShardGeometry(
|
||||
tp_size=tp_size,
|
||||
org_vocab_start=int(lm_head.shard_indices.org_vocab_start_index),
|
||||
org_vocab_end=int(lm_head.shard_indices.org_vocab_end_index),
|
||||
num_embeddings_per_partition=per_partition,
|
||||
num_embeddings_padded=num_padded,
|
||||
)
|
||||
self.vocab_shard = VocabShard(
|
||||
start=self._tp_shard.org_vocab_start, gather=vocab_gather
|
||||
)
|
||||
|
||||
@@ -587,8 +587,10 @@ class DecoderReplayTail:
|
||||
hidden_states = state = hc_boundary.materialize(
|
||||
hidden_states, forward_batch, next=next_boundary
|
||||
)
|
||||
from sglang.kernels.ops.layernorm.mhc import mhc_stream_mean
|
||||
|
||||
# The draft head reads the attention input of its target layers.
|
||||
aux.append(state.residual.mean(dim=1))
|
||||
aux.append(mhc_stream_mean(state.residual))
|
||||
ctx = (
|
||||
nullcontext()
|
||||
if check_cuda_graph_backend(Phase.PREFILL, Backend.TC_PIECEWISE)
|
||||
|
||||
Reference in New Issue
Block a user