[Spec][PP] Fix hybrid recurrent-state commit and micro-batch pairing under PP x speculative decoding (#40001)
This commit is contained in:
@@ -225,6 +225,7 @@ def commit_gdn_replayssm_fold_after_verify(
|
||||
last_correct_step_indices: torch.Tensor, # [B] conv rollback target step
|
||||
mamba_track_indices: torch.Tensor | None = None,
|
||||
mamba_steps_to_track: torch.Tensor | None = None,
|
||||
src_indices_raw: torch.Tensor | None = None,
|
||||
null_block_id: int = -1,
|
||||
) -> None:
|
||||
"""Fold each layer's accepted window into ``temporal``, then do the usual
|
||||
@@ -253,9 +254,17 @@ def commit_gdn_replayssm_fold_after_verify(
|
||||
spec_state.conv, spec_state.intermediate_conv_window
|
||||
):
|
||||
fused_conv_window_scatter_with_mask(
|
||||
conv_states, interm_conv, state_batch_indices, last_correct_step_indices
|
||||
conv_states,
|
||||
interm_conv,
|
||||
state_batch_indices,
|
||||
last_correct_step_indices,
|
||||
src_indices_raw,
|
||||
)
|
||||
if mamba_track_indices is not None and mamba_steps_to_track is not None:
|
||||
fused_conv_window_scatter_with_mask(
|
||||
conv_states, interm_conv, mamba_track_indices, mamba_steps_to_track
|
||||
conv_states,
|
||||
interm_conv,
|
||||
mamba_track_indices,
|
||||
mamba_steps_to_track,
|
||||
src_indices_raw,
|
||||
)
|
||||
|
||||
@@ -341,6 +341,7 @@ def commit_kda_replayssm_after_verify(
|
||||
last_correct_step_indices: torch.Tensor, # [B] conv rollback target step
|
||||
mamba_track_indices: torch.Tensor | None = None,
|
||||
mamba_steps_to_track: torch.Tensor | None = None,
|
||||
src_indices_raw: torch.Tensor | None = None,
|
||||
null_block_id: int = -1,
|
||||
) -> None:
|
||||
"""Fold each layer's accepted window into `temporal` and roll back conv.
|
||||
@@ -378,9 +379,17 @@ def commit_kda_replayssm_after_verify(
|
||||
spec_state.conv, spec_state.intermediate_conv_window
|
||||
):
|
||||
fused_conv_window_scatter_with_mask(
|
||||
conv_states, interm_conv, state_batch_indices, last_correct_step_indices
|
||||
conv_states,
|
||||
interm_conv,
|
||||
state_batch_indices,
|
||||
last_correct_step_indices,
|
||||
src_indices_raw,
|
||||
)
|
||||
if mamba_track_indices is not None and mamba_steps_to_track is not None:
|
||||
fused_conv_window_scatter_with_mask(
|
||||
conv_states, interm_conv, mamba_track_indices, mamba_steps_to_track
|
||||
conv_states,
|
||||
interm_conv,
|
||||
mamba_track_indices,
|
||||
mamba_steps_to_track,
|
||||
src_indices_raw,
|
||||
)
|
||||
|
||||
@@ -153,6 +153,7 @@ def _fused_mamba_state_scatter_with_mask_kernel(
|
||||
# Raw index arrays (before index_select)
|
||||
dst_indices_raw_ptr, # [total_requests] - state_indices_tensor
|
||||
step_indices_raw_ptr, # [total_requests] - last_correct_step_indices or mamba_steps_to_track
|
||||
src_indices_raw_ptr, # [total_requests] - verify scratch row for each request
|
||||
elem_per_entry: tl.constexpr,
|
||||
src_layer_stride,
|
||||
src_req_stride,
|
||||
@@ -189,8 +190,7 @@ def _fused_mamba_state_scatter_with_mask_kernel(
|
||||
# Load destination index
|
||||
dst_idx = tl.load(dst_indices_raw_ptr + pid_req).to(tl.int64)
|
||||
|
||||
# Source index is just the request index itself
|
||||
src_idx = pid_req
|
||||
src_idx = tl.load(src_indices_raw_ptr + pid_req).to(tl.int64)
|
||||
|
||||
# Bounds check to avoid illegal memory access
|
||||
if not (
|
||||
@@ -220,11 +220,22 @@ def _fused_mamba_state_scatter_with_mask_kernel(
|
||||
tl.store(dst_ptr + dst_offset + offsets, data, mask=mask)
|
||||
|
||||
|
||||
def _prepare_src_rows(src: torch.Tensor | None, steps: torch.Tensor) -> torch.Tensor:
|
||||
if src is None:
|
||||
src = torch.arange(steps.shape[0], dtype=torch.int32, device=steps.device)
|
||||
if src.ndim != 1 or src.shape != steps.shape:
|
||||
raise ValueError(
|
||||
f"source indices length mismatch: {src.shape=} vs {steps.shape=}"
|
||||
)
|
||||
return src.to(torch.int32).contiguous()
|
||||
|
||||
|
||||
def fused_mamba_state_scatter_with_mask(
|
||||
dst: torch.Tensor, # [num_layers, cache_size, *state_shape]
|
||||
src: torch.Tensor, # [num_layers, spec_size, draft_tokens, *state_shape]
|
||||
dst_indices_raw: torch.Tensor, # [total_requests] - raw indices (e.g., state_indices_tensor)
|
||||
step_indices_raw: torch.Tensor, # [total_requests] - raw step indices (step >= 0 means valid)
|
||||
src_indices_raw: torch.Tensor | None = None, # [total_requests] - scratch rows
|
||||
):
|
||||
"""
|
||||
Fully fused gather-scatter with built-in masking for mamba state updates.
|
||||
@@ -291,6 +302,7 @@ def fused_mamba_state_scatter_with_mask(
|
||||
# Ensure indices are int32 and contiguous
|
||||
dst_indices_raw = dst_indices_raw.to(torch.int32).contiguous()
|
||||
step_indices_raw = step_indices_raw.to(torch.int32).contiguous()
|
||||
src_indices_raw = _prepare_src_rows(src_indices_raw, step_indices_raw)
|
||||
|
||||
_require_entry_contiguous_dst(dst, 2, "fused_mamba_state_scatter_with_mask")
|
||||
if not src.is_contiguous():
|
||||
@@ -307,6 +319,7 @@ def fused_mamba_state_scatter_with_mask(
|
||||
dst,
|
||||
dst_indices_raw,
|
||||
step_indices_raw,
|
||||
src_indices_raw,
|
||||
elem_per_entry,
|
||||
src_layer_stride,
|
||||
src_req_stride,
|
||||
@@ -326,6 +339,7 @@ def _fused_conv_window_scatter_with_mask_kernel(
|
||||
dst_ptr,
|
||||
dst_indices_raw_ptr, # [total_requests]
|
||||
step_indices_raw_ptr, # [total_requests], entry >= 0 means valid
|
||||
src_indices_raw_ptr, # [total_requests]
|
||||
elem_per_entry: tl.constexpr, # dim * (K-1)
|
||||
KM1: tl.constexpr, # K-1 (conv window width)
|
||||
src_layer_stride,
|
||||
@@ -359,7 +373,7 @@ def _fused_conv_window_scatter_with_mask_kernel(
|
||||
return
|
||||
|
||||
dst_idx = tl.load(dst_indices_raw_ptr + pid_req).to(tl.int64)
|
||||
src_idx = pid_req
|
||||
src_idx = tl.load(src_indices_raw_ptr + pid_req).to(tl.int64)
|
||||
|
||||
if not (
|
||||
(dst_idx >= 0)
|
||||
@@ -396,6 +410,7 @@ def fused_conv_window_scatter_with_mask(
|
||||
src: torch.Tensor, # deduped conv-window view [num_layers, spec_size, draft_tokens, dim, K-1]
|
||||
dst_indices_raw: torch.Tensor, # [total_requests]
|
||||
step_indices_raw: torch.Tensor, # [total_requests], entry >= 0 means valid
|
||||
src_indices_raw: torch.Tensor | None = None, # [total_requests]
|
||||
):
|
||||
"""Conv-window variant of :func:`fused_mamba_state_scatter_with_mask`.
|
||||
|
||||
@@ -446,6 +461,7 @@ def fused_conv_window_scatter_with_mask(
|
||||
|
||||
dst_indices_raw = dst_indices_raw.to(torch.int32).contiguous()
|
||||
step_indices_raw = step_indices_raw.to(torch.int32).contiguous()
|
||||
src_indices_raw = _prepare_src_rows(src_indices_raw, step_indices_raw)
|
||||
|
||||
BLOCK_SIZE = 1024
|
||||
grid = (total_requests, num_layers, triton.cdiv(elem_per_entry, BLOCK_SIZE))
|
||||
@@ -455,6 +471,7 @@ def fused_conv_window_scatter_with_mask(
|
||||
dst,
|
||||
dst_indices_raw,
|
||||
step_indices_raw,
|
||||
src_indices_raw,
|
||||
elem_per_entry,
|
||||
km1,
|
||||
src.stride(0),
|
||||
@@ -681,6 +698,7 @@ def scatter_mamba_states_after_mtp_verify(
|
||||
last_correct_step_indices: torch.Tensor,
|
||||
mamba_track_indices: torch.Tensor | None,
|
||||
mamba_steps_to_track: torch.Tensor | None,
|
||||
src_indices_raw: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""Scatter per-step verify states (ssm + all conv types) into the
|
||||
persistent caches, plus the interval-crossing track slots."""
|
||||
@@ -693,6 +711,7 @@ def scatter_mamba_states_after_mtp_verify(
|
||||
intermediate_state_cache,
|
||||
state_indices_tensor,
|
||||
last_correct_step_indices,
|
||||
src_indices_raw,
|
||||
)
|
||||
if mamba_track_indices is not None:
|
||||
assert mamba_steps_to_track is not None
|
||||
@@ -701,6 +720,7 @@ def scatter_mamba_states_after_mtp_verify(
|
||||
intermediate_state_cache,
|
||||
mamba_track_indices,
|
||||
mamba_steps_to_track,
|
||||
src_indices_raw,
|
||||
)
|
||||
|
||||
pairs = list(zip(mamba_caches.conv, mamba_caches.intermediate_conv_window))
|
||||
@@ -708,7 +728,7 @@ def scatter_mamba_states_after_mtp_verify(
|
||||
return
|
||||
if mamba_track_indices is not None:
|
||||
assert mamba_steps_to_track is not None
|
||||
if _conv_multi_eligible(pairs):
|
||||
if src_indices_raw is None and _conv_multi_eligible(pairs):
|
||||
fused_conv_window_scatter_multi(
|
||||
pairs,
|
||||
state_indices_tensor,
|
||||
@@ -723,6 +743,7 @@ def scatter_mamba_states_after_mtp_verify(
|
||||
intermediate_conv_window_cache,
|
||||
state_indices_tensor,
|
||||
last_correct_step_indices,
|
||||
src_indices_raw,
|
||||
)
|
||||
if mamba_track_indices is not None:
|
||||
for conv_states, intermediate_conv_window_cache in pairs:
|
||||
@@ -731,6 +752,7 @@ def scatter_mamba_states_after_mtp_verify(
|
||||
intermediate_conv_window_cache,
|
||||
mamba_track_indices,
|
||||
mamba_steps_to_track,
|
||||
src_indices_raw,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -27,8 +27,10 @@ from sglang.srt.arg_groups.overrides import (
|
||||
resolving_view,
|
||||
run_post_process_pass,
|
||||
)
|
||||
from sglang.srt.configs.model_config import uses_kda_attention
|
||||
from sglang.srt.connector import ConnectorType
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.attention.linear.utils import pp_spec_stable_rows_enabled
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
|
||||
from sglang.srt.runtime_context import get_platform
|
||||
from sglang.srt.utils.common import (
|
||||
@@ -275,6 +277,23 @@ def handle_linear_attn_backend(server_args: Any):
|
||||
verify = cfg.linear_attn_verify_backend
|
||||
if verify is None and decode == "flashinfer":
|
||||
verify = "flashinfer"
|
||||
if (
|
||||
pp_spec_stable_rows_enabled()
|
||||
and verify == "flashinfer"
|
||||
and uses_kda_attention(model_config_of(server_args).hf_config)
|
||||
):
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"_handle_linear_attn_backend",
|
||||
linear_attn_verify_backend="triton",
|
||||
)
|
||||
verify = "triton"
|
||||
logger.warning(
|
||||
"SGLANG_ENABLE_PP_SPEC with KDA does not support the FlashInfer "
|
||||
"target-verify row layout; falling back "
|
||||
"--linear-attn-verify-backend to triton. FlashInfer decode is "
|
||||
"unchanged."
|
||||
)
|
||||
if (
|
||||
verify == "flashinfer"
|
||||
and cfg.mamba_ssm_dtype != "bfloat16"
|
||||
|
||||
@@ -21,6 +21,7 @@ from sglang.srt.layers.attention.base_attn_backend import (
|
||||
AttentionBackend,
|
||||
SharedReadEnds,
|
||||
)
|
||||
from sglang.srt.layers.attention.linear.utils import pp_spec_stable_rows_enabled
|
||||
from sglang.srt.layers.attention.mamba.mamba import MambaMixer2
|
||||
from sglang.srt.layers.attention.mamba.mamba2_metadata import (
|
||||
ForwardMetadata,
|
||||
@@ -1425,14 +1426,11 @@ class HybridLinearAttnBackend(AttentionBackend):
|
||||
model,
|
||||
req_pool_indices: Optional[torch.Tensor] = None,
|
||||
):
|
||||
"""Update mamba states after MTP verify via a fused gather-scatter kernel.
|
||||
|
||||
``req_pool_indices`` serves implementations that must re-derive the state
|
||||
slot ids instead of reusing this step's ``forward_metadata``; the scatter
|
||||
below reads the metadata it just planned.
|
||||
"""
|
||||
del req_pool_indices
|
||||
"""Commit accepted Mamba states; PP rows identify the delayed batch."""
|
||||
request_number = last_correct_step_indices.shape[0]
|
||||
src_indices_raw = None
|
||||
if pp_spec_stable_rows_enabled() and req_pool_indices is not None:
|
||||
src_indices_raw = req_pool_indices[:request_number]
|
||||
|
||||
# `mamba_track_indices` is VIRTUAL; the scatter writes physical views.
|
||||
if mamba_track_indices is not None:
|
||||
@@ -1440,11 +1438,18 @@ class HybridLinearAttnBackend(AttentionBackend):
|
||||
mamba_track_indices
|
||||
)
|
||||
|
||||
state_indices_tensor = (
|
||||
self.linear_attn_backend.forward_metadata.mamba_cache_indices[
|
||||
:request_number
|
||||
]
|
||||
)
|
||||
if src_indices_raw is not None:
|
||||
state_indices_tensor = self.linear_attn_backend._translate_mamba_indices(
|
||||
self.linear_attn_backend.req_to_token_pool.get_mamba_indices(
|
||||
req_pool_indices[:request_number]
|
||||
)
|
||||
)
|
||||
else:
|
||||
state_indices_tensor = (
|
||||
self.linear_attn_backend.forward_metadata.mamba_cache_indices[
|
||||
:request_number
|
||||
]
|
||||
)
|
||||
|
||||
req_pool = self.linear_attn_backend.req_to_token_pool
|
||||
mamba_caches = req_pool.get_speculative_mamba2_params_all_layers()
|
||||
@@ -1469,6 +1474,7 @@ class HybridLinearAttnBackend(AttentionBackend):
|
||||
last_correct_step_indices=last_correct_step_indices,
|
||||
mamba_track_indices=mamba_track_indices,
|
||||
mamba_steps_to_track=mamba_steps_to_track,
|
||||
src_indices_raw=src_indices_raw,
|
||||
null_block_id=-1,
|
||||
)
|
||||
return
|
||||
@@ -1492,6 +1498,7 @@ class HybridLinearAttnBackend(AttentionBackend):
|
||||
intermediate_conv_window,
|
||||
state_indices_tensor,
|
||||
last_correct_step_indices,
|
||||
src_indices_raw,
|
||||
)
|
||||
accept_lens_pool[state_indices_tensor.to(torch.int64)] = (
|
||||
last_correct_step_indices.to(torch.int32) + 1
|
||||
@@ -1504,6 +1511,7 @@ class HybridLinearAttnBackend(AttentionBackend):
|
||||
last_correct_step_indices,
|
||||
mamba_track_indices,
|
||||
mamba_steps_to_track,
|
||||
src_indices_raw=src_indices_raw,
|
||||
)
|
||||
|
||||
self._update_ple_state_after_mtp_verify(
|
||||
@@ -1511,6 +1519,7 @@ class HybridLinearAttnBackend(AttentionBackend):
|
||||
last_correct_step_indices,
|
||||
mamba_track_indices,
|
||||
mamba_steps_to_track,
|
||||
src_indices_raw,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -1519,19 +1528,28 @@ class HybridLinearAttnBackend(AttentionBackend):
|
||||
src: torch.Tensor,
|
||||
dst_indices_raw: torch.Tensor,
|
||||
step_indices_raw: torch.Tensor,
|
||||
src_indices_raw: Optional[torch.Tensor] = None,
|
||||
):
|
||||
if dst is None or src is None or step_indices_raw.numel() == 0:
|
||||
return
|
||||
if dst.is_cuda and src.is_cuda:
|
||||
fused_mamba_state_scatter_with_mask(
|
||||
dst, src, dst_indices_raw, step_indices_raw
|
||||
dst,
|
||||
src,
|
||||
dst_indices_raw,
|
||||
step_indices_raw,
|
||||
src_indices_raw,
|
||||
)
|
||||
return
|
||||
|
||||
device = dst.device
|
||||
dst_indices = dst_indices_raw.to(device=device, dtype=torch.long)
|
||||
steps = step_indices_raw.to(device=device, dtype=torch.long)
|
||||
src_indices = torch.arange(steps.shape[0], device=device, dtype=torch.long)
|
||||
src_indices = (
|
||||
torch.arange(steps.shape[0], device=device, dtype=torch.long)
|
||||
if src_indices_raw is None
|
||||
else src_indices_raw.to(device=device, dtype=torch.long)
|
||||
)
|
||||
valid = (
|
||||
(steps >= 0)
|
||||
& (steps < src.shape[2])
|
||||
@@ -1552,6 +1570,7 @@ class HybridLinearAttnBackend(AttentionBackend):
|
||||
last_correct_step_indices: torch.Tensor,
|
||||
mamba_track_indices: Optional[torch.Tensor],
|
||||
mamba_steps_to_track: Optional[torch.Tensor],
|
||||
src_indices_raw: Optional[torch.Tensor] = None,
|
||||
):
|
||||
"""Roll the accepted per-step PLE side states into their main slots."""
|
||||
req_to_token_pool = self.linear_attn_backend.req_to_token_pool
|
||||
@@ -1589,6 +1608,7 @@ class HybridLinearAttnBackend(AttentionBackend):
|
||||
intermediate_state,
|
||||
state_indices_tensor,
|
||||
last_correct_step_indices,
|
||||
src_indices_raw,
|
||||
)
|
||||
if mamba_track_indices is not None:
|
||||
self._scatter_speculative_state_with_mask(
|
||||
@@ -1596,6 +1616,7 @@ class HybridLinearAttnBackend(AttentionBackend):
|
||||
intermediate_state,
|
||||
mamba_track_indices,
|
||||
mamba_steps_to_track,
|
||||
src_indices_raw,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -16,6 +16,8 @@ from sglang.srt.layers.attention.linear.utils import (
|
||||
LinearAttnBackends,
|
||||
LinearAttnKernelBackend,
|
||||
build_verify_intermediate_state_indices,
|
||||
pp_spec_stable_rows_enabled,
|
||||
select_verify_intermediate_state_indices,
|
||||
)
|
||||
from sglang.srt.layers.radix_linear_attention import RadixLinearAttention
|
||||
from sglang.srt.mem_cache.memory_pool import MambaPool
|
||||
@@ -953,7 +955,12 @@ class GDNAttnBackend(MambaAttnBackendBase):
|
||||
intermediate_conv_window_cache = (
|
||||
mamba_cache_params.intermediate_conv_window[0]
|
||||
)
|
||||
intermediate_state_indices = self.verify_intermediate_state_indices
|
||||
intermediate_state_indices = select_verify_intermediate_state_indices(
|
||||
self.verify_intermediate_state_indices,
|
||||
forward_batch.req_pool_indices,
|
||||
cache_indices[: query_start_loc.shape[0] - 1] >= 0,
|
||||
self.req_to_token_pool.size,
|
||||
)
|
||||
mamba_pool = self.req_to_token_pool.mamba_pool
|
||||
use_replayssm_fold = (
|
||||
mamba_cache_params.replayssm_rawv is not None
|
||||
@@ -1117,6 +1124,7 @@ class GDNAttnBackend(MambaAttnBackendBase):
|
||||
intermediate_state_indices=intermediate_state_indices,
|
||||
cache_steps=forward_batch.spec_info.draft_token_num,
|
||||
retrieve_parent_token=retrieve_parent_token,
|
||||
stable_rows=pp_spec_stable_rows_enabled(),
|
||||
)
|
||||
else:
|
||||
g, beta = self._prefill_gates(layer, a, b)
|
||||
|
||||
@@ -17,6 +17,7 @@ from sglang.srt.layers.attention.linear.kernels.kda_triton import TritonKDAKerne
|
||||
from sglang.srt.layers.attention.linear.utils import (
|
||||
LinearAttnKernelBackend,
|
||||
build_verify_intermediate_state_indices,
|
||||
select_verify_intermediate_state_indices,
|
||||
)
|
||||
from sglang.srt.layers.radix_linear_attention import RadixLinearAttention
|
||||
from sglang.srt.utils import is_cpu, is_cuda, is_npu
|
||||
@@ -1027,7 +1028,12 @@ class KDAAttnBackend(MambaAttnBackendBase):
|
||||
replayssm_beta=replayssm_beta,
|
||||
)
|
||||
intermediate_conv_window_cache = mamba_cache_params.intermediate_conv_window[0]
|
||||
intermediate_state_indices = self.verify_intermediate_state_indices
|
||||
intermediate_state_indices = select_verify_intermediate_state_indices(
|
||||
self.verify_intermediate_state_indices,
|
||||
forward_batch.req_pool_indices,
|
||||
cache_indices[: query_start_loc.shape[0] - 1] >= 0,
|
||||
self.req_to_token_pool.size,
|
||||
)
|
||||
|
||||
draft_token_num = forward_batch.spec_info.draft_token_num
|
||||
ragged_layout = forward_batch.spec_info.ragged_verify_layout
|
||||
|
||||
@@ -33,6 +33,20 @@ logger = logging.getLogger(__name__)
|
||||
_FLASHINFER_GDN_ALIGNMENT = 32
|
||||
|
||||
|
||||
def copy_verify_intermediate_rows(
|
||||
destination: torch.Tensor,
|
||||
positional_source: torch.Tensor,
|
||||
dst_indices_raw: torch.Tensor,
|
||||
) -> None:
|
||||
"""Move FlashInfer's positional verify snapshots to their owned rows."""
|
||||
count = dst_indices_raw.shape[0]
|
||||
destination.index_copy_(
|
||||
0,
|
||||
dst_indices_raw.to(device=destination.device, dtype=torch.long),
|
||||
positional_source[:count],
|
||||
)
|
||||
|
||||
|
||||
def _empty_aligned_like(
|
||||
tensor: torch.Tensor, alignment: int = _FLASHINFER_GDN_ALIGNMENT
|
||||
) -> torch.Tensor:
|
||||
@@ -340,12 +354,14 @@ class FlashInferGDNKernel(LinearAttnKernelBase):
|
||||
self,
|
||||
intermediate_states_buffer: torch.Tensor,
|
||||
batch_size: int,
|
||||
force_scratch: bool = False,
|
||||
) -> tuple[torch.Tensor, bool]:
|
||||
# FlashInfer requires exact capture B, which may exceed the pool-scoped
|
||||
# buffer; padded tiers use stable scratch and copy owned rows back.
|
||||
direct = intermediate_states_buffer[:batch_size]
|
||||
if direct.shape[0] == batch_size:
|
||||
return direct, False
|
||||
if not force_scratch:
|
||||
direct = intermediate_states_buffer[:batch_size]
|
||||
if direct.shape[0] == batch_size:
|
||||
return direct, False
|
||||
|
||||
stream_key = (
|
||||
torch.cuda.current_stream(intermediate_states_buffer.device).cuda_stream
|
||||
@@ -598,6 +614,7 @@ class FlashInferGDNKernel(LinearAttnKernelBase):
|
||||
intermediate_state_indices: torch.Tensor,
|
||||
cache_steps: int,
|
||||
retrieve_parent_token: torch.Tensor,
|
||||
stable_rows: bool = False,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
# MTP verify using FlashInfer gated_delta_rule_mtp kernel (SM90 + SM100+).
|
||||
@@ -623,16 +640,17 @@ class FlashInferGDNKernel(LinearAttnKernelBase):
|
||||
|
||||
intermediate_states_buffer_mtp = intermediate_states_buffer
|
||||
copy_verify_intermediate_back = False
|
||||
if self.use_state_pool and intermediate_states_buffer is not None:
|
||||
# The SM100 bf16 MTP kernel indexes this scratch buffer by the
|
||||
# per-call batch id, while SGLang's speculative state cache is
|
||||
# Graph padding can exceed the pool-scoped scratch; use exact-B storage
|
||||
# and copy owned rows back before post-verify commit reads the pool.
|
||||
if intermediate_states_buffer is not None and (
|
||||
self.use_state_pool or stable_rows
|
||||
):
|
||||
# FlashInfer writes positional rows; isolate them in exact-B scratch.
|
||||
(
|
||||
intermediate_states_buffer_mtp,
|
||||
copy_verify_intermediate_back,
|
||||
) = self._prepare_verify_intermediate_buffer(
|
||||
intermediate_states_buffer, batch_size
|
||||
intermediate_states_buffer,
|
||||
batch_size,
|
||||
force_scratch=stable_rows,
|
||||
)
|
||||
if not self._mutable_inputs_are_aligned(
|
||||
("ssm_states", ssm_states),
|
||||
@@ -697,7 +715,14 @@ class FlashInferGDNKernel(LinearAttnKernelBase):
|
||||
use_qk_l2norm=True,
|
||||
)
|
||||
|
||||
if copy_verify_intermediate_back:
|
||||
if stable_rows and intermediate_states_buffer is not None:
|
||||
# Persist positional output under PP-stable request rows.
|
||||
copy_verify_intermediate_rows(
|
||||
intermediate_states_buffer,
|
||||
intermediate_states_buffer_mtp,
|
||||
intermediate_state_indices[:batch_size],
|
||||
)
|
||||
elif copy_verify_intermediate_back:
|
||||
intermediate_states_buffer.copy_(
|
||||
intermediate_states_buffer_mtp[: intermediate_states_buffer.shape[0]]
|
||||
)
|
||||
|
||||
@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Optional
|
||||
|
||||
import msgspec
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.runtime_context import get_exec
|
||||
from sglang.srt.utils.common import rank0_log
|
||||
|
||||
@@ -12,6 +13,10 @@ if TYPE_CHECKING:
|
||||
pass
|
||||
|
||||
|
||||
def pp_spec_stable_rows_enabled() -> bool:
|
||||
return envs.SGLANG_ENABLE_PP_SPEC.get()
|
||||
|
||||
|
||||
class LinearAttnKernelBackend(Enum):
|
||||
TRITON = "triton"
|
||||
CUTEDSL = "cutedsl"
|
||||
@@ -103,6 +108,20 @@ def resolve_linear_attn_backends(
|
||||
return backends
|
||||
|
||||
|
||||
def select_verify_intermediate_state_indices(
|
||||
default_indices, req_pool_indices, valid, pool_size: int
|
||||
):
|
||||
if not pp_spec_stable_rows_enabled():
|
||||
return default_indices
|
||||
|
||||
import torch
|
||||
|
||||
req_rows = req_pool_indices[: valid.shape[0]]
|
||||
return torch.where(valid, req_rows, torch.full_like(req_rows, pool_size)).to(
|
||||
torch.int32
|
||||
)
|
||||
|
||||
|
||||
def build_verify_intermediate_state_indices(pool_size: int, device):
|
||||
"""Per-request row index into the speculative intermediate scratch
|
||||
(`intermediate_ssm` / `intermediate_conv_window`) for the MTP /
|
||||
|
||||
@@ -14,6 +14,7 @@ from sglang.srt.disaggregation.utils import poll_and_all_reduce_attn_cp_tp_group
|
||||
from sglang.srt.distributed.communication_op import attn_cp_tp_broadcast_pyobj
|
||||
from sglang.srt.distributed.parallel_state import P2PWork
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.attention.linear.utils import pp_spec_stable_rows_enabled
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
|
||||
from sglang.srt.managers.overlap_utils import RelayPayload
|
||||
from sglang.srt.managers.schedule_batch import FINISH_ABORT, Req, ScheduleBatch
|
||||
@@ -55,6 +56,14 @@ def _pp_can_skip_output_comm(batch: ScheduleBatch) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _pp_snapshot_forward_batch(batch: ScheduleBatch) -> Optional[ScheduleBatch]:
|
||||
if batch.spec_algorithm.is_none():
|
||||
return None
|
||||
fwd_batch = batch.copy()
|
||||
fwd_batch.req_pool_indices = batch.req_pool_indices.clone()
|
||||
return fwd_batch
|
||||
|
||||
|
||||
def _pp_exchange_outputs_before_forward(
|
||||
cur_batch: Optional[ScheduleBatch],
|
||||
spec_relay: bool,
|
||||
@@ -592,7 +601,7 @@ class SchedulerPPMixin:
|
||||
self.pp_outputs: Optional[PPProxyTensors] = None
|
||||
self.last_rank_comm_queue: deque[Tuple[torch.Event, PPProxyTensors]] = deque()
|
||||
self._pp_spec_relay = (
|
||||
envs.SGLANG_ENABLE_PP_SPEC.get()
|
||||
pp_spec_stable_rows_enabled()
|
||||
and get_parallel().pp_size > 1
|
||||
and not self.spec_algorithm.is_none()
|
||||
)
|
||||
@@ -879,12 +888,8 @@ class SchedulerPPMixin:
|
||||
tensor_dict["spec_accept_lens"] = result.accept_lens
|
||||
tensor_dict["spec_new_seq_lens"] = result.new_seq_lens
|
||||
tensor_dict["spec_bonus_tokens"] = result.next_draft_input.bonus_tokens
|
||||
if (
|
||||
result.accept_index is not None
|
||||
and get_spec().speculative_eagle_topk > 1
|
||||
):
|
||||
# Only a tree needs it: a chain's accepted path is already the
|
||||
# front of each block, so compacting it is an identity.
|
||||
if result.accept_index is not None:
|
||||
# Relayed recurrent commits also need chain accept indices.
|
||||
tensor_dict["spec_accept_index"] = result.accept_index
|
||||
if result.next_verify_chain is not None:
|
||||
# Tail-drafted tree for the next verify round (root = bonus),
|
||||
@@ -1093,7 +1098,7 @@ class SchedulerPPMixin:
|
||||
new_seq_lens = pp_outputs["spec_new_seq_lens"]
|
||||
fwd_rids = [req.rid for req in fwd_batch.reqs]
|
||||
live_rids = [req.rid for req in batch.reqs]
|
||||
self._pp_spec_compact_accept_kv(
|
||||
self._pp_spec_commit_relayed_accept(
|
||||
batch,
|
||||
fwd_batch,
|
||||
fwd_rids,
|
||||
@@ -1229,7 +1234,7 @@ class SchedulerPPMixin:
|
||||
):
|
||||
self.process_batch_result(batch, output_result)
|
||||
|
||||
def _pp_spec_compact_accept_kv(
|
||||
def _pp_spec_commit_relayed_accept(
|
||||
self: Scheduler,
|
||||
batch: ScheduleBatch,
|
||||
fwd_batch: ScheduleBatch,
|
||||
@@ -1238,47 +1243,60 @@ class SchedulerPPMixin:
|
||||
verify_out_cache_loc: Optional[torch.Tensor],
|
||||
pp_outputs,
|
||||
) -> None:
|
||||
"""Move this stage's accepted-path KV to the front of each request block.
|
||||
"""Commit relayed recurrent state, then compact tree KV when present.
|
||||
|
||||
The verify forward writes one KV slot per tree node, in node order. The
|
||||
committed prefix that every later read assumes is the accepted path laid
|
||||
out contiguously, so the two have to be reconciled once per round -- and
|
||||
each stage has to do it for its own layers, since KV is not relayed.
|
||||
The last stage does it inside verify (_finalize_accept_tree_path); this
|
||||
is the same step for the stages that only ran the target forward.
|
||||
|
||||
Must run before seq_lens advances: the move writes into the block that
|
||||
starts at the pre-advance length.
|
||||
Must run before the live batch advances ``seq_lens``.
|
||||
"""
|
||||
accept_index = pp_outputs.tensors.get("spec_accept_index")
|
||||
if accept_index is None or fwd_batch.forward_mode.is_idle():
|
||||
return
|
||||
if verify_out_cache_loc is None:
|
||||
return
|
||||
from sglang.srt.speculative.spec_utils import (
|
||||
commit_mamba_states_after_verify,
|
||||
move_accept_tokens_to_target_kvcache,
|
||||
)
|
||||
|
||||
# The destination base is the length each request had when the forward
|
||||
# ran. ScheduleBatch.copy() drops seq_lens but keeps seq_lens_cpu, and
|
||||
# that snapshot is already in the forward's row order -- the live batch
|
||||
# may have been filtered or merged since, and reindexing it would skip
|
||||
# exactly the rounds whose composition changed.
|
||||
device = verify_out_cache_loc.device
|
||||
# Preserve the forward batch's row order after live-batch recomposition.
|
||||
device = (
|
||||
verify_out_cache_loc.device
|
||||
if verify_out_cache_loc is not None
|
||||
else batch.seq_lens.device
|
||||
)
|
||||
if fwd_batch.seq_lens_cpu is not None:
|
||||
seq_lens = fwd_batch.seq_lens_cpu.to(device=device, dtype=torch.int64)
|
||||
elif live_rids == fwd_rids:
|
||||
seq_lens = batch.seq_lens
|
||||
else:
|
||||
return
|
||||
raise RuntimeError(
|
||||
"PP-spec delayed relay cannot commit a recomposed micro-batch "
|
||||
"without its forward-time seq_lens_cpu snapshot"
|
||||
)
|
||||
fwd_batch.seq_lens = seq_lens
|
||||
# copy() drops tree_cache, which the tracking-grid commit needs.
|
||||
fwd_batch.tree_cache = batch.tree_cache
|
||||
accept_index = accept_index.to(device)
|
||||
accept_lens = pp_outputs["spec_accept_lens"].to(device)
|
||||
|
||||
# The last stage already commits inside run_eagle_verify.
|
||||
if not self.pp_group.is_last_rank:
|
||||
commit_mamba_states_after_verify(
|
||||
self.tp_worker,
|
||||
fwd_batch,
|
||||
accept_lens,
|
||||
accept_index,
|
||||
get_spec().speculative_num_draft_tokens,
|
||||
)
|
||||
|
||||
if verify_out_cache_loc is None:
|
||||
return
|
||||
fwd_batch.out_cache_loc = verify_out_cache_loc
|
||||
move_accept_tokens_to_target_kvcache(
|
||||
fwd_batch,
|
||||
accept_index.to(device),
|
||||
pp_outputs["spec_accept_lens"].to(device) - 1,
|
||||
self.token_to_kv_pool_allocator,
|
||||
)
|
||||
|
||||
if get_spec().speculative_eagle_topk > 1:
|
||||
move_accept_tokens_to_target_kvcache(
|
||||
fwd_batch,
|
||||
accept_index,
|
||||
accept_lens - 1,
|
||||
self.token_to_kv_pool_allocator,
|
||||
)
|
||||
|
||||
def _pp_spec_adopt_relayed_tree(
|
||||
self: Scheduler,
|
||||
@@ -1733,11 +1751,7 @@ class SchedulerPPMixin:
|
||||
)
|
||||
mb_metadata[mb_id] = PPBatchMetadata(
|
||||
can_run_cuda_graph=result.can_run_cuda_graph,
|
||||
fwd_batch=(
|
||||
cur_batch.copy()
|
||||
if not cur_batch.spec_algorithm.is_none()
|
||||
else None
|
||||
),
|
||||
fwd_batch=_pp_snapshot_forward_batch(cur_batch),
|
||||
verify_out_cache_loc=result.spec_verify_out_cache_loc,
|
||||
)
|
||||
event = self.device_module.Event()
|
||||
|
||||
@@ -1553,8 +1553,10 @@ class HybridReqToTokenPool(ReqToTokenPool):
|
||||
) -> None:
|
||||
self.ngram_pool.set_context(ngram_indices, context)
|
||||
|
||||
def set_ngram_intermediate_context(self, context: torch.Tensor) -> None:
|
||||
self.ngram_pool.set_intermediate_context(context)
|
||||
def set_ngram_intermediate_context(
|
||||
self, context: torch.Tensor, indices: Optional[torch.Tensor] = None
|
||||
) -> None:
|
||||
self.ngram_pool.set_intermediate_context(context, indices)
|
||||
|
||||
def copy_mamba_state(
|
||||
self, src_index: torch.Tensor, dst_index: torch.Tensor
|
||||
|
||||
@@ -202,11 +202,20 @@ class NGramPool:
|
||||
device=self.context.device, dtype=self.context.dtype
|
||||
)
|
||||
|
||||
def set_intermediate_context(self, context: torch.Tensor):
|
||||
def set_intermediate_context(
|
||||
self, context: torch.Tensor, indices: Optional[torch.Tensor] = None
|
||||
):
|
||||
if self.intermediate_context is not None and context.numel() > 0:
|
||||
self.intermediate_context[: context.shape[0], : context.shape[1]].copy_(
|
||||
context.to(device=self.context.device, dtype=self.context.dtype)
|
||||
)
|
||||
context = context.to(device=self.context.device, dtype=self.context.dtype)
|
||||
if indices is None:
|
||||
self.intermediate_context[: context.shape[0], : context.shape[1]].copy_(
|
||||
context
|
||||
)
|
||||
else:
|
||||
self.intermediate_context[
|
||||
indices.to(device=self.context.device, dtype=torch.long),
|
||||
: context.shape[1],
|
||||
] = context
|
||||
|
||||
def clear(self):
|
||||
if self.context is not None:
|
||||
|
||||
@@ -21,6 +21,9 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.attention.linear.utils import (
|
||||
select_verify_intermediate_state_indices,
|
||||
)
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_tp_all_gather,
|
||||
@@ -315,12 +318,19 @@ def _commit_ple_batch(batch: Optional[_PLEBatch], forward_batch: ForwardBatch) -
|
||||
valid_steps = batch.valid_tokens.reshape(
|
||||
batch.lengths.shape[0], batch.row_width
|
||||
)
|
||||
dst_indices_raw = select_verify_intermediate_state_indices(
|
||||
None,
|
||||
forward_batch.req_pool_indices,
|
||||
batch.lengths.ne(0),
|
||||
pool.size,
|
||||
)
|
||||
pool.set_ngram_intermediate_context(
|
||||
torch.where(
|
||||
valid_steps.unsqueeze(-1),
|
||||
step_contexts,
|
||||
torch.full_like(step_contexts, batch.ngram_eos_token_id),
|
||||
)
|
||||
),
|
||||
dst_indices_raw,
|
||||
)
|
||||
return
|
||||
|
||||
@@ -1097,9 +1107,23 @@ class Qwen4ExpPLELayer(nn.Module):
|
||||
intermediate_state,
|
||||
torch.zeros_like(intermediate_state),
|
||||
)
|
||||
intermediate_cache[: batch.lengths.shape[0], : batch.row_width].copy_(
|
||||
intermediate_state.to(dtype=intermediate_cache.dtype)
|
||||
intermediate_state = intermediate_state.to(
|
||||
dtype=intermediate_cache.dtype
|
||||
)
|
||||
dst_indices_raw = select_verify_intermediate_state_indices(
|
||||
None,
|
||||
forward_batch.req_pool_indices,
|
||||
batch.lengths.ne(0),
|
||||
get_req_to_token_pool().size,
|
||||
)
|
||||
if dst_indices_raw is None:
|
||||
intermediate_cache[
|
||||
: batch.lengths.shape[0], : batch.row_width
|
||||
].copy_(intermediate_state)
|
||||
else:
|
||||
intermediate_cache[
|
||||
dst_indices_raw.to(dtype=torch.long), : batch.row_width
|
||||
] = intermediate_state
|
||||
else:
|
||||
state_cols = torch.arange(
|
||||
self.short_conv_state_len, device=x.device, dtype=torch.long
|
||||
|
||||
@@ -41,6 +41,7 @@ from sglang.srt.distributed.parallel_state import (
|
||||
patch_tensor_parallel_group,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.attention.linear.utils import pp_spec_stable_rows_enabled
|
||||
from sglang.srt.managers.schedule_batch import set_mamba_track_indices_from_reqs
|
||||
from sglang.srt.managers.utils import _async_d2h
|
||||
from sglang.srt.mem_cache.allocation import (
|
||||
@@ -950,6 +951,10 @@ def commit_mamba_states_after_verify(
|
||||
# ring is allocated only then; KDA never allocates the cursors.
|
||||
req_pool = model_runner.req_to_token_pool
|
||||
mamba_pool = getattr(req_pool, "mamba_pool", None)
|
||||
bs = accept_lens.shape[0]
|
||||
src_indices_raw = (
|
||||
batch.req_pool_indices[:bs] if pp_spec_stable_rows_enabled() else None
|
||||
)
|
||||
|
||||
# Fold-every-commit: replay the accepted prefix from the ring into
|
||||
# `temporal`; the same fold stores the interval-crossing state to the
|
||||
@@ -980,6 +985,7 @@ def commit_mamba_states_after_verify(
|
||||
last_correct_step_indices=last_correct_step_indices,
|
||||
mamba_track_indices=batch.mamba_track_indices,
|
||||
mamba_steps_to_track=mamba_steps_to_track,
|
||||
src_indices_raw=src_indices_raw,
|
||||
null_block_id=-1,
|
||||
)
|
||||
return
|
||||
@@ -1000,7 +1006,6 @@ def commit_mamba_states_after_verify(
|
||||
)
|
||||
|
||||
spec_state = req_pool.get_speculative_mamba2_params_all_layers()
|
||||
bs = accept_lens.shape[0]
|
||||
state_batch_indices = req_pool.get_mamba_indices(batch.req_pool_indices)
|
||||
replay_indices = batch.req_pool_indices
|
||||
last_correct_step_indices, mamba_steps_to_track = _verify_commit_step_indices(
|
||||
@@ -1047,6 +1052,7 @@ def commit_mamba_states_after_verify(
|
||||
spec_state.intermediate_conv_window[0],
|
||||
state_batch_indices,
|
||||
last_correct_step_indices,
|
||||
src_indices_raw,
|
||||
)
|
||||
if batch.mamba_track_indices is not None:
|
||||
fused_conv_window_scatter_with_mask(
|
||||
@@ -1054,6 +1060,7 @@ def commit_mamba_states_after_verify(
|
||||
spec_state.intermediate_conv_window[0],
|
||||
batch.mamba_track_indices,
|
||||
mamba_steps_to_track,
|
||||
src_indices_raw,
|
||||
)
|
||||
return
|
||||
|
||||
@@ -1075,7 +1082,6 @@ def commit_mamba_states_after_verify(
|
||||
)
|
||||
|
||||
spec_state = req_pool.get_speculative_mamba2_params_all_layers()
|
||||
bs = accept_lens.shape[0]
|
||||
state_batch_indices = req_pool.get_mamba_indices(batch.req_pool_indices)
|
||||
accept_indices_offset = torch.arange(
|
||||
0,
|
||||
@@ -1115,13 +1121,13 @@ def commit_mamba_states_after_verify(
|
||||
last_correct_step_indices=last_correct_step_indices,
|
||||
mamba_track_indices=mamba_track_indices,
|
||||
mamba_steps_to_track=mamba_steps_to_track,
|
||||
src_indices_raw=src_indices_raw,
|
||||
null_block_id=-1, # SGLang: valid slots >= 0, padding == -1
|
||||
)
|
||||
return
|
||||
|
||||
attn_backend = model_runner.attn_backend
|
||||
|
||||
bs = accept_lens.shape[0]
|
||||
# `accept_lens` already includes the bonus token (drafts + 1 per req).
|
||||
if not batch.forward_mode.is_idle() and accept_index.numel() > 0:
|
||||
last_correct_step_indices, mamba_steps_to_track = _verify_commit_step_indices(
|
||||
|
||||
Reference in New Issue
Block a user