[Spec][PP] Fix hybrid recurrent-state commit and micro-batch pairing under PP x speculative decoding (#40001)

This commit is contained in:
YAMY
2026-09-30 13:05:39 -07:00
committed by GitHub
parent 547286bb5a
commit 4407a21f3c
18 changed files with 527 additions and 93 deletions
@@ -225,6 +225,7 @@ def commit_gdn_replayssm_fold_after_verify(
last_correct_step_indices: torch.Tensor, # [B] conv rollback target step
mamba_track_indices: torch.Tensor | None = None,
mamba_steps_to_track: torch.Tensor | None = None,
src_indices_raw: torch.Tensor | None = None,
null_block_id: int = -1,
) -> None:
"""Fold each layer's accepted window into ``temporal``, then do the usual
@@ -253,9 +254,17 @@ def commit_gdn_replayssm_fold_after_verify(
spec_state.conv, spec_state.intermediate_conv_window
):
fused_conv_window_scatter_with_mask(
conv_states, interm_conv, state_batch_indices, last_correct_step_indices
conv_states,
interm_conv,
state_batch_indices,
last_correct_step_indices,
src_indices_raw,
)
if mamba_track_indices is not None and mamba_steps_to_track is not None:
fused_conv_window_scatter_with_mask(
conv_states, interm_conv, mamba_track_indices, mamba_steps_to_track
conv_states,
interm_conv,
mamba_track_indices,
mamba_steps_to_track,
src_indices_raw,
)
@@ -341,6 +341,7 @@ def commit_kda_replayssm_after_verify(
last_correct_step_indices: torch.Tensor, # [B] conv rollback target step
mamba_track_indices: torch.Tensor | None = None,
mamba_steps_to_track: torch.Tensor | None = None,
src_indices_raw: torch.Tensor | None = None,
null_block_id: int = -1,
) -> None:
"""Fold each layer's accepted window into `temporal` and roll back conv.
@@ -378,9 +379,17 @@ def commit_kda_replayssm_after_verify(
spec_state.conv, spec_state.intermediate_conv_window
):
fused_conv_window_scatter_with_mask(
conv_states, interm_conv, state_batch_indices, last_correct_step_indices
conv_states,
interm_conv,
state_batch_indices,
last_correct_step_indices,
src_indices_raw,
)
if mamba_track_indices is not None and mamba_steps_to_track is not None:
fused_conv_window_scatter_with_mask(
conv_states, interm_conv, mamba_track_indices, mamba_steps_to_track
conv_states,
interm_conv,
mamba_track_indices,
mamba_steps_to_track,
src_indices_raw,
)
@@ -153,6 +153,7 @@ def _fused_mamba_state_scatter_with_mask_kernel(
# Raw index arrays (before index_select)
dst_indices_raw_ptr, # [total_requests] - state_indices_tensor
step_indices_raw_ptr, # [total_requests] - last_correct_step_indices or mamba_steps_to_track
src_indices_raw_ptr, # [total_requests] - verify scratch row for each request
elem_per_entry: tl.constexpr,
src_layer_stride,
src_req_stride,
@@ -189,8 +190,7 @@ def _fused_mamba_state_scatter_with_mask_kernel(
# Load destination index
dst_idx = tl.load(dst_indices_raw_ptr + pid_req).to(tl.int64)
# Source index is just the request index itself
src_idx = pid_req
src_idx = tl.load(src_indices_raw_ptr + pid_req).to(tl.int64)
# Bounds check to avoid illegal memory access
if not (
@@ -220,11 +220,22 @@ def _fused_mamba_state_scatter_with_mask_kernel(
tl.store(dst_ptr + dst_offset + offsets, data, mask=mask)
def _prepare_src_rows(src: torch.Tensor | None, steps: torch.Tensor) -> torch.Tensor:
if src is None:
src = torch.arange(steps.shape[0], dtype=torch.int32, device=steps.device)
if src.ndim != 1 or src.shape != steps.shape:
raise ValueError(
f"source indices length mismatch: {src.shape=} vs {steps.shape=}"
)
return src.to(torch.int32).contiguous()
def fused_mamba_state_scatter_with_mask(
dst: torch.Tensor, # [num_layers, cache_size, *state_shape]
src: torch.Tensor, # [num_layers, spec_size, draft_tokens, *state_shape]
dst_indices_raw: torch.Tensor, # [total_requests] - raw indices (e.g., state_indices_tensor)
step_indices_raw: torch.Tensor, # [total_requests] - raw step indices (step >= 0 means valid)
src_indices_raw: torch.Tensor | None = None, # [total_requests] - scratch rows
):
"""
Fully fused gather-scatter with built-in masking for mamba state updates.
@@ -291,6 +302,7 @@ def fused_mamba_state_scatter_with_mask(
# Ensure indices are int32 and contiguous
dst_indices_raw = dst_indices_raw.to(torch.int32).contiguous()
step_indices_raw = step_indices_raw.to(torch.int32).contiguous()
src_indices_raw = _prepare_src_rows(src_indices_raw, step_indices_raw)
_require_entry_contiguous_dst(dst, 2, "fused_mamba_state_scatter_with_mask")
if not src.is_contiguous():
@@ -307,6 +319,7 @@ def fused_mamba_state_scatter_with_mask(
dst,
dst_indices_raw,
step_indices_raw,
src_indices_raw,
elem_per_entry,
src_layer_stride,
src_req_stride,
@@ -326,6 +339,7 @@ def _fused_conv_window_scatter_with_mask_kernel(
dst_ptr,
dst_indices_raw_ptr, # [total_requests]
step_indices_raw_ptr, # [total_requests], entry >= 0 means valid
src_indices_raw_ptr, # [total_requests]
elem_per_entry: tl.constexpr, # dim * (K-1)
KM1: tl.constexpr, # K-1 (conv window width)
src_layer_stride,
@@ -359,7 +373,7 @@ def _fused_conv_window_scatter_with_mask_kernel(
return
dst_idx = tl.load(dst_indices_raw_ptr + pid_req).to(tl.int64)
src_idx = pid_req
src_idx = tl.load(src_indices_raw_ptr + pid_req).to(tl.int64)
if not (
(dst_idx >= 0)
@@ -396,6 +410,7 @@ def fused_conv_window_scatter_with_mask(
src: torch.Tensor, # deduped conv-window view [num_layers, spec_size, draft_tokens, dim, K-1]
dst_indices_raw: torch.Tensor, # [total_requests]
step_indices_raw: torch.Tensor, # [total_requests], entry >= 0 means valid
src_indices_raw: torch.Tensor | None = None, # [total_requests]
):
"""Conv-window variant of :func:`fused_mamba_state_scatter_with_mask`.
@@ -446,6 +461,7 @@ def fused_conv_window_scatter_with_mask(
dst_indices_raw = dst_indices_raw.to(torch.int32).contiguous()
step_indices_raw = step_indices_raw.to(torch.int32).contiguous()
src_indices_raw = _prepare_src_rows(src_indices_raw, step_indices_raw)
BLOCK_SIZE = 1024
grid = (total_requests, num_layers, triton.cdiv(elem_per_entry, BLOCK_SIZE))
@@ -455,6 +471,7 @@ def fused_conv_window_scatter_with_mask(
dst,
dst_indices_raw,
step_indices_raw,
src_indices_raw,
elem_per_entry,
km1,
src.stride(0),
@@ -681,6 +698,7 @@ def scatter_mamba_states_after_mtp_verify(
last_correct_step_indices: torch.Tensor,
mamba_track_indices: torch.Tensor | None,
mamba_steps_to_track: torch.Tensor | None,
src_indices_raw: torch.Tensor | None = None,
) -> None:
"""Scatter per-step verify states (ssm + all conv types) into the
persistent caches, plus the interval-crossing track slots."""
@@ -693,6 +711,7 @@ def scatter_mamba_states_after_mtp_verify(
intermediate_state_cache,
state_indices_tensor,
last_correct_step_indices,
src_indices_raw,
)
if mamba_track_indices is not None:
assert mamba_steps_to_track is not None
@@ -701,6 +720,7 @@ def scatter_mamba_states_after_mtp_verify(
intermediate_state_cache,
mamba_track_indices,
mamba_steps_to_track,
src_indices_raw,
)
pairs = list(zip(mamba_caches.conv, mamba_caches.intermediate_conv_window))
@@ -708,7 +728,7 @@ def scatter_mamba_states_after_mtp_verify(
return
if mamba_track_indices is not None:
assert mamba_steps_to_track is not None
if _conv_multi_eligible(pairs):
if src_indices_raw is None and _conv_multi_eligible(pairs):
fused_conv_window_scatter_multi(
pairs,
state_indices_tensor,
@@ -723,6 +743,7 @@ def scatter_mamba_states_after_mtp_verify(
intermediate_conv_window_cache,
state_indices_tensor,
last_correct_step_indices,
src_indices_raw,
)
if mamba_track_indices is not None:
for conv_states, intermediate_conv_window_cache in pairs:
@@ -731,6 +752,7 @@ def scatter_mamba_states_after_mtp_verify(
intermediate_conv_window_cache,
mamba_track_indices,
mamba_steps_to_track,
src_indices_raw,
)
@@ -27,8 +27,10 @@ from sglang.srt.arg_groups.overrides import (
resolving_view,
run_post_process_pass,
)
from sglang.srt.configs.model_config import uses_kda_attention
from sglang.srt.connector import ConnectorType
from sglang.srt.environ import envs
from sglang.srt.layers.attention.linear.utils import pp_spec_stable_rows_enabled
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
from sglang.srt.runtime_context import get_platform
from sglang.srt.utils.common import (
@@ -275,6 +277,23 @@ def handle_linear_attn_backend(server_args: Any):
verify = cfg.linear_attn_verify_backend
if verify is None and decode == "flashinfer":
verify = "flashinfer"
if (
pp_spec_stable_rows_enabled()
and verify == "flashinfer"
and uses_kda_attention(model_config_of(server_args).hf_config)
):
declare_resolution(
server_args,
"_handle_linear_attn_backend",
linear_attn_verify_backend="triton",
)
verify = "triton"
logger.warning(
"SGLANG_ENABLE_PP_SPEC with KDA does not support the FlashInfer "
"target-verify row layout; falling back "
"--linear-attn-verify-backend to triton. FlashInfer decode is "
"unchanged."
)
if (
verify == "flashinfer"
and cfg.mamba_ssm_dtype != "bfloat16"
@@ -21,6 +21,7 @@ from sglang.srt.layers.attention.base_attn_backend import (
AttentionBackend,
SharedReadEnds,
)
from sglang.srt.layers.attention.linear.utils import pp_spec_stable_rows_enabled
from sglang.srt.layers.attention.mamba.mamba import MambaMixer2
from sglang.srt.layers.attention.mamba.mamba2_metadata import (
ForwardMetadata,
@@ -1425,14 +1426,11 @@ class HybridLinearAttnBackend(AttentionBackend):
model,
req_pool_indices: Optional[torch.Tensor] = None,
):
"""Update mamba states after MTP verify via a fused gather-scatter kernel.
``req_pool_indices`` serves implementations that must re-derive the state
slot ids instead of reusing this step's ``forward_metadata``; the scatter
below reads the metadata it just planned.
"""
del req_pool_indices
"""Commit accepted Mamba states; PP rows identify the delayed batch."""
request_number = last_correct_step_indices.shape[0]
src_indices_raw = None
if pp_spec_stable_rows_enabled() and req_pool_indices is not None:
src_indices_raw = req_pool_indices[:request_number]
# `mamba_track_indices` is VIRTUAL; the scatter writes physical views.
if mamba_track_indices is not None:
@@ -1440,11 +1438,18 @@ class HybridLinearAttnBackend(AttentionBackend):
mamba_track_indices
)
state_indices_tensor = (
self.linear_attn_backend.forward_metadata.mamba_cache_indices[
:request_number
]
)
if src_indices_raw is not None:
state_indices_tensor = self.linear_attn_backend._translate_mamba_indices(
self.linear_attn_backend.req_to_token_pool.get_mamba_indices(
req_pool_indices[:request_number]
)
)
else:
state_indices_tensor = (
self.linear_attn_backend.forward_metadata.mamba_cache_indices[
:request_number
]
)
req_pool = self.linear_attn_backend.req_to_token_pool
mamba_caches = req_pool.get_speculative_mamba2_params_all_layers()
@@ -1469,6 +1474,7 @@ class HybridLinearAttnBackend(AttentionBackend):
last_correct_step_indices=last_correct_step_indices,
mamba_track_indices=mamba_track_indices,
mamba_steps_to_track=mamba_steps_to_track,
src_indices_raw=src_indices_raw,
null_block_id=-1,
)
return
@@ -1492,6 +1498,7 @@ class HybridLinearAttnBackend(AttentionBackend):
intermediate_conv_window,
state_indices_tensor,
last_correct_step_indices,
src_indices_raw,
)
accept_lens_pool[state_indices_tensor.to(torch.int64)] = (
last_correct_step_indices.to(torch.int32) + 1
@@ -1504,6 +1511,7 @@ class HybridLinearAttnBackend(AttentionBackend):
last_correct_step_indices,
mamba_track_indices,
mamba_steps_to_track,
src_indices_raw=src_indices_raw,
)
self._update_ple_state_after_mtp_verify(
@@ -1511,6 +1519,7 @@ class HybridLinearAttnBackend(AttentionBackend):
last_correct_step_indices,
mamba_track_indices,
mamba_steps_to_track,
src_indices_raw,
)
@staticmethod
@@ -1519,19 +1528,28 @@ class HybridLinearAttnBackend(AttentionBackend):
src: torch.Tensor,
dst_indices_raw: torch.Tensor,
step_indices_raw: torch.Tensor,
src_indices_raw: Optional[torch.Tensor] = None,
):
if dst is None or src is None or step_indices_raw.numel() == 0:
return
if dst.is_cuda and src.is_cuda:
fused_mamba_state_scatter_with_mask(
dst, src, dst_indices_raw, step_indices_raw
dst,
src,
dst_indices_raw,
step_indices_raw,
src_indices_raw,
)
return
device = dst.device
dst_indices = dst_indices_raw.to(device=device, dtype=torch.long)
steps = step_indices_raw.to(device=device, dtype=torch.long)
src_indices = torch.arange(steps.shape[0], device=device, dtype=torch.long)
src_indices = (
torch.arange(steps.shape[0], device=device, dtype=torch.long)
if src_indices_raw is None
else src_indices_raw.to(device=device, dtype=torch.long)
)
valid = (
(steps >= 0)
& (steps < src.shape[2])
@@ -1552,6 +1570,7 @@ class HybridLinearAttnBackend(AttentionBackend):
last_correct_step_indices: torch.Tensor,
mamba_track_indices: Optional[torch.Tensor],
mamba_steps_to_track: Optional[torch.Tensor],
src_indices_raw: Optional[torch.Tensor] = None,
):
"""Roll the accepted per-step PLE side states into their main slots."""
req_to_token_pool = self.linear_attn_backend.req_to_token_pool
@@ -1589,6 +1608,7 @@ class HybridLinearAttnBackend(AttentionBackend):
intermediate_state,
state_indices_tensor,
last_correct_step_indices,
src_indices_raw,
)
if mamba_track_indices is not None:
self._scatter_speculative_state_with_mask(
@@ -1596,6 +1616,7 @@ class HybridLinearAttnBackend(AttentionBackend):
intermediate_state,
mamba_track_indices,
mamba_steps_to_track,
src_indices_raw,
)
@@ -16,6 +16,8 @@ from sglang.srt.layers.attention.linear.utils import (
LinearAttnBackends,
LinearAttnKernelBackend,
build_verify_intermediate_state_indices,
pp_spec_stable_rows_enabled,
select_verify_intermediate_state_indices,
)
from sglang.srt.layers.radix_linear_attention import RadixLinearAttention
from sglang.srt.mem_cache.memory_pool import MambaPool
@@ -953,7 +955,12 @@ class GDNAttnBackend(MambaAttnBackendBase):
intermediate_conv_window_cache = (
mamba_cache_params.intermediate_conv_window[0]
)
intermediate_state_indices = self.verify_intermediate_state_indices
intermediate_state_indices = select_verify_intermediate_state_indices(
self.verify_intermediate_state_indices,
forward_batch.req_pool_indices,
cache_indices[: query_start_loc.shape[0] - 1] >= 0,
self.req_to_token_pool.size,
)
mamba_pool = self.req_to_token_pool.mamba_pool
use_replayssm_fold = (
mamba_cache_params.replayssm_rawv is not None
@@ -1117,6 +1124,7 @@ class GDNAttnBackend(MambaAttnBackendBase):
intermediate_state_indices=intermediate_state_indices,
cache_steps=forward_batch.spec_info.draft_token_num,
retrieve_parent_token=retrieve_parent_token,
stable_rows=pp_spec_stable_rows_enabled(),
)
else:
g, beta = self._prefill_gates(layer, a, b)
@@ -17,6 +17,7 @@ from sglang.srt.layers.attention.linear.kernels.kda_triton import TritonKDAKerne
from sglang.srt.layers.attention.linear.utils import (
LinearAttnKernelBackend,
build_verify_intermediate_state_indices,
select_verify_intermediate_state_indices,
)
from sglang.srt.layers.radix_linear_attention import RadixLinearAttention
from sglang.srt.utils import is_cpu, is_cuda, is_npu
@@ -1027,7 +1028,12 @@ class KDAAttnBackend(MambaAttnBackendBase):
replayssm_beta=replayssm_beta,
)
intermediate_conv_window_cache = mamba_cache_params.intermediate_conv_window[0]
intermediate_state_indices = self.verify_intermediate_state_indices
intermediate_state_indices = select_verify_intermediate_state_indices(
self.verify_intermediate_state_indices,
forward_batch.req_pool_indices,
cache_indices[: query_start_loc.shape[0] - 1] >= 0,
self.req_to_token_pool.size,
)
draft_token_num = forward_batch.spec_info.draft_token_num
ragged_layout = forward_batch.spec_info.ragged_verify_layout
@@ -33,6 +33,20 @@ logger = logging.getLogger(__name__)
_FLASHINFER_GDN_ALIGNMENT = 32
def copy_verify_intermediate_rows(
destination: torch.Tensor,
positional_source: torch.Tensor,
dst_indices_raw: torch.Tensor,
) -> None:
"""Move FlashInfer's positional verify snapshots to their owned rows."""
count = dst_indices_raw.shape[0]
destination.index_copy_(
0,
dst_indices_raw.to(device=destination.device, dtype=torch.long),
positional_source[:count],
)
def _empty_aligned_like(
tensor: torch.Tensor, alignment: int = _FLASHINFER_GDN_ALIGNMENT
) -> torch.Tensor:
@@ -340,12 +354,14 @@ class FlashInferGDNKernel(LinearAttnKernelBase):
self,
intermediate_states_buffer: torch.Tensor,
batch_size: int,
force_scratch: bool = False,
) -> tuple[torch.Tensor, bool]:
# FlashInfer requires exact capture B, which may exceed the pool-scoped
# buffer; padded tiers use stable scratch and copy owned rows back.
direct = intermediate_states_buffer[:batch_size]
if direct.shape[0] == batch_size:
return direct, False
if not force_scratch:
direct = intermediate_states_buffer[:batch_size]
if direct.shape[0] == batch_size:
return direct, False
stream_key = (
torch.cuda.current_stream(intermediate_states_buffer.device).cuda_stream
@@ -598,6 +614,7 @@ class FlashInferGDNKernel(LinearAttnKernelBase):
intermediate_state_indices: torch.Tensor,
cache_steps: int,
retrieve_parent_token: torch.Tensor,
stable_rows: bool = False,
**kwargs,
) -> torch.Tensor:
# MTP verify using FlashInfer gated_delta_rule_mtp kernel (SM90 + SM100+).
@@ -623,16 +640,17 @@ class FlashInferGDNKernel(LinearAttnKernelBase):
intermediate_states_buffer_mtp = intermediate_states_buffer
copy_verify_intermediate_back = False
if self.use_state_pool and intermediate_states_buffer is not None:
# The SM100 bf16 MTP kernel indexes this scratch buffer by the
# per-call batch id, while SGLang's speculative state cache is
# Graph padding can exceed the pool-scoped scratch; use exact-B storage
# and copy owned rows back before post-verify commit reads the pool.
if intermediate_states_buffer is not None and (
self.use_state_pool or stable_rows
):
# FlashInfer writes positional rows; isolate them in exact-B scratch.
(
intermediate_states_buffer_mtp,
copy_verify_intermediate_back,
) = self._prepare_verify_intermediate_buffer(
intermediate_states_buffer, batch_size
intermediate_states_buffer,
batch_size,
force_scratch=stable_rows,
)
if not self._mutable_inputs_are_aligned(
("ssm_states", ssm_states),
@@ -697,7 +715,14 @@ class FlashInferGDNKernel(LinearAttnKernelBase):
use_qk_l2norm=True,
)
if copy_verify_intermediate_back:
if stable_rows and intermediate_states_buffer is not None:
# Persist positional output under PP-stable request rows.
copy_verify_intermediate_rows(
intermediate_states_buffer,
intermediate_states_buffer_mtp,
intermediate_state_indices[:batch_size],
)
elif copy_verify_intermediate_back:
intermediate_states_buffer.copy_(
intermediate_states_buffer_mtp[: intermediate_states_buffer.shape[0]]
)
@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Optional
import msgspec
from sglang.srt.environ import envs
from sglang.srt.runtime_context import get_exec
from sglang.srt.utils.common import rank0_log
@@ -12,6 +13,10 @@ if TYPE_CHECKING:
pass
def pp_spec_stable_rows_enabled() -> bool:
return envs.SGLANG_ENABLE_PP_SPEC.get()
class LinearAttnKernelBackend(Enum):
TRITON = "triton"
CUTEDSL = "cutedsl"
@@ -103,6 +108,20 @@ def resolve_linear_attn_backends(
return backends
def select_verify_intermediate_state_indices(
default_indices, req_pool_indices, valid, pool_size: int
):
if not pp_spec_stable_rows_enabled():
return default_indices
import torch
req_rows = req_pool_indices[: valid.shape[0]]
return torch.where(valid, req_rows, torch.full_like(req_rows, pool_size)).to(
torch.int32
)
def build_verify_intermediate_state_indices(pool_size: int, device):
"""Per-request row index into the speculative intermediate scratch
(`intermediate_ssm` / `intermediate_conv_window`) for the MTP /
@@ -14,6 +14,7 @@ from sglang.srt.disaggregation.utils import poll_and_all_reduce_attn_cp_tp_group
from sglang.srt.distributed.communication_op import attn_cp_tp_broadcast_pyobj
from sglang.srt.distributed.parallel_state import P2PWork
from sglang.srt.environ import envs
from sglang.srt.layers.attention.linear.utils import pp_spec_stable_rows_enabled
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
from sglang.srt.managers.overlap_utils import RelayPayload
from sglang.srt.managers.schedule_batch import FINISH_ABORT, Req, ScheduleBatch
@@ -55,6 +56,14 @@ def _pp_can_skip_output_comm(batch: ScheduleBatch) -> bool:
)
def _pp_snapshot_forward_batch(batch: ScheduleBatch) -> Optional[ScheduleBatch]:
if batch.spec_algorithm.is_none():
return None
fwd_batch = batch.copy()
fwd_batch.req_pool_indices = batch.req_pool_indices.clone()
return fwd_batch
def _pp_exchange_outputs_before_forward(
cur_batch: Optional[ScheduleBatch],
spec_relay: bool,
@@ -592,7 +601,7 @@ class SchedulerPPMixin:
self.pp_outputs: Optional[PPProxyTensors] = None
self.last_rank_comm_queue: deque[Tuple[torch.Event, PPProxyTensors]] = deque()
self._pp_spec_relay = (
envs.SGLANG_ENABLE_PP_SPEC.get()
pp_spec_stable_rows_enabled()
and get_parallel().pp_size > 1
and not self.spec_algorithm.is_none()
)
@@ -879,12 +888,8 @@ class SchedulerPPMixin:
tensor_dict["spec_accept_lens"] = result.accept_lens
tensor_dict["spec_new_seq_lens"] = result.new_seq_lens
tensor_dict["spec_bonus_tokens"] = result.next_draft_input.bonus_tokens
if (
result.accept_index is not None
and get_spec().speculative_eagle_topk > 1
):
# Only a tree needs it: a chain's accepted path is already the
# front of each block, so compacting it is an identity.
if result.accept_index is not None:
# Relayed recurrent commits also need chain accept indices.
tensor_dict["spec_accept_index"] = result.accept_index
if result.next_verify_chain is not None:
# Tail-drafted tree for the next verify round (root = bonus),
@@ -1093,7 +1098,7 @@ class SchedulerPPMixin:
new_seq_lens = pp_outputs["spec_new_seq_lens"]
fwd_rids = [req.rid for req in fwd_batch.reqs]
live_rids = [req.rid for req in batch.reqs]
self._pp_spec_compact_accept_kv(
self._pp_spec_commit_relayed_accept(
batch,
fwd_batch,
fwd_rids,
@@ -1229,7 +1234,7 @@ class SchedulerPPMixin:
):
self.process_batch_result(batch, output_result)
def _pp_spec_compact_accept_kv(
def _pp_spec_commit_relayed_accept(
self: Scheduler,
batch: ScheduleBatch,
fwd_batch: ScheduleBatch,
@@ -1238,47 +1243,60 @@ class SchedulerPPMixin:
verify_out_cache_loc: Optional[torch.Tensor],
pp_outputs,
) -> None:
"""Move this stage's accepted-path KV to the front of each request block.
"""Commit relayed recurrent state, then compact tree KV when present.
The verify forward writes one KV slot per tree node, in node order. The
committed prefix that every later read assumes is the accepted path laid
out contiguously, so the two have to be reconciled once per round -- and
each stage has to do it for its own layers, since KV is not relayed.
The last stage does it inside verify (_finalize_accept_tree_path); this
is the same step for the stages that only ran the target forward.
Must run before seq_lens advances: the move writes into the block that
starts at the pre-advance length.
Must run before the live batch advances ``seq_lens``.
"""
accept_index = pp_outputs.tensors.get("spec_accept_index")
if accept_index is None or fwd_batch.forward_mode.is_idle():
return
if verify_out_cache_loc is None:
return
from sglang.srt.speculative.spec_utils import (
commit_mamba_states_after_verify,
move_accept_tokens_to_target_kvcache,
)
# The destination base is the length each request had when the forward
# ran. ScheduleBatch.copy() drops seq_lens but keeps seq_lens_cpu, and
# that snapshot is already in the forward's row order -- the live batch
# may have been filtered or merged since, and reindexing it would skip
# exactly the rounds whose composition changed.
device = verify_out_cache_loc.device
# Preserve the forward batch's row order after live-batch recomposition.
device = (
verify_out_cache_loc.device
if verify_out_cache_loc is not None
else batch.seq_lens.device
)
if fwd_batch.seq_lens_cpu is not None:
seq_lens = fwd_batch.seq_lens_cpu.to(device=device, dtype=torch.int64)
elif live_rids == fwd_rids:
seq_lens = batch.seq_lens
else:
return
raise RuntimeError(
"PP-spec delayed relay cannot commit a recomposed micro-batch "
"without its forward-time seq_lens_cpu snapshot"
)
fwd_batch.seq_lens = seq_lens
# copy() drops tree_cache, which the tracking-grid commit needs.
fwd_batch.tree_cache = batch.tree_cache
accept_index = accept_index.to(device)
accept_lens = pp_outputs["spec_accept_lens"].to(device)
# The last stage already commits inside run_eagle_verify.
if not self.pp_group.is_last_rank:
commit_mamba_states_after_verify(
self.tp_worker,
fwd_batch,
accept_lens,
accept_index,
get_spec().speculative_num_draft_tokens,
)
if verify_out_cache_loc is None:
return
fwd_batch.out_cache_loc = verify_out_cache_loc
move_accept_tokens_to_target_kvcache(
fwd_batch,
accept_index.to(device),
pp_outputs["spec_accept_lens"].to(device) - 1,
self.token_to_kv_pool_allocator,
)
if get_spec().speculative_eagle_topk > 1:
move_accept_tokens_to_target_kvcache(
fwd_batch,
accept_index,
accept_lens - 1,
self.token_to_kv_pool_allocator,
)
def _pp_spec_adopt_relayed_tree(
self: Scheduler,
@@ -1733,11 +1751,7 @@ class SchedulerPPMixin:
)
mb_metadata[mb_id] = PPBatchMetadata(
can_run_cuda_graph=result.can_run_cuda_graph,
fwd_batch=(
cur_batch.copy()
if not cur_batch.spec_algorithm.is_none()
else None
),
fwd_batch=_pp_snapshot_forward_batch(cur_batch),
verify_out_cache_loc=result.spec_verify_out_cache_loc,
)
event = self.device_module.Event()
+4 -2
View File
@@ -1553,8 +1553,10 @@ class HybridReqToTokenPool(ReqToTokenPool):
) -> None:
self.ngram_pool.set_context(ngram_indices, context)
def set_ngram_intermediate_context(self, context: torch.Tensor) -> None:
self.ngram_pool.set_intermediate_context(context)
def set_ngram_intermediate_context(
self, context: torch.Tensor, indices: Optional[torch.Tensor] = None
) -> None:
self.ngram_pool.set_intermediate_context(context, indices)
def copy_mamba_state(
self, src_index: torch.Tensor, dst_index: torch.Tensor
+13 -4
View File
@@ -202,11 +202,20 @@ class NGramPool:
device=self.context.device, dtype=self.context.dtype
)
def set_intermediate_context(self, context: torch.Tensor):
def set_intermediate_context(
self, context: torch.Tensor, indices: Optional[torch.Tensor] = None
):
if self.intermediate_context is not None and context.numel() > 0:
self.intermediate_context[: context.shape[0], : context.shape[1]].copy_(
context.to(device=self.context.device, dtype=self.context.dtype)
)
context = context.to(device=self.context.device, dtype=self.context.dtype)
if indices is None:
self.intermediate_context[: context.shape[0], : context.shape[1]].copy_(
context
)
else:
self.intermediate_context[
indices.to(device=self.context.device, dtype=torch.long),
: context.shape[1],
] = context
def clear(self):
if self.context is not None:
+27 -3
View File
@@ -21,6 +21,9 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.attention.linear.utils import (
select_verify_intermediate_state_indices,
)
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.dp_attention import (
attn_tp_all_gather,
@@ -315,12 +318,19 @@ def _commit_ple_batch(batch: Optional[_PLEBatch], forward_batch: ForwardBatch) -
valid_steps = batch.valid_tokens.reshape(
batch.lengths.shape[0], batch.row_width
)
dst_indices_raw = select_verify_intermediate_state_indices(
None,
forward_batch.req_pool_indices,
batch.lengths.ne(0),
pool.size,
)
pool.set_ngram_intermediate_context(
torch.where(
valid_steps.unsqueeze(-1),
step_contexts,
torch.full_like(step_contexts, batch.ngram_eos_token_id),
)
),
dst_indices_raw,
)
return
@@ -1097,9 +1107,23 @@ class Qwen4ExpPLELayer(nn.Module):
intermediate_state,
torch.zeros_like(intermediate_state),
)
intermediate_cache[: batch.lengths.shape[0], : batch.row_width].copy_(
intermediate_state.to(dtype=intermediate_cache.dtype)
intermediate_state = intermediate_state.to(
dtype=intermediate_cache.dtype
)
dst_indices_raw = select_verify_intermediate_state_indices(
None,
forward_batch.req_pool_indices,
batch.lengths.ne(0),
get_req_to_token_pool().size,
)
if dst_indices_raw is None:
intermediate_cache[
: batch.lengths.shape[0], : batch.row_width
].copy_(intermediate_state)
else:
intermediate_cache[
dst_indices_raw.to(dtype=torch.long), : batch.row_width
] = intermediate_state
else:
state_cols = torch.arange(
self.short_conv_state_len, device=x.device, dtype=torch.long
+9 -3
View File
@@ -41,6 +41,7 @@ from sglang.srt.distributed.parallel_state import (
patch_tensor_parallel_group,
)
from sglang.srt.environ import envs
from sglang.srt.layers.attention.linear.utils import pp_spec_stable_rows_enabled
from sglang.srt.managers.schedule_batch import set_mamba_track_indices_from_reqs
from sglang.srt.managers.utils import _async_d2h
from sglang.srt.mem_cache.allocation import (
@@ -950,6 +951,10 @@ def commit_mamba_states_after_verify(
# ring is allocated only then; KDA never allocates the cursors.
req_pool = model_runner.req_to_token_pool
mamba_pool = getattr(req_pool, "mamba_pool", None)
bs = accept_lens.shape[0]
src_indices_raw = (
batch.req_pool_indices[:bs] if pp_spec_stable_rows_enabled() else None
)
# Fold-every-commit: replay the accepted prefix from the ring into
# `temporal`; the same fold stores the interval-crossing state to the
@@ -980,6 +985,7 @@ def commit_mamba_states_after_verify(
last_correct_step_indices=last_correct_step_indices,
mamba_track_indices=batch.mamba_track_indices,
mamba_steps_to_track=mamba_steps_to_track,
src_indices_raw=src_indices_raw,
null_block_id=-1,
)
return
@@ -1000,7 +1006,6 @@ def commit_mamba_states_after_verify(
)
spec_state = req_pool.get_speculative_mamba2_params_all_layers()
bs = accept_lens.shape[0]
state_batch_indices = req_pool.get_mamba_indices(batch.req_pool_indices)
replay_indices = batch.req_pool_indices
last_correct_step_indices, mamba_steps_to_track = _verify_commit_step_indices(
@@ -1047,6 +1052,7 @@ def commit_mamba_states_after_verify(
spec_state.intermediate_conv_window[0],
state_batch_indices,
last_correct_step_indices,
src_indices_raw,
)
if batch.mamba_track_indices is not None:
fused_conv_window_scatter_with_mask(
@@ -1054,6 +1060,7 @@ def commit_mamba_states_after_verify(
spec_state.intermediate_conv_window[0],
batch.mamba_track_indices,
mamba_steps_to_track,
src_indices_raw,
)
return
@@ -1075,7 +1082,6 @@ def commit_mamba_states_after_verify(
)
spec_state = req_pool.get_speculative_mamba2_params_all_layers()
bs = accept_lens.shape[0]
state_batch_indices = req_pool.get_mamba_indices(batch.req_pool_indices)
accept_indices_offset = torch.arange(
0,
@@ -1115,13 +1121,13 @@ def commit_mamba_states_after_verify(
last_correct_step_indices=last_correct_step_indices,
mamba_track_indices=mamba_track_indices,
mamba_steps_to_track=mamba_steps_to_track,
src_indices_raw=src_indices_raw,
null_block_id=-1, # SGLang: valid slots >= 0, padding == -1
)
return
attn_backend = model_runner.attn_backend
bs = accept_lens.shape[0]
# `accept_lens` already includes the bonus token (drafts + 1 per req).
if not batch.forward_mode.is_idle() and accept_index.numel() > 0:
last_correct_step_indices, mamba_steps_to_track = _verify_commit_step_indices(