diff --git a/python/sglang/kernels/ops/attention/fla/gdn_replayssm_spec_fold.py b/python/sglang/kernels/ops/attention/fla/gdn_replayssm_spec_fold.py index 2803ad6e82..7f9b9afe48 100644 --- a/python/sglang/kernels/ops/attention/fla/gdn_replayssm_spec_fold.py +++ b/python/sglang/kernels/ops/attention/fla/gdn_replayssm_spec_fold.py @@ -225,6 +225,7 @@ def commit_gdn_replayssm_fold_after_verify( last_correct_step_indices: torch.Tensor, # [B] conv rollback target step mamba_track_indices: torch.Tensor | None = None, mamba_steps_to_track: torch.Tensor | None = None, + src_indices_raw: torch.Tensor | None = None, null_block_id: int = -1, ) -> None: """Fold each layer's accepted window into ``temporal``, then do the usual @@ -253,9 +254,17 @@ def commit_gdn_replayssm_fold_after_verify( spec_state.conv, spec_state.intermediate_conv_window ): fused_conv_window_scatter_with_mask( - conv_states, interm_conv, state_batch_indices, last_correct_step_indices + conv_states, + interm_conv, + state_batch_indices, + last_correct_step_indices, + src_indices_raw, ) if mamba_track_indices is not None and mamba_steps_to_track is not None: fused_conv_window_scatter_with_mask( - conv_states, interm_conv, mamba_track_indices, mamba_steps_to_track + conv_states, + interm_conv, + mamba_track_indices, + mamba_steps_to_track, + src_indices_raw, ) diff --git a/python/sglang/kernels/ops/attention/fla/kda_replayssm_spec_decode.py b/python/sglang/kernels/ops/attention/fla/kda_replayssm_spec_decode.py index d5108fbc88..8007ee12f3 100644 --- a/python/sglang/kernels/ops/attention/fla/kda_replayssm_spec_decode.py +++ b/python/sglang/kernels/ops/attention/fla/kda_replayssm_spec_decode.py @@ -341,6 +341,7 @@ def commit_kda_replayssm_after_verify( last_correct_step_indices: torch.Tensor, # [B] conv rollback target step mamba_track_indices: torch.Tensor | None = None, mamba_steps_to_track: torch.Tensor | None = None, + src_indices_raw: torch.Tensor | None = None, null_block_id: int = -1, ) -> None: """Fold each layer's accepted window into `temporal` and roll back conv. @@ -378,9 +379,17 @@ def commit_kda_replayssm_after_verify( spec_state.conv, spec_state.intermediate_conv_window ): fused_conv_window_scatter_with_mask( - conv_states, interm_conv, state_batch_indices, last_correct_step_indices + conv_states, + interm_conv, + state_batch_indices, + last_correct_step_indices, + src_indices_raw, ) if mamba_track_indices is not None and mamba_steps_to_track is not None: fused_conv_window_scatter_with_mask( - conv_states, interm_conv, mamba_track_indices, mamba_steps_to_track + conv_states, + interm_conv, + mamba_track_indices, + mamba_steps_to_track, + src_indices_raw, ) diff --git a/python/sglang/kernels/ops/mamba/mamba_state_scatter_triton.py b/python/sglang/kernels/ops/mamba/mamba_state_scatter_triton.py index 7896812278..ae30ad674e 100644 --- a/python/sglang/kernels/ops/mamba/mamba_state_scatter_triton.py +++ b/python/sglang/kernels/ops/mamba/mamba_state_scatter_triton.py @@ -153,6 +153,7 @@ def _fused_mamba_state_scatter_with_mask_kernel( # Raw index arrays (before index_select) dst_indices_raw_ptr, # [total_requests] - state_indices_tensor step_indices_raw_ptr, # [total_requests] - last_correct_step_indices or mamba_steps_to_track + src_indices_raw_ptr, # [total_requests] - verify scratch row for each request elem_per_entry: tl.constexpr, src_layer_stride, src_req_stride, @@ -189,8 +190,7 @@ def _fused_mamba_state_scatter_with_mask_kernel( # Load destination index dst_idx = tl.load(dst_indices_raw_ptr + pid_req).to(tl.int64) - # Source index is just the request index itself - src_idx = pid_req + src_idx = tl.load(src_indices_raw_ptr + pid_req).to(tl.int64) # Bounds check to avoid illegal memory access if not ( @@ -220,11 +220,22 @@ def _fused_mamba_state_scatter_with_mask_kernel( tl.store(dst_ptr + dst_offset + offsets, data, mask=mask) +def _prepare_src_rows(src: torch.Tensor | None, steps: torch.Tensor) -> torch.Tensor: + if src is None: + src = torch.arange(steps.shape[0], dtype=torch.int32, device=steps.device) + if src.ndim != 1 or src.shape != steps.shape: + raise ValueError( + f"source indices length mismatch: {src.shape=} vs {steps.shape=}" + ) + return src.to(torch.int32).contiguous() + + def fused_mamba_state_scatter_with_mask( dst: torch.Tensor, # [num_layers, cache_size, *state_shape] src: torch.Tensor, # [num_layers, spec_size, draft_tokens, *state_shape] dst_indices_raw: torch.Tensor, # [total_requests] - raw indices (e.g., state_indices_tensor) step_indices_raw: torch.Tensor, # [total_requests] - raw step indices (step >= 0 means valid) + src_indices_raw: torch.Tensor | None = None, # [total_requests] - scratch rows ): """ Fully fused gather-scatter with built-in masking for mamba state updates. @@ -291,6 +302,7 @@ def fused_mamba_state_scatter_with_mask( # Ensure indices are int32 and contiguous dst_indices_raw = dst_indices_raw.to(torch.int32).contiguous() step_indices_raw = step_indices_raw.to(torch.int32).contiguous() + src_indices_raw = _prepare_src_rows(src_indices_raw, step_indices_raw) _require_entry_contiguous_dst(dst, 2, "fused_mamba_state_scatter_with_mask") if not src.is_contiguous(): @@ -307,6 +319,7 @@ def fused_mamba_state_scatter_with_mask( dst, dst_indices_raw, step_indices_raw, + src_indices_raw, elem_per_entry, src_layer_stride, src_req_stride, @@ -326,6 +339,7 @@ def _fused_conv_window_scatter_with_mask_kernel( dst_ptr, dst_indices_raw_ptr, # [total_requests] step_indices_raw_ptr, # [total_requests], entry >= 0 means valid + src_indices_raw_ptr, # [total_requests] elem_per_entry: tl.constexpr, # dim * (K-1) KM1: tl.constexpr, # K-1 (conv window width) src_layer_stride, @@ -359,7 +373,7 @@ def _fused_conv_window_scatter_with_mask_kernel( return dst_idx = tl.load(dst_indices_raw_ptr + pid_req).to(tl.int64) - src_idx = pid_req + src_idx = tl.load(src_indices_raw_ptr + pid_req).to(tl.int64) if not ( (dst_idx >= 0) @@ -396,6 +410,7 @@ def fused_conv_window_scatter_with_mask( src: torch.Tensor, # deduped conv-window view [num_layers, spec_size, draft_tokens, dim, K-1] dst_indices_raw: torch.Tensor, # [total_requests] step_indices_raw: torch.Tensor, # [total_requests], entry >= 0 means valid + src_indices_raw: torch.Tensor | None = None, # [total_requests] ): """Conv-window variant of :func:`fused_mamba_state_scatter_with_mask`. @@ -446,6 +461,7 @@ def fused_conv_window_scatter_with_mask( dst_indices_raw = dst_indices_raw.to(torch.int32).contiguous() step_indices_raw = step_indices_raw.to(torch.int32).contiguous() + src_indices_raw = _prepare_src_rows(src_indices_raw, step_indices_raw) BLOCK_SIZE = 1024 grid = (total_requests, num_layers, triton.cdiv(elem_per_entry, BLOCK_SIZE)) @@ -455,6 +471,7 @@ def fused_conv_window_scatter_with_mask( dst, dst_indices_raw, step_indices_raw, + src_indices_raw, elem_per_entry, km1, src.stride(0), @@ -681,6 +698,7 @@ def scatter_mamba_states_after_mtp_verify( last_correct_step_indices: torch.Tensor, mamba_track_indices: torch.Tensor | None, mamba_steps_to_track: torch.Tensor | None, + src_indices_raw: torch.Tensor | None = None, ) -> None: """Scatter per-step verify states (ssm + all conv types) into the persistent caches, plus the interval-crossing track slots.""" @@ -693,6 +711,7 @@ def scatter_mamba_states_after_mtp_verify( intermediate_state_cache, state_indices_tensor, last_correct_step_indices, + src_indices_raw, ) if mamba_track_indices is not None: assert mamba_steps_to_track is not None @@ -701,6 +720,7 @@ def scatter_mamba_states_after_mtp_verify( intermediate_state_cache, mamba_track_indices, mamba_steps_to_track, + src_indices_raw, ) pairs = list(zip(mamba_caches.conv, mamba_caches.intermediate_conv_window)) @@ -708,7 +728,7 @@ def scatter_mamba_states_after_mtp_verify( return if mamba_track_indices is not None: assert mamba_steps_to_track is not None - if _conv_multi_eligible(pairs): + if src_indices_raw is None and _conv_multi_eligible(pairs): fused_conv_window_scatter_multi( pairs, state_indices_tensor, @@ -723,6 +743,7 @@ def scatter_mamba_states_after_mtp_verify( intermediate_conv_window_cache, state_indices_tensor, last_correct_step_indices, + src_indices_raw, ) if mamba_track_indices is not None: for conv_states, intermediate_conv_window_cache in pairs: @@ -731,6 +752,7 @@ def scatter_mamba_states_after_mtp_verify( intermediate_conv_window_cache, mamba_track_indices, mamba_steps_to_track, + src_indices_raw, ) diff --git a/python/sglang/srt/arg_groups/attention_hook.py b/python/sglang/srt/arg_groups/attention_hook.py index b5ef4ed97f..5dc9aae06f 100644 --- a/python/sglang/srt/arg_groups/attention_hook.py +++ b/python/sglang/srt/arg_groups/attention_hook.py @@ -27,8 +27,10 @@ from sglang.srt.arg_groups.overrides import ( resolving_view, run_post_process_pass, ) +from sglang.srt.configs.model_config import uses_kda_attention from sglang.srt.connector import ConnectorType from sglang.srt.environ import envs +from sglang.srt.layers.attention.linear.utils import pp_spec_stable_rows_enabled from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase from sglang.srt.runtime_context import get_platform from sglang.srt.utils.common import ( @@ -275,6 +277,23 @@ def handle_linear_attn_backend(server_args: Any): verify = cfg.linear_attn_verify_backend if verify is None and decode == "flashinfer": verify = "flashinfer" + if ( + pp_spec_stable_rows_enabled() + and verify == "flashinfer" + and uses_kda_attention(model_config_of(server_args).hf_config) + ): + declare_resolution( + server_args, + "_handle_linear_attn_backend", + linear_attn_verify_backend="triton", + ) + verify = "triton" + logger.warning( + "SGLANG_ENABLE_PP_SPEC with KDA does not support the FlashInfer " + "target-verify row layout; falling back " + "--linear-attn-verify-backend to triton. FlashInfer decode is " + "unchanged." + ) if ( verify == "flashinfer" and cfg.mamba_ssm_dtype != "bfloat16" diff --git a/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py b/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py index e04f2fe354..0c5f4c49a6 100644 --- a/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py +++ b/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py @@ -21,6 +21,7 @@ from sglang.srt.layers.attention.base_attn_backend import ( AttentionBackend, SharedReadEnds, ) +from sglang.srt.layers.attention.linear.utils import pp_spec_stable_rows_enabled from sglang.srt.layers.attention.mamba.mamba import MambaMixer2 from sglang.srt.layers.attention.mamba.mamba2_metadata import ( ForwardMetadata, @@ -1425,14 +1426,11 @@ class HybridLinearAttnBackend(AttentionBackend): model, req_pool_indices: Optional[torch.Tensor] = None, ): - """Update mamba states after MTP verify via a fused gather-scatter kernel. - - ``req_pool_indices`` serves implementations that must re-derive the state - slot ids instead of reusing this step's ``forward_metadata``; the scatter - below reads the metadata it just planned. - """ - del req_pool_indices + """Commit accepted Mamba states; PP rows identify the delayed batch.""" request_number = last_correct_step_indices.shape[0] + src_indices_raw = None + if pp_spec_stable_rows_enabled() and req_pool_indices is not None: + src_indices_raw = req_pool_indices[:request_number] # `mamba_track_indices` is VIRTUAL; the scatter writes physical views. if mamba_track_indices is not None: @@ -1440,11 +1438,18 @@ class HybridLinearAttnBackend(AttentionBackend): mamba_track_indices ) - state_indices_tensor = ( - self.linear_attn_backend.forward_metadata.mamba_cache_indices[ - :request_number - ] - ) + if src_indices_raw is not None: + state_indices_tensor = self.linear_attn_backend._translate_mamba_indices( + self.linear_attn_backend.req_to_token_pool.get_mamba_indices( + req_pool_indices[:request_number] + ) + ) + else: + state_indices_tensor = ( + self.linear_attn_backend.forward_metadata.mamba_cache_indices[ + :request_number + ] + ) req_pool = self.linear_attn_backend.req_to_token_pool mamba_caches = req_pool.get_speculative_mamba2_params_all_layers() @@ -1469,6 +1474,7 @@ class HybridLinearAttnBackend(AttentionBackend): last_correct_step_indices=last_correct_step_indices, mamba_track_indices=mamba_track_indices, mamba_steps_to_track=mamba_steps_to_track, + src_indices_raw=src_indices_raw, null_block_id=-1, ) return @@ -1492,6 +1498,7 @@ class HybridLinearAttnBackend(AttentionBackend): intermediate_conv_window, state_indices_tensor, last_correct_step_indices, + src_indices_raw, ) accept_lens_pool[state_indices_tensor.to(torch.int64)] = ( last_correct_step_indices.to(torch.int32) + 1 @@ -1504,6 +1511,7 @@ class HybridLinearAttnBackend(AttentionBackend): last_correct_step_indices, mamba_track_indices, mamba_steps_to_track, + src_indices_raw=src_indices_raw, ) self._update_ple_state_after_mtp_verify( @@ -1511,6 +1519,7 @@ class HybridLinearAttnBackend(AttentionBackend): last_correct_step_indices, mamba_track_indices, mamba_steps_to_track, + src_indices_raw, ) @staticmethod @@ -1519,19 +1528,28 @@ class HybridLinearAttnBackend(AttentionBackend): src: torch.Tensor, dst_indices_raw: torch.Tensor, step_indices_raw: torch.Tensor, + src_indices_raw: Optional[torch.Tensor] = None, ): if dst is None or src is None or step_indices_raw.numel() == 0: return if dst.is_cuda and src.is_cuda: fused_mamba_state_scatter_with_mask( - dst, src, dst_indices_raw, step_indices_raw + dst, + src, + dst_indices_raw, + step_indices_raw, + src_indices_raw, ) return device = dst.device dst_indices = dst_indices_raw.to(device=device, dtype=torch.long) steps = step_indices_raw.to(device=device, dtype=torch.long) - src_indices = torch.arange(steps.shape[0], device=device, dtype=torch.long) + src_indices = ( + torch.arange(steps.shape[0], device=device, dtype=torch.long) + if src_indices_raw is None + else src_indices_raw.to(device=device, dtype=torch.long) + ) valid = ( (steps >= 0) & (steps < src.shape[2]) @@ -1552,6 +1570,7 @@ class HybridLinearAttnBackend(AttentionBackend): last_correct_step_indices: torch.Tensor, mamba_track_indices: Optional[torch.Tensor], mamba_steps_to_track: Optional[torch.Tensor], + src_indices_raw: Optional[torch.Tensor] = None, ): """Roll the accepted per-step PLE side states into their main slots.""" req_to_token_pool = self.linear_attn_backend.req_to_token_pool @@ -1589,6 +1608,7 @@ class HybridLinearAttnBackend(AttentionBackend): intermediate_state, state_indices_tensor, last_correct_step_indices, + src_indices_raw, ) if mamba_track_indices is not None: self._scatter_speculative_state_with_mask( @@ -1596,6 +1616,7 @@ class HybridLinearAttnBackend(AttentionBackend): intermediate_state, mamba_track_indices, mamba_steps_to_track, + src_indices_raw, ) diff --git a/python/sglang/srt/layers/attention/linear/gdn_backend.py b/python/sglang/srt/layers/attention/linear/gdn_backend.py index 5045bf5809..1dfffdbb2c 100644 --- a/python/sglang/srt/layers/attention/linear/gdn_backend.py +++ b/python/sglang/srt/layers/attention/linear/gdn_backend.py @@ -16,6 +16,8 @@ from sglang.srt.layers.attention.linear.utils import ( LinearAttnBackends, LinearAttnKernelBackend, build_verify_intermediate_state_indices, + pp_spec_stable_rows_enabled, + select_verify_intermediate_state_indices, ) from sglang.srt.layers.radix_linear_attention import RadixLinearAttention from sglang.srt.mem_cache.memory_pool import MambaPool @@ -953,7 +955,12 @@ class GDNAttnBackend(MambaAttnBackendBase): intermediate_conv_window_cache = ( mamba_cache_params.intermediate_conv_window[0] ) - intermediate_state_indices = self.verify_intermediate_state_indices + intermediate_state_indices = select_verify_intermediate_state_indices( + self.verify_intermediate_state_indices, + forward_batch.req_pool_indices, + cache_indices[: query_start_loc.shape[0] - 1] >= 0, + self.req_to_token_pool.size, + ) mamba_pool = self.req_to_token_pool.mamba_pool use_replayssm_fold = ( mamba_cache_params.replayssm_rawv is not None @@ -1117,6 +1124,7 @@ class GDNAttnBackend(MambaAttnBackendBase): intermediate_state_indices=intermediate_state_indices, cache_steps=forward_batch.spec_info.draft_token_num, retrieve_parent_token=retrieve_parent_token, + stable_rows=pp_spec_stable_rows_enabled(), ) else: g, beta = self._prefill_gates(layer, a, b) diff --git a/python/sglang/srt/layers/attention/linear/kda_backend.py b/python/sglang/srt/layers/attention/linear/kda_backend.py index be4f804ba0..29fdbbed6b 100644 --- a/python/sglang/srt/layers/attention/linear/kda_backend.py +++ b/python/sglang/srt/layers/attention/linear/kda_backend.py @@ -17,6 +17,7 @@ from sglang.srt.layers.attention.linear.kernels.kda_triton import TritonKDAKerne from sglang.srt.layers.attention.linear.utils import ( LinearAttnKernelBackend, build_verify_intermediate_state_indices, + select_verify_intermediate_state_indices, ) from sglang.srt.layers.radix_linear_attention import RadixLinearAttention from sglang.srt.utils import is_cpu, is_cuda, is_npu @@ -1027,7 +1028,12 @@ class KDAAttnBackend(MambaAttnBackendBase): replayssm_beta=replayssm_beta, ) intermediate_conv_window_cache = mamba_cache_params.intermediate_conv_window[0] - intermediate_state_indices = self.verify_intermediate_state_indices + intermediate_state_indices = select_verify_intermediate_state_indices( + self.verify_intermediate_state_indices, + forward_batch.req_pool_indices, + cache_indices[: query_start_loc.shape[0] - 1] >= 0, + self.req_to_token_pool.size, + ) draft_token_num = forward_batch.spec_info.draft_token_num ragged_layout = forward_batch.spec_info.ragged_verify_layout diff --git a/python/sglang/srt/layers/attention/linear/kernels/gdn_flashinfer.py b/python/sglang/srt/layers/attention/linear/kernels/gdn_flashinfer.py index e66cee0ab5..135d53c9e3 100644 --- a/python/sglang/srt/layers/attention/linear/kernels/gdn_flashinfer.py +++ b/python/sglang/srt/layers/attention/linear/kernels/gdn_flashinfer.py @@ -33,6 +33,20 @@ logger = logging.getLogger(__name__) _FLASHINFER_GDN_ALIGNMENT = 32 +def copy_verify_intermediate_rows( + destination: torch.Tensor, + positional_source: torch.Tensor, + dst_indices_raw: torch.Tensor, +) -> None: + """Move FlashInfer's positional verify snapshots to their owned rows.""" + count = dst_indices_raw.shape[0] + destination.index_copy_( + 0, + dst_indices_raw.to(device=destination.device, dtype=torch.long), + positional_source[:count], + ) + + def _empty_aligned_like( tensor: torch.Tensor, alignment: int = _FLASHINFER_GDN_ALIGNMENT ) -> torch.Tensor: @@ -340,12 +354,14 @@ class FlashInferGDNKernel(LinearAttnKernelBase): self, intermediate_states_buffer: torch.Tensor, batch_size: int, + force_scratch: bool = False, ) -> tuple[torch.Tensor, bool]: # FlashInfer requires exact capture B, which may exceed the pool-scoped # buffer; padded tiers use stable scratch and copy owned rows back. - direct = intermediate_states_buffer[:batch_size] - if direct.shape[0] == batch_size: - return direct, False + if not force_scratch: + direct = intermediate_states_buffer[:batch_size] + if direct.shape[0] == batch_size: + return direct, False stream_key = ( torch.cuda.current_stream(intermediate_states_buffer.device).cuda_stream @@ -598,6 +614,7 @@ class FlashInferGDNKernel(LinearAttnKernelBase): intermediate_state_indices: torch.Tensor, cache_steps: int, retrieve_parent_token: torch.Tensor, + stable_rows: bool = False, **kwargs, ) -> torch.Tensor: # MTP verify using FlashInfer gated_delta_rule_mtp kernel (SM90 + SM100+). @@ -623,16 +640,17 @@ class FlashInferGDNKernel(LinearAttnKernelBase): intermediate_states_buffer_mtp = intermediate_states_buffer copy_verify_intermediate_back = False - if self.use_state_pool and intermediate_states_buffer is not None: - # The SM100 bf16 MTP kernel indexes this scratch buffer by the - # per-call batch id, while SGLang's speculative state cache is - # Graph padding can exceed the pool-scoped scratch; use exact-B storage - # and copy owned rows back before post-verify commit reads the pool. + if intermediate_states_buffer is not None and ( + self.use_state_pool or stable_rows + ): + # FlashInfer writes positional rows; isolate them in exact-B scratch. ( intermediate_states_buffer_mtp, copy_verify_intermediate_back, ) = self._prepare_verify_intermediate_buffer( - intermediate_states_buffer, batch_size + intermediate_states_buffer, + batch_size, + force_scratch=stable_rows, ) if not self._mutable_inputs_are_aligned( ("ssm_states", ssm_states), @@ -697,7 +715,14 @@ class FlashInferGDNKernel(LinearAttnKernelBase): use_qk_l2norm=True, ) - if copy_verify_intermediate_back: + if stable_rows and intermediate_states_buffer is not None: + # Persist positional output under PP-stable request rows. + copy_verify_intermediate_rows( + intermediate_states_buffer, + intermediate_states_buffer_mtp, + intermediate_state_indices[:batch_size], + ) + elif copy_verify_intermediate_back: intermediate_states_buffer.copy_( intermediate_states_buffer_mtp[: intermediate_states_buffer.shape[0]] ) diff --git a/python/sglang/srt/layers/attention/linear/utils.py b/python/sglang/srt/layers/attention/linear/utils.py index c8e469e542..5b0af9e510 100644 --- a/python/sglang/srt/layers/attention/linear/utils.py +++ b/python/sglang/srt/layers/attention/linear/utils.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Optional import msgspec +from sglang.srt.environ import envs from sglang.srt.runtime_context import get_exec from sglang.srt.utils.common import rank0_log @@ -12,6 +13,10 @@ if TYPE_CHECKING: pass +def pp_spec_stable_rows_enabled() -> bool: + return envs.SGLANG_ENABLE_PP_SPEC.get() + + class LinearAttnKernelBackend(Enum): TRITON = "triton" CUTEDSL = "cutedsl" @@ -103,6 +108,20 @@ def resolve_linear_attn_backends( return backends +def select_verify_intermediate_state_indices( + default_indices, req_pool_indices, valid, pool_size: int +): + if not pp_spec_stable_rows_enabled(): + return default_indices + + import torch + + req_rows = req_pool_indices[: valid.shape[0]] + return torch.where(valid, req_rows, torch.full_like(req_rows, pool_size)).to( + torch.int32 + ) + + def build_verify_intermediate_state_indices(pool_size: int, device): """Per-request row index into the speculative intermediate scratch (`intermediate_ssm` / `intermediate_conv_window`) for the MTP / diff --git a/python/sglang/srt/managers/scheduler_pp_mixin.py b/python/sglang/srt/managers/scheduler_pp_mixin.py index 71bca7dabf..0013308fa9 100644 --- a/python/sglang/srt/managers/scheduler_pp_mixin.py +++ b/python/sglang/srt/managers/scheduler_pp_mixin.py @@ -14,6 +14,7 @@ from sglang.srt.disaggregation.utils import poll_and_all_reduce_attn_cp_tp_group from sglang.srt.distributed.communication_op import attn_cp_tp_broadcast_pyobj from sglang.srt.distributed.parallel_state import P2PWork from sglang.srt.environ import envs +from sglang.srt.layers.attention.linear.utils import pp_spec_stable_rows_enabled from sglang.srt.layers.logits_processor import LogitsProcessorOutput from sglang.srt.managers.overlap_utils import RelayPayload from sglang.srt.managers.schedule_batch import FINISH_ABORT, Req, ScheduleBatch @@ -55,6 +56,14 @@ def _pp_can_skip_output_comm(batch: ScheduleBatch) -> bool: ) +def _pp_snapshot_forward_batch(batch: ScheduleBatch) -> Optional[ScheduleBatch]: + if batch.spec_algorithm.is_none(): + return None + fwd_batch = batch.copy() + fwd_batch.req_pool_indices = batch.req_pool_indices.clone() + return fwd_batch + + def _pp_exchange_outputs_before_forward( cur_batch: Optional[ScheduleBatch], spec_relay: bool, @@ -592,7 +601,7 @@ class SchedulerPPMixin: self.pp_outputs: Optional[PPProxyTensors] = None self.last_rank_comm_queue: deque[Tuple[torch.Event, PPProxyTensors]] = deque() self._pp_spec_relay = ( - envs.SGLANG_ENABLE_PP_SPEC.get() + pp_spec_stable_rows_enabled() and get_parallel().pp_size > 1 and not self.spec_algorithm.is_none() ) @@ -879,12 +888,8 @@ class SchedulerPPMixin: tensor_dict["spec_accept_lens"] = result.accept_lens tensor_dict["spec_new_seq_lens"] = result.new_seq_lens tensor_dict["spec_bonus_tokens"] = result.next_draft_input.bonus_tokens - if ( - result.accept_index is not None - and get_spec().speculative_eagle_topk > 1 - ): - # Only a tree needs it: a chain's accepted path is already the - # front of each block, so compacting it is an identity. + if result.accept_index is not None: + # Relayed recurrent commits also need chain accept indices. tensor_dict["spec_accept_index"] = result.accept_index if result.next_verify_chain is not None: # Tail-drafted tree for the next verify round (root = bonus), @@ -1093,7 +1098,7 @@ class SchedulerPPMixin: new_seq_lens = pp_outputs["spec_new_seq_lens"] fwd_rids = [req.rid for req in fwd_batch.reqs] live_rids = [req.rid for req in batch.reqs] - self._pp_spec_compact_accept_kv( + self._pp_spec_commit_relayed_accept( batch, fwd_batch, fwd_rids, @@ -1229,7 +1234,7 @@ class SchedulerPPMixin: ): self.process_batch_result(batch, output_result) - def _pp_spec_compact_accept_kv( + def _pp_spec_commit_relayed_accept( self: Scheduler, batch: ScheduleBatch, fwd_batch: ScheduleBatch, @@ -1238,47 +1243,60 @@ class SchedulerPPMixin: verify_out_cache_loc: Optional[torch.Tensor], pp_outputs, ) -> None: - """Move this stage's accepted-path KV to the front of each request block. + """Commit relayed recurrent state, then compact tree KV when present. - The verify forward writes one KV slot per tree node, in node order. The - committed prefix that every later read assumes is the accepted path laid - out contiguously, so the two have to be reconciled once per round -- and - each stage has to do it for its own layers, since KV is not relayed. - The last stage does it inside verify (_finalize_accept_tree_path); this - is the same step for the stages that only ran the target forward. - - Must run before seq_lens advances: the move writes into the block that - starts at the pre-advance length. + Must run before the live batch advances ``seq_lens``. """ accept_index = pp_outputs.tensors.get("spec_accept_index") if accept_index is None or fwd_batch.forward_mode.is_idle(): return - if verify_out_cache_loc is None: - return from sglang.srt.speculative.spec_utils import ( + commit_mamba_states_after_verify, move_accept_tokens_to_target_kvcache, ) - # The destination base is the length each request had when the forward - # ran. ScheduleBatch.copy() drops seq_lens but keeps seq_lens_cpu, and - # that snapshot is already in the forward's row order -- the live batch - # may have been filtered or merged since, and reindexing it would skip - # exactly the rounds whose composition changed. - device = verify_out_cache_loc.device + # Preserve the forward batch's row order after live-batch recomposition. + device = ( + verify_out_cache_loc.device + if verify_out_cache_loc is not None + else batch.seq_lens.device + ) if fwd_batch.seq_lens_cpu is not None: seq_lens = fwd_batch.seq_lens_cpu.to(device=device, dtype=torch.int64) elif live_rids == fwd_rids: seq_lens = batch.seq_lens else: - return + raise RuntimeError( + "PP-spec delayed relay cannot commit a recomposed micro-batch " + "without its forward-time seq_lens_cpu snapshot" + ) fwd_batch.seq_lens = seq_lens + # copy() drops tree_cache, which the tracking-grid commit needs. + fwd_batch.tree_cache = batch.tree_cache + accept_index = accept_index.to(device) + accept_lens = pp_outputs["spec_accept_lens"].to(device) + + # The last stage already commits inside run_eagle_verify. + if not self.pp_group.is_last_rank: + commit_mamba_states_after_verify( + self.tp_worker, + fwd_batch, + accept_lens, + accept_index, + get_spec().speculative_num_draft_tokens, + ) + + if verify_out_cache_loc is None: + return fwd_batch.out_cache_loc = verify_out_cache_loc - move_accept_tokens_to_target_kvcache( - fwd_batch, - accept_index.to(device), - pp_outputs["spec_accept_lens"].to(device) - 1, - self.token_to_kv_pool_allocator, - ) + + if get_spec().speculative_eagle_topk > 1: + move_accept_tokens_to_target_kvcache( + fwd_batch, + accept_index, + accept_lens - 1, + self.token_to_kv_pool_allocator, + ) def _pp_spec_adopt_relayed_tree( self: Scheduler, @@ -1733,11 +1751,7 @@ class SchedulerPPMixin: ) mb_metadata[mb_id] = PPBatchMetadata( can_run_cuda_graph=result.can_run_cuda_graph, - fwd_batch=( - cur_batch.copy() - if not cur_batch.spec_algorithm.is_none() - else None - ), + fwd_batch=_pp_snapshot_forward_batch(cur_batch), verify_out_cache_loc=result.spec_verify_out_cache_loc, ) event = self.device_module.Event() diff --git a/python/sglang/srt/mem_cache/memory_pool.py b/python/sglang/srt/mem_cache/memory_pool.py index aa8a3b44bf..7e6151c890 100644 --- a/python/sglang/srt/mem_cache/memory_pool.py +++ b/python/sglang/srt/mem_cache/memory_pool.py @@ -1553,8 +1553,10 @@ class HybridReqToTokenPool(ReqToTokenPool): ) -> None: self.ngram_pool.set_context(ngram_indices, context) - def set_ngram_intermediate_context(self, context: torch.Tensor) -> None: - self.ngram_pool.set_intermediate_context(context) + def set_ngram_intermediate_context( + self, context: torch.Tensor, indices: Optional[torch.Tensor] = None + ) -> None: + self.ngram_pool.set_intermediate_context(context, indices) def copy_mamba_state( self, src_index: torch.Tensor, dst_index: torch.Tensor diff --git a/python/sglang/srt/mem_cache/ple_state_pool.py b/python/sglang/srt/mem_cache/ple_state_pool.py index a3ad345389..f6a1e43574 100644 --- a/python/sglang/srt/mem_cache/ple_state_pool.py +++ b/python/sglang/srt/mem_cache/ple_state_pool.py @@ -202,11 +202,20 @@ class NGramPool: device=self.context.device, dtype=self.context.dtype ) - def set_intermediate_context(self, context: torch.Tensor): + def set_intermediate_context( + self, context: torch.Tensor, indices: Optional[torch.Tensor] = None + ): if self.intermediate_context is not None and context.numel() > 0: - self.intermediate_context[: context.shape[0], : context.shape[1]].copy_( - context.to(device=self.context.device, dtype=self.context.dtype) - ) + context = context.to(device=self.context.device, dtype=self.context.dtype) + if indices is None: + self.intermediate_context[: context.shape[0], : context.shape[1]].copy_( + context + ) + else: + self.intermediate_context[ + indices.to(device=self.context.device, dtype=torch.long), + : context.shape[1], + ] = context def clear(self): if self.context is not None: diff --git a/python/sglang/srt/models/qwen4_exp.py b/python/sglang/srt/models/qwen4_exp.py index 7bcc07ca5a..884806d015 100644 --- a/python/sglang/srt/models/qwen4_exp.py +++ b/python/sglang/srt/models/qwen4_exp.py @@ -21,6 +21,9 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import ( from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation +from sglang.srt.layers.attention.linear.utils import ( + select_verify_intermediate_state_indices, +) from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList from sglang.srt.layers.dp_attention import ( attn_tp_all_gather, @@ -315,12 +318,19 @@ def _commit_ple_batch(batch: Optional[_PLEBatch], forward_batch: ForwardBatch) - valid_steps = batch.valid_tokens.reshape( batch.lengths.shape[0], batch.row_width ) + dst_indices_raw = select_verify_intermediate_state_indices( + None, + forward_batch.req_pool_indices, + batch.lengths.ne(0), + pool.size, + ) pool.set_ngram_intermediate_context( torch.where( valid_steps.unsqueeze(-1), step_contexts, torch.full_like(step_contexts, batch.ngram_eos_token_id), - ) + ), + dst_indices_raw, ) return @@ -1097,9 +1107,23 @@ class Qwen4ExpPLELayer(nn.Module): intermediate_state, torch.zeros_like(intermediate_state), ) - intermediate_cache[: batch.lengths.shape[0], : batch.row_width].copy_( - intermediate_state.to(dtype=intermediate_cache.dtype) + intermediate_state = intermediate_state.to( + dtype=intermediate_cache.dtype ) + dst_indices_raw = select_verify_intermediate_state_indices( + None, + forward_batch.req_pool_indices, + batch.lengths.ne(0), + get_req_to_token_pool().size, + ) + if dst_indices_raw is None: + intermediate_cache[ + : batch.lengths.shape[0], : batch.row_width + ].copy_(intermediate_state) + else: + intermediate_cache[ + dst_indices_raw.to(dtype=torch.long), : batch.row_width + ] = intermediate_state else: state_cols = torch.arange( self.short_conv_state_len, device=x.device, dtype=torch.long diff --git a/python/sglang/srt/speculative/spec_utils.py b/python/sglang/srt/speculative/spec_utils.py index 0a836db76d..3042d78d50 100644 --- a/python/sglang/srt/speculative/spec_utils.py +++ b/python/sglang/srt/speculative/spec_utils.py @@ -41,6 +41,7 @@ from sglang.srt.distributed.parallel_state import ( patch_tensor_parallel_group, ) from sglang.srt.environ import envs +from sglang.srt.layers.attention.linear.utils import pp_spec_stable_rows_enabled from sglang.srt.managers.schedule_batch import set_mamba_track_indices_from_reqs from sglang.srt.managers.utils import _async_d2h from sglang.srt.mem_cache.allocation import ( @@ -950,6 +951,10 @@ def commit_mamba_states_after_verify( # ring is allocated only then; KDA never allocates the cursors. req_pool = model_runner.req_to_token_pool mamba_pool = getattr(req_pool, "mamba_pool", None) + bs = accept_lens.shape[0] + src_indices_raw = ( + batch.req_pool_indices[:bs] if pp_spec_stable_rows_enabled() else None + ) # Fold-every-commit: replay the accepted prefix from the ring into # `temporal`; the same fold stores the interval-crossing state to the @@ -980,6 +985,7 @@ def commit_mamba_states_after_verify( last_correct_step_indices=last_correct_step_indices, mamba_track_indices=batch.mamba_track_indices, mamba_steps_to_track=mamba_steps_to_track, + src_indices_raw=src_indices_raw, null_block_id=-1, ) return @@ -1000,7 +1006,6 @@ def commit_mamba_states_after_verify( ) spec_state = req_pool.get_speculative_mamba2_params_all_layers() - bs = accept_lens.shape[0] state_batch_indices = req_pool.get_mamba_indices(batch.req_pool_indices) replay_indices = batch.req_pool_indices last_correct_step_indices, mamba_steps_to_track = _verify_commit_step_indices( @@ -1047,6 +1052,7 @@ def commit_mamba_states_after_verify( spec_state.intermediate_conv_window[0], state_batch_indices, last_correct_step_indices, + src_indices_raw, ) if batch.mamba_track_indices is not None: fused_conv_window_scatter_with_mask( @@ -1054,6 +1060,7 @@ def commit_mamba_states_after_verify( spec_state.intermediate_conv_window[0], batch.mamba_track_indices, mamba_steps_to_track, + src_indices_raw, ) return @@ -1075,7 +1082,6 @@ def commit_mamba_states_after_verify( ) spec_state = req_pool.get_speculative_mamba2_params_all_layers() - bs = accept_lens.shape[0] state_batch_indices = req_pool.get_mamba_indices(batch.req_pool_indices) accept_indices_offset = torch.arange( 0, @@ -1115,13 +1121,13 @@ def commit_mamba_states_after_verify( last_correct_step_indices=last_correct_step_indices, mamba_track_indices=mamba_track_indices, mamba_steps_to_track=mamba_steps_to_track, + src_indices_raw=src_indices_raw, null_block_id=-1, # SGLang: valid slots >= 0, padding == -1 ) return attn_backend = model_runner.attn_backend - bs = accept_lens.shape[0] # `accept_lens` already includes the bonus token (drafts + 1 per req). if not batch.forward_mode.is_idle() and accept_index.numel() > 0: last_correct_step_indices, mamba_steps_to_track = _verify_commit_step_indices( diff --git a/test/registered/e2e/pp/test_pp_spec.py b/test/registered/e2e/pp/test_pp_spec.py index 264a6504b7..10e37461e6 100644 --- a/test/registered/e2e/pp/test_pp_spec.py +++ b/test/registered/e2e/pp/test_pp_spec.py @@ -14,14 +14,16 @@ from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_DRAFT_MODEL_EAGLE, + DEFAULT_HYBRID_GDN_SMALL_MODEL_NAME_FOR_TEST, DEFAULT_TARGET_MODEL_EAGLE, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, popen_launch_server, + try_cached_model, ) -register_cuda_ci(est_time=900, stage="extra-b", runner_config="4-gpu-h100") +register_cuda_ci(est_time=1400, stage="extra-b", runner_config="4-gpu-h100") # topk=1 chains and a topk=2 tree: the relayed topology is constant for the # former and data-dependent for the latter, so both shapes are covered. @@ -40,9 +42,12 @@ class TestPPSpecConsistency(CustomTestCase): crash. Pin both. """ - def _run(self, pp_size: int, shape: str): + model = DEFAULT_TARGET_MODEL_EAGLE + max_tokens = 512 + + def _spec_args(self, shape: str): num_steps, topk, num_draft_tokens = SPEC_SHAPES[shape] - other_args = [ + return [ "--speculative-algorithm", "EAGLE", "--speculative-draft-model-path", @@ -53,14 +58,15 @@ class TestPPSpecConsistency(CustomTestCase): topk, "--speculative-num-draft-tokens", num_draft_tokens, - "--mem-fraction-static", - "0.7", ] + + def _run(self, pp_size: int, shape: str): + other_args = self._spec_args(shape) + ["--mem-fraction-static", "0.7"] if pp_size > 1: other_args += ["--pp-size", str(pp_size), "--disable-overlap-schedule"] process = popen_launch_server( - DEFAULT_TARGET_MODEL_EAGLE, + self.model, DEFAULT_URL_FOR_TEST, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, other_args=other_args, @@ -70,10 +76,10 @@ class TestPPSpecConsistency(CustomTestCase): metrics = run_eval( SimpleNamespace( base_url=DEFAULT_URL_FOR_TEST, - model=DEFAULT_TARGET_MODEL_EAGLE, + model=self.model, eval_name="gsm8k", api="completion", - max_tokens=512, + max_tokens=self.max_tokens, num_examples=256, num_threads=32, ) @@ -121,6 +127,37 @@ class TestPPSpecConsistency(CustomTestCase): self._assert_matches("tree") +class TestPPSpecHybridConsistency(TestPPSpecConsistency): + """Hybrid GDN target with its own MTP head: every non-last stage must commit + the accepted recurrent state, or accuracy drops while acceptance looks normal.""" + + model = try_cached_model(DEFAULT_HYBRID_GDN_SMALL_MODEL_NAME_FOR_TEST) + max_tokens = 2048 + + def _spec_args(self, shape: str): + return [ + "--trust-remote-code", + "--language-only", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--mamba-radix-cache-strategy", + "extra_buffer", + "--mamba-track-interval", + "128", + "--reasoning-parser", + "qwen3", + ] + + def test_pp_matches_non_pp_tree(self): + self.skipTest("MTP drafts a chain") + + class TestPPSpecGate(CustomTestCase): """The gate is off by default, and the combinations the relay cannot reproduce identically on every stage are rejected rather than silently diff --git a/test/registered/kernels/ops/attention/test_kda_fused_verify_backend.py b/test/registered/kernels/ops/attention/test_kda_fused_verify_backend.py index e042c12fb5..0a30844fd4 100644 --- a/test/registered/kernels/ops/attention/test_kda_fused_verify_backend.py +++ b/test/registered/kernels/ops/attention/test_kda_fused_verify_backend.py @@ -88,6 +88,7 @@ class TestKDAFusedVerifyBackend(CustomTestCase): slots = torch.arange(batch_size + 1, 1, -1, device="cuda", dtype=torch.int32) batch = SimpleNamespace( forward_mode=ForwardMode.TARGET_VERIFY, + req_pool_indices=torch.arange(batch_size, device="cuda", dtype=torch.int32), spec_info=SimpleNamespace(draft_token_num=steps, ragged_verify_layout=None), ) rounds = [ @@ -132,6 +133,7 @@ class TestKDAFusedVerifyBackend(CustomTestCase): # conv rollback below all use the production backend and GPU kernels. backend = KDAAttnBackend.__new__(KDAAttnBackend) backend.req_to_token_pool = SimpleNamespace( + size=state.temporal.shape[1], mamba2_layer_cache=state.at_layer_idx, get_speculative_mamba2_params_all_layers=lambda: state, mamba_pool=SimpleNamespace(replayssm_is_kda=ring), diff --git a/test/registered/unit/layers/attention/test_kda_helion_dispatcher.py b/test/registered/unit/layers/attention/test_kda_helion_dispatcher.py index 1ce62a3df4..acfa3de586 100644 --- a/test/registered/unit/layers/attention/test_kda_helion_dispatcher.py +++ b/test/registered/unit/layers/attention/test_kda_helion_dispatcher.py @@ -1,9 +1,11 @@ import unittest +from types import SimpleNamespace from unittest.mock import ANY, MagicMock, patch import torch from sglang.srt.arg_groups.attention_hook import handle_linear_attn_backend +from sglang.srt.arg_groups.overrides import resolution_result from sglang.srt.layers.attention.linear.kda_backend import KDAKernelDispatcher from sglang.srt.layers.attention.linear.kernels.kda_helion import HelionKDAKernel from sglang.srt.layers.attention.linear.kernels.kda_triton import TritonKDAKernel @@ -159,6 +161,43 @@ class TestHelionKDADispatcher(unittest.TestCase): self.assertIsNone(args.linear_attn_decode_backend) self.assertEqual(args.linear_attn_backend, "helion") + def test_pp_spec_flashinfer_verify_fallback_is_kda_only(self): + cases = ( + ("KimiK3LinearForCausalLM", {"kda_layers": [0]}, None, "triton"), + ("KimiK3LinearForCausalLM", {"kda_layers": [0]}, "flashinfer", "triton"), + ("Qwen3NextForCausalLM", {"linear_attention_layers": [0]}, None, None), + ) + for architecture, linear_config, requested, expected in cases: + with self.subTest(architecture=architecture, requested=requested): + args = ServerArgs( + model_path="dummy", + linear_attn_decode_backend="flashinfer", + linear_attn_verify_backend=requested, + ) + config = SimpleNamespace( + hf_config=SimpleNamespace( + architectures=[architecture], linear_attn_config=linear_config + ) + ) + with ( + patch( + "sglang.srt.arg_groups.attention_hook.model_config_of", + return_value=config, + ), + patch( + "sglang.srt.arg_groups.attention_hook." + "pp_spec_stable_rows_enabled", + return_value=True, + ), + override_platform(is_sm100=False), + override_platform(is_cuda=False), + ): + handle_linear_attn_backend(args) + + self.assertEqual( + resolution_result(args, "linear_attn_verify_backend"), expected + ) + class TestKDATrackStateSnapshotDeclaration(unittest.TestCase): """Bookkeeping: every KDA prefill kernel must declare whether extend() diff --git a/test/registered/unit/spec/test_ngram_mamba_verify_update.py b/test/registered/unit/spec/test_ngram_mamba_verify_update.py index 616279b8c7..23a4aa2cb0 100644 --- a/test/registered/unit/spec/test_ngram_mamba_verify_update.py +++ b/test/registered/unit/spec/test_ngram_mamba_verify_update.py @@ -148,6 +148,169 @@ class TestNgramMambaVerifyUpdate(CustomTestCase): ) +class TestPPReplaySSMVerifySourceRows(CustomTestCase): + @staticmethod + def _spec_state(): + state = torch.empty((1, 8, 4, 2), dtype=torch.float32) + spec_state = MagicMock() + for name in "temporal replayssm_d replayssm_k replayssm_rawv replayssm_g replayssm_beta".split(): # noqa: SIM905 + setattr(spec_state, name, state) + spec_state.replayssm_rawk = state.unsqueeze(2) + spec_state.conv = spec_state.intermediate_conv_window = [state] + return spec_state + + def test_fold_helpers_read_pp_request_rows(self): + from sglang.kernels.ops.attention.fla.gdn_replayssm_spec_fold import ( + commit_gdn_replayssm_fold_after_verify, + ) + from sglang.kernels.ops.attention.fla.kda_replayssm_spec_decode import ( + commit_kda_replayssm_after_verify, + ) + + cases = ( + ( + commit_gdn_replayssm_fold_after_verify, + "sglang.kernels.ops.attention.fla.gdn_replayssm_spec_fold", + "commit_gdn_replayssm_fold_all_layers", + ), + ( + commit_kda_replayssm_after_verify, + "sglang.kernels.ops.attention.fla.kda_replayssm_spec_decode", + "commit_kda_replayssm_spec_all_layers", + ), + ) + for commit, module, fold_name in cases: + with ( + self.subTest(module=module), + patch(f"{module}.{fold_name}"), + patch( + "sglang.kernels.ops.mamba.mamba_state_scatter_triton." + "fused_conv_window_scatter_with_mask" + ) as scatter, + ): + source_rows = torch.tensor([17, 23]) + commit( + spec_state=self._spec_state(), + state_batch_indices=torch.tensor([5, 7]), + accept_lens=torch.tensor([3, 1]), + last_correct_step_indices=torch.tensor([2, 0]), + src_indices_raw=source_rows, + ) + + scatter.assert_called_once() + torch.testing.assert_close(scatter.call_args.args[4], source_rows) + + def test_circular_commit_reads_pp_request_rows(self): + from sglang.srt.speculative.spec_utils import commit_mamba_states_after_verify + + target_worker = MagicMock() + req_pool = target_worker.model_runner.req_to_token_pool + req_pool.mamba_pool.replayssm_spec_fold = False + req_pool.mamba_pool.replayssm_is_kda = False + req_pool.mamba_pool.replayssm_cache_base = torch.empty(1) + req_pool.get_mamba_indices.return_value = torch.tensor( + [5, 7], dtype=torch.int32 + ) + req_pool.get_speculative_mamba2_params_all_layers.return_value = ( + self._spec_state() + ) + batch = MagicMock() + batch.forward_mode.is_idle.return_value = False + batch.req_pool_indices = torch.tensor([17, 23], dtype=torch.int64) + batch.mamba_track_indices = None + batch.seq_lens = torch.tensor([10, 20], dtype=torch.int32) + accept_lens = torch.tensor([2, 1], dtype=torch.int32) + accept_index = torch.tensor([[0, 1, -1], [3, -1, -1]], dtype=torch.int32) + + with ( + patch( + "sglang.srt.speculative.spec_utils.mambaish_config", + return_value={"some": "config"}, + ), + patch( + "sglang.srt.speculative.spec_utils.pp_spec_stable_rows_enabled", + return_value=True, + ), + patch( + "sglang.kernels.ops.attention.fla.gdn_replayssm_spec_decode." + "commit_gdn_replayssm_spec" + ), + patch( + "sglang.kernels.ops.attention.fla.gdn_replayssm_spec_decode." + "commit_gdn_replayssm_circular" + ), + patch( + "sglang.kernels.ops.mamba.mamba_state_scatter_triton." + "fused_conv_window_scatter_with_mask" + ) as scatter, + ): + commit_mamba_states_after_verify( + target_worker, + batch, + accept_lens, + accept_index, + draft_token_num=3, + ) + + scatter.assert_called_once() + torch.testing.assert_close(scatter.call_args.args[4], batch.req_pool_indices) + + +class TestDelayedMambaCommitBatchPairing(CustomTestCase): + def test_flashinfer_gdn_positional_scratch_moves_to_request_rows(self): + from sglang.srt.layers.attention.linear.kernels.gdn_flashinfer import ( + copy_verify_intermediate_rows, + ) + + destination = torch.zeros((8, 2, 3), dtype=torch.float32) + positional = torch.arange(18, dtype=torch.float32).reshape(3, 2, 3) + rows = torch.tensor([5, 2, 7], dtype=torch.int32) + + copy_verify_intermediate_rows(destination, positional, rows) + + torch.testing.assert_close(destination[rows.long()], positional) + + def test_ple_commit_reads_stable_request_rows(self): + from sglang.srt.layers.attention.hybrid_linear_attn_backend import ( + HybridLinearAttnBackend, + ) + + dst = torch.zeros((1, 8, 2), dtype=torch.float32) + src = torch.zeros((1, 16, 4, 2), dtype=torch.float32) + src[0, 11, 2] = torch.tensor([3.0, 7.0]) + + HybridLinearAttnBackend._scatter_speculative_state_with_mask( + dst, + src, + torch.tensor([5], dtype=torch.int32), + torch.tensor([2], dtype=torch.int32), + torch.tensor([11], dtype=torch.int64), + ) + + torch.testing.assert_close(dst[0, 5], torch.tensor([3.0, 7.0])) + + def test_pp_verify_scratch_uses_stable_request_rows_for_all_backends(self): + from sglang.srt.layers.attention.linear.utils import ( + select_verify_intermediate_state_indices, + ) + + default = torch.arange(8, dtype=torch.int32) + req_rows = torch.tensor([17, 23, 31], dtype=torch.int64) + cache_indices = torch.tensor([4, -1, 9], dtype=torch.int32) + + with patch( + "sglang.srt.layers.attention.linear.utils.pp_spec_stable_rows_enabled", + return_value=True, + ): + result = select_verify_intermediate_state_indices( + default, req_rows, cache_indices >= 0, pool_size=64 + ) + + torch.testing.assert_close( + result, torch.tensor([17, 64, 31], dtype=torch.int32) + ) + + class TestMtpVerifyHookSignature(CustomTestCase): """Every ``update_mamba_state_after_mtp_verify`` override must accept the full keyword call the spec workers make, or it raises TypeError at verify time on