From df1a089cd39e02ad56eb5dfe5e561a77cfb93cbe Mon Sep 17 00:00:00 2001 From: elvischenv <219235043+elvischenv@users.noreply.github.com> Date: Wed, 14 Oct 2026 17:15:36 +0800 Subject: [PATCH] [Spec] Fix DFLASH verify mask policy for wrapped attention backends (#43054) Co-authored-by: Claude Opus 5.5 Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com> --- .../srt/layers/attention/flashinfer_backend.py | 15 +++++++++++++++ python/sglang/srt/speculative/dflash_utils.py | 12 ++++++++++++ 2 files changed, 27 insertions(+) diff --git a/python/sglang/srt/layers/attention/flashinfer_backend.py b/python/sglang/srt/layers/attention/flashinfer_backend.py index 64615260c8..ce541a9efd 100644 --- a/python/sglang/srt/layers/attention/flashinfer_backend.py +++ b/python/sglang/srt/layers/attention/flashinfer_backend.py @@ -2289,6 +2289,21 @@ class FlashInferIndicesUpdaterPrefill: token_pos_in_items_len = 0 max_item_len_ptr = None + # A cuda-graph wrapper built with custom_mask_buf always runs MaskMode.CUSTOM, + # but plan(custom_mask=None) skips refilling that buffer, so the kernel would + # silently read stale capture-time mask bits. + if ( + spec_info is not None + and use_custom_mask is None + and wrapper_paged.is_cuda_graph_enabled + and wrapper_paged._custom_mask_buf is not None + ): + raise RuntimeError( + f"{spec_info.spec_input_type} verify planned without a custom mask on " + "a cuda-graph wrapper captured with one; capture and replay must agree " + "on custom_mask." + ) + # fast_prefill_plan (installed at capture) is sync-free: it needs the # host-known qo/kv layout from the caller. Assert rather than silently # fall back to plan()'s blocking D2H on the replay hot-path. diff --git a/python/sglang/srt/speculative/dflash_utils.py b/python/sglang/srt/speculative/dflash_utils.py index b50bd12518..d0f8efb8e4 100644 --- a/python/sglang/srt/speculative/dflash_utils.py +++ b/python/sglang/srt/speculative/dflash_utils.py @@ -206,8 +206,20 @@ def scale_kv_cell_size_per_token_for_dflash( def resolve_dflash_verify_mask_policy(attn_backend: Any) -> tuple[str, bool]: + from sglang.srt.layers.attention.hybrid_attn_backend import HybridAttnBackend + from sglang.srt.layers.attention.tbo_backend import TboAttnBackend + from sglang.srt.model_executor.forward_batch_info import ForwardMode + backend = attn_backend for _ in range(4): + if isinstance(backend, HybridAttnBackend): + # Resolve wrappers down to the backend that runs TARGET_VERIFY + backend = backend._select_backend(ForwardMode.TARGET_VERIFY) + continue + if isinstance(backend, TboAttnBackend): + # primary and children come from the same creator. + backend = backend.primary + continue full_backend = getattr(backend, "full_attn_backend", None) if full_backend is None: break