[Spec] Fix DFLASH verify mask policy for wrapped attention backends (#43054)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Po-Han Huang
parent
817781b828
commit
df1a089cd3
@@ -2289,6 +2289,21 @@ class FlashInferIndicesUpdaterPrefill:
|
||||
token_pos_in_items_len = 0
|
||||
max_item_len_ptr = None
|
||||
|
||||
# A cuda-graph wrapper built with custom_mask_buf always runs MaskMode.CUSTOM,
|
||||
# but plan(custom_mask=None) skips refilling that buffer, so the kernel would
|
||||
# silently read stale capture-time mask bits.
|
||||
if (
|
||||
spec_info is not None
|
||||
and use_custom_mask is None
|
||||
and wrapper_paged.is_cuda_graph_enabled
|
||||
and wrapper_paged._custom_mask_buf is not None
|
||||
):
|
||||
raise RuntimeError(
|
||||
f"{spec_info.spec_input_type} verify planned without a custom mask on "
|
||||
"a cuda-graph wrapper captured with one; capture and replay must agree "
|
||||
"on custom_mask."
|
||||
)
|
||||
|
||||
# fast_prefill_plan (installed at capture) is sync-free: it needs the
|
||||
# host-known qo/kv layout from the caller. Assert rather than silently
|
||||
# fall back to plan()'s blocking D2H on the replay hot-path.
|
||||
|
||||
@@ -206,8 +206,20 @@ def scale_kv_cell_size_per_token_for_dflash(
|
||||
|
||||
|
||||
def resolve_dflash_verify_mask_policy(attn_backend: Any) -> tuple[str, bool]:
|
||||
from sglang.srt.layers.attention.hybrid_attn_backend import HybridAttnBackend
|
||||
from sglang.srt.layers.attention.tbo_backend import TboAttnBackend
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
|
||||
backend = attn_backend
|
||||
for _ in range(4):
|
||||
if isinstance(backend, HybridAttnBackend):
|
||||
# Resolve wrappers down to the backend that runs TARGET_VERIFY
|
||||
backend = backend._select_backend(ForwardMode.TARGET_VERIFY)
|
||||
continue
|
||||
if isinstance(backend, TboAttnBackend):
|
||||
# primary and children come from the same creator.
|
||||
backend = backend.primary
|
||||
continue
|
||||
full_backend = getattr(backend, "full_attn_backend", None)
|
||||
if full_backend is None:
|
||||
break
|
||||
|
||||
Reference in New Issue
Block a user