[Scheduler] Fix the deterministic-inference hang when a prefill chunk is shorter than its alignment (#43444)

This commit is contained in:
Liangsheng Yin
2026-10-09 19:06:18 -07:00
committed by GitHub
parent 78218194ad
commit 465253ef52
7 changed files with 200 additions and 9 deletions
@@ -270,6 +270,61 @@ def _copy_unified_indices_kernel(
tl.store(unified_kv_indices + dst_idx, vals, mask=mask)
@triton.jit
def _align_window_kv_kernel(
window_kv_indptr,
window_kv_indices,
aligned_kv_indptr,
pad,
aligned_kv_indices,
BLOCK: tl.constexpr,
):
pid = tl.program_id(0)
src_start = tl.load(window_kv_indptr + pid)
num_keys = tl.load(window_kv_indptr + pid + 1) - src_start
dst_start = tl.load(aligned_kv_indptr + pid)
cur_pad = tl.load(pad + pid)
for off in range(0, num_keys + cur_pad, BLOCK):
j = off + tl.arange(0, BLOCK)
mask = j < num_keys + cur_pad
val = tl.load(
window_kv_indices + src_start + tl.maximum(j - cur_pad, 0), mask=mask
)
tl.store(aligned_kv_indices + dst_start + j, val, mask=mask)
def align_window_kv_to_tiles(
window_kv_indptr: torch.Tensor,
window_kv_indices: torch.Tensor,
window_start_pos: torch.Tensor,
bs: int,
tile: int = 128,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Pad each request's window KV list with copies of its first key so the list
starts on an absolute multiple of ``tile`` (a multiple of every BLOCK_N). The
unified kernel then sums a chunked prefill's keys in the same tiles as an
unchunked one; the copies sit before the window and are masked out."""
pad = window_start_pos[:bs] % tile
lens = window_kv_indptr[1 : bs + 1] - window_kv_indptr[:bs] + pad
aligned_kv_indptr = torch.zeros_like(window_kv_indptr[: bs + 1])
aligned_kv_indptr[1:] = torch.cumsum(lens, dim=0)
# Upper bound on the padded length, so no device-to-host sync is needed.
aligned_kv_indices = torch.empty(
window_kv_indices.numel() + bs * (tile - 1),
dtype=window_kv_indices.dtype,
device=window_kv_indices.device,
)
_align_window_kv_kernel[(bs,)](
window_kv_indptr,
window_kv_indices,
aligned_kv_indptr,
pad,
aligned_kv_indices,
BLOCK=128,
)
return aligned_kv_indptr, aligned_kv_indices
def build_unified_kv_indices(
prefix_kv_indptr: torch.Tensor,
prefix_kv_indices: torch.Tensor,
+3 -1
View File
@@ -1183,7 +1183,6 @@ class Envs:
# the batch fits the chunk budget.
SGLANG_TRITON_DENSE_PREFILL_ATTN = EnvBool(True)
SGLANG_ENABLE_TORCH_COMPILE = EnvBool(False)
SGLANG_TRITON_PREFILL_TRUNCATION_ALIGN_SIZE = EnvInt(4096)
SGLANG_TRITON_DECODE_SPLIT_TILE_SIZE = EnvInt(256)
# ===================================================================
@@ -1993,6 +1992,9 @@ _DEPRECATED_ENVS: Dict[str, _DeprecatedEnv] = {
note="Buffer-mode anchor pinning is always on; set "
"SGLANG_HICACHE_BUFFER_ANCHOR_LOCK_CAP=0 to disable it."
),
"SGLANG_TRITON_PREFILL_TRUNCATION_ALIGN_SIZE": _DeprecatedEnv(
note="Deterministic inference on triton no longer aligns prefill chunks."
),
# Replaced by CLI flags.
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": _DeprecatedEnv(
note="Please use '--enable-prefill-delayer' instead."
@@ -132,6 +132,9 @@ class ForwardMetadata:
swa_attn_logits: Optional[torch.Tensor] = None
# full->SWA translated out_cache_loc (SWA KV-store write target)
swa_out_cache_loc: Optional[torch.Tensor] = None
# Deterministic extend: window KV lists padded to absolute tile boundaries
aligned_window_kv_indptr: Optional[torch.Tensor] = None
aligned_window_kv_indices: Optional[torch.Tensor] = None
# Lean decode (persistent-grid partial-result buffers)
lean_Mp: Optional[torch.Tensor] = None
lean_Lp: Optional[torch.Tensor] = None
@@ -165,6 +168,7 @@ class TritonAttnBackend(AttentionBackend):
lean_decode_seqlen_gate,
)
from sglang.kernels.ops.attention.extend_attention import (
align_window_kv_to_tiles,
build_unified_kv_indices,
can_use_dense_prefill_fp8,
dense_prefill_attention_fwd,
@@ -187,6 +191,7 @@ class TritonAttnBackend(AttentionBackend):
extend_attention_fwd_unified
)
self.build_unified_kv_indices = torch.compiler.disable(build_unified_kv_indices)
self.align_window_kv_to_tiles = torch.compiler.disable(align_window_kv_to_tiles)
# Dense (non-absorbed) MLA prefill over a materialized prefix; see
# handle_attention_triton for when the dispatcher selects it.
self.dense_prefill_attention_fwd = torch.compiler.disable(
@@ -829,6 +834,8 @@ class TritonAttnBackend(AttentionBackend):
window_kv_indices = None
window_num_kv_splits = None
window_kv_offsets = None
aligned_window_kv_indptr = None
aligned_window_kv_indices = None
swa_attn_logits = None
spec_info = forward_batch.spec_info
# Lean decode buffers are only allocated on the decode path below; default
@@ -1036,6 +1043,15 @@ class TritonAttnBackend(AttentionBackend):
self.device,
self.token_to_kv_pool,
)
if self.enable_deterministic:
aligned_window_kv_indptr, aligned_window_kv_indices = (
self.align_window_kv_to_tiles(
window_kv_indptr,
window_kv_indices,
forward_batch.extend_prefix_lens[:bs] - window_kv_lens,
bs,
)
)
qo_indptr = self.qo_indptr
qo_indptr[1 : bs + 1] = torch.cumsum(forward_batch.extend_seq_lens, dim=0)
@@ -1073,6 +1089,8 @@ class TritonAttnBackend(AttentionBackend):
window_kv_offsets,
swa_attn_logits=swa_attn_logits,
swa_out_cache_loc=swa_out_cache_loc,
aligned_window_kv_indptr=aligned_window_kv_indptr,
aligned_window_kv_indices=aligned_window_kv_indices,
lean_Mp=lean_Mp,
lean_Lp=lean_Lp,
lean_Op=lean_Op,
@@ -2084,6 +2102,17 @@ class TritonAttnBackend(AttentionBackend):
window_start_pos = forward_batch.seq_lens[:bs] - window_kv_lens
else:
window_start_pos = None
# The padding is masked only by the sliding-window mask, so a layer
# that attends bidirectionally keeps the unpadded list.
if (
sliding_window_size > 0
and self.forward_metadata.aligned_window_kv_indptr is not None
):
prefix_kv_indptr = self.forward_metadata.aligned_window_kv_indptr
prefix_kv_indices = self.forward_metadata.aligned_window_kv_indices
window_start_pos = forward_batch.extend_prefix_lens[:bs] - (
prefix_kv_indptr[1 : bs + 1] - prefix_kv_indptr[:bs]
)
else:
sliding_window_size = -1
prefix_kv_indptr = self.forward_metadata.kv_indptr
+28 -3
View File
@@ -671,6 +671,7 @@ class Scheduler(
# Init prefill kv split size when deterministic inference is enabled with various attention backends
self.init_deterministic_inference_config()
self.init_dsa_kpool_truncation_align()
self.check_truncation_align_fits_chunk()
self.init_weight_updater()
@@ -1671,12 +1672,11 @@ class Scheduler(
self.truncation_align_size = None
return
# Only flashinfer needs chunks on its prefill split tiles; triton tiles
# keys at absolute positions (align_window_kv_to_tiles).
backend_sizes = {
"flashinfer": ("SGLANG_FLASHINFER_PREFILL_SPLIT_TILE_SIZE", 4096),
"triton": ("SGLANG_TRITON_PREFILL_TRUNCATION_ALIGN_SIZE", 4096),
}
# Both entries are prefill knobs (SPLIT_TILE / PREFILL_TRUNCATION):
# the prefill half decides.
prefill_backend, _ = attention_backends()
env_var, default_size = backend_sizes.get(prefill_backend, (None, None))
self.truncation_align_size = (
@@ -1705,6 +1705,31 @@ class Scheduler(
self.truncation_align_size, dsa_index_kpool
)
def check_truncation_align_fits_chunk(self):
"""A chunk shorter than the alignment truncates to zero tokens, so a prompt
longer than the chunk would wait in the queue forever."""
align = self.truncation_align_size
chunk = self.chunked_prefill_size
if align is None or chunk is None or chunk >= align:
return
if self.disaggregation_mode == DisaggregationMode.DECODE:
# A decode server never prefills a prompt.
return
attn_dp_size = get_parallel().attn_dp_size
dp_note = (
f" (--chunked-prefill-size is split across {attn_dp_size} DP attention ranks)"
if attn_dp_size > 1
else ""
)
raise ValueError(
f"Chunked prefill is aligned to {align} tokens, but each rank's "
f"chunked prefill size is {chunk}{dp_note}; a prompt longer than "
f"{chunk} tokens could never be scheduled. Raise --chunked-prefill-size "
f"to at least {align * attn_dp_size}, or, with the flashinfer backend "
f"under deterministic inference, lower "
f"SGLANG_FLASHINFER_PREFILL_SPLIT_TILE_SIZE."
)
def init_request_dispatcher(self):
self._request_dispatcher = TypeBasedDispatcher(
[