[Scheduler] Fix the deterministic-inference hang when a prefill chunk is shorter than its alignment (#43444)

This commit is contained in:
Liangsheng Yin
2026-10-09 19:06:18 -07:00
committed by GitHub
parent 78218194ad
commit 465253ef52
7 changed files with 200 additions and 9 deletions
@@ -1725,11 +1725,6 @@ SGLang supports various environment variables that can be used to configure its
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>FlashInfer decode split-tile size for deterministic attention.</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>2048</code></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>SGLANG_TRITON_PREFILL_TRUNCATION_ALIGN_SIZE</code></td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Triton prefill truncation alignment size for deterministic attention.</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>4096</code></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>SGLANG_TRITON_DECODE_SPLIT_TILE_SIZE</code></td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Triton decode split-tile size for deterministic attention.</td>
@@ -270,6 +270,61 @@ def _copy_unified_indices_kernel(
tl.store(unified_kv_indices + dst_idx, vals, mask=mask)
@triton.jit
def _align_window_kv_kernel(
window_kv_indptr,
window_kv_indices,
aligned_kv_indptr,
pad,
aligned_kv_indices,
BLOCK: tl.constexpr,
):
pid = tl.program_id(0)
src_start = tl.load(window_kv_indptr + pid)
num_keys = tl.load(window_kv_indptr + pid + 1) - src_start
dst_start = tl.load(aligned_kv_indptr + pid)
cur_pad = tl.load(pad + pid)
for off in range(0, num_keys + cur_pad, BLOCK):
j = off + tl.arange(0, BLOCK)
mask = j < num_keys + cur_pad
val = tl.load(
window_kv_indices + src_start + tl.maximum(j - cur_pad, 0), mask=mask
)
tl.store(aligned_kv_indices + dst_start + j, val, mask=mask)
def align_window_kv_to_tiles(
window_kv_indptr: torch.Tensor,
window_kv_indices: torch.Tensor,
window_start_pos: torch.Tensor,
bs: int,
tile: int = 128,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Pad each request's window KV list with copies of its first key so the list
starts on an absolute multiple of ``tile`` (a multiple of every BLOCK_N). The
unified kernel then sums a chunked prefill's keys in the same tiles as an
unchunked one; the copies sit before the window and are masked out."""
pad = window_start_pos[:bs] % tile
lens = window_kv_indptr[1 : bs + 1] - window_kv_indptr[:bs] + pad
aligned_kv_indptr = torch.zeros_like(window_kv_indptr[: bs + 1])
aligned_kv_indptr[1:] = torch.cumsum(lens, dim=0)
# Upper bound on the padded length, so no device-to-host sync is needed.
aligned_kv_indices = torch.empty(
window_kv_indices.numel() + bs * (tile - 1),
dtype=window_kv_indices.dtype,
device=window_kv_indices.device,
)
_align_window_kv_kernel[(bs,)](
window_kv_indptr,
window_kv_indices,
aligned_kv_indptr,
pad,
aligned_kv_indices,
BLOCK=128,
)
return aligned_kv_indptr, aligned_kv_indices
def build_unified_kv_indices(
prefix_kv_indptr: torch.Tensor,
prefix_kv_indices: torch.Tensor,
+3 -1
View File
@@ -1183,7 +1183,6 @@ class Envs:
# the batch fits the chunk budget.
SGLANG_TRITON_DENSE_PREFILL_ATTN = EnvBool(True)
SGLANG_ENABLE_TORCH_COMPILE = EnvBool(False)
SGLANG_TRITON_PREFILL_TRUNCATION_ALIGN_SIZE = EnvInt(4096)
SGLANG_TRITON_DECODE_SPLIT_TILE_SIZE = EnvInt(256)
# ===================================================================
@@ -1993,6 +1992,9 @@ _DEPRECATED_ENVS: Dict[str, _DeprecatedEnv] = {
note="Buffer-mode anchor pinning is always on; set "
"SGLANG_HICACHE_BUFFER_ANCHOR_LOCK_CAP=0 to disable it."
),
"SGLANG_TRITON_PREFILL_TRUNCATION_ALIGN_SIZE": _DeprecatedEnv(
note="Deterministic inference on triton no longer aligns prefill chunks."
),
# Replaced by CLI flags.
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": _DeprecatedEnv(
note="Please use '--enable-prefill-delayer' instead."
@@ -132,6 +132,9 @@ class ForwardMetadata:
swa_attn_logits: Optional[torch.Tensor] = None
# full->SWA translated out_cache_loc (SWA KV-store write target)
swa_out_cache_loc: Optional[torch.Tensor] = None
# Deterministic extend: window KV lists padded to absolute tile boundaries
aligned_window_kv_indptr: Optional[torch.Tensor] = None
aligned_window_kv_indices: Optional[torch.Tensor] = None
# Lean decode (persistent-grid partial-result buffers)
lean_Mp: Optional[torch.Tensor] = None
lean_Lp: Optional[torch.Tensor] = None
@@ -165,6 +168,7 @@ class TritonAttnBackend(AttentionBackend):
lean_decode_seqlen_gate,
)
from sglang.kernels.ops.attention.extend_attention import (
align_window_kv_to_tiles,
build_unified_kv_indices,
can_use_dense_prefill_fp8,
dense_prefill_attention_fwd,
@@ -187,6 +191,7 @@ class TritonAttnBackend(AttentionBackend):
extend_attention_fwd_unified
)
self.build_unified_kv_indices = torch.compiler.disable(build_unified_kv_indices)
self.align_window_kv_to_tiles = torch.compiler.disable(align_window_kv_to_tiles)
# Dense (non-absorbed) MLA prefill over a materialized prefix; see
# handle_attention_triton for when the dispatcher selects it.
self.dense_prefill_attention_fwd = torch.compiler.disable(
@@ -829,6 +834,8 @@ class TritonAttnBackend(AttentionBackend):
window_kv_indices = None
window_num_kv_splits = None
window_kv_offsets = None
aligned_window_kv_indptr = None
aligned_window_kv_indices = None
swa_attn_logits = None
spec_info = forward_batch.spec_info
# Lean decode buffers are only allocated on the decode path below; default
@@ -1036,6 +1043,15 @@ class TritonAttnBackend(AttentionBackend):
self.device,
self.token_to_kv_pool,
)
if self.enable_deterministic:
aligned_window_kv_indptr, aligned_window_kv_indices = (
self.align_window_kv_to_tiles(
window_kv_indptr,
window_kv_indices,
forward_batch.extend_prefix_lens[:bs] - window_kv_lens,
bs,
)
)
qo_indptr = self.qo_indptr
qo_indptr[1 : bs + 1] = torch.cumsum(forward_batch.extend_seq_lens, dim=0)
@@ -1073,6 +1089,8 @@ class TritonAttnBackend(AttentionBackend):
window_kv_offsets,
swa_attn_logits=swa_attn_logits,
swa_out_cache_loc=swa_out_cache_loc,
aligned_window_kv_indptr=aligned_window_kv_indptr,
aligned_window_kv_indices=aligned_window_kv_indices,
lean_Mp=lean_Mp,
lean_Lp=lean_Lp,
lean_Op=lean_Op,
@@ -2084,6 +2102,17 @@ class TritonAttnBackend(AttentionBackend):
window_start_pos = forward_batch.seq_lens[:bs] - window_kv_lens
else:
window_start_pos = None
# The padding is masked only by the sliding-window mask, so a layer
# that attends bidirectionally keeps the unpadded list.
if (
sliding_window_size > 0
and self.forward_metadata.aligned_window_kv_indptr is not None
):
prefix_kv_indptr = self.forward_metadata.aligned_window_kv_indptr
prefix_kv_indices = self.forward_metadata.aligned_window_kv_indices
window_start_pos = forward_batch.extend_prefix_lens[:bs] - (
prefix_kv_indptr[1 : bs + 1] - prefix_kv_indptr[:bs]
)
else:
sliding_window_size = -1
prefix_kv_indptr = self.forward_metadata.kv_indptr
+28 -3
View File
@@ -671,6 +671,7 @@ class Scheduler(
# Init prefill kv split size when deterministic inference is enabled with various attention backends
self.init_deterministic_inference_config()
self.init_dsa_kpool_truncation_align()
self.check_truncation_align_fits_chunk()
self.init_weight_updater()
@@ -1671,12 +1672,11 @@ class Scheduler(
self.truncation_align_size = None
return
# Only flashinfer needs chunks on its prefill split tiles; triton tiles
# keys at absolute positions (align_window_kv_to_tiles).
backend_sizes = {
"flashinfer": ("SGLANG_FLASHINFER_PREFILL_SPLIT_TILE_SIZE", 4096),
"triton": ("SGLANG_TRITON_PREFILL_TRUNCATION_ALIGN_SIZE", 4096),
}
# Both entries are prefill knobs (SPLIT_TILE / PREFILL_TRUNCATION):
# the prefill half decides.
prefill_backend, _ = attention_backends()
env_var, default_size = backend_sizes.get(prefill_backend, (None, None))
self.truncation_align_size = (
@@ -1705,6 +1705,31 @@ class Scheduler(
self.truncation_align_size, dsa_index_kpool
)
def check_truncation_align_fits_chunk(self):
"""A chunk shorter than the alignment truncates to zero tokens, so a prompt
longer than the chunk would wait in the queue forever."""
align = self.truncation_align_size
chunk = self.chunked_prefill_size
if align is None or chunk is None or chunk >= align:
return
if self.disaggregation_mode == DisaggregationMode.DECODE:
# A decode server never prefills a prompt.
return
attn_dp_size = get_parallel().attn_dp_size
dp_note = (
f" (--chunked-prefill-size is split across {attn_dp_size} DP attention ranks)"
if attn_dp_size > 1
else ""
)
raise ValueError(
f"Chunked prefill is aligned to {align} tokens, but each rank's "
f"chunked prefill size is {chunk}{dp_note}; a prompt longer than "
f"{chunk} tokens could never be scheduled. Raise --chunked-prefill-size "
f"to at least {align * attn_dp_size}, or, with the flashinfer backend "
f"under deterministic inference, lower "
f"SGLANG_FLASHINFER_PREFILL_SPLIT_TILE_SIZE."
)
def init_request_dispatcher(self):
self._request_dispatcher = TypeBasedDispatcher(
[
@@ -11,6 +11,7 @@ from sglang.kernels.ops.attention.decode_attention import (
)
from sglang.kernels.ops.attention.extend_attention import (
_compact_extend_q_tiles_per_head,
align_window_kv_to_tiles,
build_unified_kv_indices,
extend_attention_fwd,
extend_attention_fwd_unified,
@@ -1130,6 +1131,51 @@ class TestTritonAttention(CustomTestCase):
unified_seq = unified_kv_indices[start_idx:end_idx]
self.assertEqual(len(unified_seq), prefix_len + extend_len)
def test_unified_sliding_window_chunked_matches_unchunked(self):
"""Deterministic inference: a chunk whose window starts off a tile boundary
must match the unchunked prefill bit for bit."""
torch.manual_seed(0)
device = get_device()
seq_len, window, prefix = 3000, 128, 2192 # window starts at 2064
q = torch.randn(seq_len, 8, 64, dtype=torch.bfloat16, device=device)
k = torch.randn(seq_len, 8, 64, dtype=torch.bfloat16, device=device)
v = torch.randn_like(k)
i32 = lambda *x: torch.tensor(x, dtype=torch.int32, device=device)
arange = lambda lo, hi: torch.arange(lo, hi, device=device)
def run(prefix_len):
win = min(prefix_len, window)
indptr, indices = align_window_kv_to_tiles(
i32(0, win),
arange(prefix_len - win, prefix_len),
i32(prefix_len - win),
1,
)
extend_len = seq_len - prefix_len
kv_indptr, kv_indices, kv_prefix_lens = build_unified_kv_indices(
indptr, indices, i32(0), i32(extend_len), arange(prefix_len, seq_len), 1
)
o = torch.empty_like(q[prefix_len:])
extend_attention_fwd_unified(
q[prefix_len:],
o,
k,
v,
1.0,
1.0,
i32(0, extend_len),
kv_indptr,
kv_indices,
kv_prefix_lens.to(torch.int32),
extend_len,
sm_scale=0.125,
sliding_window_size=window,
window_start_pos=i32(prefix_len) - (indptr[1:] - indptr[:-1]),
)
return o
self.assertTrue(torch.equal(run(prefix), run(0)[prefix:]))
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,39 @@
"""A per-rank chunk shorter than the deterministic prefill alignment must fail at
startup: it truncates every chunk to zero tokens and the prompt waits forever."""
import unittest
from types import SimpleNamespace
from unittest.mock import patch
from sglang.srt.disaggregation.utils import DisaggregationMode
from sglang.srt.managers.scheduler import Scheduler
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
_MOD = "sglang.srt.managers.scheduler"
def _check(align, chunk, attn_dp_size=1, mode=DisaggregationMode.NULL):
scheduler = Scheduler.__new__(Scheduler)
scheduler.disaggregation_mode = mode
scheduler.truncation_align_size = align
scheduler.chunked_prefill_size = chunk
parallel = SimpleNamespace(attn_dp_size=attn_dp_size)
with patch(f"{_MOD}.get_parallel", return_value=parallel):
scheduler.check_truncation_align_fits_chunk()
class TestDeterministicChunkAlign(CustomTestCase):
def test_chunk_shorter_than_alignment_fails_at_startup(self):
with self.assertRaisesRegex(ValueError, "at least 8192"):
_check(align=4096, chunk=2048, attn_dp_size=2)
def test_chunk_covering_alignment_starts(self):
_check(align=4096, chunk=4096)
_check(align=4096, chunk=2048, mode=DisaggregationMode.DECODE)
if __name__ == "__main__":
unittest.main()