[Scheduler] Fix the deterministic-inference hang when a prefill chunk is shorter than its alignment (#43444)
This commit is contained in:
@@ -270,6 +270,61 @@ def _copy_unified_indices_kernel(
|
||||
tl.store(unified_kv_indices + dst_idx, vals, mask=mask)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _align_window_kv_kernel(
|
||||
window_kv_indptr,
|
||||
window_kv_indices,
|
||||
aligned_kv_indptr,
|
||||
pad,
|
||||
aligned_kv_indices,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
src_start = tl.load(window_kv_indptr + pid)
|
||||
num_keys = tl.load(window_kv_indptr + pid + 1) - src_start
|
||||
dst_start = tl.load(aligned_kv_indptr + pid)
|
||||
cur_pad = tl.load(pad + pid)
|
||||
for off in range(0, num_keys + cur_pad, BLOCK):
|
||||
j = off + tl.arange(0, BLOCK)
|
||||
mask = j < num_keys + cur_pad
|
||||
val = tl.load(
|
||||
window_kv_indices + src_start + tl.maximum(j - cur_pad, 0), mask=mask
|
||||
)
|
||||
tl.store(aligned_kv_indices + dst_start + j, val, mask=mask)
|
||||
|
||||
|
||||
def align_window_kv_to_tiles(
|
||||
window_kv_indptr: torch.Tensor,
|
||||
window_kv_indices: torch.Tensor,
|
||||
window_start_pos: torch.Tensor,
|
||||
bs: int,
|
||||
tile: int = 128,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Pad each request's window KV list with copies of its first key so the list
|
||||
starts on an absolute multiple of ``tile`` (a multiple of every BLOCK_N). The
|
||||
unified kernel then sums a chunked prefill's keys in the same tiles as an
|
||||
unchunked one; the copies sit before the window and are masked out."""
|
||||
pad = window_start_pos[:bs] % tile
|
||||
lens = window_kv_indptr[1 : bs + 1] - window_kv_indptr[:bs] + pad
|
||||
aligned_kv_indptr = torch.zeros_like(window_kv_indptr[: bs + 1])
|
||||
aligned_kv_indptr[1:] = torch.cumsum(lens, dim=0)
|
||||
# Upper bound on the padded length, so no device-to-host sync is needed.
|
||||
aligned_kv_indices = torch.empty(
|
||||
window_kv_indices.numel() + bs * (tile - 1),
|
||||
dtype=window_kv_indices.dtype,
|
||||
device=window_kv_indices.device,
|
||||
)
|
||||
_align_window_kv_kernel[(bs,)](
|
||||
window_kv_indptr,
|
||||
window_kv_indices,
|
||||
aligned_kv_indptr,
|
||||
pad,
|
||||
aligned_kv_indices,
|
||||
BLOCK=128,
|
||||
)
|
||||
return aligned_kv_indptr, aligned_kv_indices
|
||||
|
||||
|
||||
def build_unified_kv_indices(
|
||||
prefix_kv_indptr: torch.Tensor,
|
||||
prefix_kv_indices: torch.Tensor,
|
||||
|
||||
@@ -1183,7 +1183,6 @@ class Envs:
|
||||
# the batch fits the chunk budget.
|
||||
SGLANG_TRITON_DENSE_PREFILL_ATTN = EnvBool(True)
|
||||
SGLANG_ENABLE_TORCH_COMPILE = EnvBool(False)
|
||||
SGLANG_TRITON_PREFILL_TRUNCATION_ALIGN_SIZE = EnvInt(4096)
|
||||
SGLANG_TRITON_DECODE_SPLIT_TILE_SIZE = EnvInt(256)
|
||||
|
||||
# ===================================================================
|
||||
@@ -1993,6 +1992,9 @@ _DEPRECATED_ENVS: Dict[str, _DeprecatedEnv] = {
|
||||
note="Buffer-mode anchor pinning is always on; set "
|
||||
"SGLANG_HICACHE_BUFFER_ANCHOR_LOCK_CAP=0 to disable it."
|
||||
),
|
||||
"SGLANG_TRITON_PREFILL_TRUNCATION_ALIGN_SIZE": _DeprecatedEnv(
|
||||
note="Deterministic inference on triton no longer aligns prefill chunks."
|
||||
),
|
||||
# Replaced by CLI flags.
|
||||
"SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": _DeprecatedEnv(
|
||||
note="Please use '--enable-prefill-delayer' instead."
|
||||
|
||||
@@ -132,6 +132,9 @@ class ForwardMetadata:
|
||||
swa_attn_logits: Optional[torch.Tensor] = None
|
||||
# full->SWA translated out_cache_loc (SWA KV-store write target)
|
||||
swa_out_cache_loc: Optional[torch.Tensor] = None
|
||||
# Deterministic extend: window KV lists padded to absolute tile boundaries
|
||||
aligned_window_kv_indptr: Optional[torch.Tensor] = None
|
||||
aligned_window_kv_indices: Optional[torch.Tensor] = None
|
||||
# Lean decode (persistent-grid partial-result buffers)
|
||||
lean_Mp: Optional[torch.Tensor] = None
|
||||
lean_Lp: Optional[torch.Tensor] = None
|
||||
@@ -165,6 +168,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
lean_decode_seqlen_gate,
|
||||
)
|
||||
from sglang.kernels.ops.attention.extend_attention import (
|
||||
align_window_kv_to_tiles,
|
||||
build_unified_kv_indices,
|
||||
can_use_dense_prefill_fp8,
|
||||
dense_prefill_attention_fwd,
|
||||
@@ -187,6 +191,7 @@ class TritonAttnBackend(AttentionBackend):
|
||||
extend_attention_fwd_unified
|
||||
)
|
||||
self.build_unified_kv_indices = torch.compiler.disable(build_unified_kv_indices)
|
||||
self.align_window_kv_to_tiles = torch.compiler.disable(align_window_kv_to_tiles)
|
||||
# Dense (non-absorbed) MLA prefill over a materialized prefix; see
|
||||
# handle_attention_triton for when the dispatcher selects it.
|
||||
self.dense_prefill_attention_fwd = torch.compiler.disable(
|
||||
@@ -829,6 +834,8 @@ class TritonAttnBackend(AttentionBackend):
|
||||
window_kv_indices = None
|
||||
window_num_kv_splits = None
|
||||
window_kv_offsets = None
|
||||
aligned_window_kv_indptr = None
|
||||
aligned_window_kv_indices = None
|
||||
swa_attn_logits = None
|
||||
spec_info = forward_batch.spec_info
|
||||
# Lean decode buffers are only allocated on the decode path below; default
|
||||
@@ -1036,6 +1043,15 @@ class TritonAttnBackend(AttentionBackend):
|
||||
self.device,
|
||||
self.token_to_kv_pool,
|
||||
)
|
||||
if self.enable_deterministic:
|
||||
aligned_window_kv_indptr, aligned_window_kv_indices = (
|
||||
self.align_window_kv_to_tiles(
|
||||
window_kv_indptr,
|
||||
window_kv_indices,
|
||||
forward_batch.extend_prefix_lens[:bs] - window_kv_lens,
|
||||
bs,
|
||||
)
|
||||
)
|
||||
|
||||
qo_indptr = self.qo_indptr
|
||||
qo_indptr[1 : bs + 1] = torch.cumsum(forward_batch.extend_seq_lens, dim=0)
|
||||
@@ -1073,6 +1089,8 @@ class TritonAttnBackend(AttentionBackend):
|
||||
window_kv_offsets,
|
||||
swa_attn_logits=swa_attn_logits,
|
||||
swa_out_cache_loc=swa_out_cache_loc,
|
||||
aligned_window_kv_indptr=aligned_window_kv_indptr,
|
||||
aligned_window_kv_indices=aligned_window_kv_indices,
|
||||
lean_Mp=lean_Mp,
|
||||
lean_Lp=lean_Lp,
|
||||
lean_Op=lean_Op,
|
||||
@@ -2084,6 +2102,17 @@ class TritonAttnBackend(AttentionBackend):
|
||||
window_start_pos = forward_batch.seq_lens[:bs] - window_kv_lens
|
||||
else:
|
||||
window_start_pos = None
|
||||
# The padding is masked only by the sliding-window mask, so a layer
|
||||
# that attends bidirectionally keeps the unpadded list.
|
||||
if (
|
||||
sliding_window_size > 0
|
||||
and self.forward_metadata.aligned_window_kv_indptr is not None
|
||||
):
|
||||
prefix_kv_indptr = self.forward_metadata.aligned_window_kv_indptr
|
||||
prefix_kv_indices = self.forward_metadata.aligned_window_kv_indices
|
||||
window_start_pos = forward_batch.extend_prefix_lens[:bs] - (
|
||||
prefix_kv_indptr[1 : bs + 1] - prefix_kv_indptr[:bs]
|
||||
)
|
||||
else:
|
||||
sliding_window_size = -1
|
||||
prefix_kv_indptr = self.forward_metadata.kv_indptr
|
||||
|
||||
@@ -671,6 +671,7 @@ class Scheduler(
|
||||
# Init prefill kv split size when deterministic inference is enabled with various attention backends
|
||||
self.init_deterministic_inference_config()
|
||||
self.init_dsa_kpool_truncation_align()
|
||||
self.check_truncation_align_fits_chunk()
|
||||
|
||||
self.init_weight_updater()
|
||||
|
||||
@@ -1671,12 +1672,11 @@ class Scheduler(
|
||||
self.truncation_align_size = None
|
||||
return
|
||||
|
||||
# Only flashinfer needs chunks on its prefill split tiles; triton tiles
|
||||
# keys at absolute positions (align_window_kv_to_tiles).
|
||||
backend_sizes = {
|
||||
"flashinfer": ("SGLANG_FLASHINFER_PREFILL_SPLIT_TILE_SIZE", 4096),
|
||||
"triton": ("SGLANG_TRITON_PREFILL_TRUNCATION_ALIGN_SIZE", 4096),
|
||||
}
|
||||
# Both entries are prefill knobs (SPLIT_TILE / PREFILL_TRUNCATION):
|
||||
# the prefill half decides.
|
||||
prefill_backend, _ = attention_backends()
|
||||
env_var, default_size = backend_sizes.get(prefill_backend, (None, None))
|
||||
self.truncation_align_size = (
|
||||
@@ -1705,6 +1705,31 @@ class Scheduler(
|
||||
self.truncation_align_size, dsa_index_kpool
|
||||
)
|
||||
|
||||
def check_truncation_align_fits_chunk(self):
|
||||
"""A chunk shorter than the alignment truncates to zero tokens, so a prompt
|
||||
longer than the chunk would wait in the queue forever."""
|
||||
align = self.truncation_align_size
|
||||
chunk = self.chunked_prefill_size
|
||||
if align is None or chunk is None or chunk >= align:
|
||||
return
|
||||
if self.disaggregation_mode == DisaggregationMode.DECODE:
|
||||
# A decode server never prefills a prompt.
|
||||
return
|
||||
attn_dp_size = get_parallel().attn_dp_size
|
||||
dp_note = (
|
||||
f" (--chunked-prefill-size is split across {attn_dp_size} DP attention ranks)"
|
||||
if attn_dp_size > 1
|
||||
else ""
|
||||
)
|
||||
raise ValueError(
|
||||
f"Chunked prefill is aligned to {align} tokens, but each rank's "
|
||||
f"chunked prefill size is {chunk}{dp_note}; a prompt longer than "
|
||||
f"{chunk} tokens could never be scheduled. Raise --chunked-prefill-size "
|
||||
f"to at least {align * attn_dp_size}, or, with the flashinfer backend "
|
||||
f"under deterministic inference, lower "
|
||||
f"SGLANG_FLASHINFER_PREFILL_SPLIT_TILE_SIZE."
|
||||
)
|
||||
|
||||
def init_request_dispatcher(self):
|
||||
self._request_dispatcher = TypeBasedDispatcher(
|
||||
[
|
||||
|
||||
Reference in New Issue
Block a user