[Scheduler] Replace extend_range with extend_end derived from prefix_len (#42822)

This commit is contained in:
Liangsheng Yin
2026-10-07 16:07:16 -07:00
committed by GitHub
parent 5349d3e9d2
commit 1066decce6
42 changed files with 215 additions and 297 deletions
+3 -3
View File
@@ -424,7 +424,7 @@ def prepare_inputs_for_correctness_test(bench_args, tokenizer, custom_prompts):
)
req.full_untruncated_fill_ids = req.origin_input_ids
req.logprob_start_len = -1
req.set_extend_range(req.prefix_len, len(req.origin_input_ids))
req.extend_end = len(req.origin_input_ids)
reqs.append(req)
return input_ids, reqs
@@ -441,7 +441,7 @@ def prepare_extend_inputs_for_correctness_test(
# The cut prefix is already in the request's row from the first extend.
req.prefix_len = bench_args.cut_len
req.logprob_start_len = -1
req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids))
req.extend_end = len(req.full_untruncated_fill_ids)
return reqs
@@ -468,7 +468,7 @@ def prepare_synthetic_inputs_for_latency_test(
)
req.full_untruncated_fill_ids = req.origin_input_ids
req.logprob_start_len = -1
req.set_extend_range(req.prefix_len, len(req.origin_input_ids))
req.extend_end = len(req.origin_input_ids)
reqs.append(req)
return reqs
+3 -3
View File
@@ -1841,7 +1841,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
@property
def num_tokens_pre_allocated(self):
return sum(
decode_req.req.extend_range.end
decode_req.req.extend_end
for decode_req in self.transfer_queue.queue
if not decode_req.host_staged
)
@@ -2205,7 +2205,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
# Decode-disagg allocates before batch assembly, so it binds the prefix
# here instead of in init_next_round_input.
req.prefix_len = prefix_len
req.set_extend_range(total_prefix_len, req.kv.kv_committed_len)
req.extend_end = req.kv.kv_committed_len
self.tree_cache.maybe_hand_to_session(req)
# Return the transfer destination indices:
@@ -3158,7 +3158,7 @@ class SchedulerDisaggregationDecodeMixin:
# only sees committed KV (full array includes one uncommitted
# token because init_next_round_input rebuilt it as full).
if req.kv.kv_committed_len is not None:
req.set_extend_range(req.prefix_len, req.kv.kv_committed_len)
req.extend_end = req.kv.kv_committed_len
else:
waiting_queue.append(req)
@@ -36,7 +36,7 @@ class ScheduleBatchDisaggregationDecodeMixin:
req_pool_indices = []
# Pre-calculate total size
total_size = sum(req.extend_range.length for req in reqs)
total_size = sum(req.extend_len for req in reqs)
extend_num_tokens = total_size
out_cache_loc = torch.empty(total_size, dtype=torch.int64, device=self.device)
@@ -47,19 +47,19 @@ class ScheduleBatchDisaggregationDecodeMixin:
pre_len = req.prefix_len
chunk = self.req_to_token_pool.req_to_token[req.kv.req_pool_idx][
pre_len : pre_len + req.extend_range.length
pre_len : pre_len + req.extend_len
]
assert offset + req.extend_range.length <= total_size, (
f"Exceeds total size: offset={offset}, req.extend_range.length={req.extend_range.length}, total_size={total_size}"
assert offset + req.extend_len <= total_size, (
f"Exceeds total size: offset={offset}, req.extend_len={req.extend_len}, total_size={total_size}"
)
out_cache_loc[offset : offset + req.extend_range.length] = chunk
offset += req.extend_range.length
out_cache_loc[offset : offset + req.extend_len] = chunk
offset += req.extend_len
seq_len = len(req.origin_input_ids) + max(0, len(req.output_ids) - 1)
seq_lens.append(seq_len)
if len(req.output_ids) == 0:
assert seq_len - pre_len == req.extend_range.length, (
f"seq_len={seq_len}, pre_len={pre_len}, req.extend_range.length={req.extend_range.length}"
assert seq_len - pre_len == req.extend_len, (
f"seq_len={seq_len}, pre_len={pre_len}, req.extend_len={req.extend_len}"
)
if not req.retracted_stain:
@@ -99,7 +99,7 @@ class ScheduleBatchDisaggregationDecodeMixin:
self.extend_num_tokens = extend_num_tokens
self.prefix_lens = [r.prefix_len for r in reqs]
self.extend_lens = [r.extend_range.length for r in reqs]
self.extend_lens = [r.extend_len for r in reqs]
self.extend_logprob_start_lens = None
self.extend_input_logprob_token_ids = None
self.multimodal_inputs = [r.multimodal_inputs for r in reqs]
+8 -10
View File
@@ -593,7 +593,7 @@ class SchedulerDisaggregationPrefillMixin:
cache = self.tree_cache
if req.pending_bootstrap and _uses_write_through_cache(cache):
cache.advance_unpublished_req(req)
req.prefix_len = req.extend_range.end
req.prefix_len = req.extend_end
return
checkpoint_kv_cache(req, cache)
@@ -1060,12 +1060,12 @@ class SchedulerDisaggregationPrefillMixin:
req.inflight_middle_chunks -= 1
# Still chunking iff its next chunk was launched: either it is
# still self.chunked_req, or its final chunk (extend_range
# still self.chunked_req, or its final chunk (extend_end
# reaching the end of the input) is in flight. A yielded req
# is neither, so do its deferred release here.
still_chunking = self.chunked_req is req or (
req.extend_range is not None
and req.extend_range.end >= len(req.origin_input_ids)
req.extend_end is not None
and req.extend_end >= len(req.origin_input_ids)
)
# Abort is terminal. Do not requeue an aborted optimistic
# request merely because bootstrap is still pending.
@@ -1399,7 +1399,7 @@ class SchedulerDisaggregationPrefillMixin:
elif self.enable_overlap:
# Delay KV transfer to process_batch_result_disagg_prefill when overlap is enabled to ensure results are resolved
req.tmp_end_idx = min(
req.extend_range.end,
req.extend_end,
len(req.origin_input_ids),
)
else:
@@ -1468,7 +1468,7 @@ class SchedulerDisaggregationPrefillMixin:
value = 0
else:
if end_idx is None:
end_idx = min(req.extend_range.end, len(req.origin_input_ids))
end_idx = min(req.extend_end, len(req.origin_input_ids))
page_indices_gpu = page_indices_for_request(self, req, end_idx)
state_indices = state_indices_for_request(self, req, end_idx)
value = computer.compute(page_indices_gpu, state_indices)
@@ -1488,9 +1488,7 @@ class SchedulerDisaggregationPrefillMixin:
start_idx = req.start_send_idx
transfer_input_len = len(req.origin_input_ids)
end_idx = (
end_idx
if end_idx is not None
else min(req.extend_range.end, transfer_input_len)
end_idx if end_idx is not None else min(req.extend_end, transfer_input_len)
)
if not last_chunk:
@@ -1523,7 +1521,7 @@ class SchedulerDisaggregationPrefillMixin:
# range actually materialized on prefill. C128 state is request
# scoped, so its transfer index must use the logical input length
# that decode used to register the destination row.
seq_len = min(req.extend_range.end, transfer_input_len)
seq_len = min(req.extend_end, transfer_input_len)
c128_seq_len = transfer_input_len
def _mamba_payload():
+1 -1
View File
@@ -74,7 +74,7 @@ class ReqDllmMixin:
self.full_untruncated_fill_ids = (
self.full_untruncated_fill_ids[:prefix_len] + self.dllm_incomplete_ids
)
# extend_range is (re)computed by the staging adder
# extend_end is (re)computed by the staging adder
# (add_dllm_staging_req) before this req is scheduled, mirroring the
# non-incomplete path which also defers it to the adder.
return
+6 -6
View File
@@ -112,7 +112,7 @@ class SchedulerDllmMixin:
continue
req.full_untruncated_fill_ids[
req.extend_range.end - new_tokens : req.extend_range.end
req.extend_end - new_tokens : req.extend_end
] = array("q", next_token_ids)
self.metrics_reporter.num_generated_tokens += new_tokens
@@ -137,16 +137,16 @@ class SchedulerDllmMixin:
# Mirror the resolved block into the committed fill ids so the
# prefix cache keys on the real tokens, not the mask block, next
# round. Index relative to extend_range.end (the truncated/
# round. Index relative to extend_end (the truncated/
# committed length), which can be shorter than
# full_untruncated_fill_ids when the staging adder truncates the
# block to the KV budget.
req.full_untruncated_fill_ids[
req.extend_range.end - block_size : req.extend_range.end
req.extend_end - block_size : req.extend_end
] = array("q", next_token_ids)
len_input = len(req.origin_input_ids)
len_fill = req.extend_range.end
len_fill = req.extend_end
if len_fill <= len_input:
continue
@@ -186,7 +186,7 @@ class SchedulerDllmMixin:
def _stash_dllm_context(self: Scheduler, req: Req) -> None:
context_len = req.dllm_block_offset
block_size = self.dllm_config.block_size
assert req.extend_range.end == context_len + block_size
assert req.extend_end == context_len + block_size
assert req.kv is not None and req.kv.req_pool_idx is not None
allocated_len = req.kv.kv_allocated_len
@@ -201,7 +201,7 @@ class SchedulerDllmMixin:
req.kv.kv_committed_len = context_len
req.kv.kv_allocated_len = context_len
assert req.kv.max_evicted_seqlen <= context_len
req.set_extend_range(context_len, context_len)
req.extend_end = context_len
self.stash_chunked_request(req)
def finish_dllm_forward(self: Scheduler, req: Req) -> None:
@@ -171,10 +171,9 @@ class SchedulerMlxOverlapMixin:
extend_logprob_start_len_per_req = None
if batch.return_logprob:
# Mirror Scheduler.run_batch's launch-time copy.
extend_input_len_per_req = [
req.extend_range.length if req.extend_range is not None else 0
for req in batch.reqs
]
extend_input_len_per_req = (
list(batch.extend_lens) if batch.forward_mode.is_extend() else None
)
extend_logprob_start_len_per_req = batch.extend_logprob_start_lens
return MlxPendingJob(
launch=launch,
@@ -205,9 +205,9 @@ class MlxTpModelWorker(TpModelWorker):
discarded (the runner pops it as the stale intermediate token), so
the runner may skip the logit head for it.
"""
if req.extend_range is None:
if req.extend_end is None:
return True
return req.extend_range.end >= len(req.full_untruncated_fill_ids)
return req.extend_end >= len(req.full_untruncated_fill_ids)
@staticmethod
def _sampling_active(batch: ScheduleBatch) -> bool:
@@ -407,7 +407,7 @@ class HiSparseCoordinator:
req.hisparse_staging = True
full_kv_indices = self.req_to_token_pool.req_to_token[
req.kv.req_pool_idx, : req.extend_range.end
req.kv.req_pool_idx, : req.extend_end
].to(dtype=torch.int64, copy=True)
device_indices = (
self.mem_pool_device.translate_loc_from_full_to_hisparse_device(
@@ -503,7 +503,7 @@ class HiSparseCoordinator:
def alloc_device_buffer(self, req: Req) -> None:
if self.is_dsv4_hisparse:
allocated_len = req.extend_range.end
allocated_len = req.extend_end
alloc_size = self.padded_buffer_size
else:
allocated_len = req.kv.kv_allocated_len
@@ -920,7 +920,7 @@ class HiSparseCoordinator:
# Wait for any in-flight staging DMA to complete before freeing
self.write_staging_stream.synchronize()
prefill_len = req.extend_range.end
prefill_len = req.extend_end
allocated_locs = self.req_to_token_pool.req_to_token[
req.kv.req_pool_idx, :prefill_len
]
+32 -37
View File
@@ -14,7 +14,6 @@ from sglang.srt.runtime_context import (
mamba_track_grid,
)
from sglang.srt.utils.common import (
Range,
ceil_align,
flatten_arrays_to_pinned_cpu,
is_hip,
@@ -1070,9 +1069,10 @@ class Req(ReqDllmMixin):
self.output_ids = array("q")
# Full untruncated sequence: origin + output (+ DLLM mask block).
# Kept in sync by refresh_fill_ids; admission only updates
# extend_range, never mutates this array's length.
# extend_end, never mutates this array's length.
self.full_untruncated_fill_ids = array("q")
self.extend_range: Optional[Range] = None
# This round extends fill ids [prefix_len, extend_end).
self.extend_end: Optional[int] = None
self.dllm_initialized: bool = False
self.session = session
@@ -1371,8 +1371,8 @@ class Req(ReqDllmMixin):
# the start index of the sent kv cache
# We want to send it chunk by chunk for chunked prefill.
# After every chunk forward, we do the following:
# kv_send(req.input_ids[req.start_send_idx:req.extend_range.end])
# start_send_idx = req.extend_range.end
# kv_send(req.input_ids[req.start_send_idx:req.extend_end])
# start_send_idx = req.extend_end
self.start_send_idx: int = 0
self.disagg_decode_prefix_len: int = 0
@@ -1563,11 +1563,12 @@ class Req(ReqDllmMixin):
# Whether request reached finished condition
return self.finished_reason is not None
def set_extend_range(self, start: int, end: int) -> None:
self.extend_range = Range(start, end)
@property
def extend_len(self) -> int:
return self.extend_end - self.prefix_len
def get_fill_ids(self) -> array:
return self.full_untruncated_fill_ids[: self.extend_range.end]
return self.full_untruncated_fill_ids[: self.extend_end]
def refresh_fill_ids(self) -> None:
"""Keep full_untruncated_fill_ids == origin_input_ids + output_ids by
@@ -1935,7 +1936,7 @@ class Req(ReqDllmMixin):
self.num_matched_prefix_tokens = 0
self.lock = None
self.swa_branching_seqlen = None
self.extend_range = None
self.extend_end = None
self.dllm_initialized = False
self.is_retracted = True
self.retracted_stain = True
@@ -2295,7 +2296,7 @@ def _compute_chunked_req_next_prompt_token(
multimodal placeholder (hash) tokens that lie outside the model vocab."""
if chunked_req is None:
return None
fill_len = chunked_req.extend_range.end
fill_len = chunked_req.extend_end
origin_ids = chunked_req.origin_input_ids
if fill_len >= len(origin_ids):
return None
@@ -2618,17 +2619,17 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
input_ids[i] = input_ids[i][encoder_len:]
encoder_out_cache_loc.append(self.out_cache_loc[pt : pt + encoder_len])
decoder_out_cache_loc.append(
self.out_cache_loc[pt + encoder_len : pt + req.extend_range.length]
self.out_cache_loc[pt + encoder_len : pt + req.extend_len]
)
extend_lens[i] -= encoder_len
self.extend_num_tokens = self.extend_num_tokens - encoder_len
else:
decoder_out_cache_loc.append(
self.out_cache_loc[pt : pt + req.extend_range.length]
self.out_cache_loc[pt : pt + req.extend_len]
)
prefix_lens[i] -= encoder_len
pt += req.extend_range.length
pt += req.extend_len
self.extend_lens = extend_lens
self.prefix_lens = prefix_lens
@@ -2666,7 +2667,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
for i, req in enumerate(self.reqs):
encoder_len = self.encoder_lens_cpu[i]
old_start_len = extend_logprob_start_lens[i]
old_contribution = req.extend_range.length - old_start_len
old_contribution = req.extend_len - old_start_len
if req.prefix_len < encoder_len:
tokens_to_strip = max(0, encoder_len - old_start_len)
@@ -2695,11 +2696,6 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
encoder_len = self.encoder_lens_cpu[i]
if encoder_len == 0:
continue
if req.prefix_len < encoder_len:
assert req.prefix_len == 0
req.extend_range = req.extend_range._replace(
start=req.extend_range.start + encoder_len
)
req.logprob_start_len = max(req.logprob_start_len, encoder_len)
def prepare_for_extend(self):
@@ -2713,10 +2709,10 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
reqs = self.reqs
input_ids = [r.get_fill_ids()[r.prefix_len :] for r in reqs]
extend_num_tokens = sum(len(ids) for ids in input_ids)
seq_lens = [r.extend_range.end for r in reqs]
orig_seq_lens = [max(r.extend_range.end, len(r.origin_input_ids)) for r in reqs]
seq_lens = [r.extend_end for r in reqs]
orig_seq_lens = [max(r.extend_end, len(r.origin_input_ids)) for r in reqs]
prefix_lens = [r.prefix_len for r in reqs]
extend_lens = [r.extend_range.length for r in reqs]
extend_lens = [r.extend_len for r in reqs]
extend_logprob_start_lens = [
compute_extend_logprob_start_len(
logprob_start_len=r.logprob_start_len,
@@ -2785,8 +2781,6 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
mamba_track_seqlens_cpu = []
for i, (req, seq_len, pre_len) in enumerate(zip(reqs, seq_lens, prefix_lens)):
assert seq_len - pre_len == req.extend_range.length
req.extend_batch_idx += 1
# If input_embeds are available, store them
@@ -2794,7 +2788,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
# Slice to match extend_input_len — PrefillAdder truncates
# fill_len/extend_input_len on chunk overflow but not input_embeds.
input_embeds.extend(
req.input_embeds[pre_len : pre_len + req.extend_range.length]
req.input_embeds[pre_len : pre_len + req.extend_len]
)
if req.positional_embed_overrides is not None:
@@ -2806,7 +2800,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
req.positional_embed_overrides.positions
):
extend_pos = pos - pre_len
if extend_pos < 0 or extend_pos >= req.extend_range.length:
if extend_pos < 0 or extend_pos >= req.extend_len:
continue # Outside current extend chunk, skip
embeds_to_add.append((embed_idx, input_id_pointer + extend_pos))
if embeds_to_add:
@@ -2870,7 +2864,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
# extend_input_logprob_token_id = [4, 0]
global_start_idx, global_end_idx = (
req.prefix_len,
req.extend_range.end,
req.extend_end,
)
if req.logprob_start_len == -1:
logprob_start_len = len(req.origin_input_ids)
@@ -2885,12 +2879,12 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
]
extend_input_logprob_token_ids.extend(logprob_token_ids)
# We will need req.extend_range.length - extend_logprob_start_lens[i] number of
# We will need req.extend_len - extend_logprob_start_lens[i] number of
# tokens, and logprob_token_ids is for input logprob, so pad the rest of them by 0.
extend_input_logprob_token_ids.extend(
[0]
* (
req.extend_range.length
req.extend_len
- extend_logprob_start_lens[i]
- len(logprob_token_ids)
)
@@ -3015,7 +3009,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
return i + 1
prefix_len = req.prefix_len
seq_end = prefix_len + req.extend_range.length
seq_end = req.extend_end
if get_parallel().dcp_enabled:
# DCP widens radix pages beyond scheduler chunk boundaries. Pick an
# absolute page depth only when the kernel produced an h snapshot.
@@ -3028,10 +3022,9 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
# Chunked prefill can leave an active request off the absolute
# checkpoint grid. Without DCP, keep tracking snapshots relative to
# that request prefix so later chunks can continue donating states.
mask = req.extend_range.length >= checkpoint_grid
mask = req.extend_len >= checkpoint_grid
mamba_track_seqlen_aligned = (
prefix_len
+ (req.extend_range.length // checkpoint_grid) * checkpoint_grid
prefix_len + (req.extend_len // checkpoint_grid) * checkpoint_grid
)
track_index = req.kv.mamba_ping_pong_track_buffer[
req.kv.mamba_next_track_idx
@@ -3050,8 +3043,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
# A coarser checkpoint grid may not be a model-state boundary, so
# force retrieval from the intermediate h state in that case.
mamba_track_fla_chunk_aligned = (
req.prefix_len
+ (req.extend_range.length // state_chunk_size) * state_chunk_size
req.prefix_len + (req.extend_len // state_chunk_size) * state_chunk_size
)
if mamba_track_fla_chunk_aligned != mamba_track_seqlen_aligned:
# We want to track mamba_track_seqlen_aligned, and it's not the last position,
@@ -3134,7 +3126,9 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
running_batch.reqs, running_prefix_lens, strict=True
):
req.refresh_fill_ids()
req.set_extend_range(prefix_len, prefix_len + 1)
# req.prefix_len still ends at the last prefill, so req.extend_len is
# not 1 here; this row's prefix and extend length live on the batch.
req.extend_end = prefix_len + 1
# Decode tokens of the running portion live in future_map.output_tokens_buf.
self.input_ids = None
@@ -3214,7 +3208,8 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
req.refresh_fill_ids()
# end runs one past full_untruncated_fill_ids while output_ids
# trails; safe only while decoding_reqs suppresses the checkpoint insert.
req.set_extend_range(seq_len - 1, seq_len)
# As in mix_with_running, read prefix and extend length off the batch.
req.extend_end = seq_len
self.prefix_lens = [seq_len - 1 for seq_len in seq_lens]
self.extend_lens = [1] * bs
+14 -17
View File
@@ -945,9 +945,10 @@ class PrefillAdder:
return trunc_len
def _add_dllm_req(self, req: Req, prefix_len: int):
def _add_dllm_req(self, req: Req):
prefix_len = req.prefix_len
trunc_len = self._get_dllm_extend_len(req, prefix_len)
req.set_extend_range(prefix_len, prefix_len + trunc_len)
req.extend_end = prefix_len + trunc_len
self.can_run_list.append(req)
@@ -1019,7 +1020,7 @@ class PrefillAdder:
return AddReqResult.NO_TOKEN
truncated = cand_extend_input_len > _rem_tokens
new_len = min(cand_extend_input_len, _rem_tokens)
req.set_extend_range(req.prefix_len, req.prefix_len + new_len)
req.extend_end = req.prefix_len + new_len
self.can_run_list.append(req)
# Update budget: reserve max_new_tokens only if not truncated
@@ -1030,7 +1031,7 @@ class PrefillAdder:
)
self._update_prefill_budget(
0,
req.extend_range.length,
req.extend_len,
max_new_tokens,
req.retracted_stain,
mamba_gap_reserve=self._mamba_gap_budget_for_req(req),
@@ -1092,11 +1093,11 @@ class PrefillAdder:
)
if not reserved:
raise RuntimeError("chunked request exceeds the sharded assembly scratch")
req.set_extend_range(req.prefix_len, req.prefix_len + new_len)
req.extend_end = req.prefix_len + new_len
self.can_run_list.append(req)
self._update_prefill_budget(
0,
req.extend_range.length,
req.extend_len,
(
min(req.sampling_params.max_new_tokens, CLIP_MAX_NEW_TOKENS)
if not truncated
@@ -1105,7 +1106,7 @@ class PrefillAdder:
req.retracted_stain,
mamba_gap_reserve=self._mamba_gap_budget_for_req(req),
is_chunked_continuation=True,
compute_charge=req.extend_range.length if self.exact_chunk_fill else None,
compute_charge=req.extend_len if self.exact_chunk_fill else None,
)
# Return if chunked prefill not finished
@@ -1212,7 +1213,7 @@ class PrefillAdder:
) is not None:
return tile_stop
self._add_dllm_req(req, 0)
self._add_dllm_req(req)
elif (
self.rem_chunk_tokens is None # chunked prefill is disabled
or cand_extend_input_len <= self.rem_chunk_tokens # it is the last chunk
@@ -1228,17 +1229,15 @@ class PrefillAdder:
extend_len=cand_extend_input_len,
):
return AddReqResult.OTHER
req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids))
req.extend_end = len(req.full_untruncated_fill_ids)
self.can_run_list.append(req)
self._update_prefill_budget(
0,
req.extend_range.length,
req.extend_len,
min(req.sampling_params.max_new_tokens, CLIP_MAX_NEW_TOKENS),
req.retracted_stain,
mamba_gap_reserve=self._mamba_gap_budget_for_req(req),
compute_charge=(
req.extend_range.length if self.exact_chunk_fill else None
),
compute_charge=(req.extend_len if self.exact_chunk_fill else None),
)
else:
if self.rem_chunk_tokens <= 0:
@@ -1255,7 +1254,7 @@ class PrefillAdder:
return AddReqResult.OTHER
assert req.prefix_len == 0
req.set_extend_range(req.prefix_len, req.prefix_len + trunc_len)
req.extend_end = req.prefix_len + trunc_len
self.can_run_list.append(req)
self.new_chunked_req = req
self._update_prefill_budget(
@@ -1513,9 +1512,7 @@ class PrefillAdder:
self, req: Req, admission: _PrefillAdmission, mamba_gap_reserve: int
) -> None:
assert req.prefix_len == admission.prefix_len
req.set_extend_range(
admission.prefix_len, admission.prefix_len + admission.extend_len
)
req.extend_end = admission.prefix_len + admission.extend_len
req.lock = self.tree_cache.lock(req.last_node)
self.can_run_list.append(req)
if admission.is_chunked:
+8 -11
View File
@@ -3677,8 +3677,8 @@ class Scheduler(
chunked_req_to_exclude.add(self.chunked_req)
# A parked chunk (add_chunked_req hybrid-SWA early-return) leaves
# extend_range.end at prefix_len: it computed no new KV, so nothing to stash.
if self.chunked_req.extend_range.end > self.chunked_req.prefix_len:
# extend_end at prefix_len: it computed no new KV, so nothing to stash.
if self.chunked_req.extend_end > self.chunked_req.prefix_len:
self.stash_chunked_request(self.chunked_req)
# HiSparse has its own prefill-to-decode transition; skip last_batch merge.
@@ -4120,7 +4120,7 @@ class Scheduler(
if self.tp_worker.model_runner.prefill_aware_swa:
for req in can_run_list:
req.kv.swa_evict_floor = req.extend_range.end
req.kv.swa_evict_floor = req.extend_end
# Record prefill stats for logging after forward.
new_batch.prefill_stats = PrefillStats.from_adder(
@@ -4129,9 +4129,7 @@ class Scheduler(
self.enable_priority_scheduling,
num_pending_tokens=self.load_inquirer._get_num_pending_tokens(
chunk_deduct=(
self.chunked_req.extend_range.length
if self.chunked_req is not None
else 0
self.chunked_req.extend_len if self.chunked_req is not None else 0
),
),
)
@@ -4613,10 +4611,9 @@ class Scheduler(
# modified by overlap schedule. So we have to copy them here so that
# we can use the correct values in output processing.
if batch.return_logprob or batch.return_hidden_states:
batch_result.extend_input_len_per_req = [
req.extend_range.length if req.extend_range is not None else 0
for req in batch.reqs
]
batch_result.extend_input_len_per_req = (
list(batch.extend_lens) if batch.forward_mode.is_extend() else None
)
else:
batch_result.extend_input_len_per_req = None
@@ -5470,7 +5467,7 @@ class Scheduler(
and (req := self.chunked_req) is not None
):
# Retract skips the chunk step that sets this result's KV send boundary.
req.tmp_end_idx = min(req.extend_range.end, len(req.origin_input_ids))
req.tmp_end_idx = min(req.extend_end, len(req.origin_input_ids))
tmp_batch, tmp_result = self.result_queue.popleft()
self.process_batch_result(tmp_batch, tmp_result)
@@ -170,7 +170,7 @@ class DynamicChunkSizer:
# request so release_kv_cache can release it symmetrically.
req.init_next_round_input(self.tree_cache)
req.lock = self.tree_cache.lock(req.last_node)
req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids))
req.extend_end = len(req.full_untruncated_fill_ids)
# Prepare batch
batch = ScheduleBatch.init_new(
@@ -183,7 +183,7 @@ class DynamicChunkSizer:
self.spec_algorithm,
)
current_seq_len = req.extend_range.end
current_seq_len = req.extend_end
if is_dp_attention_enabled():
# Profiling runs one request on this rank; other DP ranks report 0.
+2 -2
View File
@@ -243,9 +243,9 @@ def checkpoint_kv_cache(req: Req, tree_cache: BasePrefixCache) -> None:
# insert; a finished request belongs in release_kv_cache.
assert not req.finished(), f"checkpointing finished request {req.rid}"
if not req.skip_radix_cache_insert:
tree_cache.checkpoint(req, up_to=req.extend_range.end)
tree_cache.checkpoint(req, up_to=req.extend_end)
# The next extend resumes after this one, published or not.
req.prefix_len = req.extend_range.end
req.prefix_len = req.extend_end
def evict_from_tree_cache(
@@ -237,7 +237,7 @@ Insert a request's KV into the tree: at every checkpoint while it runs, and once
| Aspect | Detail |
|--------|--------|
| **Purpose** | Publish the request's KV `[cache_protected_len, up_to)` so other requests can match it; nodes past `req.kv.cache_inserted_len` count one hit (a request counts each node once). `checkpoint_kv_cache` calls it with `req.extend_range.end` while the request runs; `release_kv_cache` calls it with the request-owned length when it finishes (`req.finished()`), then frees `[cache_protected_len, up_to)` and everything past it and unlocks `req.lock`; with `checkpoint=False` it skips the insert and calls `on_release(req, checkpointed=False)` for component cleanup |
| **Purpose** | Publish the request's KV `[cache_protected_len, up_to)` so other requests can match it; nodes past `req.kv.cache_inserted_len` count one hit (a request counts each node once). `checkpoint_kv_cache` calls it with `req.extend_end` while the request runs; `release_kv_cache` calls it with the request-owned length when it finishes (`req.finished()`), then frees `[cache_protected_len, up_to)` and everything past it and unlocks `req.lock`; with `checkpoint=False` it skips the insert and calls `on_release(req, checkpointed=False)` for component cleanup |
| **Inputs** | `req` — the request; `up_to` — row position the insert may read up to |
| **Output** | `None` |
| **Mutation** | Component hooks → `insert` → re-match → writes the tree's indices back into the row → moves `req.lock` to the node the insert ended on → updates `req.kv.cache_protected_len`, `req.kv.cache_inserted_len`, `req.last_node` (`checkpoint_kv_cache` then advances `req.prefix_len`) → component cleanup. A finished request's component state (Mamba) is handed to the tree instead of forked, and what it still held is freed. Frees no KV slot: `release_kv_cache` does that afterwards. |
@@ -123,9 +123,10 @@ class NgramEmbeddingManager:
all_tokens = []
column_starts = []
request_lengths = []
for req in batch.reqs:
start = req.prefix_len
end = start + req.extend_range.length
for req, start, extend_len in zip(
batch.reqs, batch.prefix_lens, batch.extend_lens, strict=True
):
end = start + extend_len
fill_ids = req.origin_input_ids + req.output_ids
if start == 0:
tokens = fill_ids[start:end]
@@ -196,8 +197,7 @@ class NgramEmbeddingManager:
if not batch.forward_mode.is_extend_without_speculative():
return
rows = []
for req in batch.reqs:
start = req.extend_range.start
for req, start in zip(batch.reqs, batch.prefix_lens, strict=True):
lo = max(0, start - n1)
ids = req.full_untruncated_fill_ids[lo:start]
rows.append([0] * (n1 - len(ids)) + list(ids))
-10
View File
@@ -69,7 +69,6 @@ from typing import (
Generic,
Iterator,
List,
NamedTuple,
Optional,
Protocol,
Sequence,
@@ -1335,15 +1334,6 @@ def get_current_device_stream_fast():
# ==============================================================================
class Range(NamedTuple):
start: int
end: int
@property
def length(self) -> int:
return self.end - self.start
def assert_int64_array(values: array, name: str) -> None:
"""Require a signed int64 array suitable for zero-copy tensor views."""
assert (