[Scheduler] Replace extend_range with extend_end derived from prefix_len (#42822)
This commit is contained in:
@@ -424,7 +424,7 @@ def prepare_inputs_for_correctness_test(bench_args, tokenizer, custom_prompts):
|
||||
)
|
||||
req.full_untruncated_fill_ids = req.origin_input_ids
|
||||
req.logprob_start_len = -1
|
||||
req.set_extend_range(req.prefix_len, len(req.origin_input_ids))
|
||||
req.extend_end = len(req.origin_input_ids)
|
||||
reqs.append(req)
|
||||
|
||||
return input_ids, reqs
|
||||
@@ -441,7 +441,7 @@ def prepare_extend_inputs_for_correctness_test(
|
||||
# The cut prefix is already in the request's row from the first extend.
|
||||
req.prefix_len = bench_args.cut_len
|
||||
req.logprob_start_len = -1
|
||||
req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids))
|
||||
req.extend_end = len(req.full_untruncated_fill_ids)
|
||||
return reqs
|
||||
|
||||
|
||||
@@ -468,7 +468,7 @@ def prepare_synthetic_inputs_for_latency_test(
|
||||
)
|
||||
req.full_untruncated_fill_ids = req.origin_input_ids
|
||||
req.logprob_start_len = -1
|
||||
req.set_extend_range(req.prefix_len, len(req.origin_input_ids))
|
||||
req.extend_end = len(req.origin_input_ids)
|
||||
reqs.append(req)
|
||||
|
||||
return reqs
|
||||
|
||||
@@ -1841,7 +1841,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
@property
|
||||
def num_tokens_pre_allocated(self):
|
||||
return sum(
|
||||
decode_req.req.extend_range.end
|
||||
decode_req.req.extend_end
|
||||
for decode_req in self.transfer_queue.queue
|
||||
if not decode_req.host_staged
|
||||
)
|
||||
@@ -2205,7 +2205,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
# Decode-disagg allocates before batch assembly, so it binds the prefix
|
||||
# here instead of in init_next_round_input.
|
||||
req.prefix_len = prefix_len
|
||||
req.set_extend_range(total_prefix_len, req.kv.kv_committed_len)
|
||||
req.extend_end = req.kv.kv_committed_len
|
||||
self.tree_cache.maybe_hand_to_session(req)
|
||||
|
||||
# Return the transfer destination indices:
|
||||
@@ -3158,7 +3158,7 @@ class SchedulerDisaggregationDecodeMixin:
|
||||
# only sees committed KV (full array includes one uncommitted
|
||||
# token because init_next_round_input rebuilt it as full).
|
||||
if req.kv.kv_committed_len is not None:
|
||||
req.set_extend_range(req.prefix_len, req.kv.kv_committed_len)
|
||||
req.extend_end = req.kv.kv_committed_len
|
||||
else:
|
||||
waiting_queue.append(req)
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ class ScheduleBatchDisaggregationDecodeMixin:
|
||||
req_pool_indices = []
|
||||
|
||||
# Pre-calculate total size
|
||||
total_size = sum(req.extend_range.length for req in reqs)
|
||||
total_size = sum(req.extend_len for req in reqs)
|
||||
extend_num_tokens = total_size
|
||||
out_cache_loc = torch.empty(total_size, dtype=torch.int64, device=self.device)
|
||||
|
||||
@@ -47,19 +47,19 @@ class ScheduleBatchDisaggregationDecodeMixin:
|
||||
pre_len = req.prefix_len
|
||||
|
||||
chunk = self.req_to_token_pool.req_to_token[req.kv.req_pool_idx][
|
||||
pre_len : pre_len + req.extend_range.length
|
||||
pre_len : pre_len + req.extend_len
|
||||
]
|
||||
assert offset + req.extend_range.length <= total_size, (
|
||||
f"Exceeds total size: offset={offset}, req.extend_range.length={req.extend_range.length}, total_size={total_size}"
|
||||
assert offset + req.extend_len <= total_size, (
|
||||
f"Exceeds total size: offset={offset}, req.extend_len={req.extend_len}, total_size={total_size}"
|
||||
)
|
||||
out_cache_loc[offset : offset + req.extend_range.length] = chunk
|
||||
offset += req.extend_range.length
|
||||
out_cache_loc[offset : offset + req.extend_len] = chunk
|
||||
offset += req.extend_len
|
||||
|
||||
seq_len = len(req.origin_input_ids) + max(0, len(req.output_ids) - 1)
|
||||
seq_lens.append(seq_len)
|
||||
if len(req.output_ids) == 0:
|
||||
assert seq_len - pre_len == req.extend_range.length, (
|
||||
f"seq_len={seq_len}, pre_len={pre_len}, req.extend_range.length={req.extend_range.length}"
|
||||
assert seq_len - pre_len == req.extend_len, (
|
||||
f"seq_len={seq_len}, pre_len={pre_len}, req.extend_len={req.extend_len}"
|
||||
)
|
||||
|
||||
if not req.retracted_stain:
|
||||
@@ -99,7 +99,7 @@ class ScheduleBatchDisaggregationDecodeMixin:
|
||||
|
||||
self.extend_num_tokens = extend_num_tokens
|
||||
self.prefix_lens = [r.prefix_len for r in reqs]
|
||||
self.extend_lens = [r.extend_range.length for r in reqs]
|
||||
self.extend_lens = [r.extend_len for r in reqs]
|
||||
self.extend_logprob_start_lens = None
|
||||
self.extend_input_logprob_token_ids = None
|
||||
self.multimodal_inputs = [r.multimodal_inputs for r in reqs]
|
||||
|
||||
@@ -593,7 +593,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
cache = self.tree_cache
|
||||
if req.pending_bootstrap and _uses_write_through_cache(cache):
|
||||
cache.advance_unpublished_req(req)
|
||||
req.prefix_len = req.extend_range.end
|
||||
req.prefix_len = req.extend_end
|
||||
return
|
||||
|
||||
checkpoint_kv_cache(req, cache)
|
||||
@@ -1060,12 +1060,12 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
req.inflight_middle_chunks -= 1
|
||||
|
||||
# Still chunking iff its next chunk was launched: either it is
|
||||
# still self.chunked_req, or its final chunk (extend_range
|
||||
# still self.chunked_req, or its final chunk (extend_end
|
||||
# reaching the end of the input) is in flight. A yielded req
|
||||
# is neither, so do its deferred release here.
|
||||
still_chunking = self.chunked_req is req or (
|
||||
req.extend_range is not None
|
||||
and req.extend_range.end >= len(req.origin_input_ids)
|
||||
req.extend_end is not None
|
||||
and req.extend_end >= len(req.origin_input_ids)
|
||||
)
|
||||
# Abort is terminal. Do not requeue an aborted optimistic
|
||||
# request merely because bootstrap is still pending.
|
||||
@@ -1399,7 +1399,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
elif self.enable_overlap:
|
||||
# Delay KV transfer to process_batch_result_disagg_prefill when overlap is enabled to ensure results are resolved
|
||||
req.tmp_end_idx = min(
|
||||
req.extend_range.end,
|
||||
req.extend_end,
|
||||
len(req.origin_input_ids),
|
||||
)
|
||||
else:
|
||||
@@ -1468,7 +1468,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
value = 0
|
||||
else:
|
||||
if end_idx is None:
|
||||
end_idx = min(req.extend_range.end, len(req.origin_input_ids))
|
||||
end_idx = min(req.extend_end, len(req.origin_input_ids))
|
||||
page_indices_gpu = page_indices_for_request(self, req, end_idx)
|
||||
state_indices = state_indices_for_request(self, req, end_idx)
|
||||
value = computer.compute(page_indices_gpu, state_indices)
|
||||
@@ -1488,9 +1488,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
start_idx = req.start_send_idx
|
||||
transfer_input_len = len(req.origin_input_ids)
|
||||
end_idx = (
|
||||
end_idx
|
||||
if end_idx is not None
|
||||
else min(req.extend_range.end, transfer_input_len)
|
||||
end_idx if end_idx is not None else min(req.extend_end, transfer_input_len)
|
||||
)
|
||||
|
||||
if not last_chunk:
|
||||
@@ -1523,7 +1521,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
# range actually materialized on prefill. C128 state is request
|
||||
# scoped, so its transfer index must use the logical input length
|
||||
# that decode used to register the destination row.
|
||||
seq_len = min(req.extend_range.end, transfer_input_len)
|
||||
seq_len = min(req.extend_end, transfer_input_len)
|
||||
c128_seq_len = transfer_input_len
|
||||
|
||||
def _mamba_payload():
|
||||
|
||||
@@ -74,7 +74,7 @@ class ReqDllmMixin:
|
||||
self.full_untruncated_fill_ids = (
|
||||
self.full_untruncated_fill_ids[:prefix_len] + self.dllm_incomplete_ids
|
||||
)
|
||||
# extend_range is (re)computed by the staging adder
|
||||
# extend_end is (re)computed by the staging adder
|
||||
# (add_dllm_staging_req) before this req is scheduled, mirroring the
|
||||
# non-incomplete path which also defers it to the adder.
|
||||
return
|
||||
|
||||
@@ -112,7 +112,7 @@ class SchedulerDllmMixin:
|
||||
continue
|
||||
|
||||
req.full_untruncated_fill_ids[
|
||||
req.extend_range.end - new_tokens : req.extend_range.end
|
||||
req.extend_end - new_tokens : req.extend_end
|
||||
] = array("q", next_token_ids)
|
||||
self.metrics_reporter.num_generated_tokens += new_tokens
|
||||
|
||||
@@ -137,16 +137,16 @@ class SchedulerDllmMixin:
|
||||
|
||||
# Mirror the resolved block into the committed fill ids so the
|
||||
# prefix cache keys on the real tokens, not the mask block, next
|
||||
# round. Index relative to extend_range.end (the truncated/
|
||||
# round. Index relative to extend_end (the truncated/
|
||||
# committed length), which can be shorter than
|
||||
# full_untruncated_fill_ids when the staging adder truncates the
|
||||
# block to the KV budget.
|
||||
req.full_untruncated_fill_ids[
|
||||
req.extend_range.end - block_size : req.extend_range.end
|
||||
req.extend_end - block_size : req.extend_end
|
||||
] = array("q", next_token_ids)
|
||||
|
||||
len_input = len(req.origin_input_ids)
|
||||
len_fill = req.extend_range.end
|
||||
len_fill = req.extend_end
|
||||
if len_fill <= len_input:
|
||||
continue
|
||||
|
||||
@@ -186,7 +186,7 @@ class SchedulerDllmMixin:
|
||||
def _stash_dllm_context(self: Scheduler, req: Req) -> None:
|
||||
context_len = req.dllm_block_offset
|
||||
block_size = self.dllm_config.block_size
|
||||
assert req.extend_range.end == context_len + block_size
|
||||
assert req.extend_end == context_len + block_size
|
||||
assert req.kv is not None and req.kv.req_pool_idx is not None
|
||||
|
||||
allocated_len = req.kv.kv_allocated_len
|
||||
@@ -201,7 +201,7 @@ class SchedulerDllmMixin:
|
||||
req.kv.kv_committed_len = context_len
|
||||
req.kv.kv_allocated_len = context_len
|
||||
assert req.kv.max_evicted_seqlen <= context_len
|
||||
req.set_extend_range(context_len, context_len)
|
||||
req.extend_end = context_len
|
||||
self.stash_chunked_request(req)
|
||||
|
||||
def finish_dllm_forward(self: Scheduler, req: Req) -> None:
|
||||
|
||||
@@ -171,10 +171,9 @@ class SchedulerMlxOverlapMixin:
|
||||
extend_logprob_start_len_per_req = None
|
||||
if batch.return_logprob:
|
||||
# Mirror Scheduler.run_batch's launch-time copy.
|
||||
extend_input_len_per_req = [
|
||||
req.extend_range.length if req.extend_range is not None else 0
|
||||
for req in batch.reqs
|
||||
]
|
||||
extend_input_len_per_req = (
|
||||
list(batch.extend_lens) if batch.forward_mode.is_extend() else None
|
||||
)
|
||||
extend_logprob_start_len_per_req = batch.extend_logprob_start_lens
|
||||
return MlxPendingJob(
|
||||
launch=launch,
|
||||
|
||||
@@ -205,9 +205,9 @@ class MlxTpModelWorker(TpModelWorker):
|
||||
discarded (the runner pops it as the stale intermediate token), so
|
||||
the runner may skip the logit head for it.
|
||||
"""
|
||||
if req.extend_range is None:
|
||||
if req.extend_end is None:
|
||||
return True
|
||||
return req.extend_range.end >= len(req.full_untruncated_fill_ids)
|
||||
return req.extend_end >= len(req.full_untruncated_fill_ids)
|
||||
|
||||
@staticmethod
|
||||
def _sampling_active(batch: ScheduleBatch) -> bool:
|
||||
|
||||
@@ -407,7 +407,7 @@ class HiSparseCoordinator:
|
||||
req.hisparse_staging = True
|
||||
|
||||
full_kv_indices = self.req_to_token_pool.req_to_token[
|
||||
req.kv.req_pool_idx, : req.extend_range.end
|
||||
req.kv.req_pool_idx, : req.extend_end
|
||||
].to(dtype=torch.int64, copy=True)
|
||||
device_indices = (
|
||||
self.mem_pool_device.translate_loc_from_full_to_hisparse_device(
|
||||
@@ -503,7 +503,7 @@ class HiSparseCoordinator:
|
||||
|
||||
def alloc_device_buffer(self, req: Req) -> None:
|
||||
if self.is_dsv4_hisparse:
|
||||
allocated_len = req.extend_range.end
|
||||
allocated_len = req.extend_end
|
||||
alloc_size = self.padded_buffer_size
|
||||
else:
|
||||
allocated_len = req.kv.kv_allocated_len
|
||||
@@ -920,7 +920,7 @@ class HiSparseCoordinator:
|
||||
# Wait for any in-flight staging DMA to complete before freeing
|
||||
self.write_staging_stream.synchronize()
|
||||
|
||||
prefill_len = req.extend_range.end
|
||||
prefill_len = req.extend_end
|
||||
allocated_locs = self.req_to_token_pool.req_to_token[
|
||||
req.kv.req_pool_idx, :prefill_len
|
||||
]
|
||||
|
||||
@@ -14,7 +14,6 @@ from sglang.srt.runtime_context import (
|
||||
mamba_track_grid,
|
||||
)
|
||||
from sglang.srt.utils.common import (
|
||||
Range,
|
||||
ceil_align,
|
||||
flatten_arrays_to_pinned_cpu,
|
||||
is_hip,
|
||||
@@ -1070,9 +1069,10 @@ class Req(ReqDllmMixin):
|
||||
self.output_ids = array("q")
|
||||
# Full untruncated sequence: origin + output (+ DLLM mask block).
|
||||
# Kept in sync by refresh_fill_ids; admission only updates
|
||||
# extend_range, never mutates this array's length.
|
||||
# extend_end, never mutates this array's length.
|
||||
self.full_untruncated_fill_ids = array("q")
|
||||
self.extend_range: Optional[Range] = None
|
||||
# This round extends fill ids [prefix_len, extend_end).
|
||||
self.extend_end: Optional[int] = None
|
||||
self.dllm_initialized: bool = False
|
||||
|
||||
self.session = session
|
||||
@@ -1371,8 +1371,8 @@ class Req(ReqDllmMixin):
|
||||
# the start index of the sent kv cache
|
||||
# We want to send it chunk by chunk for chunked prefill.
|
||||
# After every chunk forward, we do the following:
|
||||
# kv_send(req.input_ids[req.start_send_idx:req.extend_range.end])
|
||||
# start_send_idx = req.extend_range.end
|
||||
# kv_send(req.input_ids[req.start_send_idx:req.extend_end])
|
||||
# start_send_idx = req.extend_end
|
||||
self.start_send_idx: int = 0
|
||||
self.disagg_decode_prefix_len: int = 0
|
||||
|
||||
@@ -1563,11 +1563,12 @@ class Req(ReqDllmMixin):
|
||||
# Whether request reached finished condition
|
||||
return self.finished_reason is not None
|
||||
|
||||
def set_extend_range(self, start: int, end: int) -> None:
|
||||
self.extend_range = Range(start, end)
|
||||
@property
|
||||
def extend_len(self) -> int:
|
||||
return self.extend_end - self.prefix_len
|
||||
|
||||
def get_fill_ids(self) -> array:
|
||||
return self.full_untruncated_fill_ids[: self.extend_range.end]
|
||||
return self.full_untruncated_fill_ids[: self.extend_end]
|
||||
|
||||
def refresh_fill_ids(self) -> None:
|
||||
"""Keep full_untruncated_fill_ids == origin_input_ids + output_ids by
|
||||
@@ -1935,7 +1936,7 @@ class Req(ReqDllmMixin):
|
||||
self.num_matched_prefix_tokens = 0
|
||||
self.lock = None
|
||||
self.swa_branching_seqlen = None
|
||||
self.extend_range = None
|
||||
self.extend_end = None
|
||||
self.dllm_initialized = False
|
||||
self.is_retracted = True
|
||||
self.retracted_stain = True
|
||||
@@ -2295,7 +2296,7 @@ def _compute_chunked_req_next_prompt_token(
|
||||
multimodal placeholder (hash) tokens that lie outside the model vocab."""
|
||||
if chunked_req is None:
|
||||
return None
|
||||
fill_len = chunked_req.extend_range.end
|
||||
fill_len = chunked_req.extend_end
|
||||
origin_ids = chunked_req.origin_input_ids
|
||||
if fill_len >= len(origin_ids):
|
||||
return None
|
||||
@@ -2618,17 +2619,17 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
input_ids[i] = input_ids[i][encoder_len:]
|
||||
encoder_out_cache_loc.append(self.out_cache_loc[pt : pt + encoder_len])
|
||||
decoder_out_cache_loc.append(
|
||||
self.out_cache_loc[pt + encoder_len : pt + req.extend_range.length]
|
||||
self.out_cache_loc[pt + encoder_len : pt + req.extend_len]
|
||||
)
|
||||
extend_lens[i] -= encoder_len
|
||||
self.extend_num_tokens = self.extend_num_tokens - encoder_len
|
||||
else:
|
||||
decoder_out_cache_loc.append(
|
||||
self.out_cache_loc[pt : pt + req.extend_range.length]
|
||||
self.out_cache_loc[pt : pt + req.extend_len]
|
||||
)
|
||||
prefix_lens[i] -= encoder_len
|
||||
|
||||
pt += req.extend_range.length
|
||||
pt += req.extend_len
|
||||
self.extend_lens = extend_lens
|
||||
self.prefix_lens = prefix_lens
|
||||
|
||||
@@ -2666,7 +2667,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
for i, req in enumerate(self.reqs):
|
||||
encoder_len = self.encoder_lens_cpu[i]
|
||||
old_start_len = extend_logprob_start_lens[i]
|
||||
old_contribution = req.extend_range.length - old_start_len
|
||||
old_contribution = req.extend_len - old_start_len
|
||||
|
||||
if req.prefix_len < encoder_len:
|
||||
tokens_to_strip = max(0, encoder_len - old_start_len)
|
||||
@@ -2695,11 +2696,6 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
encoder_len = self.encoder_lens_cpu[i]
|
||||
if encoder_len == 0:
|
||||
continue
|
||||
if req.prefix_len < encoder_len:
|
||||
assert req.prefix_len == 0
|
||||
req.extend_range = req.extend_range._replace(
|
||||
start=req.extend_range.start + encoder_len
|
||||
)
|
||||
req.logprob_start_len = max(req.logprob_start_len, encoder_len)
|
||||
|
||||
def prepare_for_extend(self):
|
||||
@@ -2713,10 +2709,10 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
reqs = self.reqs
|
||||
input_ids = [r.get_fill_ids()[r.prefix_len :] for r in reqs]
|
||||
extend_num_tokens = sum(len(ids) for ids in input_ids)
|
||||
seq_lens = [r.extend_range.end for r in reqs]
|
||||
orig_seq_lens = [max(r.extend_range.end, len(r.origin_input_ids)) for r in reqs]
|
||||
seq_lens = [r.extend_end for r in reqs]
|
||||
orig_seq_lens = [max(r.extend_end, len(r.origin_input_ids)) for r in reqs]
|
||||
prefix_lens = [r.prefix_len for r in reqs]
|
||||
extend_lens = [r.extend_range.length for r in reqs]
|
||||
extend_lens = [r.extend_len for r in reqs]
|
||||
extend_logprob_start_lens = [
|
||||
compute_extend_logprob_start_len(
|
||||
logprob_start_len=r.logprob_start_len,
|
||||
@@ -2785,8 +2781,6 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
mamba_track_seqlens_cpu = []
|
||||
|
||||
for i, (req, seq_len, pre_len) in enumerate(zip(reqs, seq_lens, prefix_lens)):
|
||||
assert seq_len - pre_len == req.extend_range.length
|
||||
|
||||
req.extend_batch_idx += 1
|
||||
|
||||
# If input_embeds are available, store them
|
||||
@@ -2794,7 +2788,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
# Slice to match extend_input_len — PrefillAdder truncates
|
||||
# fill_len/extend_input_len on chunk overflow but not input_embeds.
|
||||
input_embeds.extend(
|
||||
req.input_embeds[pre_len : pre_len + req.extend_range.length]
|
||||
req.input_embeds[pre_len : pre_len + req.extend_len]
|
||||
)
|
||||
|
||||
if req.positional_embed_overrides is not None:
|
||||
@@ -2806,7 +2800,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
req.positional_embed_overrides.positions
|
||||
):
|
||||
extend_pos = pos - pre_len
|
||||
if extend_pos < 0 or extend_pos >= req.extend_range.length:
|
||||
if extend_pos < 0 or extend_pos >= req.extend_len:
|
||||
continue # Outside current extend chunk, skip
|
||||
embeds_to_add.append((embed_idx, input_id_pointer + extend_pos))
|
||||
if embeds_to_add:
|
||||
@@ -2870,7 +2864,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
# extend_input_logprob_token_id = [4, 0]
|
||||
global_start_idx, global_end_idx = (
|
||||
req.prefix_len,
|
||||
req.extend_range.end,
|
||||
req.extend_end,
|
||||
)
|
||||
if req.logprob_start_len == -1:
|
||||
logprob_start_len = len(req.origin_input_ids)
|
||||
@@ -2885,12 +2879,12 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
]
|
||||
extend_input_logprob_token_ids.extend(logprob_token_ids)
|
||||
|
||||
# We will need req.extend_range.length - extend_logprob_start_lens[i] number of
|
||||
# We will need req.extend_len - extend_logprob_start_lens[i] number of
|
||||
# tokens, and logprob_token_ids is for input logprob, so pad the rest of them by 0.
|
||||
extend_input_logprob_token_ids.extend(
|
||||
[0]
|
||||
* (
|
||||
req.extend_range.length
|
||||
req.extend_len
|
||||
- extend_logprob_start_lens[i]
|
||||
- len(logprob_token_ids)
|
||||
)
|
||||
@@ -3015,7 +3009,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
return i + 1
|
||||
|
||||
prefix_len = req.prefix_len
|
||||
seq_end = prefix_len + req.extend_range.length
|
||||
seq_end = req.extend_end
|
||||
if get_parallel().dcp_enabled:
|
||||
# DCP widens radix pages beyond scheduler chunk boundaries. Pick an
|
||||
# absolute page depth only when the kernel produced an h snapshot.
|
||||
@@ -3028,10 +3022,9 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
# Chunked prefill can leave an active request off the absolute
|
||||
# checkpoint grid. Without DCP, keep tracking snapshots relative to
|
||||
# that request prefix so later chunks can continue donating states.
|
||||
mask = req.extend_range.length >= checkpoint_grid
|
||||
mask = req.extend_len >= checkpoint_grid
|
||||
mamba_track_seqlen_aligned = (
|
||||
prefix_len
|
||||
+ (req.extend_range.length // checkpoint_grid) * checkpoint_grid
|
||||
prefix_len + (req.extend_len // checkpoint_grid) * checkpoint_grid
|
||||
)
|
||||
track_index = req.kv.mamba_ping_pong_track_buffer[
|
||||
req.kv.mamba_next_track_idx
|
||||
@@ -3050,8 +3043,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
# A coarser checkpoint grid may not be a model-state boundary, so
|
||||
# force retrieval from the intermediate h state in that case.
|
||||
mamba_track_fla_chunk_aligned = (
|
||||
req.prefix_len
|
||||
+ (req.extend_range.length // state_chunk_size) * state_chunk_size
|
||||
req.prefix_len + (req.extend_len // state_chunk_size) * state_chunk_size
|
||||
)
|
||||
if mamba_track_fla_chunk_aligned != mamba_track_seqlen_aligned:
|
||||
# We want to track mamba_track_seqlen_aligned, and it's not the last position,
|
||||
@@ -3134,7 +3126,9 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
running_batch.reqs, running_prefix_lens, strict=True
|
||||
):
|
||||
req.refresh_fill_ids()
|
||||
req.set_extend_range(prefix_len, prefix_len + 1)
|
||||
# req.prefix_len still ends at the last prefill, so req.extend_len is
|
||||
# not 1 here; this row's prefix and extend length live on the batch.
|
||||
req.extend_end = prefix_len + 1
|
||||
|
||||
# Decode tokens of the running portion live in future_map.output_tokens_buf.
|
||||
self.input_ids = None
|
||||
@@ -3214,7 +3208,8 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
req.refresh_fill_ids()
|
||||
# end runs one past full_untruncated_fill_ids while output_ids
|
||||
# trails; safe only while decoding_reqs suppresses the checkpoint insert.
|
||||
req.set_extend_range(seq_len - 1, seq_len)
|
||||
# As in mix_with_running, read prefix and extend length off the batch.
|
||||
req.extend_end = seq_len
|
||||
|
||||
self.prefix_lens = [seq_len - 1 for seq_len in seq_lens]
|
||||
self.extend_lens = [1] * bs
|
||||
|
||||
@@ -945,9 +945,10 @@ class PrefillAdder:
|
||||
|
||||
return trunc_len
|
||||
|
||||
def _add_dllm_req(self, req: Req, prefix_len: int):
|
||||
def _add_dllm_req(self, req: Req):
|
||||
prefix_len = req.prefix_len
|
||||
trunc_len = self._get_dllm_extend_len(req, prefix_len)
|
||||
req.set_extend_range(prefix_len, prefix_len + trunc_len)
|
||||
req.extend_end = prefix_len + trunc_len
|
||||
|
||||
self.can_run_list.append(req)
|
||||
|
||||
@@ -1019,7 +1020,7 @@ class PrefillAdder:
|
||||
return AddReqResult.NO_TOKEN
|
||||
truncated = cand_extend_input_len > _rem_tokens
|
||||
new_len = min(cand_extend_input_len, _rem_tokens)
|
||||
req.set_extend_range(req.prefix_len, req.prefix_len + new_len)
|
||||
req.extend_end = req.prefix_len + new_len
|
||||
self.can_run_list.append(req)
|
||||
|
||||
# Update budget: reserve max_new_tokens only if not truncated
|
||||
@@ -1030,7 +1031,7 @@ class PrefillAdder:
|
||||
)
|
||||
self._update_prefill_budget(
|
||||
0,
|
||||
req.extend_range.length,
|
||||
req.extend_len,
|
||||
max_new_tokens,
|
||||
req.retracted_stain,
|
||||
mamba_gap_reserve=self._mamba_gap_budget_for_req(req),
|
||||
@@ -1092,11 +1093,11 @@ class PrefillAdder:
|
||||
)
|
||||
if not reserved:
|
||||
raise RuntimeError("chunked request exceeds the sharded assembly scratch")
|
||||
req.set_extend_range(req.prefix_len, req.prefix_len + new_len)
|
||||
req.extend_end = req.prefix_len + new_len
|
||||
self.can_run_list.append(req)
|
||||
self._update_prefill_budget(
|
||||
0,
|
||||
req.extend_range.length,
|
||||
req.extend_len,
|
||||
(
|
||||
min(req.sampling_params.max_new_tokens, CLIP_MAX_NEW_TOKENS)
|
||||
if not truncated
|
||||
@@ -1105,7 +1106,7 @@ class PrefillAdder:
|
||||
req.retracted_stain,
|
||||
mamba_gap_reserve=self._mamba_gap_budget_for_req(req),
|
||||
is_chunked_continuation=True,
|
||||
compute_charge=req.extend_range.length if self.exact_chunk_fill else None,
|
||||
compute_charge=req.extend_len if self.exact_chunk_fill else None,
|
||||
)
|
||||
|
||||
# Return if chunked prefill not finished
|
||||
@@ -1212,7 +1213,7 @@ class PrefillAdder:
|
||||
) is not None:
|
||||
return tile_stop
|
||||
|
||||
self._add_dllm_req(req, 0)
|
||||
self._add_dllm_req(req)
|
||||
elif (
|
||||
self.rem_chunk_tokens is None # chunked prefill is disabled
|
||||
or cand_extend_input_len <= self.rem_chunk_tokens # it is the last chunk
|
||||
@@ -1228,17 +1229,15 @@ class PrefillAdder:
|
||||
extend_len=cand_extend_input_len,
|
||||
):
|
||||
return AddReqResult.OTHER
|
||||
req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids))
|
||||
req.extend_end = len(req.full_untruncated_fill_ids)
|
||||
self.can_run_list.append(req)
|
||||
self._update_prefill_budget(
|
||||
0,
|
||||
req.extend_range.length,
|
||||
req.extend_len,
|
||||
min(req.sampling_params.max_new_tokens, CLIP_MAX_NEW_TOKENS),
|
||||
req.retracted_stain,
|
||||
mamba_gap_reserve=self._mamba_gap_budget_for_req(req),
|
||||
compute_charge=(
|
||||
req.extend_range.length if self.exact_chunk_fill else None
|
||||
),
|
||||
compute_charge=(req.extend_len if self.exact_chunk_fill else None),
|
||||
)
|
||||
else:
|
||||
if self.rem_chunk_tokens <= 0:
|
||||
@@ -1255,7 +1254,7 @@ class PrefillAdder:
|
||||
return AddReqResult.OTHER
|
||||
|
||||
assert req.prefix_len == 0
|
||||
req.set_extend_range(req.prefix_len, req.prefix_len + trunc_len)
|
||||
req.extend_end = req.prefix_len + trunc_len
|
||||
self.can_run_list.append(req)
|
||||
self.new_chunked_req = req
|
||||
self._update_prefill_budget(
|
||||
@@ -1513,9 +1512,7 @@ class PrefillAdder:
|
||||
self, req: Req, admission: _PrefillAdmission, mamba_gap_reserve: int
|
||||
) -> None:
|
||||
assert req.prefix_len == admission.prefix_len
|
||||
req.set_extend_range(
|
||||
admission.prefix_len, admission.prefix_len + admission.extend_len
|
||||
)
|
||||
req.extend_end = admission.prefix_len + admission.extend_len
|
||||
req.lock = self.tree_cache.lock(req.last_node)
|
||||
self.can_run_list.append(req)
|
||||
if admission.is_chunked:
|
||||
|
||||
@@ -3677,8 +3677,8 @@ class Scheduler(
|
||||
chunked_req_to_exclude.add(self.chunked_req)
|
||||
|
||||
# A parked chunk (add_chunked_req hybrid-SWA early-return) leaves
|
||||
# extend_range.end at prefix_len: it computed no new KV, so nothing to stash.
|
||||
if self.chunked_req.extend_range.end > self.chunked_req.prefix_len:
|
||||
# extend_end at prefix_len: it computed no new KV, so nothing to stash.
|
||||
if self.chunked_req.extend_end > self.chunked_req.prefix_len:
|
||||
self.stash_chunked_request(self.chunked_req)
|
||||
|
||||
# HiSparse has its own prefill-to-decode transition; skip last_batch merge.
|
||||
@@ -4120,7 +4120,7 @@ class Scheduler(
|
||||
|
||||
if self.tp_worker.model_runner.prefill_aware_swa:
|
||||
for req in can_run_list:
|
||||
req.kv.swa_evict_floor = req.extend_range.end
|
||||
req.kv.swa_evict_floor = req.extend_end
|
||||
|
||||
# Record prefill stats for logging after forward.
|
||||
new_batch.prefill_stats = PrefillStats.from_adder(
|
||||
@@ -4129,9 +4129,7 @@ class Scheduler(
|
||||
self.enable_priority_scheduling,
|
||||
num_pending_tokens=self.load_inquirer._get_num_pending_tokens(
|
||||
chunk_deduct=(
|
||||
self.chunked_req.extend_range.length
|
||||
if self.chunked_req is not None
|
||||
else 0
|
||||
self.chunked_req.extend_len if self.chunked_req is not None else 0
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -4613,10 +4611,9 @@ class Scheduler(
|
||||
# modified by overlap schedule. So we have to copy them here so that
|
||||
# we can use the correct values in output processing.
|
||||
if batch.return_logprob or batch.return_hidden_states:
|
||||
batch_result.extend_input_len_per_req = [
|
||||
req.extend_range.length if req.extend_range is not None else 0
|
||||
for req in batch.reqs
|
||||
]
|
||||
batch_result.extend_input_len_per_req = (
|
||||
list(batch.extend_lens) if batch.forward_mode.is_extend() else None
|
||||
)
|
||||
else:
|
||||
batch_result.extend_input_len_per_req = None
|
||||
|
||||
@@ -5470,7 +5467,7 @@ class Scheduler(
|
||||
and (req := self.chunked_req) is not None
|
||||
):
|
||||
# Retract skips the chunk step that sets this result's KV send boundary.
|
||||
req.tmp_end_idx = min(req.extend_range.end, len(req.origin_input_ids))
|
||||
req.tmp_end_idx = min(req.extend_end, len(req.origin_input_ids))
|
||||
tmp_batch, tmp_result = self.result_queue.popleft()
|
||||
self.process_batch_result(tmp_batch, tmp_result)
|
||||
|
||||
|
||||
@@ -170,7 +170,7 @@ class DynamicChunkSizer:
|
||||
# request so release_kv_cache can release it symmetrically.
|
||||
req.init_next_round_input(self.tree_cache)
|
||||
req.lock = self.tree_cache.lock(req.last_node)
|
||||
req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids))
|
||||
req.extend_end = len(req.full_untruncated_fill_ids)
|
||||
|
||||
# Prepare batch
|
||||
batch = ScheduleBatch.init_new(
|
||||
@@ -183,7 +183,7 @@ class DynamicChunkSizer:
|
||||
self.spec_algorithm,
|
||||
)
|
||||
|
||||
current_seq_len = req.extend_range.end
|
||||
current_seq_len = req.extend_end
|
||||
|
||||
if is_dp_attention_enabled():
|
||||
# Profiling runs one request on this rank; other DP ranks report 0.
|
||||
|
||||
@@ -243,9 +243,9 @@ def checkpoint_kv_cache(req: Req, tree_cache: BasePrefixCache) -> None:
|
||||
# insert; a finished request belongs in release_kv_cache.
|
||||
assert not req.finished(), f"checkpointing finished request {req.rid}"
|
||||
if not req.skip_radix_cache_insert:
|
||||
tree_cache.checkpoint(req, up_to=req.extend_range.end)
|
||||
tree_cache.checkpoint(req, up_to=req.extend_end)
|
||||
# The next extend resumes after this one, published or not.
|
||||
req.prefix_len = req.extend_range.end
|
||||
req.prefix_len = req.extend_end
|
||||
|
||||
|
||||
def evict_from_tree_cache(
|
||||
|
||||
@@ -237,7 +237,7 @@ Insert a request's KV into the tree: at every checkpoint while it runs, and once
|
||||
|
||||
| Aspect | Detail |
|
||||
|--------|--------|
|
||||
| **Purpose** | Publish the request's KV `[cache_protected_len, up_to)` so other requests can match it; nodes past `req.kv.cache_inserted_len` count one hit (a request counts each node once). `checkpoint_kv_cache` calls it with `req.extend_range.end` while the request runs; `release_kv_cache` calls it with the request-owned length when it finishes (`req.finished()`), then frees `[cache_protected_len, up_to)` and everything past it and unlocks `req.lock`; with `checkpoint=False` it skips the insert and calls `on_release(req, checkpointed=False)` for component cleanup |
|
||||
| **Purpose** | Publish the request's KV `[cache_protected_len, up_to)` so other requests can match it; nodes past `req.kv.cache_inserted_len` count one hit (a request counts each node once). `checkpoint_kv_cache` calls it with `req.extend_end` while the request runs; `release_kv_cache` calls it with the request-owned length when it finishes (`req.finished()`), then frees `[cache_protected_len, up_to)` and everything past it and unlocks `req.lock`; with `checkpoint=False` it skips the insert and calls `on_release(req, checkpointed=False)` for component cleanup |
|
||||
| **Inputs** | `req` — the request; `up_to` — row position the insert may read up to |
|
||||
| **Output** | `None` |
|
||||
| **Mutation** | Component hooks → `insert` → re-match → writes the tree's indices back into the row → moves `req.lock` to the node the insert ended on → updates `req.kv.cache_protected_len`, `req.kv.cache_inserted_len`, `req.last_node` (`checkpoint_kv_cache` then advances `req.prefix_len`) → component cleanup. A finished request's component state (Mamba) is handed to the tree instead of forked, and what it still held is freed. Frees no KV slot: `release_kv_cache` does that afterwards. |
|
||||
|
||||
@@ -123,9 +123,10 @@ class NgramEmbeddingManager:
|
||||
all_tokens = []
|
||||
column_starts = []
|
||||
request_lengths = []
|
||||
for req in batch.reqs:
|
||||
start = req.prefix_len
|
||||
end = start + req.extend_range.length
|
||||
for req, start, extend_len in zip(
|
||||
batch.reqs, batch.prefix_lens, batch.extend_lens, strict=True
|
||||
):
|
||||
end = start + extend_len
|
||||
fill_ids = req.origin_input_ids + req.output_ids
|
||||
if start == 0:
|
||||
tokens = fill_ids[start:end]
|
||||
@@ -196,8 +197,7 @@ class NgramEmbeddingManager:
|
||||
if not batch.forward_mode.is_extend_without_speculative():
|
||||
return
|
||||
rows = []
|
||||
for req in batch.reqs:
|
||||
start = req.extend_range.start
|
||||
for req, start in zip(batch.reqs, batch.prefix_lens, strict=True):
|
||||
lo = max(0, start - n1)
|
||||
ids = req.full_untruncated_fill_ids[lo:start]
|
||||
rows.append([0] * (n1 - len(ids)) + list(ids))
|
||||
|
||||
@@ -69,7 +69,6 @@ from typing import (
|
||||
Generic,
|
||||
Iterator,
|
||||
List,
|
||||
NamedTuple,
|
||||
Optional,
|
||||
Protocol,
|
||||
Sequence,
|
||||
@@ -1335,15 +1334,6 @@ def get_current_device_stream_fast():
|
||||
# ==============================================================================
|
||||
|
||||
|
||||
class Range(NamedTuple):
|
||||
start: int
|
||||
end: int
|
||||
|
||||
@property
|
||||
def length(self) -> int:
|
||||
return self.end - self.start
|
||||
|
||||
|
||||
def assert_int64_array(values: array, name: str) -> None:
|
||||
"""Require a signed int64 array suitable for zero-copy tensor views."""
|
||||
assert (
|
||||
|
||||
Reference in New Issue
Block a user