From 1066decce698d8fcca214946ad7d4f8356805b0b Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Wed, 7 Oct 2026 16:07:16 -0700 Subject: [PATCH] [Scheduler] Replace `extend_range` with `extend_end` derived from `prefix_len` (#42822) --- python/sglang/benchmark/one_batch.py | 6 +- python/sglang/srt/disaggregation/decode.py | 6 +- .../decode_schedule_batch_mixin.py | 18 ++-- python/sglang/srt/disaggregation/prefill.py | 18 ++-- python/sglang/srt/dllm/mixin/req.py | 2 +- python/sglang/srt/dllm/mixin/scheduler.py | 12 +-- .../hardware_backend/mlx/scheduler_mixin.py | 7 +- .../srt/hardware_backend/mlx/tp_worker.py | 4 +- .../srt/managers/hisparse_coordinator.py | 6 +- python/sglang/srt/managers/schedule_batch.py | 69 +++++++------- python/sglang/srt/managers/schedule_policy.py | 31 +++---- python/sglang/srt/managers/scheduler.py | 19 ++-- .../dynamic_chunk_sizer.py | 4 +- python/sglang/srt/mem_cache/common.py | 4 +- .../unified_cache/components/README.md | 2 +- .../ngram_embedding_manager.py | 10 +- python/sglang/srt/utils/common.py | 10 -- .../test_scripted_hybrid_swa.py | 2 +- .../chunked_prefill/test_scripted_radix.py | 4 +- .../test_scripted_special_case.py | 24 +++-- test/manual/test_forward_split_prefill.py | 2 +- .../test_disaggregation_optimistic_prefill.py | 2 +- .../unit/disaggregation/test_dcp_pack.py | 2 +- .../test_disaggregation_wire.py | 2 +- .../test_prefill_abort_result_cleanup.py | 6 +- .../test_prefill_result_polling.py | 6 +- .../dllm/test_gemma4_uniform_lifecycle.py | 27 +++--- .../mlx/test_tp_worker_routing.py | 12 +-- .../unit/managers/test_hisparse_unit.py | 8 +- .../managers/test_mamba_checkpoint_depth.py | 2 +- .../unit/managers/test_prefill_adder.py | 93 +++++++------------ ...test_priority_scheduling_disaggregation.py | 3 +- ...schedule_batch_convert_decode_to_extend.py | 5 +- .../test_schedule_batch_out_of_place.py | 10 +- .../test_scheduler_chunked_req_gate.py | 5 +- .../mem_cache/test_decode_radix_lock_ref.py | 13 ++- .../test_decode_retraction_backup.py | 6 +- .../unit/mem_cache/test_hisparse_allocator.py | 7 +- .../mem_cache/test_prefill_memory_budget.py | 2 +- .../test_unified_radix_cache_bench.py | 2 +- .../test_unified_radix_cache_unittest.py | 36 +++---- .../simulation/sglang/scheduler.py | 3 +- 42 files changed, 215 insertions(+), 297 deletions(-) diff --git a/python/sglang/benchmark/one_batch.py b/python/sglang/benchmark/one_batch.py index 236dfd91b8..50fc4053a9 100644 --- a/python/sglang/benchmark/one_batch.py +++ b/python/sglang/benchmark/one_batch.py @@ -424,7 +424,7 @@ def prepare_inputs_for_correctness_test(bench_args, tokenizer, custom_prompts): ) req.full_untruncated_fill_ids = req.origin_input_ids req.logprob_start_len = -1 - req.set_extend_range(req.prefix_len, len(req.origin_input_ids)) + req.extend_end = len(req.origin_input_ids) reqs.append(req) return input_ids, reqs @@ -441,7 +441,7 @@ def prepare_extend_inputs_for_correctness_test( # The cut prefix is already in the request's row from the first extend. req.prefix_len = bench_args.cut_len req.logprob_start_len = -1 - req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids)) + req.extend_end = len(req.full_untruncated_fill_ids) return reqs @@ -468,7 +468,7 @@ def prepare_synthetic_inputs_for_latency_test( ) req.full_untruncated_fill_ids = req.origin_input_ids req.logprob_start_len = -1 - req.set_extend_range(req.prefix_len, len(req.origin_input_ids)) + req.extend_end = len(req.origin_input_ids) reqs.append(req) return reqs diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index 464be0aca1..aecf19b1ca 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -1841,7 +1841,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): @property def num_tokens_pre_allocated(self): return sum( - decode_req.req.extend_range.end + decode_req.req.extend_end for decode_req in self.transfer_queue.queue if not decode_req.host_staged ) @@ -2205,7 +2205,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): # Decode-disagg allocates before batch assembly, so it binds the prefix # here instead of in init_next_round_input. req.prefix_len = prefix_len - req.set_extend_range(total_prefix_len, req.kv.kv_committed_len) + req.extend_end = req.kv.kv_committed_len self.tree_cache.maybe_hand_to_session(req) # Return the transfer destination indices: @@ -3158,7 +3158,7 @@ class SchedulerDisaggregationDecodeMixin: # only sees committed KV (full array includes one uncommitted # token because init_next_round_input rebuilt it as full). if req.kv.kv_committed_len is not None: - req.set_extend_range(req.prefix_len, req.kv.kv_committed_len) + req.extend_end = req.kv.kv_committed_len else: waiting_queue.append(req) diff --git a/python/sglang/srt/disaggregation/decode_schedule_batch_mixin.py b/python/sglang/srt/disaggregation/decode_schedule_batch_mixin.py index 0698334b8f..11602ed6cb 100644 --- a/python/sglang/srt/disaggregation/decode_schedule_batch_mixin.py +++ b/python/sglang/srt/disaggregation/decode_schedule_batch_mixin.py @@ -36,7 +36,7 @@ class ScheduleBatchDisaggregationDecodeMixin: req_pool_indices = [] # Pre-calculate total size - total_size = sum(req.extend_range.length for req in reqs) + total_size = sum(req.extend_len for req in reqs) extend_num_tokens = total_size out_cache_loc = torch.empty(total_size, dtype=torch.int64, device=self.device) @@ -47,19 +47,19 @@ class ScheduleBatchDisaggregationDecodeMixin: pre_len = req.prefix_len chunk = self.req_to_token_pool.req_to_token[req.kv.req_pool_idx][ - pre_len : pre_len + req.extend_range.length + pre_len : pre_len + req.extend_len ] - assert offset + req.extend_range.length <= total_size, ( - f"Exceeds total size: offset={offset}, req.extend_range.length={req.extend_range.length}, total_size={total_size}" + assert offset + req.extend_len <= total_size, ( + f"Exceeds total size: offset={offset}, req.extend_len={req.extend_len}, total_size={total_size}" ) - out_cache_loc[offset : offset + req.extend_range.length] = chunk - offset += req.extend_range.length + out_cache_loc[offset : offset + req.extend_len] = chunk + offset += req.extend_len seq_len = len(req.origin_input_ids) + max(0, len(req.output_ids) - 1) seq_lens.append(seq_len) if len(req.output_ids) == 0: - assert seq_len - pre_len == req.extend_range.length, ( - f"seq_len={seq_len}, pre_len={pre_len}, req.extend_range.length={req.extend_range.length}" + assert seq_len - pre_len == req.extend_len, ( + f"seq_len={seq_len}, pre_len={pre_len}, req.extend_len={req.extend_len}" ) if not req.retracted_stain: @@ -99,7 +99,7 @@ class ScheduleBatchDisaggregationDecodeMixin: self.extend_num_tokens = extend_num_tokens self.prefix_lens = [r.prefix_len for r in reqs] - self.extend_lens = [r.extend_range.length for r in reqs] + self.extend_lens = [r.extend_len for r in reqs] self.extend_logprob_start_lens = None self.extend_input_logprob_token_ids = None self.multimodal_inputs = [r.multimodal_inputs for r in reqs] diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index baa4bf3959..0f7f8e6ecb 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -593,7 +593,7 @@ class SchedulerDisaggregationPrefillMixin: cache = self.tree_cache if req.pending_bootstrap and _uses_write_through_cache(cache): cache.advance_unpublished_req(req) - req.prefix_len = req.extend_range.end + req.prefix_len = req.extend_end return checkpoint_kv_cache(req, cache) @@ -1060,12 +1060,12 @@ class SchedulerDisaggregationPrefillMixin: req.inflight_middle_chunks -= 1 # Still chunking iff its next chunk was launched: either it is - # still self.chunked_req, or its final chunk (extend_range + # still self.chunked_req, or its final chunk (extend_end # reaching the end of the input) is in flight. A yielded req # is neither, so do its deferred release here. still_chunking = self.chunked_req is req or ( - req.extend_range is not None - and req.extend_range.end >= len(req.origin_input_ids) + req.extend_end is not None + and req.extend_end >= len(req.origin_input_ids) ) # Abort is terminal. Do not requeue an aborted optimistic # request merely because bootstrap is still pending. @@ -1399,7 +1399,7 @@ class SchedulerDisaggregationPrefillMixin: elif self.enable_overlap: # Delay KV transfer to process_batch_result_disagg_prefill when overlap is enabled to ensure results are resolved req.tmp_end_idx = min( - req.extend_range.end, + req.extend_end, len(req.origin_input_ids), ) else: @@ -1468,7 +1468,7 @@ class SchedulerDisaggregationPrefillMixin: value = 0 else: if end_idx is None: - end_idx = min(req.extend_range.end, len(req.origin_input_ids)) + end_idx = min(req.extend_end, len(req.origin_input_ids)) page_indices_gpu = page_indices_for_request(self, req, end_idx) state_indices = state_indices_for_request(self, req, end_idx) value = computer.compute(page_indices_gpu, state_indices) @@ -1488,9 +1488,7 @@ class SchedulerDisaggregationPrefillMixin: start_idx = req.start_send_idx transfer_input_len = len(req.origin_input_ids) end_idx = ( - end_idx - if end_idx is not None - else min(req.extend_range.end, transfer_input_len) + end_idx if end_idx is not None else min(req.extend_end, transfer_input_len) ) if not last_chunk: @@ -1523,7 +1521,7 @@ class SchedulerDisaggregationPrefillMixin: # range actually materialized on prefill. C128 state is request # scoped, so its transfer index must use the logical input length # that decode used to register the destination row. - seq_len = min(req.extend_range.end, transfer_input_len) + seq_len = min(req.extend_end, transfer_input_len) c128_seq_len = transfer_input_len def _mamba_payload(): diff --git a/python/sglang/srt/dllm/mixin/req.py b/python/sglang/srt/dllm/mixin/req.py index 589e16044d..bff77f5789 100644 --- a/python/sglang/srt/dllm/mixin/req.py +++ b/python/sglang/srt/dllm/mixin/req.py @@ -74,7 +74,7 @@ class ReqDllmMixin: self.full_untruncated_fill_ids = ( self.full_untruncated_fill_ids[:prefix_len] + self.dllm_incomplete_ids ) - # extend_range is (re)computed by the staging adder + # extend_end is (re)computed by the staging adder # (add_dllm_staging_req) before this req is scheduled, mirroring the # non-incomplete path which also defers it to the adder. return diff --git a/python/sglang/srt/dllm/mixin/scheduler.py b/python/sglang/srt/dllm/mixin/scheduler.py index b64d867c0a..4512cbad7c 100644 --- a/python/sglang/srt/dllm/mixin/scheduler.py +++ b/python/sglang/srt/dllm/mixin/scheduler.py @@ -112,7 +112,7 @@ class SchedulerDllmMixin: continue req.full_untruncated_fill_ids[ - req.extend_range.end - new_tokens : req.extend_range.end + req.extend_end - new_tokens : req.extend_end ] = array("q", next_token_ids) self.metrics_reporter.num_generated_tokens += new_tokens @@ -137,16 +137,16 @@ class SchedulerDllmMixin: # Mirror the resolved block into the committed fill ids so the # prefix cache keys on the real tokens, not the mask block, next - # round. Index relative to extend_range.end (the truncated/ + # round. Index relative to extend_end (the truncated/ # committed length), which can be shorter than # full_untruncated_fill_ids when the staging adder truncates the # block to the KV budget. req.full_untruncated_fill_ids[ - req.extend_range.end - block_size : req.extend_range.end + req.extend_end - block_size : req.extend_end ] = array("q", next_token_ids) len_input = len(req.origin_input_ids) - len_fill = req.extend_range.end + len_fill = req.extend_end if len_fill <= len_input: continue @@ -186,7 +186,7 @@ class SchedulerDllmMixin: def _stash_dllm_context(self: Scheduler, req: Req) -> None: context_len = req.dllm_block_offset block_size = self.dllm_config.block_size - assert req.extend_range.end == context_len + block_size + assert req.extend_end == context_len + block_size assert req.kv is not None and req.kv.req_pool_idx is not None allocated_len = req.kv.kv_allocated_len @@ -201,7 +201,7 @@ class SchedulerDllmMixin: req.kv.kv_committed_len = context_len req.kv.kv_allocated_len = context_len assert req.kv.max_evicted_seqlen <= context_len - req.set_extend_range(context_len, context_len) + req.extend_end = context_len self.stash_chunked_request(req) def finish_dllm_forward(self: Scheduler, req: Req) -> None: diff --git a/python/sglang/srt/hardware_backend/mlx/scheduler_mixin.py b/python/sglang/srt/hardware_backend/mlx/scheduler_mixin.py index 2619ec4d71..0e754870df 100644 --- a/python/sglang/srt/hardware_backend/mlx/scheduler_mixin.py +++ b/python/sglang/srt/hardware_backend/mlx/scheduler_mixin.py @@ -171,10 +171,9 @@ class SchedulerMlxOverlapMixin: extend_logprob_start_len_per_req = None if batch.return_logprob: # Mirror Scheduler.run_batch's launch-time copy. - extend_input_len_per_req = [ - req.extend_range.length if req.extend_range is not None else 0 - for req in batch.reqs - ] + extend_input_len_per_req = ( + list(batch.extend_lens) if batch.forward_mode.is_extend() else None + ) extend_logprob_start_len_per_req = batch.extend_logprob_start_lens return MlxPendingJob( launch=launch, diff --git a/python/sglang/srt/hardware_backend/mlx/tp_worker.py b/python/sglang/srt/hardware_backend/mlx/tp_worker.py index e8eb576833..be7e6f3133 100644 --- a/python/sglang/srt/hardware_backend/mlx/tp_worker.py +++ b/python/sglang/srt/hardware_backend/mlx/tp_worker.py @@ -205,9 +205,9 @@ class MlxTpModelWorker(TpModelWorker): discarded (the runner pops it as the stale intermediate token), so the runner may skip the logit head for it. """ - if req.extend_range is None: + if req.extend_end is None: return True - return req.extend_range.end >= len(req.full_untruncated_fill_ids) + return req.extend_end >= len(req.full_untruncated_fill_ids) @staticmethod def _sampling_active(batch: ScheduleBatch) -> bool: diff --git a/python/sglang/srt/managers/hisparse_coordinator.py b/python/sglang/srt/managers/hisparse_coordinator.py index 37e87d29db..fcc7d9bec5 100644 --- a/python/sglang/srt/managers/hisparse_coordinator.py +++ b/python/sglang/srt/managers/hisparse_coordinator.py @@ -407,7 +407,7 @@ class HiSparseCoordinator: req.hisparse_staging = True full_kv_indices = self.req_to_token_pool.req_to_token[ - req.kv.req_pool_idx, : req.extend_range.end + req.kv.req_pool_idx, : req.extend_end ].to(dtype=torch.int64, copy=True) device_indices = ( self.mem_pool_device.translate_loc_from_full_to_hisparse_device( @@ -503,7 +503,7 @@ class HiSparseCoordinator: def alloc_device_buffer(self, req: Req) -> None: if self.is_dsv4_hisparse: - allocated_len = req.extend_range.end + allocated_len = req.extend_end alloc_size = self.padded_buffer_size else: allocated_len = req.kv.kv_allocated_len @@ -920,7 +920,7 @@ class HiSparseCoordinator: # Wait for any in-flight staging DMA to complete before freeing self.write_staging_stream.synchronize() - prefill_len = req.extend_range.end + prefill_len = req.extend_end allocated_locs = self.req_to_token_pool.req_to_token[ req.kv.req_pool_idx, :prefill_len ] diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index dbcb736d00..71e3cfde19 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -14,7 +14,6 @@ from sglang.srt.runtime_context import ( mamba_track_grid, ) from sglang.srt.utils.common import ( - Range, ceil_align, flatten_arrays_to_pinned_cpu, is_hip, @@ -1070,9 +1069,10 @@ class Req(ReqDllmMixin): self.output_ids = array("q") # Full untruncated sequence: origin + output (+ DLLM mask block). # Kept in sync by refresh_fill_ids; admission only updates - # extend_range, never mutates this array's length. + # extend_end, never mutates this array's length. self.full_untruncated_fill_ids = array("q") - self.extend_range: Optional[Range] = None + # This round extends fill ids [prefix_len, extend_end). + self.extend_end: Optional[int] = None self.dllm_initialized: bool = False self.session = session @@ -1371,8 +1371,8 @@ class Req(ReqDllmMixin): # the start index of the sent kv cache # We want to send it chunk by chunk for chunked prefill. # After every chunk forward, we do the following: - # kv_send(req.input_ids[req.start_send_idx:req.extend_range.end]) - # start_send_idx = req.extend_range.end + # kv_send(req.input_ids[req.start_send_idx:req.extend_end]) + # start_send_idx = req.extend_end self.start_send_idx: int = 0 self.disagg_decode_prefix_len: int = 0 @@ -1563,11 +1563,12 @@ class Req(ReqDllmMixin): # Whether request reached finished condition return self.finished_reason is not None - def set_extend_range(self, start: int, end: int) -> None: - self.extend_range = Range(start, end) + @property + def extend_len(self) -> int: + return self.extend_end - self.prefix_len def get_fill_ids(self) -> array: - return self.full_untruncated_fill_ids[: self.extend_range.end] + return self.full_untruncated_fill_ids[: self.extend_end] def refresh_fill_ids(self) -> None: """Keep full_untruncated_fill_ids == origin_input_ids + output_ids by @@ -1935,7 +1936,7 @@ class Req(ReqDllmMixin): self.num_matched_prefix_tokens = 0 self.lock = None self.swa_branching_seqlen = None - self.extend_range = None + self.extend_end = None self.dllm_initialized = False self.is_retracted = True self.retracted_stain = True @@ -2295,7 +2296,7 @@ def _compute_chunked_req_next_prompt_token( multimodal placeholder (hash) tokens that lie outside the model vocab.""" if chunked_req is None: return None - fill_len = chunked_req.extend_range.end + fill_len = chunked_req.extend_end origin_ids = chunked_req.origin_input_ids if fill_len >= len(origin_ids): return None @@ -2618,17 +2619,17 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): input_ids[i] = input_ids[i][encoder_len:] encoder_out_cache_loc.append(self.out_cache_loc[pt : pt + encoder_len]) decoder_out_cache_loc.append( - self.out_cache_loc[pt + encoder_len : pt + req.extend_range.length] + self.out_cache_loc[pt + encoder_len : pt + req.extend_len] ) extend_lens[i] -= encoder_len self.extend_num_tokens = self.extend_num_tokens - encoder_len else: decoder_out_cache_loc.append( - self.out_cache_loc[pt : pt + req.extend_range.length] + self.out_cache_loc[pt : pt + req.extend_len] ) prefix_lens[i] -= encoder_len - pt += req.extend_range.length + pt += req.extend_len self.extend_lens = extend_lens self.prefix_lens = prefix_lens @@ -2666,7 +2667,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): for i, req in enumerate(self.reqs): encoder_len = self.encoder_lens_cpu[i] old_start_len = extend_logprob_start_lens[i] - old_contribution = req.extend_range.length - old_start_len + old_contribution = req.extend_len - old_start_len if req.prefix_len < encoder_len: tokens_to_strip = max(0, encoder_len - old_start_len) @@ -2695,11 +2696,6 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): encoder_len = self.encoder_lens_cpu[i] if encoder_len == 0: continue - if req.prefix_len < encoder_len: - assert req.prefix_len == 0 - req.extend_range = req.extend_range._replace( - start=req.extend_range.start + encoder_len - ) req.logprob_start_len = max(req.logprob_start_len, encoder_len) def prepare_for_extend(self): @@ -2713,10 +2709,10 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): reqs = self.reqs input_ids = [r.get_fill_ids()[r.prefix_len :] for r in reqs] extend_num_tokens = sum(len(ids) for ids in input_ids) - seq_lens = [r.extend_range.end for r in reqs] - orig_seq_lens = [max(r.extend_range.end, len(r.origin_input_ids)) for r in reqs] + seq_lens = [r.extend_end for r in reqs] + orig_seq_lens = [max(r.extend_end, len(r.origin_input_ids)) for r in reqs] prefix_lens = [r.prefix_len for r in reqs] - extend_lens = [r.extend_range.length for r in reqs] + extend_lens = [r.extend_len for r in reqs] extend_logprob_start_lens = [ compute_extend_logprob_start_len( logprob_start_len=r.logprob_start_len, @@ -2785,8 +2781,6 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): mamba_track_seqlens_cpu = [] for i, (req, seq_len, pre_len) in enumerate(zip(reqs, seq_lens, prefix_lens)): - assert seq_len - pre_len == req.extend_range.length - req.extend_batch_idx += 1 # If input_embeds are available, store them @@ -2794,7 +2788,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): # Slice to match extend_input_len — PrefillAdder truncates # fill_len/extend_input_len on chunk overflow but not input_embeds. input_embeds.extend( - req.input_embeds[pre_len : pre_len + req.extend_range.length] + req.input_embeds[pre_len : pre_len + req.extend_len] ) if req.positional_embed_overrides is not None: @@ -2806,7 +2800,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): req.positional_embed_overrides.positions ): extend_pos = pos - pre_len - if extend_pos < 0 or extend_pos >= req.extend_range.length: + if extend_pos < 0 or extend_pos >= req.extend_len: continue # Outside current extend chunk, skip embeds_to_add.append((embed_idx, input_id_pointer + extend_pos)) if embeds_to_add: @@ -2870,7 +2864,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): # extend_input_logprob_token_id = [4, 0] global_start_idx, global_end_idx = ( req.prefix_len, - req.extend_range.end, + req.extend_end, ) if req.logprob_start_len == -1: logprob_start_len = len(req.origin_input_ids) @@ -2885,12 +2879,12 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): ] extend_input_logprob_token_ids.extend(logprob_token_ids) - # We will need req.extend_range.length - extend_logprob_start_lens[i] number of + # We will need req.extend_len - extend_logprob_start_lens[i] number of # tokens, and logprob_token_ids is for input logprob, so pad the rest of them by 0. extend_input_logprob_token_ids.extend( [0] * ( - req.extend_range.length + req.extend_len - extend_logprob_start_lens[i] - len(logprob_token_ids) ) @@ -3015,7 +3009,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): return i + 1 prefix_len = req.prefix_len - seq_end = prefix_len + req.extend_range.length + seq_end = req.extend_end if get_parallel().dcp_enabled: # DCP widens radix pages beyond scheduler chunk boundaries. Pick an # absolute page depth only when the kernel produced an h snapshot. @@ -3028,10 +3022,9 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): # Chunked prefill can leave an active request off the absolute # checkpoint grid. Without DCP, keep tracking snapshots relative to # that request prefix so later chunks can continue donating states. - mask = req.extend_range.length >= checkpoint_grid + mask = req.extend_len >= checkpoint_grid mamba_track_seqlen_aligned = ( - prefix_len - + (req.extend_range.length // checkpoint_grid) * checkpoint_grid + prefix_len + (req.extend_len // checkpoint_grid) * checkpoint_grid ) track_index = req.kv.mamba_ping_pong_track_buffer[ req.kv.mamba_next_track_idx @@ -3050,8 +3043,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): # A coarser checkpoint grid may not be a model-state boundary, so # force retrieval from the intermediate h state in that case. mamba_track_fla_chunk_aligned = ( - req.prefix_len - + (req.extend_range.length // state_chunk_size) * state_chunk_size + req.prefix_len + (req.extend_len // state_chunk_size) * state_chunk_size ) if mamba_track_fla_chunk_aligned != mamba_track_seqlen_aligned: # We want to track mamba_track_seqlen_aligned, and it's not the last position, @@ -3134,7 +3126,9 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): running_batch.reqs, running_prefix_lens, strict=True ): req.refresh_fill_ids() - req.set_extend_range(prefix_len, prefix_len + 1) + # req.prefix_len still ends at the last prefill, so req.extend_len is + # not 1 here; this row's prefix and extend length live on the batch. + req.extend_end = prefix_len + 1 # Decode tokens of the running portion live in future_map.output_tokens_buf. self.input_ids = None @@ -3214,7 +3208,8 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): req.refresh_fill_ids() # end runs one past full_untruncated_fill_ids while output_ids # trails; safe only while decoding_reqs suppresses the checkpoint insert. - req.set_extend_range(seq_len - 1, seq_len) + # As in mix_with_running, read prefix and extend length off the batch. + req.extend_end = seq_len self.prefix_lens = [seq_len - 1 for seq_len in seq_lens] self.extend_lens = [1] * bs diff --git a/python/sglang/srt/managers/schedule_policy.py b/python/sglang/srt/managers/schedule_policy.py index b15c226f3b..fe41d2d675 100644 --- a/python/sglang/srt/managers/schedule_policy.py +++ b/python/sglang/srt/managers/schedule_policy.py @@ -945,9 +945,10 @@ class PrefillAdder: return trunc_len - def _add_dllm_req(self, req: Req, prefix_len: int): + def _add_dllm_req(self, req: Req): + prefix_len = req.prefix_len trunc_len = self._get_dllm_extend_len(req, prefix_len) - req.set_extend_range(prefix_len, prefix_len + trunc_len) + req.extend_end = prefix_len + trunc_len self.can_run_list.append(req) @@ -1019,7 +1020,7 @@ class PrefillAdder: return AddReqResult.NO_TOKEN truncated = cand_extend_input_len > _rem_tokens new_len = min(cand_extend_input_len, _rem_tokens) - req.set_extend_range(req.prefix_len, req.prefix_len + new_len) + req.extend_end = req.prefix_len + new_len self.can_run_list.append(req) # Update budget: reserve max_new_tokens only if not truncated @@ -1030,7 +1031,7 @@ class PrefillAdder: ) self._update_prefill_budget( 0, - req.extend_range.length, + req.extend_len, max_new_tokens, req.retracted_stain, mamba_gap_reserve=self._mamba_gap_budget_for_req(req), @@ -1092,11 +1093,11 @@ class PrefillAdder: ) if not reserved: raise RuntimeError("chunked request exceeds the sharded assembly scratch") - req.set_extend_range(req.prefix_len, req.prefix_len + new_len) + req.extend_end = req.prefix_len + new_len self.can_run_list.append(req) self._update_prefill_budget( 0, - req.extend_range.length, + req.extend_len, ( min(req.sampling_params.max_new_tokens, CLIP_MAX_NEW_TOKENS) if not truncated @@ -1105,7 +1106,7 @@ class PrefillAdder: req.retracted_stain, mamba_gap_reserve=self._mamba_gap_budget_for_req(req), is_chunked_continuation=True, - compute_charge=req.extend_range.length if self.exact_chunk_fill else None, + compute_charge=req.extend_len if self.exact_chunk_fill else None, ) # Return if chunked prefill not finished @@ -1212,7 +1213,7 @@ class PrefillAdder: ) is not None: return tile_stop - self._add_dllm_req(req, 0) + self._add_dllm_req(req) elif ( self.rem_chunk_tokens is None # chunked prefill is disabled or cand_extend_input_len <= self.rem_chunk_tokens # it is the last chunk @@ -1228,17 +1229,15 @@ class PrefillAdder: extend_len=cand_extend_input_len, ): return AddReqResult.OTHER - req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids)) + req.extend_end = len(req.full_untruncated_fill_ids) self.can_run_list.append(req) self._update_prefill_budget( 0, - req.extend_range.length, + req.extend_len, min(req.sampling_params.max_new_tokens, CLIP_MAX_NEW_TOKENS), req.retracted_stain, mamba_gap_reserve=self._mamba_gap_budget_for_req(req), - compute_charge=( - req.extend_range.length if self.exact_chunk_fill else None - ), + compute_charge=(req.extend_len if self.exact_chunk_fill else None), ) else: if self.rem_chunk_tokens <= 0: @@ -1255,7 +1254,7 @@ class PrefillAdder: return AddReqResult.OTHER assert req.prefix_len == 0 - req.set_extend_range(req.prefix_len, req.prefix_len + trunc_len) + req.extend_end = req.prefix_len + trunc_len self.can_run_list.append(req) self.new_chunked_req = req self._update_prefill_budget( @@ -1513,9 +1512,7 @@ class PrefillAdder: self, req: Req, admission: _PrefillAdmission, mamba_gap_reserve: int ) -> None: assert req.prefix_len == admission.prefix_len - req.set_extend_range( - admission.prefix_len, admission.prefix_len + admission.extend_len - ) + req.extend_end = admission.prefix_len + admission.extend_len req.lock = self.tree_cache.lock(req.last_node) self.can_run_list.append(req) if admission.is_chunked: diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 8122547aef..dad4a86c27 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -3677,8 +3677,8 @@ class Scheduler( chunked_req_to_exclude.add(self.chunked_req) # A parked chunk (add_chunked_req hybrid-SWA early-return) leaves - # extend_range.end at prefix_len: it computed no new KV, so nothing to stash. - if self.chunked_req.extend_range.end > self.chunked_req.prefix_len: + # extend_end at prefix_len: it computed no new KV, so nothing to stash. + if self.chunked_req.extend_end > self.chunked_req.prefix_len: self.stash_chunked_request(self.chunked_req) # HiSparse has its own prefill-to-decode transition; skip last_batch merge. @@ -4120,7 +4120,7 @@ class Scheduler( if self.tp_worker.model_runner.prefill_aware_swa: for req in can_run_list: - req.kv.swa_evict_floor = req.extend_range.end + req.kv.swa_evict_floor = req.extend_end # Record prefill stats for logging after forward. new_batch.prefill_stats = PrefillStats.from_adder( @@ -4129,9 +4129,7 @@ class Scheduler( self.enable_priority_scheduling, num_pending_tokens=self.load_inquirer._get_num_pending_tokens( chunk_deduct=( - self.chunked_req.extend_range.length - if self.chunked_req is not None - else 0 + self.chunked_req.extend_len if self.chunked_req is not None else 0 ), ), ) @@ -4613,10 +4611,9 @@ class Scheduler( # modified by overlap schedule. So we have to copy them here so that # we can use the correct values in output processing. if batch.return_logprob or batch.return_hidden_states: - batch_result.extend_input_len_per_req = [ - req.extend_range.length if req.extend_range is not None else 0 - for req in batch.reqs - ] + batch_result.extend_input_len_per_req = ( + list(batch.extend_lens) if batch.forward_mode.is_extend() else None + ) else: batch_result.extend_input_len_per_req = None @@ -5470,7 +5467,7 @@ class Scheduler( and (req := self.chunked_req) is not None ): # Retract skips the chunk step that sets this result's KV send boundary. - req.tmp_end_idx = min(req.extend_range.end, len(req.origin_input_ids)) + req.tmp_end_idx = min(req.extend_end, len(req.origin_input_ids)) tmp_batch, tmp_result = self.result_queue.popleft() self.process_batch_result(tmp_batch, tmp_result) diff --git a/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py b/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py index ad27b6e758..93b67c4b9a 100644 --- a/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py +++ b/python/sglang/srt/managers/scheduler_components/dynamic_chunk_sizer.py @@ -170,7 +170,7 @@ class DynamicChunkSizer: # request so release_kv_cache can release it symmetrically. req.init_next_round_input(self.tree_cache) req.lock = self.tree_cache.lock(req.last_node) - req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids)) + req.extend_end = len(req.full_untruncated_fill_ids) # Prepare batch batch = ScheduleBatch.init_new( @@ -183,7 +183,7 @@ class DynamicChunkSizer: self.spec_algorithm, ) - current_seq_len = req.extend_range.end + current_seq_len = req.extend_end if is_dp_attention_enabled(): # Profiling runs one request on this rank; other DP ranks report 0. diff --git a/python/sglang/srt/mem_cache/common.py b/python/sglang/srt/mem_cache/common.py index b01ae03028..8f8b948ff9 100644 --- a/python/sglang/srt/mem_cache/common.py +++ b/python/sglang/srt/mem_cache/common.py @@ -243,9 +243,9 @@ def checkpoint_kv_cache(req: Req, tree_cache: BasePrefixCache) -> None: # insert; a finished request belongs in release_kv_cache. assert not req.finished(), f"checkpointing finished request {req.rid}" if not req.skip_radix_cache_insert: - tree_cache.checkpoint(req, up_to=req.extend_range.end) + tree_cache.checkpoint(req, up_to=req.extend_end) # The next extend resumes after this one, published or not. - req.prefix_len = req.extend_range.end + req.prefix_len = req.extend_end def evict_from_tree_cache( diff --git a/python/sglang/srt/mem_cache/unified_cache/components/README.md b/python/sglang/srt/mem_cache/unified_cache/components/README.md index 3fb021564b..b4a765db10 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/README.md +++ b/python/sglang/srt/mem_cache/unified_cache/components/README.md @@ -237,7 +237,7 @@ Insert a request's KV into the tree: at every checkpoint while it runs, and once | Aspect | Detail | |--------|--------| -| **Purpose** | Publish the request's KV `[cache_protected_len, up_to)` so other requests can match it; nodes past `req.kv.cache_inserted_len` count one hit (a request counts each node once). `checkpoint_kv_cache` calls it with `req.extend_range.end` while the request runs; `release_kv_cache` calls it with the request-owned length when it finishes (`req.finished()`), then frees `[cache_protected_len, up_to)` and everything past it and unlocks `req.lock`; with `checkpoint=False` it skips the insert and calls `on_release(req, checkpointed=False)` for component cleanup | +| **Purpose** | Publish the request's KV `[cache_protected_len, up_to)` so other requests can match it; nodes past `req.kv.cache_inserted_len` count one hit (a request counts each node once). `checkpoint_kv_cache` calls it with `req.extend_end` while the request runs; `release_kv_cache` calls it with the request-owned length when it finishes (`req.finished()`), then frees `[cache_protected_len, up_to)` and everything past it and unlocks `req.lock`; with `checkpoint=False` it skips the insert and calls `on_release(req, checkpointed=False)` for component cleanup | | **Inputs** | `req` — the request; `up_to` — row position the insert may read up to | | **Output** | `None` | | **Mutation** | Component hooks → `insert` → re-match → writes the tree's indices back into the row → moves `req.lock` to the node the insert ended on → updates `req.kv.cache_protected_len`, `req.kv.cache_inserted_len`, `req.last_node` (`checkpoint_kv_cache` then advances `req.prefix_len`) → component cleanup. A finished request's component state (Mamba) is handed to the tree instead of forked, and what it still held is freed. Frees no KV slot: `release_kv_cache` does that afterwards. | diff --git a/python/sglang/srt/model_executor/model_runner_components/ngram_embedding_manager.py b/python/sglang/srt/model_executor/model_runner_components/ngram_embedding_manager.py index c0e5389cc0..cd4055faf8 100644 --- a/python/sglang/srt/model_executor/model_runner_components/ngram_embedding_manager.py +++ b/python/sglang/srt/model_executor/model_runner_components/ngram_embedding_manager.py @@ -123,9 +123,10 @@ class NgramEmbeddingManager: all_tokens = [] column_starts = [] request_lengths = [] - for req in batch.reqs: - start = req.prefix_len - end = start + req.extend_range.length + for req, start, extend_len in zip( + batch.reqs, batch.prefix_lens, batch.extend_lens, strict=True + ): + end = start + extend_len fill_ids = req.origin_input_ids + req.output_ids if start == 0: tokens = fill_ids[start:end] @@ -196,8 +197,7 @@ class NgramEmbeddingManager: if not batch.forward_mode.is_extend_without_speculative(): return rows = [] - for req in batch.reqs: - start = req.extend_range.start + for req, start in zip(batch.reqs, batch.prefix_lens, strict=True): lo = max(0, start - n1) ids = req.full_untruncated_fill_ids[lo:start] rows.append([0] * (n1 - len(ids)) + list(ids)) diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index 7b289705d4..b33b6d71b4 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -69,7 +69,6 @@ from typing import ( Generic, Iterator, List, - NamedTuple, Optional, Protocol, Sequence, @@ -1335,15 +1334,6 @@ def get_current_device_stream_fast(): # ============================================================================== -class Range(NamedTuple): - start: int - end: int - - @property - def length(self) -> int: - return self.end - self.start - - def assert_int64_array(values: array, name: str) -> None: """Require a signed int64 array suitable for zero-copy tensor views.""" assert ( diff --git a/test/manual/chunked_prefill/test_scripted_hybrid_swa.py b/test/manual/chunked_prefill/test_scripted_hybrid_swa.py index 99f79f50cb..f2a35d75d0 100644 --- a/test/manual/chunked_prefill/test_scripted_hybrid_swa.py +++ b/test/manual/chunked_prefill/test_scripted_hybrid_swa.py @@ -97,7 +97,7 @@ class TestSWABasic(ScriptedTestCase): for _ in range(400): if r.is_chunking: assert r.req.prefix_len <= r.req.kv.kv_committed_len, ( - f"prefix_indices must be bounded by kv_committed_len, " + f"prefix_len must be bounded by kv_committed_len, " f"got prefix_indices_len={r.req.prefix_len}, " f"kv_committed_len={r.req.kv.kv_committed_len}" ) diff --git a/test/manual/chunked_prefill/test_scripted_radix.py b/test/manual/chunked_prefill/test_scripted_radix.py index 0584ce87a1..7808c7b530 100644 --- a/test/manual/chunked_prefill/test_scripted_radix.py +++ b/test/manual/chunked_prefill/test_scripted_radix.py @@ -307,7 +307,7 @@ class TestRadixNoTailChunked(ScriptedTestCase): protected_len: int = req.kv.cache_protected_len assert prefix_len == protected_len, ( f"page_size=1 must take the no-tail else branch: " - f"len(prefix_indices)={prefix_len} != " + f"prefix_len={prefix_len} != " f"cache_protected_len={protected_len} (a partial-page tail " f"was appended, which only happens for page_size > 1)" ) @@ -462,7 +462,7 @@ class TestRadixPartialPage(ScriptedTestCase): prefix_len: int = req.prefix_len protected_len: int = req.kv.cache_protected_len assert prefix_len >= protected_len, ( - f"len(prefix_indices)={prefix_len} dropped below " + f"prefix_len={prefix_len} dropped below " f"cache_protected_len={protected_len}: tail was freed " f"prematurely" ) diff --git a/test/manual/chunked_prefill/test_scripted_special_case.py b/test/manual/chunked_prefill/test_scripted_special_case.py index c7ab81196c..f3700a1914 100644 --- a/test/manual/chunked_prefill/test_scripted_special_case.py +++ b/test/manual/chunked_prefill/test_scripted_special_case.py @@ -332,7 +332,7 @@ class TestSpecialCaseBasic(ScriptedTestCase): expected = chunked.seqlen - chunked.prefix_len assert pending == expected, ( f"load_inquirer chunked contribution must equal the prefix-" - f"subtracting formula seqlen - len(prefix_indices) = {expected}; " + f"subtracting formula seqlen - prefix_len = {expected}; " f"got {pending}" ) assert pending <= chunked.seqlen, ( @@ -377,7 +377,7 @@ class TestSpecialCaseBasic(ScriptedTestCase): observed = s.load_inquirer._get_num_pending_tokens() assert observed == expected, ( f"chunked contribution must equal remainder " - f"seqlen - len(prefix_indices) = {expected}, got {observed}; " + f"seqlen - prefix_len = {expected}, got {observed}; " f"a value of {chunked.seqlen} would mean the committed prefix " "is being double-counted" ) @@ -407,9 +407,9 @@ class TestSpecialCaseBasic(ScriptedTestCase): r.is_chunking and chunked is not None and chunked.rid == r.rid - and chunked.extend_range.length > 0 + and chunked.extend_len > 0 ): - deduct = chunked.extend_range.length + deduct = chunked.extend_len base = s.load_inquirer._get_num_pending_tokens() deducted = s.load_inquirer._get_num_pending_tokens(chunk_deduct=deduct) assert deducted == base - deduct, ( @@ -474,17 +474,15 @@ class TestSpecialCaseBasic(ScriptedTestCase): r.is_chunking and r.chunks_done >= 1 and req is not None - and req.extend_range is not None + and req.extend_end is not None ): saw_mid_chunk = True - assert ( - req.extend_range.end == req.prefix_len + req.extend_range.length - ), ( + assert req.extend_end == req.prefix_len + req.extend_len, ( f"init_next_round_input must rebuild fill_ids to the committed " f"prefix plus the in-flight chunk; " - f"fill_ids_len={req.extend_range.end}, " + f"fill_ids_len={req.extend_end}, " f"prefix_indices_len={req.prefix_len}, " - f"extend_input_len={req.extend_range.length}, " + f"extend_input_len={req.extend_len}, " f"chunks_done={r.chunks_done}" ) if r.finished: @@ -779,11 +777,11 @@ class TestSpecialCaseDeterministicFlashInfer(ScriptedTestCase): page_size = 16 saw_chunking = False for _ in range(DEFAULT_MAX_STEPS): - if r.is_chunking and r.req.extend_range is not None: + if r.is_chunking and r.req.extend_end is not None: saw_chunking = True - assert r.req.extend_range.length % page_size == 0, ( + assert r.req.extend_len % page_size == 0, ( f"deterministic chunk boundary must be page-aligned; " - f"got extend_input_len={r.req.extend_range.length}, page_size={page_size}" + f"got extend_input_len={r.req.extend_len}, page_size={page_size}" ) if r.finished: break diff --git a/test/manual/test_forward_split_prefill.py b/test/manual/test_forward_split_prefill.py index 6b950e18b0..4a6f3a1993 100644 --- a/test/manual/test_forward_split_prefill.py +++ b/test/manual/test_forward_split_prefill.py @@ -113,7 +113,7 @@ class TestForwardSplitPrefill(CustomTestCase): ) req.full_untruncated_fill_ids = req.origin_input_ids req.logprob_start_len = -1 - req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids)) + req.extend_end = len(req.full_untruncated_fill_ids) reqs.append(req) # Create dummy tree_cache for tests (no prefix caching, just allocation) diff --git a/test/registered/disaggregation/test_disaggregation_optimistic_prefill.py b/test/registered/disaggregation/test_disaggregation_optimistic_prefill.py index 1982ee9bcd..80fc3a2842 100644 --- a/test/registered/disaggregation/test_disaggregation_optimistic_prefill.py +++ b/test/registered/disaggregation/test_disaggregation_optimistic_prefill.py @@ -457,7 +457,7 @@ class TestOptimisticPrefillMambaRetryRelease(CustomTestCase): (req.kv.req_pool_idx, slice(0, len(self.PROMPT))), kv_indices ) req.full_untruncated_fill_ids = array("q", self.PROMPT) - req.set_extend_range(0, len(self.PROMPT)) + req.extend_end = len(self.PROMPT) req.kv.kv_committed_len = len(self.PROMPT) req.kv.kv_allocated_len = len(self.PROMPT) req.kv.mamba_last_track_seqlen = self.TRACK_SEQLEN diff --git a/test/registered/unit/disaggregation/test_dcp_pack.py b/test/registered/unit/disaggregation/test_dcp_pack.py index 01e2e78f9f..493e6c87b7 100644 --- a/test/registered/unit/disaggregation/test_dcp_pack.py +++ b/test/registered/unit/disaggregation/test_dcp_pack.py @@ -250,7 +250,7 @@ class TestDcpCachedPrefixSend(CustomTestCase): rid="cached-prefix", kv=SimpleNamespace(req_pool_idx=0), origin_input_ids=[0] * total, - extend_range=SimpleNamespace(end=total), + extend_end=total, start_send_idx=prefix, disagg_decode_prefix_len=prefix, disagg_kv_sender=sender, diff --git a/test/registered/unit/disaggregation/test_disaggregation_wire.py b/test/registered/unit/disaggregation/test_disaggregation_wire.py index 7258e87b86..fd3dad1af6 100644 --- a/test/registered/unit/disaggregation/test_disaggregation_wire.py +++ b/test/registered/unit/disaggregation/test_disaggregation_wire.py @@ -666,7 +666,7 @@ class TestDisaggregationWire(unittest.TestCase): req = SimpleNamespace( kv=ReqKvInfo(req_pool_idx=0), prefix_len=2, - extend_range=SimpleNamespace(length=3), + extend_len=3, origin_input_ids=[0, 1, 2, 3, 4], output_ids=[], retracted_stain=True, diff --git a/test/registered/unit/disaggregation/test_prefill_abort_result_cleanup.py b/test/registered/unit/disaggregation/test_prefill_abort_result_cleanup.py index 2a12b110a5..03229cfb88 100644 --- a/test/registered/unit/disaggregation/test_prefill_abort_result_cleanup.py +++ b/test/registered/unit/disaggregation/test_prefill_abort_result_cleanup.py @@ -42,7 +42,7 @@ class _Req: self.grammar = None self.output_ids = [] self.origin_input_ids = list(range(100)) - self.extend_range = None + self.extend_end = None self.time_stats = SimpleNamespace( set_prefill_finished_time=Mock(), set_last_chunked_prefill_finish_time=Mock(), @@ -128,7 +128,7 @@ def test_aborted_final_result_releases_hybrid_cache( def test_aborted_middle_result_releases_after_last_chunk(release_kv_cache): scheduler = _Scheduler() req = _Req(inflight_middle_chunks=1) - req.extend_range = SimpleNamespace(end=50) + req.extend_end = 50 scheduler.process_batch_result_disagg_prefill(_batch(req), _result()) @@ -143,7 +143,7 @@ def test_aborted_middle_result_releases_after_last_chunk(release_kv_cache): def test_aborted_middle_result_waits_for_inflight_chunk(release_kv_cache): scheduler = _Scheduler() req = _Req(inflight_middle_chunks=1) - req.extend_range = SimpleNamespace(end=len(req.origin_input_ids)) + req.extend_end = len(req.origin_input_ids) scheduler.process_batch_result_disagg_prefill(_batch(req), _result()) diff --git a/test/registered/unit/disaggregation/test_prefill_result_polling.py b/test/registered/unit/disaggregation/test_prefill_result_polling.py index 7e55ca2c2c..0f005a4625 100644 --- a/test/registered/unit/disaggregation/test_prefill_result_polling.py +++ b/test/registered/unit/disaggregation/test_prefill_result_polling.py @@ -1256,7 +1256,7 @@ class TestPrefillResultPolling(CustomTestCase): class ChunkScheduler(_Scheduler): def run_batch(self, batch): end = int(batch.forward_iter) - req.extend_range = SimpleNamespace(end=end) + req.extend_end = end self.chunked_req = req if end < 12 else None batch.chunked_req = self.chunked_req if self.chunked_req is not None: @@ -1264,7 +1264,7 @@ class TestPrefillResultPolling(CustomTestCase): return super().run_batch(batch) def checkpoint_disagg_prefill(self, req): - checkpoint_ends.append(req.extend_range.end) + checkpoint_ends.append(req.extend_end) super().checkpoint_disagg_prefill(req) if pause_at == 1: @@ -1335,7 +1335,7 @@ class TestPrefillResultPolling(CustomTestCase): def run_batch(self, batch): if req in batch.reqs: - req.extend_range = SimpleNamespace(end=batch.forward_iter) + req.extend_end = batch.forward_iter self.chunked_req = batch.chunked_req = req req.inflight_middle_chunks += 1 return super().run_batch(batch) diff --git a/test/registered/unit/dllm/test_gemma4_uniform_lifecycle.py b/test/registered/unit/dllm/test_gemma4_uniform_lifecycle.py index 66b6574d95..c254d250f4 100644 --- a/test/registered/unit/dllm/test_gemma4_uniform_lifecycle.py +++ b/test/registered/unit/dllm/test_gemma4_uniform_lifecycle.py @@ -32,9 +32,7 @@ class _Req: self.dllm_incomplete_ids = array("q") self.dllm_algo_state = None self.dllm_phase_prefill = prefill - self.extend_range = SimpleNamespace( - start=context_len, end=context_len + block_size, length=block_size - ) + self.extend_end = context_len + block_size self.kv = ReqKvInfo( req_pool_idx=1, kv_allocated_len=context_len + block_size, @@ -50,12 +48,11 @@ class _Req: def seqlen(self): return len(self.origin_input_ids) + len(self.output_ids) + extend_len = Req.extend_len + def is_dllm_prefill(self): return self.dllm_phase_prefill - def set_extend_range(self, start, end): - self.extend_range = SimpleNamespace(start=start, end=end, length=end - start) - def update_finish_state(self, new_accepted_len=1): if self.finish_on_update: self.finished_reason = object() @@ -210,12 +207,13 @@ class TestGemma4ContextLifecycle(unittest.TestCase): truncation_align_size=None, ) self.assertEqual((admission.prefix_len, admission.extend_len), (0, 300)) - PrefillAdder._add_dllm_req(adder, req, 0) - self.assertEqual((req.extend_range.start, req.extend_range.end), (0, 300)) + PrefillAdder._add_dllm_req(adder, req) + self.assertEqual((req.prefix_len, req.extend_end), (0, 300)) req.dllm_phase_prefill = False - PrefillAdder._add_dllm_req(adder, req, 300) - self.assertEqual((req.extend_range.start, req.extend_range.end), (300, 556)) + req.prefix_len = 300 + PrefillAdder._add_dllm_req(adder, req) + self.assertEqual((req.prefix_len, req.extend_end), (300, 556)) def test_chunked_context_prefill_stops_at_context_boundary(self): adder = object.__new__(PrefillAdder) @@ -231,7 +229,7 @@ class TestGemma4ContextLifecycle(unittest.TestCase): result = PrefillAdder.add_dllm_staging_req(adder, req) self.assertEqual(result, AddReqResult.CONTINUE) - self.assertEqual((req.extend_range.start, req.extend_range.end), (7, 10)) + self.assertEqual((req.prefix_len, req.extend_end), (7, 10)) self.assertEqual(adder.can_run_list, [req]) def test_completed_canvas_frees_only_decoder_pages_and_keeps_slot(self): @@ -241,7 +239,7 @@ class TestGemma4ContextLifecycle(unittest.TestCase): ) req.full_untruncated_fill_ids.extend([0] * block_size) req.dllm_block_offset = context_len - req.set_extend_range(context_len, context_len + block_size) + req.extend_end = context_len + block_size req.kv.req_pool_idx = 3 req.kv.kv_allocated_len = context_len + block_size req.kv.kv_committed_len = context_len + block_size @@ -262,10 +260,7 @@ class TestGemma4ContextLifecycle(unittest.TestCase): self.assertEqual(req.kv.req_pool_idx, 3) self.assertEqual(req.kv.kv_committed_len, context_len) self.assertEqual(req.kv.kv_allocated_len, context_len) - self.assertEqual( - (req.extend_range.start, req.extend_range.end), - (context_len, context_len), - ) + self.assertEqual(req.extend_end, context_len) scheduler.stash_chunked_request.assert_called_once_with(req) def test_unresolved_fdfo_preserves_state_and_reuses_exact_slots(self): diff --git a/test/registered/unit/hardware_backend/mlx/test_tp_worker_routing.py b/test/registered/unit/hardware_backend/mlx/test_tp_worker_routing.py index 7a24e2a1de..bb2a2bbcdb 100644 --- a/test/registered/unit/hardware_backend/mlx/test_tp_worker_routing.py +++ b/test/registered/unit/hardware_backend/mlx/test_tp_worker_routing.py @@ -171,9 +171,9 @@ class _FakeReq: self.fill_ids = [0] self.kv = ReqKvInfo(req_pool_idx=req_pool_idx) # Mirrors Req's chunk-finality contract read by - # MlxTpModelWorker._chunk_needs_logits: extend_range=None means + # MlxTpModelWorker._chunk_needs_logits: extend_end=None means # "not truncated" (final chunk / plain prefill). - self.extend_range = None + self.extend_end = None self.full_untruncated_fill_ids = self.fill_ids def get_fill_ids(self): @@ -267,18 +267,18 @@ class TestMlxExtendRouting(CustomTestCase): """THE REGRESSION (sync): a 1-token continuation must extend, not decode.""" runner = self._run_sync([_FakeReq("r1")], [1], {"r1"}, None, ForwardMode.EXTEND) self.assertEqual(runner.ops_for("r1"), ["extend_start"]) - # Untruncated (extend_range None) => final chunk => logits required. + # Untruncated (extend_end None) => final chunk => logits required. self.assertIs(runner.logits_flags[("extend_start", "r1")], True) def test_sync_non_final_chunk_skips_logits(self): - """Head-skip derivation: a scheduler-truncated chunk (extend_range.end + """Head-skip derivation: a scheduler-truncated chunk (extend_end below the request's full untruncated length) reaches the runner with needs_logits=False; its next-token output is popped as the stale intermediate token, so computing the vocab head for it is pure waste. Everything else about routing is unchanged.""" req = _FakeReq("r1") req.full_untruncated_fill_ids = list(range(8)) - req.extend_range = SimpleNamespace(start=0, end=4) # 4 < 8: non-final + req.extend_end = 4 # 4 < 8: non-final runner = self._run_sync([req], [4], {"r1"}, None, ForwardMode.EXTEND) self.assertEqual(runner.ops_for("r1"), ["extend_start"]) self.assertIs(runner.logits_flags[("extend_start", "r1")], False) @@ -314,7 +314,7 @@ class TestMlxExtendRouting(CustomTestCase): """Async twin of the head-skip derivation guard.""" req = _FakeReq("r1") req.full_untruncated_fill_ids = list(range(8)) - req.extend_range = SimpleNamespace(start=0, end=4) + req.extend_end = 4 runner, _ = self._run_async([req], [4], {"r1"}, None, ForwardMode.EXTEND) self.assertEqual(runner.ops_for("r1"), ["extend_start"]) self.assertIs(runner.logits_flags[("extend_start", "r1")], False) diff --git a/test/registered/unit/managers/test_hisparse_unit.py b/test/registered/unit/managers/test_hisparse_unit.py index c296d999d0..7dd5dc8517 100644 --- a/test/registered/unit/managers/test_hisparse_unit.py +++ b/test/registered/unit/managers/test_hisparse_unit.py @@ -24,7 +24,6 @@ from sglang.srt.utils import ( is_hip, is_xpu, ) -from sglang.srt.utils.common import Range from sglang.test.ci.ci_register import ( register_amd_ci, register_cuda_ci, @@ -73,9 +72,6 @@ def _make_req(rid="test-req-0", origin_input_ids=None, output_ids=None): inflight_middle_chunks=0, ) req.finished = lambda: req.finished_reason is not None - req.set_extend_range = lambda start, end: setattr( - req, "extend_range", Range(start, end) - ) return req @@ -241,7 +237,7 @@ class TestHiSparseUnit(unittest.TestCase): req.kv.kv_allocated_len = fill_len req.kv.kv_committed_len = fill_len req.full_untruncated_fill_ids = array("q", range(fill_len)) - req.extend_range = Range(0, fill_len) + req.extend_end = fill_len return kv_loc # ================================================================== @@ -791,7 +787,7 @@ class TestHiSparseUnit(unittest.TestCase): ) self.assertEqual(req.kv.kv_allocated_len, fill_len) self.assertEqual(req.kv.kv_committed_len, fill_len) - self.assertEqual(req.extend_range.length, fill_len) + self.assertEqual(req.extend_end - req.prefix_len, fill_len) rounded_len = (fill_len + self.page_size - 1) // self.page_size * self.page_size self.assertEqual( diff --git a/test/registered/unit/managers/test_mamba_checkpoint_depth.py b/test/registered/unit/managers/test_mamba_checkpoint_depth.py index cf74339972..addb5d90ea 100644 --- a/test/registered/unit/managers/test_mamba_checkpoint_depth.py +++ b/test/registered/unit/managers/test_mamba_checkpoint_depth.py @@ -54,7 +54,7 @@ def _track_seqlen( vocab_size=128, ) req.prefix_len = prefix_len - req.set_extend_range(prefix_len, prefix_len + extend_len) + req.extend_end = prefix_len + extend_len req.kv.mamba_ping_pong_track_buffer = torch.tensor([0, 1], dtype=torch.int64) req.kv.mamba_next_track_idx = 0 req.mamba_branching_seqlen = None diff --git a/test/registered/unit/managers/test_prefill_adder.py b/test/registered/unit/managers/test_prefill_adder.py index 7a94fe897c..45c72e4c47 100644 --- a/test/registered/unit/managers/test_prefill_adder.py +++ b/test/registered/unit/managers/test_prefill_adder.py @@ -32,7 +32,6 @@ from sglang.srt.mem_cache.unified_cache.component_type import ComponentType from sglang.srt.mem_cache.unified_memory_pool import init_unified_swa_pools from sglang.srt.runtime_context import get_context from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler -from sglang.srt.utils.common import Range from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -124,6 +123,9 @@ class TestPrefillAdder(CustomTestCase): req.cache_request_handle = CacheRequestHandle(req.rid, 0) req.priority = priority req.prefix_len = 0 + # A spec=Req mock gets a per-instance class; derive extend_len like Req. + type(req).extend_len = Req.extend_len + req.extend_end = None req.full_untruncated_fill_ids = [] req.output_ids = [0] * output_len req.sampling_params = SimpleNamespace(max_new_tokens=max_new_tokens) @@ -194,11 +196,6 @@ class TestPrefillAdder(CustomTestCase): req.swa_host_hit_length = 0 req.last_node = MagicMock() req.full_untruncated_fill_ids = list(range(12)) - req.set_extend_range = MagicMock( - side_effect=lambda start, end: setattr( - req, "extend_range", Range(start, end) - ) - ) return req def create_shortest_prefill_adder(self, *, chunk_tokens=4096): @@ -234,7 +231,7 @@ class TestPrefillAdder(CustomTestCase): continuation, waiting, adder.rem_chunk_tokens, adder.page_size ) self.assertIs(adder.add_chunked_req(continuation), continuation) - self.assertEqual(continuation.extend_range.length, 2560) + self.assertEqual(continuation.extend_len, 2560) for req in waiting: adder.add_one_req(req, has_chunked_req=True, truncation_align_size=None) self.assertEqual(adder.can_run_list, [continuation, *waiting]) @@ -252,7 +249,7 @@ class TestPrefillAdder(CustomTestCase): ) self.assertEqual(adder.can_run_list, []) self.assertIsNone(adder.new_chunked_req) - req.set_extend_range.assert_not_called() + self.assertIsNone(req.extend_end) self.mock_tree_cache.init_load_back.assert_not_called() def test_shortest_prefill_rechecks_chunk_limit_after_host_miss(self): @@ -273,7 +270,7 @@ class TestPrefillAdder(CustomTestCase): ) self.mock_tree_cache.init_load_back.assert_called_once() self.assertEqual(adder.can_run_list, []) - req.set_extend_range.assert_not_called() + self.assertIsNone(req.extend_end) def test_shortest_prefill_preserves_memory_admission(self): adder = self.create_shortest_prefill_adder() @@ -291,7 +288,7 @@ class TestPrefillAdder(CustomTestCase): req = self.create_shared_req("continuation") req.full_untruncated_fill_ids = list(range(8192)) self.assertIs(adder.add_chunked_req(req), req) - self.assertEqual(req.extend_range.length, 4096) + self.assertEqual(req.extend_len, 4096) def test_exact_chunk_fill_keeps_mamba_chunks_page_aligned(self): # A Mamba checkpoint only lands on a page-aligned chunk end, so an @@ -321,7 +318,7 @@ class TestPrefillAdder(CustomTestCase): adder.add_one_req( req, has_chunked_req=False, truncation_align_size=None ) - self.assertEqual(req.extend_range.length, expected) + self.assertEqual(req.extend_len, expected) def test_shared_admission_reserves_all_pending_requests(self): adder = self.create_shared_adder() @@ -371,7 +368,7 @@ class TestPrefillAdder(CustomTestCase): ) self.assertIs(adder.add_chunked_req(req), req) self.assertEqual(adder.can_run_list, []) - req.set_extend_range.assert_not_called() + self.assertIsNone(req.extend_end) self.assertEqual( (adder.memory_budget.total_offset, adder.memory_budget.swa_offset), before, @@ -381,7 +378,7 @@ class TestPrefillAdder(CustomTestCase): adder = self.create_shared_adder() req = self.create_shared_req("continuation", max_new_tokens=80) self.assertIs(adder.add_chunked_req(req), req) - self.assertEqual(req.extend_range.length, 8) + self.assertEqual(req.extend_len, 8) self.assertEqual(adder.memory_budget.total_offset, 12) self.assertEqual(adder.memory_budget.swa_offset, 12) @@ -787,11 +784,6 @@ class TestPrefillAdder(CustomTestCase): req.origin_input_ids = list(range(16)) req.full_untruncated_fill_ids = list(range(16)) req.last_node = MagicMock() - req.set_extend_range = MagicMock( - side_effect=lambda start, end: setattr( - req, "extend_range", Range(start, end) - ) - ) result = adder.add_one_req( req, has_chunked_req=False, truncation_align_size=None @@ -799,7 +791,7 @@ class TestPrefillAdder(CustomTestCase): self.assertEqual(result, AddReqResult.NO_TOKEN) self.assertEqual(adder.can_run_list, []) - req.set_extend_range.assert_not_called() + self.assertIsNone(req.extend_end) def create_sharded_adder( self, @@ -862,7 +854,7 @@ class TestPrefillAdder(CustomTestCase): AddReqResult.OTHER, ) self.assertEqual(adder.can_run_list, [first]) - second.set_extend_range.assert_not_called() + self.assertIsNone(second.extend_end) self.assertEqual((adder.rem_total_tokens, adder.rem_chunk_tokens), remaining) self.assertEqual(adder.kv_shard_block_bound_pages, 2) self.assertEqual(adder.kv_shard_chunk_pages, 1) @@ -941,7 +933,7 @@ class TestPrefillAdder(CustomTestCase): AddReqResult.OTHER, ) self.assertEqual(adder.can_run_list, reqs[:8]) - reqs[8].set_extend_range.assert_not_called() + self.assertIsNone(reqs[8].extend_end) self.assertEqual( ( adder.rem_total_tokens, @@ -986,7 +978,7 @@ class TestPrefillAdder(CustomTestCase): ) self.assertEqual(adder.can_run_list, [req]) self.assertIs(adder.new_chunked_req, req) - self.assertEqual(req.extend_range, Range(12, 12 + selected)) + self.assertEqual((req.prefix_len, req.extend_end), (12, 12 + selected)) self.assertEqual(adder.kv_shard_block_bound_pages, 2) self.assertEqual(adder.kv_shard_chunk_pages, selected // 4) @@ -1000,7 +992,7 @@ class TestPrefillAdder(CustomTestCase): AddReqResult.OTHER, ) self.assertEqual(adder.can_run_list, []) - req.set_extend_range.assert_not_called() + self.assertIsNone(req.extend_end) self.assertEqual(adder.kv_shard_block_bound_pages, 0) self.assertEqual(adder.kv_shard_chunk_pages, 0) @@ -1041,15 +1033,6 @@ class TestPrefillAdder(CustomTestCase): req = self.create_mock_req("chunked", priority=0, max_new_tokens=128) req.prefix_len = 0 req.full_untruncated_fill_ids = list(range(extend_input_len)) - # set_extend_range is the only writer of extend_range; the production - # path reads req.extend_range.length right after calling it, so the mock - # must actually set the attribute (a spec=Req mock has the method but - # not the instance attribute). - req.set_extend_range = MagicMock( - side_effect=lambda start, end: setattr( - req, "extend_range", Range(start, end) - ) - ) return adder, req def test_add_chunked_req_hybrid_swa_reserves_page_for_alloc_extend(self): @@ -1066,8 +1049,7 @@ class TestPrefillAdder(CustomTestCase): result = adder.add_chunked_req(req) self.assertIs(result, req) # truncated → chunked prefill continues - req.set_extend_range.assert_called_once() - start, end = req.set_extend_range.call_args.args + start, end = req.prefix_len, req.extend_end new_len = end - start self.assertLessEqual(new_len + PAGE_SIZE, REM_SWA) self.assertEqual(new_len, REM_SWA - PAGE_SIZE) @@ -1084,7 +1066,7 @@ class TestPrefillAdder(CustomTestCase): result = adder.add_chunked_req(req) self.assertIs(result, req) - req.set_extend_range.assert_not_called() + self.assertIsNone(req.extend_end) self.assertEqual(len(adder.can_run_list), 0) def test_swa_budget_for_req(self): @@ -1156,11 +1138,6 @@ class TestPrefillAdder(CustomTestCase): req.host_hit_length = 0 req.swa_host_hit_length = 0 req.last_node = MagicMock() - req.set_extend_range = MagicMock( - side_effect=lambda start, end: setattr( - req, "extend_range", Range(start, end) - ) - ) req.sampling_params = SimpleNamespace(max_new_tokens=40, ignore_eos=False) # Pre-fix: a constant sliding-window reservation rejects the resume. @@ -1210,10 +1187,6 @@ class TestPrefillAdder(CustomTestCase): req.best_match_node = MagicMock() req.kv = SimpleNamespace(cache_protected_len=0) - def set_extend_range(start, end): - req.extend_range = Range(start, end) - - req.set_extend_range = MagicMock(side_effect=set_extend_range) req.sampling_params = SimpleNamespace(max_new_tokens=8, ignore_eos=False) def load_back(params): @@ -1245,7 +1218,7 @@ class TestPrefillAdder(CustomTestCase): self.assertEqual(len(admitted), 1) self.assertEqual(req.prefix_len, HOST_HIT + 4) self.assertEqual(req.kv.cache_protected_len, HOST_HIT + 4) - req.set_extend_range.assert_called_once_with(HOST_HIT + 4, SPAN) + self.assertEqual((req.prefix_len, req.extend_end), (HOST_HIT + 4, SPAN)) # A partial FULL load stays fatal. with self.assertRaisesRegex(RuntimeError, "promised"): run(HOST_HIT // 2) @@ -1335,7 +1308,7 @@ class TestPrefillAdder(CustomTestCase): self.assertIs(result, AddReqResult.NO_TOKEN) self.assertEqual(adder.can_run_list, admitted_before) self.assertIsNone(adder.new_chunked_req) - req.set_extend_range.assert_not_called() + self.assertIsNone(req.extend_end) self.assertEqual( ( adder.memory_budget.remaining_swa, @@ -1366,7 +1339,7 @@ class TestPrefillAdder(CustomTestCase): self.assertIs(result, AddReqResult.CONTINUE) self.assertEqual(adder.can_run_list, [req]) - self.assertEqual(req.extend_range, Range(0, 8192)) + self.assertEqual((req.prefix_len, req.extend_end), (0, 8192)) self.assertGreaterEqual(adder.memory_budget.remaining_swa, 0) self.assertEqual(self.mock_tree_cache.inc_lock_ref.call_count, 3) self.assertEqual(self.mock_tree_cache.dec_lock_ref.call_count, 2) @@ -1378,28 +1351,28 @@ class TestPrefillAdder(CustomTestCase): dict( restored_tokens=6144, restored_swa_tokens=1024, available_swa=3328 ), - Range(6144, 8192), + (6144, 8192), 128, AddReqResult.CONTINUE, ), ( "exact_fit_ring", dict(ring=True), - Range(0, 8192), + (0, 8192), 0, AddReqResult.NO_TOKEN, ), ( "fitting_chunk", dict(chunk_tokens=2048), - Range(0, 2048), + (0, 2048), 1920, AddReqResult.OTHER, ), ( "never_fits", dict(size_swa=4096), - Range(0, 2944), + (0, 2944), 1024, AddReqResult.CONTINUE, ), @@ -1414,10 +1387,11 @@ class TestPrefillAdder(CustomTestCase): self.assertIs(result, expected_result) self.mock_tree_cache.init_load_back.assert_called_once() self.assertEqual(adder.can_run_list, [req]) - self.assertEqual(req.extend_range, expected_range) - self.assertEqual(req.host_loaded_length, expected_range.start) + expected_start, expected_end = expected_range + self.assertEqual((req.prefix_len, req.extend_end), expected_range) + self.assertEqual(req.host_loaded_length, expected_start) self.assertIs( - adder.new_chunked_req, req if expected_range.end < 8192 else None + adder.new_chunked_req, req if expected_end < 8192 else None ) self.assertEqual(adder.memory_budget.remaining_swa, remaining_swa) @@ -1430,7 +1404,7 @@ class TestPrefillAdder(CustomTestCase): adder.add_one_req(req, has_chunked_req=False, truncation_align_size=None) self.assertEqual(adder.can_run_list, []) - req.set_extend_range.assert_not_called() + self.assertIsNone(req.extend_end) def _create_host_hit_req(self, *, prefix_len=0, host_hit=8192, tail=1024): req = self._create_delayer_req(prefix_len + host_hit + tail) @@ -1514,7 +1488,7 @@ class TestPrefillAdder(CustomTestCase): tile_gate.assert_called_once() self.mock_tree_cache.init_load_back.assert_called_once() self.assertEqual(adder.can_run_list, [req]) - req.set_extend_range.assert_called_once_with(24, 24 + extend) + self.assertEqual((req.prefix_len, req.extend_end), (24, 24 + extend)) self.mock_tree_cache.inc_lock_ref.assert_any_call(restored_node) self.assertIs( self.mock_tree_cache.dec_lock_ref.call_args.args[0], old_node @@ -1680,11 +1654,6 @@ class TestPrefillAdder(CustomTestCase): req.host_hit_length = 0 req.last_node = MagicMock() req.sampling_params.ignore_eos = False - req.set_extend_range = MagicMock( - side_effect=lambda start, end: setattr( - req, "extend_range", Range(start, end) - ) - ) return req def test_add_chunked_req_non_hybrid_no_swa_reservation(self): @@ -1702,7 +1671,7 @@ class TestPrefillAdder(CustomTestCase): result = adder.add_chunked_req(req) self.assertIsNone(result) - req.set_extend_range.assert_called_once_with(0, 200) + self.assertEqual((req.prefix_len, req.extend_end), (0, 200)) self.assertIn(req, adder.can_run_list) def _adder_with_extend_lens(self, extend_lens): diff --git a/test/registered/unit/managers/test_priority_scheduling_disaggregation.py b/test/registered/unit/managers/test_priority_scheduling_disaggregation.py index da5afca8a1..ad2eab226f 100644 --- a/test/registered/unit/managers/test_priority_scheduling_disaggregation.py +++ b/test/registered/unit/managers/test_priority_scheduling_disaggregation.py @@ -33,7 +33,6 @@ from sglang.srt.observability import req_time_stats from sglang.srt.observability.req_time_stats import SchedulerReqTimeStats from sglang.srt.runtime_context import get_context, publish, reset_context # noqa: E402 from sglang.srt.server_args import ServerArgs -from sglang.srt.utils.common import Range from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.separate_buffer_allocator_double import ( bind_separate_buffer_capacity, @@ -350,7 +349,7 @@ class TestOptimisticPrefillCacheOwnership(unittest.TestCase): req = SimpleNamespace( pending_bootstrap=True, kv=SimpleNamespace(req_pool_idx=0), - extend_range=Range(0, 5), + extend_end=5, ) SchedulerDisaggregationPrefillMixin.checkpoint_disagg_prefill(scheduler, req) diff --git a/test/registered/unit/managers/test_schedule_batch_convert_decode_to_extend.py b/test/registered/unit/managers/test_schedule_batch_convert_decode_to_extend.py index 4e082a5307..e51c7df089 100644 --- a/test/registered/unit/managers/test_schedule_batch_convert_decode_to_extend.py +++ b/test/registered/unit/managers/test_schedule_batch_convert_decode_to_extend.py @@ -29,13 +29,12 @@ class _FakeReq: real Req methods so the array bookkeeping is not re-implemented here.""" refresh_fill_ids = Req.refresh_fill_ids - set_extend_range = Req.set_extend_range def __init__(self, *, num_prompt_tokens: int, num_output_tokens: int): self.origin_input_ids = array("l", range(num_prompt_tokens)) self.output_ids = list(range(num_output_tokens)) self.full_untruncated_fill_ids = array("l", self.origin_input_ids) - self.extend_range = None + self.extend_end = None self.beam_group = None @@ -68,7 +67,7 @@ class TestConvertDecodeToExtendGeometry(CustomTestCase): self.assertEqual(batch.extend_lens, [1] * len(seq_lens)) self.assertEqual(batch.extend_num_tokens, len(seq_lens)) for req, seq_len in zip(batch.reqs, seq_lens, strict=True): - self.assertEqual(tuple(req.extend_range), (seq_len - 1, seq_len)) + self.assertEqual(req.extend_end, seq_len) # What the attention path actually consumes: arange(prefix, prefix+len) # must land on the slot prepare_for_decode allocated, at seq_len - 1. for prefix_len, extend_len, seq_len in zip( diff --git a/test/registered/unit/managers/test_schedule_batch_out_of_place.py b/test/registered/unit/managers/test_schedule_batch_out_of_place.py index 5d850bb6a8..2134b78a3d 100644 --- a/test/registered/unit/managers/test_schedule_batch_out_of_place.py +++ b/test/registered/unit/managers/test_schedule_batch_out_of_place.py @@ -17,7 +17,6 @@ from sglang.srt.managers.schedule_batch import ( # noqa: E402 ) from sglang.srt.model_executor.forward_batch_info import ForwardMode # noqa: E402 from sglang.srt.speculative.spec_info import SpeculativeAlgorithm # noqa: E402 -from sglang.srt.utils.common import Range # noqa: E402 register_cpu_ci(est_time=11, suite="base-a-test-cpu") @@ -130,7 +129,7 @@ class _FakeReq: self.origin_input_ids = list(range(origin_len)) self.output_ids = list(range(output_len)) self.full_untruncated_fill_ids = list(range(origin_len + output_len)) - self.extend_range = None + self.extend_end = None def refresh_fill_ids(self): self.full_untruncated_fill_ids = self.origin_input_ids + self.output_ids @@ -139,9 +138,6 @@ class _FakeReq: def seqlen(self): return len(self.origin_input_ids) + len(self.output_ids) - def set_extend_range(self, start, end): - self.extend_range = Range(start, end) - class TestMergeBatchOutOfPlace(unittest.TestCase): def test_merge_batch_rebinds_lists_without_mutating_either_side(self): @@ -243,14 +239,14 @@ class TestPrepareEncoderInfoExtendOutOfPlace(unittest.TestCase): rid="img", multimodal_inputs=types.SimpleNamespace(num_image_tokens=2), prefix_len=0, - extend_range=Range(0, 5), + extend_len=5, logprob_start_len=0, ) req_text_only = types.SimpleNamespace( rid="txt", multimodal_inputs=None, prefix_len=0, - extend_range=Range(0, 4), + extend_len=4, logprob_start_len=0, ) batch = make_schedule_batch( diff --git a/test/registered/unit/managers/test_scheduler_chunked_req_gate.py b/test/registered/unit/managers/test_scheduler_chunked_req_gate.py index 56211d88c8..313f9fe43a 100644 --- a/test/registered/unit/managers/test_scheduler_chunked_req_gate.py +++ b/test/registered/unit/managers/test_scheduler_chunked_req_gate.py @@ -16,7 +16,6 @@ from sglang.srt.disaggregation.utils import DisaggregationMode from sglang.srt.managers.schedule_batch import NextBatchPlan, Req, ReqKvInfo from sglang.srt.managers.scheduler import Scheduler from sglang.srt.mem_cache.chunk_cache import ChunkCache -from sglang.srt.utils.common import Range register_cpu_ci(est_time=11, suite="base-a-test-cpu") @@ -26,7 +25,6 @@ def _make_req( req_pool_idx: int, fill_ids: list, prefix_len: int, - extend_input_len: int, fill_len: int, ) -> Req: req = Req.__new__(Req) @@ -35,7 +33,7 @@ def _make_req( req.output_ids = array("q") req.full_untruncated_fill_ids = array("q", fill_ids) req.prefix_len = prefix_len - req.extend_range = Range(fill_len - extend_input_len, fill_len) + req.extend_end = fill_len req.inflight_middle_chunks = 0 req.host_hit_length = 0 req.kv = ReqKvInfo(req_pool_idx=req_pool_idx) @@ -132,7 +130,6 @@ class TestStashGatePreservesPrefix(CustomTestCase): req_pool_idx=self.POOL_IDX, fill_ids=list(range(self.POST_RESET_FILL_LEN)), prefix_len=self.INITIAL_PREFIX_LEN, - extend_input_len=fill_len - self.INITIAL_PREFIX_LEN, fill_len=fill_len, ) s = _scheduler_for_get_next_batch(tree_cache=cache, chunked_req=req) diff --git a/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py b/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py index 9fa80abad7..5e36e0d4f2 100644 --- a/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py +++ b/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py @@ -53,7 +53,6 @@ from sglang.srt.mem_cache.common import release_kv_cache from sglang.srt.mem_cache.radix_cache import RadixCache, RadixKey from sglang.srt.mem_cache.unified_cache.component_type import ComponentType from sglang.srt.sampling.sampling_params import SamplingParams -from sglang.srt.utils.common import Range from sglang.test.separate_buffer_allocator_double import ( bind_separate_buffer_capacity, ) @@ -91,7 +90,7 @@ class MockReq: self, fill_ids, req_pool_idx=0, cache_protected_len=0, last_node=None, lock=None ): self.full_untruncated_fill_ids = array("q", fill_ids) - self.extend_range = Range(0, len(self.full_untruncated_fill_ids)) + self.extend_end = len(self.full_untruncated_fill_ids) self.origin_input_ids = array( "q", fill_ids[:-1] if len(fill_ids) > 1 else fill_ids ) @@ -111,7 +110,7 @@ class MockReq: self.kv_rotation_base = None def get_fill_ids(self): - return self.full_untruncated_fill_ids[: self.extend_range.end] + return self.full_untruncated_fill_ids[: self.extend_end] rid = "mock-req" skip_radix_cache_insert = False @@ -264,7 +263,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): ) # Step 2: checkpoint (dec old lock, inc new lock) - cache.checkpoint(req, up_to=req.extend_range.end) + cache.checkpoint(req, up_to=req.extend_end) # Step 3: release_kv_cache (insert, free the rest, dec lock) req.finished_reason = "finished" @@ -315,7 +314,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): ) # Step 2: checkpoint (dec root=no-op, inc new leaf) - cache.checkpoint(req, up_to=req.extend_range.end) + cache.checkpoint(req, up_to=req.extend_end) # Step 3: cache_finished_req (dec leaf) req.finished_reason = "finished" @@ -661,7 +660,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): SchedulerDisaggregationDecodeMixin._get_new_prebuilt_batch( scheduler, SimpleNamespace(batch_size=lambda: 0) ) - cache.checkpoint(req, up_to=req.extend_range.end) + cache.checkpoint(req, up_to=req.extend_end) self.assertEqual(shared.lock_ref, 2) self.assertEqual(req_to_token[0, :3].tolist(), prefix_vals) @@ -698,7 +697,7 @@ class TestDecodeLockRefScenarios(CustomTestCase): lock=lock, ) - cache.checkpoint(req, up_to=req.extend_range.end) + cache.checkpoint(req, up_to=req.extend_end) req.finished_reason = "finished" release_kv_cache(req, cache, checkpoint=True) diff --git a/test/registered/unit/mem_cache/test_decode_retraction_backup.py b/test/registered/unit/mem_cache/test_decode_retraction_backup.py index 1c9e07c0d0..0377c85bbc 100644 --- a/test/registered/unit/mem_cache/test_decode_retraction_backup.py +++ b/test/registered/unit/mem_cache/test_decode_retraction_backup.py @@ -589,7 +589,7 @@ class TestDecodeRetractionBackup(CustomTestCase): cached_indices = queue._pre_alloc(cached) self._seed_pool(env.target_pool, cached_indices, base=500) cached_values = self._snapshot_pool(env.target_pool, cached_indices) - cache.checkpoint(cached, up_to=cached.extend_range.end) + cache.checkpoint(cached, up_to=cached.extend_end) pressure = env.allocator.alloc(self.pool_size // 2 - 4) req = make_req("receiving", 8) @@ -614,11 +614,11 @@ class TestDecodeRetractionBackup(CustomTestCase): # Prebuilt preparation retains the full-transfer/root state, then # restores before normal cache insertion deduplicates the prefix. req.init_next_round_input(None) - req.set_extend_range(0, 8) + req.extend_end = 8 restore_kv_cache(req, cache, env.req_to_token_pool, env.allocator, "host_pool") self.assertEqual(cache.host_pool_group.available_size(), host_free_before) free_before_insert = env.allocator.available_size() - cache.checkpoint(req, up_to=req.extend_range.end) + cache.checkpoint(req, up_to=req.extend_end) row = env.req_to_token_pool.req_to_token[req.kv.req_pool_idx, :8] self.assertTrue(torch.equal(row[:4], cached_indices)) diff --git a/test/registered/unit/mem_cache/test_hisparse_allocator.py b/test/registered/unit/mem_cache/test_hisparse_allocator.py index 6da4199370..c2e4b58f09 100644 --- a/test/registered/unit/mem_cache/test_hisparse_allocator.py +++ b/test/registered/unit/mem_cache/test_hisparse_allocator.py @@ -242,11 +242,6 @@ class TestDeepSeekV4HiSparseAllocator(CustomTestCase): kv=ReqKvInfo(), ) - def set_extend_range(start, end): - req.extend_range = SimpleNamespace(start=start, end=end, length=end - start) - - req.set_extend_range = set_extend_range - class ReqToTokenPool: def __init__(self): self.writes = [] @@ -304,7 +299,7 @@ class TestDeepSeekV4HiSparseAllocator(CustomTestCase): ) self.assertEqual(req.kv.kv_allocated_len, fill_len) self.assertEqual(req.kv.kv_committed_len, fill_len) - self.assertEqual(req.extend_range.length, fill_len) + self.assertEqual(req.extend_end, fill_len) self.assertEqual(len(req_to_token_pool.writes), 1) coordinator.host_token_len.assert_called_once_with(fill_len) regular_host_alloc.assert_called_once_with( diff --git a/test/registered/unit/mem_cache/test_prefill_memory_budget.py b/test/registered/unit/mem_cache/test_prefill_memory_budget.py index e94c992f9a..af36a77a51 100644 --- a/test/registered/unit/mem_cache/test_prefill_memory_budget.py +++ b/test/registered/unit/mem_cache/test_prefill_memory_budget.py @@ -209,7 +209,7 @@ class TestSharedPrefillAdmission(unittest.TestCase): self.assertIsNone(adder.add_chunked_req(req)) self.assertEqual(adder.can_run_list, [req]) - self.assertEqual(req.extend_range.length, 1) + self.assertEqual(req.extend_len, 1) def test_unaligned_ignore_eos_enters_empty_pool(self): for page_size in (4, 64): diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_bench.py b/test/registered/unit/mem_cache/test_unified_radix_cache_bench.py index f7eb92939c..e11306e038 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_bench.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_bench.py @@ -634,7 +634,7 @@ def bench_release( req.origin_input_ids = array("q", seq) req.output_ids = array("q") req.full_untruncated_fill_ids = array("q", seq) - req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids)) + req.extend_end = len(req.full_untruncated_fill_ids) req.last_node = node req.kv.cache_protected_len = matched_len req.kv.kv_committed_len = len(seq) diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py index 50aa6bc94d..7622b0192b 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py @@ -1853,7 +1853,7 @@ class UnifiedRadixCacheSuite: req.lock = None req.extra_key = None req.full_untruncated_fill_ids = array("q", input_ids + output_ids) - req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids)) + req.extend_end = len(req.full_untruncated_fill_ids) if self.cfg.has_mamba: req.kv.mamba_last_track_seqlen = kv_len @@ -1899,8 +1899,8 @@ class UnifiedRadixCacheSuite: req.origin_input_ids = array("q", prompt_ids) req.output_ids = array("q", output_ids) req.full_untruncated_fill_ids = array("q", prompt_ids + output_ids) - req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids)) - kv_len = req.extend_range.end + req.extend_end = len(req.full_untruncated_fill_ids) + kv_len = req.extend_end kv_indices = self._alloc(allocator, kv_len) req_to_token_pool.write((req.kv.req_pool_idx, slice(0, kv_len)), kv_indices) req.kv.kv_committed_len = kv_len @@ -1972,7 +1972,7 @@ class UnifiedRadixCacheSuite: req.lock = None req.extra_key = None req.full_untruncated_fill_ids = array("q", tokens) - req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids)) + req.extend_end = len(req.full_untruncated_fill_ids) avail_before = allocator.available_size() release_kv_cache(req, cache, checkpoint=False) @@ -1990,7 +1990,7 @@ class UnifiedRadixCacheSuite: req.origin_input_ids = array("q", tokens) req.output_ids = array("q") req.full_untruncated_fill_ids = array("q", tokens) - req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids)) + req.extend_end = len(req.full_untruncated_fill_ids) kv_len = len(tokens) kv_indices = self._alloc(allocator, kv_len) req_to_token_pool.write((req.kv.req_pool_idx, slice(0, kv_len)), kv_indices) @@ -2002,7 +2002,7 @@ class UnifiedRadixCacheSuite: if self.cfg.has_mamba: req.kv.mamba_last_track_seqlen = kv_len - cache.checkpoint(req, up_to=req.extend_range.end) + cache.checkpoint(req, up_to=req.extend_end) self.assertGreater(req.kv.cache_protected_len, 0) self.assertIsNotNone(req.last_node) @@ -2024,7 +2024,7 @@ class UnifiedRadixCacheSuite: req.origin_input_ids = array("q", tokens) req.output_ids = [] req.full_untruncated_fill_ids = array("q", tokens) - req.set_extend_range(0, len(req.full_untruncated_fill_ids)) + req.extend_end = len(req.full_untruncated_fill_ids) kv_indices = self._alloc(allocator, len(tokens)) req_to_token_pool.write( (req.kv.req_pool_idx, slice(0, len(tokens))), kv_indices @@ -2036,7 +2036,7 @@ class UnifiedRadixCacheSuite: req.extra_key = None req.kv.set_evicted_seqlen(ComponentType.SWA, evicted_len) - cache.checkpoint(req, up_to=req.extend_range.end) + cache.checkpoint(req, up_to=req.extend_end) (first,) = _node_children(cache, cache.root_node_handle()) self.assertEqual(_node_key_length(cache, first), evicted_len) @@ -2134,7 +2134,7 @@ class UnifiedRadixCacheSuite: req.lock = None req.extra_key = None req.full_untruncated_fill_ids = array("q", input_ids) - req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids)) + req.extend_end = len(req.full_untruncated_fill_ids) if self.cfg.has_mamba: req.kv.mamba_last_track_seqlen = kv_len @@ -2245,7 +2245,7 @@ class UnifiedRadixCacheSuite: req.origin_input_ids = array("q", tokens) req.output_ids = [] req.full_untruncated_fill_ids = array("q", tokens) - req.set_extend_range(0, len(req.full_untruncated_fill_ids)) + req.extend_end = len(req.full_untruncated_fill_ids) kv_len = len(tokens) fresh_value = self._alloc(allocator, kv_len) req_to_token_pool.write((req.kv.req_pool_idx, slice(0, kv_len)), fresh_value) @@ -2258,7 +2258,7 @@ class UnifiedRadixCacheSuite: full_available_before_insert = allocator.full_attn_allocator.available_size() - cache.checkpoint(req, up_to=req.extend_range.end) + cache.checkpoint(req, up_to=req.extend_end) self.assertEqual( allocator.full_attn_allocator.available_size(), @@ -2932,7 +2932,7 @@ class UnifiedRadixCacheSuite: req.origin_input_ids = tokens req.output_ids = [] req.full_untruncated_fill_ids = array("q", tokens) - req.set_extend_range(0, len(req.full_untruncated_fill_ids)) + req.extend_end = len(req.full_untruncated_fill_ids) kv_indices = self._alloc(allocator, pre_len) req_to_token_pool.write((req.kv.req_pool_idx, slice(0, pre_len)), kv_indices) req.kv.kv_committed_len = pre_len @@ -2944,7 +2944,7 @@ class UnifiedRadixCacheSuite: swa_avail_before = allocator.swa_attn_allocator.available_size() with envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.override(True): - cache.checkpoint(req, up_to=req.extend_range.end) + cache.checkpoint(req, up_to=req.extend_end) cushion = max(self.cfg.sliding_window_size, self.cfg.page_size) expected_evicted = (pre_len - 1) - cushion @@ -3018,7 +3018,7 @@ class UnifiedRadixCacheSuite: req.origin_input_ids = tokens req.output_ids = [] req.full_untruncated_fill_ids = array("q", tokens) - req.set_extend_range(0, len(req.full_untruncated_fill_ids)) + req.extend_end = len(req.full_untruncated_fill_ids) kv_indices = self._alloc(allocator, pre_len) req_to_token_pool.write((req.kv.req_pool_idx, slice(0, pre_len)), kv_indices) req.kv.kv_committed_len = pre_len @@ -3028,7 +3028,7 @@ class UnifiedRadixCacheSuite: req.extra_key = None with envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.override(True): - cache.checkpoint(req, up_to=req.extend_range.end) + cache.checkpoint(req, up_to=req.extend_end) self.assertEqual( req.kv.get_evicted_seqlen(ComponentType.SWA), @@ -7424,7 +7424,7 @@ class UnifiedRadixCacheSuite: req.origin_input_ids = tokens req.output_ids = [] req.full_untruncated_fill_ids = array("q", tokens) - req.set_extend_range(0, forward_len) + req.extend_end = forward_len req.kv.cache_protected_len = branching_seqlen kv_indices = self._alloc(allocator, forward_len) @@ -10850,7 +10850,7 @@ class TestSWAWindowUnderBigramKey(CustomTestCase): req.origin_input_ids = tokens req.output_ids = [] req.full_untruncated_fill_ids = array("q", tokens) - req.set_extend_range(0, len(req.full_untruncated_fill_ids)) + req.extend_end = len(req.full_untruncated_fill_ids) kv_indices = self._alloc_paged(allocator, seq_len) req_to_token_pool.write((req.kv.req_pool_idx, slice(0, seq_len)), kv_indices) req.kv.kv_committed_len = seq_len @@ -10860,7 +10860,7 @@ class TestSWAWindowUnderBigramKey(CustomTestCase): req.extra_key = None with envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.override(True): - cache.checkpoint(req, up_to=req.extend_range.end) + cache.checkpoint(req, up_to=req.extend_end) boundary = (seq_len - 1) // page_size * page_size self.assertGreaterEqual( diff --git a/tools/sglang-simulator/src/sglang_simulator/simulation/sglang/scheduler.py b/tools/sglang-simulator/src/sglang_simulator/simulation/sglang/scheduler.py index eedf4cde7a..4a665a229b 100644 --- a/tools/sglang-simulator/src/sglang_simulator/simulation/sglang/scheduler.py +++ b/tools/sglang-simulator/src/sglang_simulator/simulation/sglang/scheduler.py @@ -427,8 +427,7 @@ class C_SchedulerHook(BaseHook): for req in batch.reqs: extend_length = getattr(req, "extend_input_len", None) if extend_length is None: - # The range API represents extend tokens as a half-open interval. - extend_length = req.extend_range.length + extend_length = req.extend_len simulation_batch.reqs.append( ScheduleRequest( extend_length=extend_length,