diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index 5d79d1e90..c6c8020a1 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -152,7 +152,8 @@ class DecodeReqToTokenPool: len(reusing) <= 1 ), "only one chunked request may reuse req_pool_idx in a batch" assert all( - reqs[i].is_chunked > 0 or reqs[i].kv_committed_len > 0 for i in reusing + reqs[i].inflight_middle_chunks > 0 or reqs[i].kv_committed_len > 0 + for i in reusing ), "reusing request must be chunked or have committed KV" need_size = len(reqs) - len(reusing) diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index a64344844..fd267a802 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -514,7 +514,7 @@ class SchedulerDisaggregationPrefillMixin: for i, (req, next_token_id) in enumerate( zip(batch.reqs, next_token_ids, strict=True) ): - if req.is_chunked <= 0: + if req.inflight_middle_chunks <= 0: req.time_stats.set_prefill_finished_time() # There is no output_ids for prefill @@ -564,7 +564,7 @@ class SchedulerDisaggregationPrefillMixin: req.grammar.finished = req.finished() else: # being chunked reqs' prefill is not finished - req.is_chunked -= 1 + req.inflight_middle_chunks -= 1 if req.return_logprob: extend_logprob_start_len = extend_logprob_start_len_per_req[i] diff --git a/python/sglang/srt/dllm/mixin/scheduler.py b/python/sglang/srt/dllm/mixin/scheduler.py index c51058582..09969976f 100644 --- a/python/sglang/srt/dllm/mixin/scheduler.py +++ b/python/sglang/srt/dllm/mixin/scheduler.py @@ -200,7 +200,7 @@ class SchedulerDllmMixin: if can_run_list: self.dllm_manager.add_staging_reqs(can_run_list) - self.dllm_manager.increment_chunked_count() + self.dllm_manager.increment_inflight_middle_chunks() self.adder = adder self.can_run_list = can_run_list @@ -338,10 +338,10 @@ class DllmManager: return True return len(self.waiting_queue) == 0 - def increment_chunked_count(self) -> None: + def increment_inflight_middle_chunks(self) -> None: """Increment chunked count for all staging requests.""" for req in self.staging_queue: - req.is_chunked += 1 + req.inflight_middle_chunks += 1 def filter_finished_reqs(self) -> None: """Remove finished requests from both queues.""" diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index e32c77b05..a7a97cdd4 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -773,7 +773,7 @@ class Req(ReqDllmMixin): # Whether or not if it is chunked. It increments whenever # it is chunked, and decrement whenever chunked request is # processed. - self.is_chunked = 0 + self.inflight_middle_chunks = 0 # For retraction self.is_retracted = False @@ -1263,7 +1263,7 @@ class Req(ReqDllmMixin): self.temp_input_top_logprobs_val = None self.temp_input_top_logprobs_idx = None self.extend_logprob_start_len = 0 - self.is_chunked = 0 + self.inflight_middle_chunks = 0 self.mamba_pool_idx = None self.mamba_ping_pong_track_buffer = None self.mamba_next_track_idx = None diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 9f5a718b6..894c28eb4 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -2577,7 +2577,7 @@ class Scheduler( self._chunked_req_scheduled_last_iter = True if self.chunked_req is not None: - self.chunked_req.is_chunked += 1 + self.chunked_req.inflight_middle_chunks += 1 set_time_batch(can_run_list, "set_forward_entry_time") diff --git a/python/sglang/srt/managers/scheduler_components/batch_result_processor.py b/python/sglang/srt/managers/scheduler_components/batch_result_processor.py index d0e316ccc..4448e2845 100644 --- a/python/sglang/srt/managers/scheduler_components/batch_result_processor.py +++ b/python/sglang/srt/managers/scheduler_components/batch_result_processor.py @@ -215,7 +215,7 @@ class SchedulerBatchResultProcessor: # decode req in mixed batch or retracted req continue - if req.is_chunked <= 0: + if req.inflight_middle_chunks <= 0: req.time_stats.set_prefill_finished_time() # req output_ids are set here @@ -264,7 +264,7 @@ class SchedulerBatchResultProcessor: else: # being chunked reqs' prefill is not finished - req.is_chunked -= 1 + req.inflight_middle_chunks -= 1 # There is only at most one request being currently chunked. # Because this request does not finish prefill, # we don't want to stream the request currently being chunked. @@ -304,7 +304,7 @@ class SchedulerBatchResultProcessor: req.embedding = embeddings[i] if req.return_pooled_hidden_states and phs is not None: req.pooled_hidden_state = phs[i] - if req.is_chunked <= 0: + if req.inflight_middle_chunks <= 0: req.time_stats.set_prefill_finished_time() # Dummy output token for embedding models req.output_ids.append(0) @@ -317,7 +317,7 @@ class SchedulerBatchResultProcessor: maybe_cache_unfinished_req(req, self.tree_cache) else: # being chunked reqs' prefill is not finished - req.is_chunked -= 1 + req.inflight_middle_chunks -= 1 req.time_stats.set_last_chunked_prefill_finish_time() self.output_streamer.stream_output( diff --git a/python/sglang/srt/mem_cache/memory_pool.py b/python/sglang/srt/mem_cache/memory_pool.py index 8d7ad26a4..d9bc8b190 100644 --- a/python/sglang/srt/mem_cache/memory_pool.py +++ b/python/sglang/srt/mem_cache/memory_pool.py @@ -165,10 +165,11 @@ class ReqToTokenPool: # https://github.com/sgl-project/sglang/pull/20476 # if not any(r.is_dllm() for r in reqs): # assert ( - # sum(1 for i in reusing if reqs[i].is_chunked > 0) <= 1 + # sum(1 for i in reusing if reqs[i].inflight_middle_chunks > 0) <= 1 # ), "only one chunked request may reuse req_pool_idx in a batch" assert all( - reqs[i].is_chunked > 0 or reqs[i].kv_committed_len > 0 for i in reusing + reqs[i].inflight_middle_chunks > 0 or reqs[i].kv_committed_len > 0 + for i in reusing ), "reusing request must be chunked or have committed KV" need_size = len(reqs) - len(reusing) diff --git a/test/registered/unit/managers/test_hisparse_unit.py b/test/registered/unit/managers/test_hisparse_unit.py index ff2c98b34..a8538c42c 100644 --- a/test/registered/unit/managers/test_hisparse_unit.py +++ b/test/registered/unit/managers/test_hisparse_unit.py @@ -52,7 +52,7 @@ def _make_req(rid="test-req-0", origin_input_ids=None, output_ids=None): finished_reason=None, hisparse_staging=False, staging=False, - is_chunked=0, + inflight_middle_chunks=0, ) req.finished = lambda: req.finished_reason is not None return req diff --git a/test/registered/unit/managers/test_scheduler_chunked_req_gate.py b/test/registered/unit/managers/test_scheduler_chunked_req_gate.py index d4a92ba82..7cfd7d843 100644 --- a/test/registered/unit/managers/test_scheduler_chunked_req_gate.py +++ b/test/registered/unit/managers/test_scheduler_chunked_req_gate.py @@ -33,7 +33,7 @@ def _make_req( req.prefix_indices = prefix_indices req.req_pool_idx = req_pool_idx req.extend_input_len = extend_input_len - req.is_chunked = 0 + req.inflight_middle_chunks = 0 req.host_hit_length = 0 req.cache_protected_len = 0 req.skip_radix_cache_insert = False