diff --git a/python/sglang/srt/batch_overlap/two_batch_overlap.py b/python/sglang/srt/batch_overlap/two_batch_overlap.py index 424c8c578d..d850cf399a 100644 --- a/python/sglang/srt/batch_overlap/two_batch_overlap.py +++ b/python/sglang/srt/batch_overlap/two_batch_overlap.py @@ -775,6 +775,7 @@ class TboForwardBatchPreparer: "dp_padding_mode", "global_forward_mode", "is_prefill_only", + "contains_last_prefill_chunk", "spec_algorithm", "capture_hidden_mode", "defer_logits_to_eager", # forward-level flag, inherited by both child batches diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index 0af731d5a2..af2c5ca9c0 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -626,6 +626,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): spec_info: Optional[SpecInput] = None # === Derived from ScheduleBatch.reqs === + # True for synthetic capture batches; init_new snapshots real request ranges. + # Running decode rows do not complete a prefill. + contains_last_prefill_chunk: bool = True # For LoRA lora_ids: Optional[List[str]] = None # For dumper: request IDs for cross-step sequence tracking @@ -1009,6 +1012,15 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): if batch.seq_lens_sum is None and seq_lens_cpu is not None: batch.seq_lens_sum = int(seq_lens_cpu.sum()) + contains_last_prefill_chunk = False + if batch.forward_mode.is_extend_without_speculative(): + decoding_req_ids = {id(req) for req in (batch.decoding_reqs or ())} + contains_last_prefill_chunk = any( + req.extend_end == len(req.full_untruncated_fill_ids) + for req in batch.reqs + if id(req) not in decoding_req_ids + ) + from sglang.srt.model_executor.model_runner import ModelRunner # Only a real ModelRunner owns watermark state; lightweight stand-ins skip it. @@ -1066,6 +1078,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): dp_prefill_cuda_graph_max_prefix_len=batch.dp_prefill_cuda_graph_max_prefix_len, global_forward_mode=batch.global_forward_mode, is_prefill_only=batch.is_prefill_only, + contains_last_prefill_chunk=contains_last_prefill_chunk, spec_algorithm=batch.spec_algorithm, capture_hidden_mode=capture_hidden_mode, return_hidden_states_before_norm=return_hidden_states_before_norm, diff --git a/test/registered/unit/batch_overlap/test_tbo_filter_batch_marker.py b/test/registered/unit/batch_overlap/test_tbo_filter_batch_marker.py index d763f78430..33b032a313 100644 --- a/test/registered/unit/batch_overlap/test_tbo_filter_batch_marker.py +++ b/test/registered/unit/batch_overlap/test_tbo_filter_batch_marker.py @@ -58,6 +58,16 @@ def _filter(batch: ForwardBatch, *, lo: int, hi: int) -> ForwardBatch: class TestTboFilterBatchMarker(CustomTestCase): + def test_filter_batch_inherits_prefill_completion(self): + for completing in (False, True): + with self.subTest(completing=completing): + parent = _make_target_verify_batch(8) + # Isolate the existing, unrelated capture-flag omission. + parent.full_prefill_cuda_graph_capture = None + parent.contains_last_prefill_chunk = completing + child = _filter(parent, lo=0, hi=4) + self.assertEqual(child.contains_last_prefill_chunk, completing) + def test_filter_batch_clears_mlp_sync_unpad_fields_on_children(self): # MLP-sync padding records _original_batch_size/_original_num_tokens # before TBO splits the batch (prepare_mlp_sync_batch pads first, then diff --git a/test/registered/unit/layers/test_mamba_prefill_track_metadata.py b/test/registered/unit/layers/test_mamba_prefill_track_metadata.py index 4b951dd427..176a3b2606 100644 --- a/test/registered/unit/layers/test_mamba_prefill_track_metadata.py +++ b/test/registered/unit/layers/test_mamba_prefill_track_metadata.py @@ -236,7 +236,15 @@ class TestMambaPrefillTrackMetadata(unittest.TestCase): override.install() self.addCleanup(override.restore) batch = ScheduleBatch( - reqs=[SimpleNamespace(rid="one", lora_id=None, token_type_ids=None)], + reqs=[ + SimpleNamespace( + rid="one", + lora_id=None, + token_type_ids=None, + extend_end=65, + full_untruncated_fill_ids=[3] * 65, + ) + ], device="cpu", forward_mode=ForwardMode.EXTEND, input_ids=torch.tensor([3]), diff --git a/test/registered/unit/managers/test_schedule_batch_out_of_place.py b/test/registered/unit/managers/test_schedule_batch_out_of_place.py index 416fe85c42..55d9ccd578 100644 --- a/test/registered/unit/managers/test_schedule_batch_out_of_place.py +++ b/test/registered/unit/managers/test_schedule_batch_out_of_place.py @@ -2,6 +2,7 @@ import dataclasses import types import unittest from array import array +from itertools import product from unittest.mock import MagicMock, patch import torch @@ -13,10 +14,20 @@ maybe_stub_sgl_kernel() from sglang.srt.managers import schedule_batch # noqa: E402 from sglang.srt.managers.schedule_batch import ( # noqa: E402 + Req, ScheduleBatch, split_cached_prefix_by_tier, ) -from sglang.srt.model_executor.forward_batch_info import ForwardMode # noqa: E402 +from sglang.srt.managers.scheduler_components.batch_result_processor import ( # noqa: E402 + SchedulerBatchResultProcessor, +) +from sglang.srt.model_executor.forward_batch_info import ( # noqa: E402 + CaptureHiddenMode, + ForwardBatch, + ForwardMode, +) +from sglang.srt.runtime_context import get_context # noqa: E402 +from sglang.srt.sampling.sampling_params import SamplingParams # noqa: E402 from sglang.srt.speculative.spec_info import SpeculativeAlgorithm # noqa: E402 register_cpu_ci(est_time=11, suite="base-a-test-cpu") @@ -140,6 +151,130 @@ class _FakeReq: return len(self.origin_input_ids) + len(self.output_ids) +class TestForwardBatchPrefillCompletion(unittest.TestCase): + def test_first_commit_or_zero_token_completion_keeps_gate_open(self): + runner = types.SimpleNamespace( + device="cpu", + is_draft_worker=False, + model_config=types.SimpleNamespace( + requires_mm_token_modalities=False, model_is_mrope=False + ), + kv_index_translator=MagicMock(), + prefill_attention_backend_str="torch_native", + ngram_embedding_manager=types.SimpleNamespace(enabled=False), + lora_manager=None, + attn_backend=MagicMock(), + attn_tp_sequence_sharded=lambda _: False, + ) + for algorithm, (output_len, end, mixed, zero_tokens, completing) in product( + ( + SpeculativeAlgorithm.NONE, + SpeculativeAlgorithm.EAGLE, + SpeculativeAlgorithm.EAGLE3, + ), + ( + (0, 4, False, False, False), + (0, 8, False, False, True), + (0, 4, True, False, False), + (0, 8, True, False, True), + (2, 8, False, False, False), + (2, 10, False, False, True), + (0, 4, False, True, False), + (0, 8, False, True, True), + (0, 4, True, True, False), + (0, 8, True, True, True), + ), + ): + with self.subTest( + algorithm=algorithm, + output_len=output_len, + end=end, + mixed=mixed, + zero_tokens=zero_tokens, + ): + incoming = Req( + "incoming", + "", + array("q", range(8)), + SamplingParams(max_new_tokens=0 if zero_tokens else 16), + ) + incoming.sampling_params.normalize(None) + incoming.output_ids.extend(range(output_len)) + incoming.refresh_fill_ids() + incoming.extend_end = end + running = Req("running", "", array("q", range(8)), SamplingParams()) + running.output_ids.extend([0, 1]) + running.refresh_fill_ids() + running.extend_end = 10 + reqs = [incoming, running] if mixed else [incoming] + prefix = [2, 9][: len(reqs)] + lengths = [req.extend_end - start for req, start in zip(reqs, prefix)] + seq_lens = torch.tensor([req.extend_end for req in reqs]) + batch = ScheduleBatch( + reqs=reqs, + decoding_reqs=[running] if mixed else None, + device="cpu", + model_config=types.SimpleNamespace( + is_encoder_decoder=False, is_matryoshka=False + ), + spec_algorithm=algorithm, + forward_mode=ForwardMode.MIXED if mixed else ForwardMode.EXTEND, + input_ids=torch.zeros(sum(lengths), dtype=torch.int64), + req_pool_indices=torch.arange(len(reqs)), + req_pool_indices_cpu=torch.arange(len(reqs)), + seq_lens=seq_lens, + seq_lens_cpu=seq_lens, + seq_lens_sum=int(seq_lens.sum()), + orig_seq_lens=torch.tensor( + [len(req.full_untruncated_fill_ids) for req in reqs] + ), + out_cache_loc=torch.arange(sum(lengths)), + prefix_lens=prefix, + extend_lens=lengths, + extend_num_tokens=sum(lengths), + extend_logprob_start_lens=[0] * len(reqs), + ) + with get_context().override_server_args(device="cpu"): + forward = ForwardBatch.init_new( + batch, + runner, + capture_hidden_mode=CaptureHiddenMode.NULL, + return_hidden_states_before_norm=False, + ) + output_starts = [len(req.output_ids) for req in reqs] + if completing: + incoming.output_ids.append(100) + incoming.update_finish_state() + if mixed: + running.output_ids.append(101) + commits = SchedulerBatchResultProcessor._build_auxiliary_commits( + batch, output_starts + ) + if completing and zero_tokens: + self.assertTrue(incoming.finished()) + self.assertEqual(incoming.finished_len, 0) + self.assertIsNone(commits[0]) + required_rows = [ + i + for i, (req, commit) in enumerate(zip(reqs, commits, strict=True)) + if (commit is not None and commit.output_index == 0) + or (req.finished() and req.finished_len == 0) + ] + self.assertEqual(bool(required_rows), completing and output_len == 0) + for i in required_rows: + self.assertTrue(forward.contains_last_prefill_chunk) + self.assertEqual( + int(forward.extend_prefix_lens[i] + forward.extend_seq_lens[i]), + int(forward.orig_seq_lens[i]), + ) + self.assertEqual(forward.contains_last_prefill_chunk, completing) + incoming.extend_end = 4 if completing else 8 + output_len + self.assertEqual(forward.contains_last_prefill_chunk, completing) + self.assertEqual( + dataclasses.replace(forward).contains_last_prefill_chunk, completing + ) + + class TestMergeBatchOutOfPlace(unittest.TestCase): def test_merge_batch_rebinds_lists_without_mutating_either_side(self): """merge_batch must build new list objects; no field of either side may be mutated in place."""