diff --git a/python/sglang/srt/dllm/algorithm/base.py b/python/sglang/srt/dllm/algorithm/base.py index 47a221ed4a..cbafc7239d 100644 --- a/python/sglang/srt/dllm/algorithm/base.py +++ b/python/sglang/srt/dllm/algorithm/base.py @@ -98,9 +98,15 @@ class DllmAlgorithm: return self._run_sync(model_runner, forward_batch) def _block_start_list(self, forward_batch: ForwardBatch) -> List[int]: + return self._prompt_mask(forward_batch).sum(dim=1).tolist() + + def _prompt_mask(self, forward_batch: ForwardBatch) -> torch.Tensor: + prompt_mask = forward_batch.dllm_prompt_mask + if prompt_mask is not None: + return prompt_mask batch_size = forward_batch.batch_size input_ids = forward_batch.input_ids.view(batch_size, self.block_size) - return (input_ids != self.mask_id).sum(dim=1).tolist() + return input_ids != self.mask_id def _run_sync( self, model_runner: ModelRunner, forward_batch: ForwardBatch diff --git a/python/sglang/srt/dllm/algorithm/joint_threshold.py b/python/sglang/srt/dllm/algorithm/joint_threshold.py index 00769a9249..ee074b3795 100644 --- a/python/sglang/srt/dllm/algorithm/joint_threshold.py +++ b/python/sglang/srt/dllm/algorithm/joint_threshold.py @@ -60,7 +60,7 @@ def joint_threshold_update_step_vectorized( -1 ) - mask_pos = input_ids.eq(mask_id) + mask_pos = input_ids.eq(mask_id) & ~prompt_masks has_mask = mask_pos.any(dim=1) # ---------- post-edit ---------- @@ -134,10 +134,9 @@ class JointThreshold(DllmAlgorithm): def init_step_state(self, forward_batch: ForwardBatch) -> List[Any]: batch_size = forward_batch.batch_size - input_ids = forward_batch.input_ids.view(batch_size, self.block_size) # Built once as a GPU tensor and reused across steps (no per-step # host/device transfer); the FDFO carry keeps it in-process. - prompt_mask = input_ids != self.mask_id + prompt_mask = self._prompt_mask(forward_batch) if self._use_shared_state: # One shared batched state, advanced in place across every step of the # synchronous loop. @@ -278,7 +277,7 @@ class JointThreshold(DllmAlgorithm): -1, ) - mask_index = curr_input_ids == self.mask_id + mask_index = (curr_input_ids == self.mask_id) & ~curr_prompt_mask has_mask = mask_index.any() # Mask to token (M2T) diff --git a/python/sglang/srt/dllm/algorithm/joint_threshold_indel.py b/python/sglang/srt/dllm/algorithm/joint_threshold_indel.py index a808ca14fe..239526d93b 100644 --- a/python/sglang/srt/dllm/algorithm/joint_threshold_indel.py +++ b/python/sglang/srt/dllm/algorithm/joint_threshold_indel.py @@ -53,16 +53,22 @@ class JointThresholdInDel(DllmAlgorithm): def init_step_state(self, forward_batch: ForwardBatch) -> List[Any]: batch_size = forward_batch.batch_size input_ids = forward_batch.input_ids.view(batch_size, self.block_size) - # The prompt is the immutable contiguous prefix before the first mask. - # is_orig_mask marks positions associated with masks in the initial - # block. DELETE/SPLIT move the marker with the corresponding token. - prompt_lens = (input_ids != self.mask_id).cumprod(dim=1).sum(dim=1).tolist() - positions = torch.arange(self.block_size, device=input_ids.device) + # Explicit metadata distinguishes literal mask IDs in committed tokens + # from masks to denoise. Batches without literal mask IDs carry no + # mask, and their committed tokens are the contiguous non-mask prefix. + prompt_mask = forward_batch.dllm_prompt_mask + if prompt_mask is None: + prompt_lens = (input_ids != self.mask_id).cumprod(dim=1).sum(dim=1) + positions = torch.arange(self.block_size, device=input_ids.device) + prompt_mask = positions[None, :] < prompt_lens[:, None] + prompt_lens = prompt_mask.sum(dim=1).tolist() + # DELETE/SPLIT move these markers with the corresponding suffix token. + orig_mask = torch.where(prompt_mask, False, input_ids == self.mask_id) return [ { "prompt_len": prompt_lens[i], - "prompt_index": positions < prompt_lens[i], - "is_orig_mask": input_ids[i] == self.mask_id, + "prompt_index": prompt_mask[i], + "is_orig_mask": orig_mask[i], "post_edit_steps": 0, "num_update_steps": 0, "finished": False, @@ -146,7 +152,9 @@ class JointThresholdInDel(DllmAlgorithm): -1, ) - mask_index = curr_input_ids == self.mask_id + mask_index = torch.where( + curr_prompt_index, False, curr_input_ids == self.mask_id + ) is_orig_mask = state["is_orig_mask"] original_mask_index = is_orig_mask & mask_index original_mask_count, total_mask_count = torch.stack( diff --git a/python/sglang/srt/dllm/algorithm/low_confidence.py b/python/sglang/srt/dllm/algorithm/low_confidence.py index 4185467816..79679fdfef 100644 --- a/python/sglang/srt/dllm/algorithm/low_confidence.py +++ b/python/sglang/srt/dllm/algorithm/low_confidence.py @@ -27,6 +27,9 @@ class LowConfidence(DllmAlgorithm): logits = full_logits.view(batch_size, self.block_size, vocab_size) input_ids = forward_batch.input_ids.view(batch_size, self.block_size) block_mask_index = input_ids == self.mask_id + prompt_mask = forward_batch.dllm_prompt_mask + if prompt_mask is not None: + block_mask_index = torch.where(prompt_mask, False, block_mask_index) done = block_mask_index.sum(dim=1) == 0 x = torch.argmax(logits, dim=-1) diff --git a/python/sglang/srt/dllm/mixin/req.py b/python/sglang/srt/dllm/mixin/req.py index bff77f5789..926d23829c 100644 --- a/python/sglang/srt/dllm/mixin/req.py +++ b/python/sglang/srt/dllm/mixin/req.py @@ -24,6 +24,7 @@ class ReqDllmMixin: self.dllm_algo_state = None self.dllm_block_offset = 0 self.dllm_config = dllm_config + self.dllm_has_literal_mask = False if self.dllm_config is not None: if self.dllm_config.requires_separate_context_encoding: @@ -60,12 +61,15 @@ class ReqDllmMixin: if len(self.full_untruncated_fill_ids) < min_required_length: return - input_block = self.full_untruncated_fill_ids[prefix_length:min_required_length] - self.dllm_phase = ( - DllmReqPhase.STAGING_PREFILL - if self.dllm_config.mask_id not in input_block - else DllmReqPhase.STAGING_DECODE - ) + # Retraction preserves output_ids while releasing KV. Re-encode all + # committed tokens, including literal mask IDs, before generating more. + committed_len = len(self.origin_input_ids) + len(self.output_ids) + is_prefill_phase = min_required_length <= committed_len + + if is_prefill_phase: + self.dllm_phase = DllmReqPhase.STAGING_PREFILL + else: + self.dllm_phase = DllmReqPhase.STAGING_DECODE def _init_fill_ids_for_dllm(self: Req): if self.dllm_incomplete_ids: @@ -79,6 +83,14 @@ class ReqDllmMixin: # non-incomplete path which also defers it to the adder. return + if not self.dllm_initialized: + # Committed tokens are immutable, so a literal mask ID among them + # needs the explicit prompt mask. Recomputed after retraction. + mask_id = self.dllm_config.mask_id + self.dllm_has_literal_mask = ( + mask_id in self.origin_input_ids or mask_id in self.output_ids + ) + if self.dllm_config.requires_separate_context_encoding: self.refresh_fill_ids() self.dllm_block_offset = self.seqlen diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py old mode 100755 new mode 100644 index 8ef50abfd7..05df1728f4 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -1711,7 +1711,6 @@ class Req(ReqDllmMixin): ): if self.is_dllm(): self._init_fill_ids_for_dllm() - self.determine_dllm_phase() else: self.refresh_fill_ids() @@ -1761,6 +1760,9 @@ class Req(ReqDllmMixin): if self.is_dllm(): self._update_block_offset_for_dllm() + if self.is_dllm(): + self.determine_dllm_phase() + if ( self.is_retracted and self.multimodal_inputs is not None @@ -1786,6 +1788,15 @@ class Req(ReqDllmMixin): max_prefix_len = min(max_prefix_len, self.dllm_block_offset) if self.return_logprob and self.logprob_start_len >= 0: max_prefix_len = min(max_prefix_len, self.logprob_start_len) + if self.is_dllm(): + # Appended masks and a partial prompt/output block are not reusable + # KV: bidirectional attention depends on the rest of that block. + # Bound the lookup by real tokens, even if another cached prompt + # happens to contain the same token IDs as our synthetic masks. + block_size = self.dllm_config.block_size + committed_len = len(self.origin_input_ids) + len(self.output_ids) + max_prefix_len = min(max_prefix_len, committed_len) + max_prefix_len = max_prefix_len // block_size * block_size return max(max_prefix_len, 0) # Based on https://github.com/vllm-project/vllm/blob/7a64d24aad69e4d2548aa0bf528d9fe63428ab01/vllm/transformers_utils/detokenizer.py#L194-L313 @@ -2932,6 +2943,11 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): # flag will always True if not req.retracted_stain: new_cached = pre_len - req.already_computed + if req.is_dllm(): + # Denoising revisits the current block while only earlier + # blocks are a reusable prefix. Recomputing that block must + # not subtract its size from cross-request cache hits. + new_cached = max(0, new_cached) req.cached_tokens += new_cached # Calculate detailed breakdown of cached tokens by source (for HiCache) diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index fb9f088293..d45a1b038e 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -653,6 +653,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): # === Forward-derived (built in init_new on the forward stream; FB-owned) === # Position information positions: torch.Tensor = None + # dLLM committed positions are immutable, including literal mask-token IDs. + # None when no request in the batch has a literal mask ID. + dllm_prompt_mask: Optional[torch.Tensor] = None # For extend extend_num_tokens: Optional[int] = None @@ -1164,14 +1167,35 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): if batch.dllm_config is not None and ret.forward_mode.is_dllm_extend(): block_size = batch.dllm_config.block_size positions_dtype = torch.int64 if is_hip() or _is_npu else torch.int32 - ret.positions = torch.tensor( - [ - i - for block_offset in (req.dllm_block_offset for req in batch.reqs) - for i in range(block_offset, block_offset + block_size) - ], - dtype=positions_dtype, - ).to(device, non_blocking=True) + positions = [ + i + for block_offset in (req.dllm_block_offset for req in batch.reqs) + for i in range(block_offset, block_offset + block_size) + ] + if any(req.dllm_has_literal_mask for req in batch.reqs): + # Committed outputs are immutable too when a retracted request + # rebuilds its KV. Transfer their lengths with the block + # positions to avoid an extra, blocking H2D copy. + committed_lens = [ + len(req.origin_input_ids) + len(req.output_ids) + for req in batch.reqs + ] + position_info = torch.tensor( + positions + committed_lens, + dtype=positions_dtype, + pin_memory=is_pin_memory_available(device), + ).to(device, non_blocking=True) + ret.positions = position_info[: len(positions)] + prompt_lens = position_info[len(positions) :] + ret.dllm_prompt_mask = ( + ret.positions.view(-1, block_size) < prompt_lens[:, None] + ) + else: + # Without literal mask IDs among committed tokens, every mask + # ID in the block is still to be denoised. + ret.positions = torch.tensor(positions, dtype=positions_dtype).to( + device, non_blocking=True + ) elif ( ret.spec_info is not None and getattr(ret.spec_info, "positions", None) is not None diff --git a/test/registered/unit/dllm/algorithm/test_joint_threshold_indel.py b/test/registered/unit/dllm/algorithm/test_joint_threshold_indel.py index 6ab5992dbc..0b7f7a11d0 100644 --- a/test/registered/unit/dllm/algorithm/test_joint_threshold_indel.py +++ b/test/registered/unit/dllm/algorithm/test_joint_threshold_indel.py @@ -57,6 +57,7 @@ def make_batch(blocks): return SimpleNamespace( batch_size=input_ids.shape[0], input_ids=input_ids.flatten(), + dllm_prompt_mask=None, ) @@ -97,6 +98,35 @@ class TestJointThresholdInDelInitialization(CustomTestCase): self.assertEqual(algorithm.max_regular_update_steps, 9) self.assertEqual(algorithm.max_steps(6), 11) + def test_literal_prompt_masks_are_immutable_and_not_original_masks(self): + for fdfo in (False, True): + with self.subTest(fdfo=fdfo): + algorithm = JointThresholdInDel(make_config(block_size=4, fdfo=fdfo)) + batch = make_batch([[MASK, 3, MASK, MASK]]) + batch.dllm_prompt_mask = torch.tensor([[True, True, False, False]]) + state = algorithm.init_step_state(batch)[0] + self.assertEqual(state["prompt_len"], 2) + self.assertEqual( + state["is_orig_mask"].tolist(), [False, False, True, True] + ) + runner = Mock() + runner.forward.side_effect = lambda *args, **kwargs: make_model_output( + make_logits([[7]] * 4) + ) + states = None + for _ in range(32): + _, output, accepted, states, _ = algorithm.run( + runner, batch, states + ) + self.assertEqual(batch.input_ids[:2].tolist(), [MASK, 3]) + if not fdfo or accepted == [4]: + break + else: + self.fail("denoising did not finish") + self.assertEqual(batch.input_ids.tolist(), [MASK, 3, 7, 7]) + if not fdfo: + self.assertEqual(output[0].tolist(), [7, 7]) + def test_prompt_is_the_contiguous_prefix_not_every_initial_non_mask(self): algorithm = JointThresholdInDel(make_config(block_size=5)) batch = make_batch([[3, MASK, 4, MASK, MASK]]) diff --git a/test/registered/unit/dllm/test_gemma4_renoise.py b/test/registered/unit/dllm/test_gemma4_renoise.py index 12335de663..7ac7756a37 100644 --- a/test/registered/unit/dllm/test_gemma4_renoise.py +++ b/test/registered/unit/dllm/test_gemma4_renoise.py @@ -46,6 +46,7 @@ def _batch(rids, block_size, *, sampling_seeds=None, encoder=False, empty=False) forward_mode=(ForwardMode.EXTEND if encoder else ForwardMode.DLLM_EXTEND), input_embeds=None, dllm_input_preparation_state=None, + dllm_prompt_mask=None, ) diff --git a/test/registered/unit/mem_cache/test_dllm_fdfo_kv_reuse.py b/test/registered/unit/mem_cache/test_dllm_fdfo_kv_reuse.py index bfd901ce3d..e153dd8d33 100644 --- a/test/registered/unit/mem_cache/test_dllm_fdfo_kv_reuse.py +++ b/test/registered/unit/mem_cache/test_dllm_fdfo_kv_reuse.py @@ -1,17 +1,23 @@ -"""Tests dLLM FDFO KV slot reuse in alloc_for_extend.""" +"""Tests dLLM KV reuse and committed-token boundaries.""" import unittest from array import array from types import SimpleNamespace -from unittest.mock import Mock +from unittest.mock import Mock, patch import torch +from sglang.srt.dllm.algorithm.joint_threshold import JointThreshold +from sglang.srt.dllm.algorithm.low_confidence import LowConfidence +from sglang.srt.dllm.config import DllmConfig from sglang.srt.dllm.mixin.scheduler import DllmManager, SchedulerDllmMixin -from sglang.srt.managers.schedule_batch import ReqKvInfo +from sglang.srt.managers.schedule_batch import Req, ReqKvInfo, ScheduleBatch from sglang.srt.mem_cache.allocation import alloc_for_extend +from sglang.srt.mem_cache.base_prefix_cache import InsertParams from sglang.srt.mem_cache.memory_pool import ReqToTokenPool +from sglang.srt.mem_cache.radix_cache import RadixCache, RadixKey from sglang.srt.runtime_context import get_context +from sglang.srt.sampling.sampling_params import SamplingParams from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=11, suite="base-a-test-cpu") @@ -270,5 +276,137 @@ class TestDllmFdfoResolvedBlockKeepsRow(unittest.TestCase): self.assertNotIn(row, pool.free_slots) +class TestDllmCommittedTokens(unittest.TestCase): + @staticmethod + def make_req(prompt, output=()): + req = Req( + rid="committed-tokens", + origin_input_text="", + origin_input_ids=array("q", prompt), + sampling_params=SamplingParams(max_new_tokens=16, temperature=0), + vocab_size=128, + dllm_config=DllmConfig("LowConfidence", {}, 4, 99, 4), + ) + req.output_ids.extend(output) + return req + + def test_cache_lookup_stops_before_partial_prompt_and_synthetic_masks(self): + # A different request legitimately cached these literal mask IDs. Only + # the first complete block belongs to this request's committed prefix. + cache = RadixCache.create_simulated(page_size=4) + cache.insert( + InsertParams( + key=RadixKey(array("q", [1, 2, 3, 4, 5, 99, 99, 99])), + value=torch.arange(8), + ) + ) + req = self.make_req([1, 2, 3, 4, 5]) + req.init_next_round_input(cache) + self.assertEqual(req.prefix_len, 4) + self.assertEqual(req.dllm_block_offset, 4) + self.assertFalse(req.is_dllm_prefill()) + self.assertFalse(req.dllm_has_literal_mask) + self.assertEqual(list(req.full_untruncated_fill_ids[4:8]), [5, 99, 99, 99]) + + def test_retraction_reencodes_committed_outputs_before_denoising(self): + req = self.make_req([1, 99, 2], [7, 99, 8, 9, 10]) + req.reset_for_retract() + req.init_next_round_input() + self.assertEqual(list(req.output_ids), [7, 99, 8, 9, 10]) + self.assertTrue(req.dllm_has_literal_mask) + self.assertTrue(req.is_dllm_prefill()) + # The second complete block consists of already-emitted output tokens. + req.prefix_len = 4 + req.determine_dllm_phase() + self.assertTrue(req.is_dllm_prefill()) + req.prefix_len = 8 + req.determine_dllm_phase() + self.assertFalse(req.is_dllm_prefill()) + + def test_repeated_denoise_passes_do_not_subtract_cache_hits(self): + req = self.make_req([1, 2, 3, 4, 5]) + req._init_fill_ids_for_dllm() + req.prefix_len = 4 + req.extend_end = 8 + req.already_computed = 8 + req.cached_tokens = 4 + req._cache_breakdown_computed = True + batch = ScheduleBatch( + reqs=[req], + device="cpu", + dllm_config=req.dllm_config, + model_config=SimpleNamespace(is_encoder_decoder=False, vocab_size=128), + ) + execution = SimpleNamespace( + features=SimpleNamespace(enable_encoder_swa_bounded_replay=False), + mamba=SimpleNamespace(enable_mamba_extra_buffer=False), + ) + # Allocation and sampling are collaborators; exercise the scheduler's + # real cache accounting on two revisits of the same fixed-size block. + with ( + patch( + "sglang.srt.managers.schedule_batch.get_exec", return_value=execution + ), + patch( + "sglang.srt.managers.schedule_batch.alloc_for_extend", + return_value=(torch.arange(4), torch.tensor([0]), torch.tensor([0])), + ), + patch( + "sglang.srt.managers.schedule_batch.SamplingBatchInfo.from_schedule_batch" + ), + ): + for _ in range(2): + batch.prepare_for_extend() + self.assertEqual(req.cached_tokens, 4) + + def test_literal_prompt_masks_are_preserved_and_not_emitted(self): + class Runner: + def forward(self, batch, **kwargs): + logits = torch.zeros(4, 16) + logits[:, 7] = 20 + return SimpleNamespace( + logits_output=SimpleNamespace(full_logits=logits), + can_run_graph=False, + ) + + for cls, vectorized in ( + (LowConfidence, False), + (JointThreshold, False), + (JointThreshold, True), + ): + for fdfo in (False, True): + with self.subTest( + algorithm=cls.__name__, vectorized=vectorized, fdfo=fdfo + ): + algorithm = cls( + DllmConfig( + cls.__name__, + {"vectorized_decoding": vectorized}, + 4, + 15, + 1, + fdfo, + ) + ) + batch = SimpleNamespace( + batch_size=1, + input_ids=torch.tensor([15, 2, 15, 15]), + dllm_prompt_mask=torch.tensor([[True, True, False, False]]), + ) + states = None + for _ in range(32): + _, output, accepted, states, _ = algorithm.run( + Runner(), batch, states + ) + self.assertEqual(batch.input_ids[:2].tolist(), [15, 2]) + if not fdfo or accepted == [4]: + break + else: + self.fail("denoising did not finish") + self.assertEqual(batch.input_ids.tolist(), [15, 2, 7, 7]) + if not fdfo: + self.assertEqual(output[0].tolist(), [7, 7]) + + if __name__ == "__main__": unittest.main()