[Fix] dLLM: Preserve prompt tails and literal mask tokens across prefix reuse (#39592)

Signed-off-by: 0z5a <dezhen.lu@student.uni-tuebingen.de>
Co-authored-by: Richard Wang <wangrichard08@gmail.com>
This commit is contained in:
0z5a
2026-10-19 14:48:37 -07:00
committed by GitHub
co-authored by Richard Wang
parent fdc2dd0d05
commit 80ea5ee638
10 changed files with 268 additions and 31 deletions
+7 -1
View File
@@ -98,9 +98,15 @@ class DllmAlgorithm:
return self._run_sync(model_runner, forward_batch)
def _block_start_list(self, forward_batch: ForwardBatch) -> List[int]:
return self._prompt_mask(forward_batch).sum(dim=1).tolist()
def _prompt_mask(self, forward_batch: ForwardBatch) -> torch.Tensor:
prompt_mask = forward_batch.dllm_prompt_mask
if prompt_mask is not None:
return prompt_mask
batch_size = forward_batch.batch_size
input_ids = forward_batch.input_ids.view(batch_size, self.block_size)
return (input_ids != self.mask_id).sum(dim=1).tolist()
return input_ids != self.mask_id
def _run_sync(
self, model_runner: ModelRunner, forward_batch: ForwardBatch
@@ -60,7 +60,7 @@ def joint_threshold_update_step_vectorized(
-1
)
mask_pos = input_ids.eq(mask_id)
mask_pos = input_ids.eq(mask_id) & ~prompt_masks
has_mask = mask_pos.any(dim=1)
# ---------- post-edit ----------
@@ -134,10 +134,9 @@ class JointThreshold(DllmAlgorithm):
def init_step_state(self, forward_batch: ForwardBatch) -> List[Any]:
batch_size = forward_batch.batch_size
input_ids = forward_batch.input_ids.view(batch_size, self.block_size)
# Built once as a GPU tensor and reused across steps (no per-step
# host/device transfer); the FDFO carry keeps it in-process.
prompt_mask = input_ids != self.mask_id
prompt_mask = self._prompt_mask(forward_batch)
if self._use_shared_state:
# One shared batched state, advanced in place across every step of the
# synchronous loop.
@@ -278,7 +277,7 @@ class JointThreshold(DllmAlgorithm):
-1,
)
mask_index = curr_input_ids == self.mask_id
mask_index = (curr_input_ids == self.mask_id) & ~curr_prompt_mask
has_mask = mask_index.any()
# Mask to token (M2T)
@@ -53,16 +53,22 @@ class JointThresholdInDel(DllmAlgorithm):
def init_step_state(self, forward_batch: ForwardBatch) -> List[Any]:
batch_size = forward_batch.batch_size
input_ids = forward_batch.input_ids.view(batch_size, self.block_size)
# The prompt is the immutable contiguous prefix before the first mask.
# is_orig_mask marks positions associated with masks in the initial
# block. DELETE/SPLIT move the marker with the corresponding token.
prompt_lens = (input_ids != self.mask_id).cumprod(dim=1).sum(dim=1).tolist()
positions = torch.arange(self.block_size, device=input_ids.device)
# Explicit metadata distinguishes literal mask IDs in committed tokens
# from masks to denoise. Batches without literal mask IDs carry no
# mask, and their committed tokens are the contiguous non-mask prefix.
prompt_mask = forward_batch.dllm_prompt_mask
if prompt_mask is None:
prompt_lens = (input_ids != self.mask_id).cumprod(dim=1).sum(dim=1)
positions = torch.arange(self.block_size, device=input_ids.device)
prompt_mask = positions[None, :] < prompt_lens[:, None]
prompt_lens = prompt_mask.sum(dim=1).tolist()
# DELETE/SPLIT move these markers with the corresponding suffix token.
orig_mask = torch.where(prompt_mask, False, input_ids == self.mask_id)
return [
{
"prompt_len": prompt_lens[i],
"prompt_index": positions < prompt_lens[i],
"is_orig_mask": input_ids[i] == self.mask_id,
"prompt_index": prompt_mask[i],
"is_orig_mask": orig_mask[i],
"post_edit_steps": 0,
"num_update_steps": 0,
"finished": False,
@@ -146,7 +152,9 @@ class JointThresholdInDel(DllmAlgorithm):
-1,
)
mask_index = curr_input_ids == self.mask_id
mask_index = torch.where(
curr_prompt_index, False, curr_input_ids == self.mask_id
)
is_orig_mask = state["is_orig_mask"]
original_mask_index = is_orig_mask & mask_index
original_mask_count, total_mask_count = torch.stack(
@@ -27,6 +27,9 @@ class LowConfidence(DllmAlgorithm):
logits = full_logits.view(batch_size, self.block_size, vocab_size)
input_ids = forward_batch.input_ids.view(batch_size, self.block_size)
block_mask_index = input_ids == self.mask_id
prompt_mask = forward_batch.dllm_prompt_mask
if prompt_mask is not None:
block_mask_index = torch.where(prompt_mask, False, block_mask_index)
done = block_mask_index.sum(dim=1) == 0
x = torch.argmax(logits, dim=-1)
+18 -6
View File
@@ -24,6 +24,7 @@ class ReqDllmMixin:
self.dllm_algo_state = None
self.dllm_block_offset = 0
self.dllm_config = dllm_config
self.dllm_has_literal_mask = False
if self.dllm_config is not None:
if self.dllm_config.requires_separate_context_encoding:
@@ -60,12 +61,15 @@ class ReqDllmMixin:
if len(self.full_untruncated_fill_ids) < min_required_length:
return
input_block = self.full_untruncated_fill_ids[prefix_length:min_required_length]
self.dllm_phase = (
DllmReqPhase.STAGING_PREFILL
if self.dllm_config.mask_id not in input_block
else DllmReqPhase.STAGING_DECODE
)
# Retraction preserves output_ids while releasing KV. Re-encode all
# committed tokens, including literal mask IDs, before generating more.
committed_len = len(self.origin_input_ids) + len(self.output_ids)
is_prefill_phase = min_required_length <= committed_len
if is_prefill_phase:
self.dllm_phase = DllmReqPhase.STAGING_PREFILL
else:
self.dllm_phase = DllmReqPhase.STAGING_DECODE
def _init_fill_ids_for_dllm(self: Req):
if self.dllm_incomplete_ids:
@@ -79,6 +83,14 @@ class ReqDllmMixin:
# non-incomplete path which also defers it to the adder.
return
if not self.dllm_initialized:
# Committed tokens are immutable, so a literal mask ID among them
# needs the explicit prompt mask. Recomputed after retraction.
mask_id = self.dllm_config.mask_id
self.dllm_has_literal_mask = (
mask_id in self.origin_input_ids or mask_id in self.output_ids
)
if self.dllm_config.requires_separate_context_encoding:
self.refresh_fill_ids()
self.dllm_block_offset = self.seqlen
+17 -1
View File
@@ -1711,7 +1711,6 @@ class Req(ReqDllmMixin):
):
if self.is_dllm():
self._init_fill_ids_for_dllm()
self.determine_dllm_phase()
else:
self.refresh_fill_ids()
@@ -1761,6 +1760,9 @@ class Req(ReqDllmMixin):
if self.is_dllm():
self._update_block_offset_for_dllm()
if self.is_dllm():
self.determine_dllm_phase()
if (
self.is_retracted
and self.multimodal_inputs is not None
@@ -1786,6 +1788,15 @@ class Req(ReqDllmMixin):
max_prefix_len = min(max_prefix_len, self.dllm_block_offset)
if self.return_logprob and self.logprob_start_len >= 0:
max_prefix_len = min(max_prefix_len, self.logprob_start_len)
if self.is_dllm():
# Appended masks and a partial prompt/output block are not reusable
# KV: bidirectional attention depends on the rest of that block.
# Bound the lookup by real tokens, even if another cached prompt
# happens to contain the same token IDs as our synthetic masks.
block_size = self.dllm_config.block_size
committed_len = len(self.origin_input_ids) + len(self.output_ids)
max_prefix_len = min(max_prefix_len, committed_len)
max_prefix_len = max_prefix_len // block_size * block_size
return max(max_prefix_len, 0)
# Based on https://github.com/vllm-project/vllm/blob/7a64d24aad69e4d2548aa0bf528d9fe63428ab01/vllm/transformers_utils/detokenizer.py#L194-L313
@@ -2932,6 +2943,11 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
# flag will always True
if not req.retracted_stain:
new_cached = pre_len - req.already_computed
if req.is_dllm():
# Denoising revisits the current block while only earlier
# blocks are a reusable prefix. Recomputing that block must
# not subtract its size from cross-request cache hits.
new_cached = max(0, new_cached)
req.cached_tokens += new_cached
# Calculate detailed breakdown of cached tokens by source (for HiCache)
@@ -653,6 +653,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
# === Forward-derived (built in init_new on the forward stream; FB-owned) ===
# Position information
positions: torch.Tensor = None
# dLLM committed positions are immutable, including literal mask-token IDs.
# None when no request in the batch has a literal mask ID.
dllm_prompt_mask: Optional[torch.Tensor] = None
# For extend
extend_num_tokens: Optional[int] = None
@@ -1164,14 +1167,35 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
if batch.dllm_config is not None and ret.forward_mode.is_dllm_extend():
block_size = batch.dllm_config.block_size
positions_dtype = torch.int64 if is_hip() or _is_npu else torch.int32
ret.positions = torch.tensor(
[
i
for block_offset in (req.dllm_block_offset for req in batch.reqs)
for i in range(block_offset, block_offset + block_size)
],
dtype=positions_dtype,
).to(device, non_blocking=True)
positions = [
i
for block_offset in (req.dllm_block_offset for req in batch.reqs)
for i in range(block_offset, block_offset + block_size)
]
if any(req.dllm_has_literal_mask for req in batch.reqs):
# Committed outputs are immutable too when a retracted request
# rebuilds its KV. Transfer their lengths with the block
# positions to avoid an extra, blocking H2D copy.
committed_lens = [
len(req.origin_input_ids) + len(req.output_ids)
for req in batch.reqs
]
position_info = torch.tensor(
positions + committed_lens,
dtype=positions_dtype,
pin_memory=is_pin_memory_available(device),
).to(device, non_blocking=True)
ret.positions = position_info[: len(positions)]
prompt_lens = position_info[len(positions) :]
ret.dllm_prompt_mask = (
ret.positions.view(-1, block_size) < prompt_lens[:, None]
)
else:
# Without literal mask IDs among committed tokens, every mask
# ID in the block is still to be denoised.
ret.positions = torch.tensor(positions, dtype=positions_dtype).to(
device, non_blocking=True
)
elif (
ret.spec_info is not None
and getattr(ret.spec_info, "positions", None) is not None