[Fix] dLLM: Preserve prompt tails and literal mask tokens across prefix reuse (#39592)
Signed-off-by: 0z5a <dezhen.lu@student.uni-tuebingen.de> Co-authored-by: Richard Wang <wangrichard08@gmail.com>
This commit is contained in:
@@ -98,9 +98,15 @@ class DllmAlgorithm:
|
||||
return self._run_sync(model_runner, forward_batch)
|
||||
|
||||
def _block_start_list(self, forward_batch: ForwardBatch) -> List[int]:
|
||||
return self._prompt_mask(forward_batch).sum(dim=1).tolist()
|
||||
|
||||
def _prompt_mask(self, forward_batch: ForwardBatch) -> torch.Tensor:
|
||||
prompt_mask = forward_batch.dllm_prompt_mask
|
||||
if prompt_mask is not None:
|
||||
return prompt_mask
|
||||
batch_size = forward_batch.batch_size
|
||||
input_ids = forward_batch.input_ids.view(batch_size, self.block_size)
|
||||
return (input_ids != self.mask_id).sum(dim=1).tolist()
|
||||
return input_ids != self.mask_id
|
||||
|
||||
def _run_sync(
|
||||
self, model_runner: ModelRunner, forward_batch: ForwardBatch
|
||||
|
||||
@@ -60,7 +60,7 @@ def joint_threshold_update_step_vectorized(
|
||||
-1
|
||||
)
|
||||
|
||||
mask_pos = input_ids.eq(mask_id)
|
||||
mask_pos = input_ids.eq(mask_id) & ~prompt_masks
|
||||
has_mask = mask_pos.any(dim=1)
|
||||
|
||||
# ---------- post-edit ----------
|
||||
@@ -134,10 +134,9 @@ class JointThreshold(DllmAlgorithm):
|
||||
|
||||
def init_step_state(self, forward_batch: ForwardBatch) -> List[Any]:
|
||||
batch_size = forward_batch.batch_size
|
||||
input_ids = forward_batch.input_ids.view(batch_size, self.block_size)
|
||||
# Built once as a GPU tensor and reused across steps (no per-step
|
||||
# host/device transfer); the FDFO carry keeps it in-process.
|
||||
prompt_mask = input_ids != self.mask_id
|
||||
prompt_mask = self._prompt_mask(forward_batch)
|
||||
if self._use_shared_state:
|
||||
# One shared batched state, advanced in place across every step of the
|
||||
# synchronous loop.
|
||||
@@ -278,7 +277,7 @@ class JointThreshold(DllmAlgorithm):
|
||||
-1,
|
||||
)
|
||||
|
||||
mask_index = curr_input_ids == self.mask_id
|
||||
mask_index = (curr_input_ids == self.mask_id) & ~curr_prompt_mask
|
||||
has_mask = mask_index.any()
|
||||
|
||||
# Mask to token (M2T)
|
||||
|
||||
@@ -53,16 +53,22 @@ class JointThresholdInDel(DllmAlgorithm):
|
||||
def init_step_state(self, forward_batch: ForwardBatch) -> List[Any]:
|
||||
batch_size = forward_batch.batch_size
|
||||
input_ids = forward_batch.input_ids.view(batch_size, self.block_size)
|
||||
# The prompt is the immutable contiguous prefix before the first mask.
|
||||
# is_orig_mask marks positions associated with masks in the initial
|
||||
# block. DELETE/SPLIT move the marker with the corresponding token.
|
||||
prompt_lens = (input_ids != self.mask_id).cumprod(dim=1).sum(dim=1).tolist()
|
||||
positions = torch.arange(self.block_size, device=input_ids.device)
|
||||
# Explicit metadata distinguishes literal mask IDs in committed tokens
|
||||
# from masks to denoise. Batches without literal mask IDs carry no
|
||||
# mask, and their committed tokens are the contiguous non-mask prefix.
|
||||
prompt_mask = forward_batch.dllm_prompt_mask
|
||||
if prompt_mask is None:
|
||||
prompt_lens = (input_ids != self.mask_id).cumprod(dim=1).sum(dim=1)
|
||||
positions = torch.arange(self.block_size, device=input_ids.device)
|
||||
prompt_mask = positions[None, :] < prompt_lens[:, None]
|
||||
prompt_lens = prompt_mask.sum(dim=1).tolist()
|
||||
# DELETE/SPLIT move these markers with the corresponding suffix token.
|
||||
orig_mask = torch.where(prompt_mask, False, input_ids == self.mask_id)
|
||||
return [
|
||||
{
|
||||
"prompt_len": prompt_lens[i],
|
||||
"prompt_index": positions < prompt_lens[i],
|
||||
"is_orig_mask": input_ids[i] == self.mask_id,
|
||||
"prompt_index": prompt_mask[i],
|
||||
"is_orig_mask": orig_mask[i],
|
||||
"post_edit_steps": 0,
|
||||
"num_update_steps": 0,
|
||||
"finished": False,
|
||||
@@ -146,7 +152,9 @@ class JointThresholdInDel(DllmAlgorithm):
|
||||
-1,
|
||||
)
|
||||
|
||||
mask_index = curr_input_ids == self.mask_id
|
||||
mask_index = torch.where(
|
||||
curr_prompt_index, False, curr_input_ids == self.mask_id
|
||||
)
|
||||
is_orig_mask = state["is_orig_mask"]
|
||||
original_mask_index = is_orig_mask & mask_index
|
||||
original_mask_count, total_mask_count = torch.stack(
|
||||
|
||||
@@ -27,6 +27,9 @@ class LowConfidence(DllmAlgorithm):
|
||||
logits = full_logits.view(batch_size, self.block_size, vocab_size)
|
||||
input_ids = forward_batch.input_ids.view(batch_size, self.block_size)
|
||||
block_mask_index = input_ids == self.mask_id
|
||||
prompt_mask = forward_batch.dllm_prompt_mask
|
||||
if prompt_mask is not None:
|
||||
block_mask_index = torch.where(prompt_mask, False, block_mask_index)
|
||||
done = block_mask_index.sum(dim=1) == 0
|
||||
|
||||
x = torch.argmax(logits, dim=-1)
|
||||
|
||||
@@ -24,6 +24,7 @@ class ReqDllmMixin:
|
||||
self.dllm_algo_state = None
|
||||
self.dllm_block_offset = 0
|
||||
self.dllm_config = dllm_config
|
||||
self.dllm_has_literal_mask = False
|
||||
|
||||
if self.dllm_config is not None:
|
||||
if self.dllm_config.requires_separate_context_encoding:
|
||||
@@ -60,12 +61,15 @@ class ReqDllmMixin:
|
||||
if len(self.full_untruncated_fill_ids) < min_required_length:
|
||||
return
|
||||
|
||||
input_block = self.full_untruncated_fill_ids[prefix_length:min_required_length]
|
||||
self.dllm_phase = (
|
||||
DllmReqPhase.STAGING_PREFILL
|
||||
if self.dllm_config.mask_id not in input_block
|
||||
else DllmReqPhase.STAGING_DECODE
|
||||
)
|
||||
# Retraction preserves output_ids while releasing KV. Re-encode all
|
||||
# committed tokens, including literal mask IDs, before generating more.
|
||||
committed_len = len(self.origin_input_ids) + len(self.output_ids)
|
||||
is_prefill_phase = min_required_length <= committed_len
|
||||
|
||||
if is_prefill_phase:
|
||||
self.dllm_phase = DllmReqPhase.STAGING_PREFILL
|
||||
else:
|
||||
self.dllm_phase = DllmReqPhase.STAGING_DECODE
|
||||
|
||||
def _init_fill_ids_for_dllm(self: Req):
|
||||
if self.dllm_incomplete_ids:
|
||||
@@ -79,6 +83,14 @@ class ReqDllmMixin:
|
||||
# non-incomplete path which also defers it to the adder.
|
||||
return
|
||||
|
||||
if not self.dllm_initialized:
|
||||
# Committed tokens are immutable, so a literal mask ID among them
|
||||
# needs the explicit prompt mask. Recomputed after retraction.
|
||||
mask_id = self.dllm_config.mask_id
|
||||
self.dllm_has_literal_mask = (
|
||||
mask_id in self.origin_input_ids or mask_id in self.output_ids
|
||||
)
|
||||
|
||||
if self.dllm_config.requires_separate_context_encoding:
|
||||
self.refresh_fill_ids()
|
||||
self.dllm_block_offset = self.seqlen
|
||||
|
||||
Executable → Regular
+17
-1
@@ -1711,7 +1711,6 @@ class Req(ReqDllmMixin):
|
||||
):
|
||||
if self.is_dllm():
|
||||
self._init_fill_ids_for_dllm()
|
||||
self.determine_dllm_phase()
|
||||
else:
|
||||
self.refresh_fill_ids()
|
||||
|
||||
@@ -1761,6 +1760,9 @@ class Req(ReqDllmMixin):
|
||||
if self.is_dllm():
|
||||
self._update_block_offset_for_dllm()
|
||||
|
||||
if self.is_dllm():
|
||||
self.determine_dllm_phase()
|
||||
|
||||
if (
|
||||
self.is_retracted
|
||||
and self.multimodal_inputs is not None
|
||||
@@ -1786,6 +1788,15 @@ class Req(ReqDllmMixin):
|
||||
max_prefix_len = min(max_prefix_len, self.dllm_block_offset)
|
||||
if self.return_logprob and self.logprob_start_len >= 0:
|
||||
max_prefix_len = min(max_prefix_len, self.logprob_start_len)
|
||||
if self.is_dllm():
|
||||
# Appended masks and a partial prompt/output block are not reusable
|
||||
# KV: bidirectional attention depends on the rest of that block.
|
||||
# Bound the lookup by real tokens, even if another cached prompt
|
||||
# happens to contain the same token IDs as our synthetic masks.
|
||||
block_size = self.dllm_config.block_size
|
||||
committed_len = len(self.origin_input_ids) + len(self.output_ids)
|
||||
max_prefix_len = min(max_prefix_len, committed_len)
|
||||
max_prefix_len = max_prefix_len // block_size * block_size
|
||||
return max(max_prefix_len, 0)
|
||||
|
||||
# Based on https://github.com/vllm-project/vllm/blob/7a64d24aad69e4d2548aa0bf528d9fe63428ab01/vllm/transformers_utils/detokenizer.py#L194-L313
|
||||
@@ -2932,6 +2943,11 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
# flag will always True
|
||||
if not req.retracted_stain:
|
||||
new_cached = pre_len - req.already_computed
|
||||
if req.is_dllm():
|
||||
# Denoising revisits the current block while only earlier
|
||||
# blocks are a reusable prefix. Recomputing that block must
|
||||
# not subtract its size from cross-request cache hits.
|
||||
new_cached = max(0, new_cached)
|
||||
req.cached_tokens += new_cached
|
||||
|
||||
# Calculate detailed breakdown of cached tokens by source (for HiCache)
|
||||
|
||||
@@ -653,6 +653,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
# === Forward-derived (built in init_new on the forward stream; FB-owned) ===
|
||||
# Position information
|
||||
positions: torch.Tensor = None
|
||||
# dLLM committed positions are immutable, including literal mask-token IDs.
|
||||
# None when no request in the batch has a literal mask ID.
|
||||
dllm_prompt_mask: Optional[torch.Tensor] = None
|
||||
|
||||
# For extend
|
||||
extend_num_tokens: Optional[int] = None
|
||||
@@ -1164,14 +1167,35 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
if batch.dllm_config is not None and ret.forward_mode.is_dllm_extend():
|
||||
block_size = batch.dllm_config.block_size
|
||||
positions_dtype = torch.int64 if is_hip() or _is_npu else torch.int32
|
||||
ret.positions = torch.tensor(
|
||||
[
|
||||
i
|
||||
for block_offset in (req.dllm_block_offset for req in batch.reqs)
|
||||
for i in range(block_offset, block_offset + block_size)
|
||||
],
|
||||
dtype=positions_dtype,
|
||||
).to(device, non_blocking=True)
|
||||
positions = [
|
||||
i
|
||||
for block_offset in (req.dllm_block_offset for req in batch.reqs)
|
||||
for i in range(block_offset, block_offset + block_size)
|
||||
]
|
||||
if any(req.dllm_has_literal_mask for req in batch.reqs):
|
||||
# Committed outputs are immutable too when a retracted request
|
||||
# rebuilds its KV. Transfer their lengths with the block
|
||||
# positions to avoid an extra, blocking H2D copy.
|
||||
committed_lens = [
|
||||
len(req.origin_input_ids) + len(req.output_ids)
|
||||
for req in batch.reqs
|
||||
]
|
||||
position_info = torch.tensor(
|
||||
positions + committed_lens,
|
||||
dtype=positions_dtype,
|
||||
pin_memory=is_pin_memory_available(device),
|
||||
).to(device, non_blocking=True)
|
||||
ret.positions = position_info[: len(positions)]
|
||||
prompt_lens = position_info[len(positions) :]
|
||||
ret.dllm_prompt_mask = (
|
||||
ret.positions.view(-1, block_size) < prompt_lens[:, None]
|
||||
)
|
||||
else:
|
||||
# Without literal mask IDs among committed tokens, every mask
|
||||
# ID in the block is still to be denoised.
|
||||
ret.positions = torch.tensor(positions, dtype=positions_dtype).to(
|
||||
device, non_blocking=True
|
||||
)
|
||||
elif (
|
||||
ret.spec_info is not None
|
||||
and getattr(ret.spec_info, "positions", None) is not None
|
||||
|
||||
Reference in New Issue
Block a user