[RL, Spec] Introduce top-p mask capture for spec and add DFlash/DSpark impl (#34201)

Co-authored-by: hnyls2002 <lsyincs@gmail.com>
This commit is contained in:
Jason Mancuso
2026-09-30 13:32:26 -07:00
committed by GitHub
co-authored by hnyls2002
parent 986cb8268b
commit 201c4b9b2a
16 changed files with 481 additions and 29 deletions
@@ -11,7 +11,7 @@ from sglang.kernels.ops.speculative.dspark.dispatch import inputs_on_cuda
from sglang.srt.speculative.dflash_info_v2 import DFlashDraftInputV2
from sglang.srt.speculative.dflash_utils import (
_get_or_create_chain_verify_buffers,
build_dflash_verify_target_probs,
build_speculative_verify_target_probs,
compute_dflash_correct_drafts_and_bonus,
)
from sglang.srt.utils import is_npu
@@ -104,7 +104,7 @@ def _accept_sampling_core(
rows_per_request=verify_num_draft_tokens,
).view(bs, verify_num_draft_tokens, -1)
else:
target_probs = build_dflash_verify_target_probs(
target_probs = build_speculative_verify_target_probs(
next_token_logits=target_logits,
sampling_info=sampling_info,
draft_token_num=verify_num_draft_tokens,
+9 -2
View File
@@ -106,6 +106,7 @@ class SamplingMaskOutput:
selected_logprobs: torch.Tensor
support_logprobs: Optional[torch.Tensor]
statuses: torch.Tensor
num_accept_tokens: Optional[torch.Tensor] = None
def map_device_tensors(self, fn) -> None:
self.token_ids = fn(self.token_ids)
@@ -114,6 +115,8 @@ class SamplingMaskOutput:
if self.support_logprobs is not None:
self.support_logprobs = fn(self.support_logprobs)
self.statuses = fn(self.statuses)
if self.num_accept_tokens is not None:
self.num_accept_tokens = fn(self.num_accept_tokens)
def _trace_e2e_logits(stage: str, **fields) -> None:
@@ -235,8 +238,12 @@ class LogitsProcessorOutput:
# Post-filter support IDs and requested behavior logprobs, bounded by server
# capacity. Logprobs are normalized over the full realized support.
sampling_mask_output: Optional[SamplingMaskOutput] = None
next_token_sampling_mask_idx: Optional[List[Optional[np.ndarray]]] = None
next_token_sampling_logprobs: Optional[List[Optional[np.ndarray]]] = None
next_token_sampling_mask_idx: Optional[
List[Optional[Union[np.ndarray, List[np.ndarray]]]]
] = None
next_token_sampling_logprobs: Optional[
List[Optional[Union[np.ndarray, List[np.ndarray]]]]
] = None
next_token_sampling_mask_status: Optional[List[Optional[int]]] = None
## Part 3: Prefill-only. This part will be assigned in python/sglang/srt/layers/logits_processor.py::LogitsProcessor
+69 -8
View File
@@ -329,6 +329,9 @@ from sglang.srt.speculative.eagle_utils import (
get_draft_recurrent_hidden_state_spec_from_config,
)
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.srt.speculative.spec_sampling_mask import (
validate_spec_sampling_mask_request,
)
from sglang.srt.speculative.uno_validation import validate_uno_request
from sglang.srt.state_capturer.indexer_topk import destroy_global_indexer_capturer
from sglang.srt.state_capturer.routed_experts import destroy_global_experts_capturer
@@ -2964,14 +2967,10 @@ class Scheduler(
return
if req.return_sampling_mask and not self.spec_algorithm.is_none():
# Spec workers do not emit one sampling support per accepted token, so
# the returned mask would not align 1:1 with generated tokens. Reject
# the combination instead of silently returning a misaligned mask.
error_msg = (
"return_sampling_mask is not supported with speculative decoding."
)
self._reject_sampling_mask_request(req, error_msg)
return
error_msg = validate_spec_sampling_mask_request(req, self.spec_algorithm)
if error_msg is not None:
self._reject_sampling_mask_request(req, error_msg)
return
if req.return_sampling_mask and get_exec().kernel.sampling_backend == "ascend":
# The ascend backend samples from logits directly and never builds the
@@ -5299,6 +5298,52 @@ class Scheduler(
return GetInternalStateReqOutput(internal_state=msgspec_to_builtins(ret))
def _has_active_sampling_mask_request(self) -> bool:
"""Whether any unfinished request holding a sampling mask is queued or
running, including PP micro-batches and PD decode queues."""
active_batches = [self.running_batch, self.last_batch]
active_batches.extend(getattr(self, "running_mbs", ()))
active_batches.extend(getattr(self, "mbs", ()))
active_batches.extend(getattr(self, "last_mbs", ()))
has_active_mask_request = any(
req.return_sampling_mask and not req.finished()
for batch in active_batches
if batch is not None
for req in batch.reqs
)
has_active_mask_request |= any(
req.return_sampling_mask and not req.finished()
for req in self.waiting_queue
)
has_active_mask_request |= any(
req.return_sampling_mask and not req.finished()
for req in self.grammar_manager.grammar_queue
)
has_active_mask_request |= (
self.chunked_req is not None
and self.chunked_req.return_sampling_mask
and not self.chunked_req.finished()
)
if self.disaggregation_mode == DisaggregationMode.DECODE:
prealloc_queue = self.disagg_decode_prealloc_queue
decode_reqs = [
*prealloc_queue.queue,
*prealloc_queue.pending_reqs,
*self.disagg_decode_transfer_queue.queue,
]
has_active_mask_request |= any(
decode_req.req.return_sampling_mask and not decode_req.req.finished()
for decode_req in decode_reqs
)
has_active_mask_request |= any(
req.return_sampling_mask and not req.finished()
for req in (
*prealloc_queue.retracted_queue,
*prealloc_queue.held_rebootstrap_reqs,
)
)
return has_active_mask_request
def set_internal_state(self, recv_req: SetInternalStateReq):
server_args_dict = recv_req.server_args
args_allow_update = set(
@@ -5325,6 +5370,22 @@ class Scheduler(
)
if_success = False
break
elif (
k
in (
"speculative_accept_threshold_single",
"speculative_accept_threshold_acc",
)
and self.spec_algorithm.is_dflash()
and float(v) != 1.0
):
if self._has_active_sampling_mask_request():
logging.warning(
f"Updating {k} is rejected while DFlash sampling-mask "
"requests are active."
)
if_success = False
break
elif k == "dspark_force_budget_frac":
if not self.spec_algorithm.is_dspark() or not hasattr(
self.draft_worker, "set_dspark_forced_budget_frac"
@@ -12,6 +12,7 @@ from typing import (
Union,
)
import numpy as np
import torch
from sglang.srt.disaggregation.utils import DisaggregationMode
@@ -1034,9 +1035,9 @@ class SchedulerBatchResultProcessor:
)
if req.return_sampling_mask:
# return_sampling_mask + speculative decoding is rejected at
# request entry, so this remains one support mask per token.
self.add_sampling_mask_return_values(i, req, logits_output)
self.add_sampling_mask_return_values(
i, req, logits_output, accept_len=new_accept_len
)
if req.return_hidden_states and logits_output.hidden_states is not None:
# hidden_states is [bs * stride, hidden_dim], one row per emitted
@@ -1161,12 +1162,68 @@ class SchedulerBatchResultProcessor:
i: int,
req: Req,
output: LogitsProcessorOutput,
*,
accept_len: int = 1,
) -> None:
"""Attach sparse sampling support metadata to the return values."""
req.sampling_mask_rows.append(
output.next_token_sampling_mask_idx[i],
output.next_token_sampling_logprobs[i],
masks = output.next_token_sampling_mask_idx[i]
logprobs = output.next_token_sampling_logprobs[i]
if not isinstance(masks, list):
req.sampling_mask_rows.append(masks, logprobs)
return
# Speculative rows carry one support per accepted token; drop the ones
# past finished_len, as output_ids is trimmed there.
step_start = len(req.output_ids) - accept_len
num_visible = max(
0, min(accept_len, self._visible_output_len(req) - step_start)
)
for token in range(num_visible):
req.sampling_mask_rows.append(masks[token], logprobs[token])
@staticmethod
def _fill_speculative_sampling_mask_rows(
*,
reqs: List[Req],
batch_indices: List[int],
num_accept_tokens: List[int],
lengths: List[List[int]],
statuses: List[List[int]],
token_ids: np.ndarray,
selected_logprobs: np.ndarray,
support_logprobs: Optional[np.ndarray],
masks: list,
logprobs: list,
status_by_batch: list,
) -> None:
"""Speculative rows carry one support per draft position; keep the
accepted prefix, as a list with one entry per accepted token."""
assert len(batch_indices) == len(num_accept_tokens)
support_row = 0
for row, batch_index in enumerate(batch_indices):
returns_support_logprobs = (
reqs[batch_index].sampling_logprobs_mode == "support"
)
num_tokens = num_accept_tokens[row]
status = max(statuses[row][:num_tokens])
status_by_batch[batch_index] = status
if status == SamplingMaskStatus.OK:
row_lengths = lengths[row][:num_tokens]
masks[batch_index] = [
token_ids[row, token, :length]
for token, length in enumerate(row_lengths)
]
if returns_support_logprobs:
logprobs[batch_index] = [
support_logprobs[support_row, token, :length]
for token, length in enumerate(row_lengths)
]
else:
logprobs[batch_index] = [
selected_logprobs[row, token : token + 1]
for token in range(num_tokens)
]
if returns_support_logprobs:
support_row += 1
@staticmethod
def materialize_sampling_mask_output(
@@ -1194,6 +1251,26 @@ class SchedulerBatchResultProcessor:
if sampling_output.support_logprobs is None
else sampling_output.support_logprobs.cpu().numpy()
)
if sampling_output.num_accept_tokens is not None:
SchedulerBatchResultProcessor._fill_speculative_sampling_mask_rows(
reqs=reqs,
batch_indices=batch_indices,
num_accept_tokens=sampling_output.num_accept_tokens.tolist(),
lengths=lengths,
statuses=statuses,
token_ids=token_ids,
selected_logprobs=selected_logprobs,
support_logprobs=support_logprobs,
masks=masks,
logprobs=logprobs,
status_by_batch=status_by_batch,
)
output.next_token_sampling_mask_idx = masks
output.next_token_sampling_logprobs = logprobs
output.next_token_sampling_mask_status = status_by_batch
output.sampling_mask_output = None
return
support_row = 0
for row, batch_index in enumerate(batch_indices):
returns_support_logprobs = (
+6
View File
@@ -312,6 +312,11 @@ def get_logprob_dict_from_result(result: GenerationBatchResult) -> dict:
"sampling_mask_statuses": (
None if sampling_mask_output is None else sampling_mask_output.statuses
),
"sampling_mask_num_accept_tokens": (
None
if sampling_mask_output is None
else sampling_mask_output.num_accept_tokens
),
"input_token_logprobs": result.logits_output.input_token_logprobs,
"input_top_logprobs_val": result.logits_output.input_top_logprobs_val,
"input_top_logprobs_idx": result.logits_output.input_top_logprobs_idx,
@@ -332,6 +337,7 @@ def get_logprob_from_pp_outputs(
selected_logprobs=next_pp_outputs["sampling_mask_selected_logprobs"],
support_logprobs=next_pp_outputs["sampling_mask_support_logprobs"],
statuses=next_pp_outputs["sampling_mask_statuses"],
num_accept_tokens=next_pp_outputs["sampling_mask_num_accept_tokens"],
)
logits_output = LogitsProcessorOutput(
# Do not send logits and hidden states because they are large
@@ -98,6 +98,8 @@ class SamplingBatchInfo:
return_sampling_support_logprobs: Optional[List[bool]] = None
sampling_mask_batch_indices: Optional[torch.Tensor] = None
sampling_support_logprobs_capture_indices: Optional[torch.Tensor] = None
# Per-row top_k of mask requests (1 elsewhere); bounds the capture width.
sampling_mask_top_ks: Optional[List[int]] = None
# Device
device: str = "cuda"
@@ -201,6 +203,9 @@ class SamplingBatchInfo:
device,
)
)
sampling_mask_top_ks = [
r.sampling_params.top_k if r.return_sampling_mask else 1 for r in reqs
]
if has_custom_logit_processor:
# Merge the same type of custom logit processors together
@@ -271,6 +276,7 @@ class SamplingBatchInfo:
sampling_support_logprobs_capture_indices=(
sampling_support_logprobs_capture_indices
),
sampling_mask_top_ks=sampling_mask_top_ks,
)
ret.adjusted_from_schedule_batch(batch, vocab_size)
return ret
@@ -435,6 +441,10 @@ class SamplingBatchInfo:
self.return_sampling_masks = [
self.return_sampling_masks[i] for i in keep_indices
]
if self.sampling_mask_top_ks is not None:
self.sampling_mask_top_ks = [
self.sampling_mask_top_ks[i] for i in keep_indices
]
if self.return_sampling_support_logprobs is not None:
self.return_sampling_support_logprobs = [
self.return_sampling_support_logprobs[i] for i in keep_indices
@@ -537,6 +547,9 @@ class SamplingBatchInfo:
self.device,
)
)
self.sampling_mask_top_ks = (
self.sampling_mask_top_ks or [1] * self_len
) + (other.sampling_mask_top_ks or [1] * other_len)
# Note: because the __len()__ operator is defined on the temperatures tensor,
# please make sure any merge operation with len(self) or len(other) is done before
@@ -1086,7 +1086,7 @@ def compute_dflash_sampling_correct_drafts_and_bonus(
dtype=torch.float32,
)
target_probs = build_dflash_verify_target_probs(
target_probs = build_speculative_verify_target_probs(
next_token_logits=next_token_logits,
sampling_info=sampling_info,
draft_token_num=draft_token_num,
@@ -1127,7 +1127,7 @@ def compute_dflash_sampling_correct_drafts_and_bonus(
return correct_len, bonus
def build_dflash_verify_target_probs(
def build_speculative_verify_target_probs(
*,
next_token_logits: torch.Tensor,
sampling_info: Any,
@@ -82,6 +82,9 @@ from sglang.srt.speculative.lilicorr_utils import (
target_input_embeddings,
)
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.srt.speculative.spec_sampling_mask import (
SpeculativeSamplingMaskCapture,
)
from sglang.srt.speculative.spec_tp_sync import SpecTpSync, SpecTpSyncSite
from sglang.srt.speculative.spec_utils import (
SIMULATE_ACC_LEN,
@@ -2810,6 +2813,19 @@ class DFlashWorkerV2(BaseSpecWorker):
# accept_len; recompute it from the forced commit_lens.
new_seq_lens = None
sampling_mask_capture = SpeculativeSamplingMaskCapture.from_logits(
sampling_info,
next_token_logits=logits_output.next_token_logits,
draft_input=draft_input,
draft_token_num=int(self.block_size),
bs=bs,
)
if sampling_mask_capture is not None:
logits_output.sampling_mask_output = sampling_mask_capture.build_output(
out_tokens=out_tokens,
commit_lens=commit_lens,
)
if batch.return_logprob:
compute_spec_logprobs(
batch,
@@ -71,6 +71,9 @@ from sglang.srt.speculative.dspark_components.dspark_verify import (
TargetVerifyExecutor,
verify_logits_adjustments_are_noop,
)
from sglang.srt.speculative.spec_sampling_mask import (
SpeculativeSamplingMaskCapture,
)
from sglang.srt.speculative.spec_tp_sync import SpecTpSync, SpecTpSyncSite
from sglang.srt.speculative.spec_utils import (
GrammarTree,
@@ -968,6 +971,19 @@ class DSparkWorkerV2(BaseSpecWorker):
req_pool_indices=batch.req_pool_indices,
commit_lens=accept.commit_lens,
)
sampling_mask_capture = SpeculativeSamplingMaskCapture.from_logits(
sampling_info,
next_token_logits=logits_output.next_token_logits,
draft_input=draft_input,
draft_token_num=self.verify_num_draft_tokens,
bs=bs,
greedy_mask=draft_block.greedy_mask,
)
if sampling_mask_capture is not None:
logits_output.sampling_mask_output = sampling_mask_capture.build_output(
out_tokens=accept.out_tokens,
commit_lens=accept.commit_lens,
)
if batch.return_logprob:
compute_spec_logprobs(
batch,
@@ -0,0 +1,179 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Optional
import msgspec
import torch
from sglang.srt.environ import envs
from sglang.srt.layers.logits_processor import SamplingMaskOutput, SamplingMaskStatus
from sglang.srt.runtime_context import get_spec
from sglang.srt.speculative.dflash_utils import (
build_speculative_verify_target_probs,
is_dflash_sampling_verify_available,
)
if TYPE_CHECKING:
from sglang.srt.managers.schedule_batch import Req
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
def validate_spec_sampling_mask_request(
req: Req, spec_algorithm: SpeculativeAlgorithm
) -> Optional[str]:
"""Reject configurations whose committed tokens are not exact samples from
the captured target distribution."""
if not spec_algorithm.is_dflash_family():
return "return_sampling_mask is not supported with speculative decoding."
# build_speculative_verify_target_probs applies top-k/top-p only.
if req.sampling_params.min_p > 0:
return "return_sampling_mask with speculative decoding does not support min_p."
if envs.SGLANG_SIMULATE_ACC_LEN.get() > 0:
return (
"return_sampling_mask is not supported with simulated speculative "
"acceptance."
)
if spec_algorithm.is_dflash():
if (
get_spec().speculative_accept_threshold_single != 1.0
or get_spec().speculative_accept_threshold_acc != 1.0
):
return (
"return_sampling_mask with DFlash requires acceptance thresholds "
"of 1.0."
)
# Without the kernel DFlash verifies non-greedy rows by greedy argmax.
if req.sampling_params.top_k > 1 and not is_dflash_sampling_verify_available():
return (
"return_sampling_mask with non-greedy DFlash decoding requires "
"sampling verification support."
)
return None
class SpeculativeSamplingMaskCapture(msgspec.Struct):
target_probs: torch.Tensor | None
batch_indices: torch.Tensor
max_top_k: int
greedy_mask: torch.Tensor | None = None
support_capture_indices: torch.Tensor | None = None
@classmethod
def from_logits(
cls,
sampling_info,
*,
next_token_logits: torch.Tensor,
draft_input,
draft_token_num: int,
bs: int,
greedy_mask: torch.Tensor | None = None,
) -> SpeculativeSamplingMaskCapture | None:
"""Rebuild the verified target policy after acceptance; returns None
when no request in the batch asked for sampling masks."""
if sampling_info is None or sampling_info.sampling_mask_batch_indices is None:
return None
target_probs = None
if not sampling_info.is_all_greedy:
target_probs = build_speculative_verify_target_probs(
next_token_logits=next_token_logits,
sampling_info=sampling_info,
draft_token_num=draft_token_num,
bs=bs,
max_top_k=draft_input.max_top_k,
uniform_top_k_value=draft_input.uniform_top_k_value,
)
return cls(
target_probs=target_probs,
batch_indices=sampling_info.sampling_mask_batch_indices,
max_top_k=max(sampling_info.sampling_mask_top_ks),
greedy_mask=greedy_mask,
support_capture_indices=sampling_info.sampling_support_logprobs_capture_indices,
)
def build_output(
self,
*,
out_tokens: torch.Tensor,
commit_lens: torch.Tensor,
) -> SamplingMaskOutput:
batch_indices = self.batch_indices
output_tokens = out_tokens.index_select(0, batch_indices).clone()
num_accept_tokens = commit_lens.index_select(0, batch_indices).clone()
greedy_mask = (
None
if self.greedy_mask is None
else self.greedy_mask.index_select(0, batch_indices).clone()
)
support_logprobs = None
if self.target_probs is None:
support_tokens = output_tokens.unsqueeze(-1).to(torch.int32)
support_lens = torch.ones_like(output_tokens, dtype=torch.int32)
selected_logprobs = torch.zeros_like(output_tokens, dtype=torch.float32)
if self.support_capture_indices is not None:
support_logprobs = selected_logprobs.index_select(
0, self.support_capture_indices
).unsqueeze(-1)
else:
max_top_k = min(int(self.max_top_k), self.target_probs.shape[-1])
if max_top_k <= 0:
raise ValueError(
"Sampling-mask capture requires a positive finite top_k."
)
target_probs = self.target_probs.index_select(0, batch_indices)
support_probs, support_tokens = torch.topk(
target_probs, k=max_top_k, dim=-1
)
support_tokens = support_tokens.to(torch.int32)
support_lens = (support_probs > 0).sum(dim=-1, dtype=torch.int32)
selected_logprobs = torch.log(
target_probs.gather(-1, output_tokens.unsqueeze(-1)).squeeze(-1)
)
if self.support_capture_indices is not None:
support_logprobs = torch.log(
support_probs.index_select(0, self.support_capture_indices)
)
if greedy_mask is not None:
greedy_rows = greedy_mask[:, None]
support_tokens = torch.where(
greedy_rows[:, :, None],
output_tokens[:, :, None],
support_tokens,
)
support_lens = torch.where(
greedy_rows,
torch.ones_like(support_lens),
support_lens,
)
selected_logprobs = torch.where(
greedy_rows,
torch.zeros_like(selected_logprobs),
selected_logprobs,
)
if support_logprobs is not None:
support_logprobs = torch.where(
greedy_rows.index_select(0, self.support_capture_indices)[
:, :, None
],
torch.zeros_like(support_logprobs),
support_logprobs,
)
statuses = torch.where(
torch.isfinite(selected_logprobs),
int(SamplingMaskStatus.OK),
int(SamplingMaskStatus.INVALID),
).to(torch.int8)
return SamplingMaskOutput(
token_ids=support_tokens,
lengths=support_lens,
selected_logprobs=selected_logprobs,
support_logprobs=support_logprobs,
statuses=statuses,
num_accept_tokens=num_accept_tokens,
)
+2 -2
View File
@@ -11,7 +11,7 @@ from sglang.kernels.ops.speculative.reject_sampling import (
)
from sglang.srt.speculative.dflash_utils import (
_get_or_create_chain_verify_buffers,
build_dflash_verify_target_probs,
build_speculative_verify_target_probs,
)
from sglang.srt.speculative.spec_utils import fast_sample
@@ -277,7 +277,7 @@ def _build_dense_probs(
uniform_top_k_value: int | None,
) -> torch.Tensor:
"""Build the dense sampling distribution used by SGLang verification."""
return build_dflash_verify_target_probs(
return build_speculative_verify_target_probs(
next_token_logits=next_token_logits,
sampling_info=sampling_info,
draft_token_num=forward_width,
@@ -9,6 +9,7 @@ Thresholds are read off ``self`` so a config can tune them as class attributes.
import concurrent.futures
import json
import math
import random
import threading
from concurrent.futures import ThreadPoolExecutor
@@ -696,3 +697,64 @@ class SpecGrammarKit:
f"got {len(output_logprobs)} logprobs vs {completion_tokens} completion tokens",
)
json.loads(out["text"])
class SpecSamplingMaskKit:
"""return_sampling_mask on a mixed greedy/sampling batch: one mask per output
token, each output token inside its mask, and greedy rows as singletons."""
def test_sampling_mask(self):
response = requests.post(
self.base_url + "/generate",
json={
"text": ["The capital of France is"] * 4,
"sampling_params": [
{
"temperature": 0.0,
"top_k": 1,
"max_new_tokens": 7,
"ignore_eos": True,
},
{
"temperature": 1.0,
"top_k": 10,
"top_p": 0.95,
"max_new_tokens": 7,
"ignore_eos": True,
},
]
* 2,
"return_sampling_mask": True,
"sampling_logprobs_mode": [
"selected",
"selected",
"support",
"support",
],
},
timeout=60,
)
self.assertEqual(response.status_code, 200, response.text)
outputs = response.json()
self.assertEqual(len(outputs), 4)
for request_idx, output in enumerate(outputs):
output_ids = output["output_ids"]
meta_info = output["meta_info"]
masks = meta_info["output_token_sampling_mask"]
logprobs = meta_info["output_token_sampling_logprobs"]
self.assertEqual(len(output_ids), 7)
self.assertEqual(len(masks), len(output_ids))
self.assertEqual(len(logprobs), len(output_ids))
for output_id, mask, logprob in zip(output_ids, masks, logprobs):
self.assertIn(output_id, mask)
if request_idx >= 2:
self.assertEqual(len(logprob), len(mask))
self.assertTrue(all(math.isfinite(value) for value in logprob))
self.assertAlmostEqual(
sum(math.exp(value) for value in logprob), 1.0, places=5
)
else:
self.assertTrue(math.isfinite(logprob))
if request_idx % 2 == 0:
self.assertEqual(mask, [output_id])
self.assertEqual(logprob, [0.0] if request_idx >= 2 else 0.0)