[Sampler] Sort only the max(top_k) leading columns in the torch sampler (#43346)
This commit is contained in:
@@ -450,6 +450,7 @@ class Sampler(nn.Module):
|
||||
sampling_info.sampling_seed,
|
||||
positions,
|
||||
return_filtered_probs=return_sampling_mask,
|
||||
max_top_k=sampling_info.max_top_k,
|
||||
)
|
||||
if return_sampling_mask:
|
||||
(
|
||||
@@ -759,6 +760,7 @@ def top_k_top_p_min_p_sampling_from_probs_torch(
|
||||
positions: torch.Tensor,
|
||||
*,
|
||||
return_filtered_probs: bool = False,
|
||||
max_top_k: Optional[int] = None,
|
||||
):
|
||||
"""
|
||||
A top-k, top-p and min-p sampling implementation with native pytorch operations.
|
||||
@@ -768,11 +770,26 @@ def top_k_top_p_min_p_sampling_from_probs_torch(
|
||||
By default, returns only sampled token IDs. With return_filtered_probs=True,
|
||||
also returns the actual filtered weights, their token-ID permutation, and
|
||||
the selected weights.
|
||||
|
||||
max_top_k, when given, must be >= every value in top_ks.
|
||||
"""
|
||||
probs_sort, probs_idx = probs.sort(dim=-1, descending=True)
|
||||
# The top-k mask below zeroes every column at rank >= top_k, so the max_top_k
|
||||
# leading columns carry all the mass that survives and sorting only those is
|
||||
# exact. The seeded path keeps the full sort because the order of tied
|
||||
# probabilities may depend on k, which is a batch-wide value, and
|
||||
# deterministic inference must not depend on the batch composition.
|
||||
if (
|
||||
max_top_k is not None
|
||||
and max_top_k < probs.shape[-1]
|
||||
and sampling_seed is None
|
||||
and not return_filtered_probs
|
||||
):
|
||||
probs_sort, probs_idx = torch.topk(probs, k=max_top_k, dim=-1, sorted=True)
|
||||
else:
|
||||
probs_sort, probs_idx = probs.sort(dim=-1, descending=True)
|
||||
probs_sum = torch.cumsum(probs_sort, dim=-1)
|
||||
probs_sort[
|
||||
torch.arange(0, probs.shape[-1], device=probs.device).view(1, -1)
|
||||
torch.arange(0, probs_sort.shape[-1], device=probs.device).view(1, -1)
|
||||
>= top_ks.view(-1, 1)
|
||||
] = 0.0
|
||||
probs_sort[(probs_sum - probs_sort) > top_ps.view(-1, 1)] = 0.0
|
||||
|
||||
@@ -111,6 +111,11 @@ class SamplingBatchInfo:
|
||||
# device avoids a scalar synchronization in the per-token sampling path.
|
||||
npu_top_k_top_p_eligible: bool = False
|
||||
|
||||
# Host-side upper bound on every row's top_k, or None when some row samples
|
||||
# from the whole vocabulary. Any value >= the true max stays exact, so
|
||||
# filter_batch keeps it unchanged.
|
||||
max_top_k: Optional[int] = None
|
||||
|
||||
@classmethod
|
||||
def from_schedule_batch(cls, batch: ScheduleBatch, vocab_size: int):
|
||||
enable_deterministic = get_exec().deterministic.enable_deterministic_inference
|
||||
@@ -263,6 +268,11 @@ class SamplingBatchInfo:
|
||||
npu_top_k_top_p_eligible=all(
|
||||
1 <= r.sampling_params.top_k <= 1024 for r in reqs
|
||||
),
|
||||
max_top_k=(
|
||||
max(r.sampling_params.top_k for r in reqs)
|
||||
if reqs and all(1 <= r.sampling_params.top_k < TOP_K_ALL for r in reqs)
|
||||
else None
|
||||
),
|
||||
vocab_size=vocab_size,
|
||||
penalizer_orchestrator=penalizer_orchestrator,
|
||||
has_custom_logit_processor=has_custom_logit_processor,
|
||||
@@ -572,6 +582,11 @@ class SamplingBatchInfo:
|
||||
self.need_top_k_sampling |= other.need_top_k_sampling
|
||||
self.need_min_p_sampling |= other.need_min_p_sampling
|
||||
self.npu_top_k_top_p_eligible &= other.npu_top_k_top_p_eligible
|
||||
self.max_top_k = (
|
||||
None
|
||||
if self.max_top_k is None or other.max_top_k is None
|
||||
else max(self.max_top_k, other.max_top_k)
|
||||
)
|
||||
|
||||
self.adjusted_merge_batch(other)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user