[feature] add per-item candidate token scoring and calibration (#40826)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Mick
2026-09-24 17:00:13 +08:00
committed by GitHub
co-authored by Mick Qian
parent 41812afe43
commit 174a5f37b9
9 changed files with 558 additions and 41 deletions
@@ -30,13 +30,15 @@ class EngineScoreMixin:
self,
query: Optional[Union[str, List[int]]] = None,
items: Optional[Union[str, List[str], List[List[int]]]] = None,
label_token_ids: Optional[List[int]] = None,
label_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
apply_softmax: bool = False,
item_first: bool = False,
embed_override_token_id: Optional[int] = None,
query_embed_overrides: Optional[List[torch.Tensor]] = None,
item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None,
return_pooled_hidden_states: bool = False,
temperature: float = 1.0,
return_token_logprobs: bool = False,
) -> ScoreResult:
"""
Score items against a query using the loaded model.
@@ -54,8 +56,8 @@ class EngineScoreMixin:
Args:
query: The query text or pre-tokenized token IDs.
items: The item text(s) or pre-tokenized token IDs.
label_token_ids: Token IDs to score (required for CausalLM; ignored for
SequenceClassification).
label_token_ids: Shared token IDs or one list per item, preserving candidate
order (required for CausalLM; ignored for SequenceClassification).
apply_softmax: Whether to normalize scores using softmax.
item_first: If True, prepend items before query (single-item mode only).
embed_override_token_id: Placeholder token ID used to locate override positions.
@@ -64,6 +66,10 @@ class EngineScoreMixin:
return_pooled_hidden_states: Whether to include raw pooled transformer
hidden states (before the task head) in the result. Only supported
for non-generation models (SequenceClassification, RewardModel).
temperature: Positive calibration temperature, applied before candidate/class
softmax. Requires apply_softmax=True when different from 1.
return_token_logprobs: Include uncalibrated full-vocabulary logprobs for
each item's candidates. Only supported for CausalLM, not raw logits.
Returns:
ScoreResult with scores (one list per item), prompt token count, and
@@ -81,6 +87,8 @@ class EngineScoreMixin:
item_embed_overrides=item_embed_overrides,
request=None,
return_pooled_hidden_states=return_pooled_hidden_states,
temperature=temperature,
return_token_logprobs=return_token_logprobs,
)
)
@@ -88,13 +96,15 @@ class EngineScoreMixin:
self,
query: Optional[Union[str, List[int]]] = None,
items: Optional[Union[str, List[str], List[List[int]]]] = None,
label_token_ids: Optional[List[int]] = None,
label_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
apply_softmax: bool = False,
item_first: bool = False,
embed_override_token_id: Optional[int] = None,
query_embed_overrides: Optional[List[torch.Tensor]] = None,
item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None,
return_pooled_hidden_states: bool = False,
temperature: float = 1.0,
return_token_logprobs: bool = False,
) -> ScoreResult:
"""Asynchronous version of score(). See score() for full documentation."""
return await self.tokenizer_manager.score_request(
@@ -108,4 +118,6 @@ class EngineScoreMixin:
item_embed_overrides=item_embed_overrides,
request=None,
return_pooled_hidden_states=return_pooled_hidden_states,
temperature=temperature,
return_token_logprobs=return_token_logprobs,
)
@@ -1398,10 +1398,12 @@ class ScoringRequest(BaseModel):
item_embed_overrides: Optional[List[Optional[List[List[float]]]]] = (
None # [num_items][num_item_embed_overrides][hidden_size]
)
label_token_ids: Optional[List[int]] = (
None # Token IDs to compute probabilities for
label_token_ids: Optional[Union[List[int], List[List[int]]]] = (
None # shared candidates or one candidate list per item
)
apply_softmax: bool = False
temperature: float = Field(default=1.0, gt=0, allow_inf_nan=False)
return_token_logprobs: bool = False
item_first: bool = False
return_pooled_hidden_states: bool = False
model: str = DEFAULT_MODEL_NAME
@@ -1412,6 +1414,7 @@ class ScoringResponse(BaseModel):
List[float]
] # List of lists of probabilities, each in the order of label_token_ids
pooled_hidden_states: Optional[List[Optional[List[float]]]] = None
token_logprobs: Optional[List[List[float]]] = None
model: str
usage: Optional[UsageInfo] = None
object: str = "scoring"
@@ -78,6 +78,8 @@ class OpenAIServingScore(OpenAIServingBase):
item_embed_overrides=item_embed_overrides,
request=raw_request,
return_pooled_hidden_states=request.return_pooled_hidden_states,
temperature=request.temperature,
return_token_logprobs=request.return_token_logprobs,
)
phs_as_lists = None
@@ -90,6 +92,7 @@ class OpenAIServingScore(OpenAIServingBase):
response = ScoringResponse(
scores=result.scores,
pooled_hidden_states=phs_as_lists,
token_logprobs=result.token_logprobs,
model=request.model,
usage=UsageInfo(
prompt_tokens=result.prompt_tokens,
@@ -24,15 +24,19 @@ class ScoreResult:
# The HTTP path converts to lists in serving_score.py before JSON serialization.
# Same layout as scores: one tensor per item (not a single packed 2D tensor).
pooled_hidden_states: Optional[List[Optional[torch.Tensor]]] = None
# uncalibrated full-vocabulary logprobs, in each item's candidate order
token_logprobs: Optional[List[List[float]]] = None
class TokenizerManagerScoreMixin:
async def score_prompts(
self,
prompts: Union[str, List[str], List[List[int]]],
label_token_ids: List[int],
label_token_ids: Union[List[int], List[List[int]]],
apply_softmax: bool = False,
request: Optional[Any] = None,
temperature: float = 1.0,
return_token_logprobs: bool = False,
) -> ScoreResult:
"""
Score probabilities of specified token IDs after each *full prompt*.
@@ -51,6 +55,8 @@ class TokenizerManagerScoreMixin:
apply_softmax=apply_softmax,
item_first=False,
request=request,
temperature=temperature,
return_token_logprobs=return_token_logprobs,
)
# Tokenized prompts
@@ -62,6 +68,8 @@ class TokenizerManagerScoreMixin:
apply_softmax=apply_softmax,
item_first=False,
request=request,
temperature=temperature,
return_token_logprobs=return_token_logprobs,
)
raise ValueError("Invalid prompts type for score_prompts.")
@@ -102,7 +110,8 @@ class TokenizerManagerScoreMixin:
items_ids = []
for item in items_list:
if isinstance(item, str):
items_ids.append(self.tokenizer.encode(item))
# items continue the query, so only the query gets special tokens
items_ids.append(self.tokenizer.encode(item, add_special_tokens=False))
else:
items_ids.append(list(item))
@@ -112,10 +121,12 @@ class TokenizerManagerScoreMixin:
self,
results: Any,
items: List,
label_token_ids: Optional[List[int]],
label_token_ids: Optional[List[List[int]]],
apply_softmax: bool,
batch_request=None,
return_pooled_hidden_states: bool = False,
temperature: float = 1.0,
return_token_logprobs: bool = False,
) -> ScoreResult:
"""
Process results from multi-item scoring request.
@@ -135,27 +146,31 @@ class TokenizerManagerScoreMixin:
# Extract per-delimiter scores from whichever field has them
input_logprobs = meta_info.get("input_token_ids_logprobs", [])
embedding = single_result.get("embedding")
token_logprobs = [] if return_token_logprobs else None
if input_logprobs:
# Generation model: extract label-token logprobs at each delimiter
per_delimiter_scores = []
for logprobs_data in input_logprobs:
logprobs = self._extract_logprobs_for_tokens(
logprobs_data, label_token_ids
if len(input_logprobs) != expected_count:
raise RuntimeError(
f"Expected {expected_count} delimiter entries, got {len(input_logprobs)}"
)
per_delimiter_scores = [[]] # the query boundary is not a decision
for logprobs_data, labels in zip(input_logprobs[1:], label_token_ids):
logprobs = self._extract_logprobs_for_tokens(logprobs_data, labels)
score_list = self._convert_logprobs_to_scores(
logprobs, label_token_ids, apply_softmax
logprobs, labels, apply_softmax, temperature
)
per_delimiter_scores.append(score_list)
if return_token_logprobs:
token_logprobs.append([logprobs[token] for token in labels])
elif embedding is not None:
# Classification model: scores are directly in 2D embedding.
if apply_softmax:
scores_tensor = (
torch.tensor(embedding)
if isinstance(embedding, list)
else embedding
scores_tensor = torch.as_tensor(embedding, dtype=torch.float64)
scores_tensor = scores_tensor - scores_tensor.amax(dim=-1, keepdim=True)
scores_tensor = torch.nn.functional.softmax(
scores_tensor / temperature, dim=-1
)
scores_tensor = torch.nn.functional.softmax(scores_tensor, dim=-1)
per_delimiter_scores = scores_tensor.tolist()
else:
per_delimiter_scores = (
@@ -188,14 +203,17 @@ class TokenizerManagerScoreMixin:
scores=scores,
prompt_tokens=prompt_tokens,
pooled_hidden_states=phs_list,
token_logprobs=token_logprobs,
)
def _process_single_item_scoring_results(
self,
results: Any,
label_token_ids: Optional[List[int]],
label_token_ids: Optional[List[List[int]]],
apply_softmax: bool,
return_pooled_hidden_states: bool = False,
temperature: float = 1.0,
return_token_logprobs: bool = False,
) -> ScoreResult:
"""
Process results from single-item scoring request.
@@ -208,10 +226,11 @@ class TokenizerManagerScoreMixin:
phs_list = []
has_phs = False
prompt_tokens = 0
token_logprobs = [] if return_token_logprobs else None
is_generation = self.is_generation
if is_generation:
for result in results:
for result, labels in zip(results, label_token_ids):
# For single-item scoring, logprobs are in output_token_ids_logprobs
output_logprobs = result["meta_info"].get(
"output_token_ids_logprobs", []
@@ -225,13 +244,13 @@ class TokenizerManagerScoreMixin:
)
# Extract logprobs for the first (and only) position
logprobs = self._extract_logprobs_for_tokens(
output_logprobs[0], label_token_ids
)
logprobs = self._extract_logprobs_for_tokens(output_logprobs[0], labels)
score_list = self._convert_logprobs_to_scores(
logprobs, label_token_ids, apply_softmax
logprobs, labels, apply_softmax, temperature
)
scores.append(score_list)
if return_token_logprobs:
token_logprobs.append([logprobs[token] for token in labels])
else:
for result in results:
embedding = result.get("embedding", None)
@@ -241,8 +260,9 @@ class TokenizerManagerScoreMixin:
prompt_tokens += result.get("meta_info", {}).get("prompt_tokens", 0)
if apply_softmax:
scores_tensor = torch.as_tensor(embedding, dtype=torch.float64)
embedding = torch.softmax(
torch.as_tensor(embedding), dim=-1
(scores_tensor - scores_tensor.max()) / temperature, dim=-1
).tolist()
# The classification head produces per-token logits, which the pooler reduces
@@ -262,6 +282,7 @@ class TokenizerManagerScoreMixin:
scores=scores,
prompt_tokens=prompt_tokens,
pooled_hidden_states=phs_list if has_phs else None,
token_logprobs=token_logprobs,
)
# ------------------------------------------------------------------
@@ -445,7 +466,7 @@ class TokenizerManagerScoreMixin:
self,
query: Optional[Union[str, List[int]]] = None,
items: Optional[Union[str, List[str], List[List[int]]]] = None,
label_token_ids: Optional[List[int]] = None,
label_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
apply_softmax: bool = False,
item_first: bool = False,
embed_override_token_id: Optional[int] = None,
@@ -453,6 +474,8 @@ class TokenizerManagerScoreMixin:
item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None,
request: Optional[Any] = None,
return_pooled_hidden_states: bool = False,
temperature: float = 1.0,
return_token_logprobs: bool = False,
) -> ScoreResult:
"""
Score the probability of specified token IDs appearing after the given (query + item) pair.
@@ -477,6 +500,15 @@ class TokenizerManagerScoreMixin:
"""
is_generation = self.is_generation
if not math.isfinite(temperature) or temperature <= 0:
raise ValueError("temperature must be finite and greater than zero")
if temperature != 1.0 and not apply_softmax:
raise ValueError("temperature requires apply_softmax=True")
if return_token_logprobs and not is_generation:
raise ValueError(
"return_token_logprobs is only supported for CausalLM models"
)
if is_generation and label_token_ids is None:
raise ValueError(
"label_token_ids is required for generation (CausalLM) models."
@@ -484,7 +516,37 @@ class TokenizerManagerScoreMixin:
if items is None:
raise ValueError("items must be provided")
if not items:
return ScoreResult(scores=[], prompt_tokens=0)
return ScoreResult(
scores=[], token_logprobs=[] if return_token_logprobs else None
)
num_items = 1 if isinstance(items, str) else len(items)
if is_generation:
if not isinstance(label_token_ids, list) or not label_token_ids:
raise ValueError("label_token_ids must be a nonempty list")
if isinstance(label_token_ids[0], list):
if len(label_token_ids) != num_items:
raise ValueError("label_token_ids must have one list per item")
else:
label_token_ids = [label_token_ids] * num_items
for labels in label_token_ids:
if not isinstance(labels, list) or not labels:
raise ValueError("each item must have a nonempty candidate list")
if any(type(token) is not int or token < 0 for token in labels):
raise ValueError(
"label_token_ids must contain nonnegative integers"
)
if len(set(labels)) != len(labels):
raise ValueError(
"label_token_ids must not contain duplicate tokens"
)
if self.tokenizer is not None:
vocab_size = self.tokenizer.vocab_size
for token_id in labels:
if token_id >= vocab_size:
raise ValueError(
f"Token ID {token_id} is out of vocabulary (vocab size: {vocab_size})"
)
has_embeds = (
query_embed_overrides is not None or item_embed_overrides is not None
@@ -501,14 +563,6 @@ class TokenizerManagerScoreMixin:
f"item_embed_overrides length ({len(item_embed_overrides)}) "
f"must match items length ({len(items)})."
)
if self.tokenizer is not None and label_token_ids is not None:
vocab_size = self.tokenizer.vocab_size
for token_id in label_token_ids:
if token_id >= vocab_size:
raise ValueError(
f"Token ID {token_id} is out of vocabulary (vocab size: {vocab_size})"
)
# Check if multi-item scoring is enabled
use_multi_item_scoring = get_exec().features.enable_mis
@@ -601,10 +655,20 @@ class TokenizerManagerScoreMixin:
# Create the appropriate request type
mis_delimiter_indices = [delimiter_indices] if use_multi_item_scoring else None
if is_generation:
# packed MIS requests gather the union, then select per item
request_labels = (
list(
dict.fromkeys(
token for labels in label_token_ids for token in labels
)
)
if use_multi_item_scoring
else label_token_ids
)
batch_request = GenerateReqInput(
text=text_prompts,
input_ids=input_ids,
token_ids_logprob=label_token_ids,
token_ids_logprob=request_labels,
return_logprob=True,
# Set logprob_start_len=0 for multi-item scoring since we want logprobs at all delimiter positions
logprob_start_len=0 if use_multi_item_scoring else -1,
@@ -633,11 +697,18 @@ class TokenizerManagerScoreMixin:
apply_softmax,
batch_request,
return_pooled_hidden_states,
temperature,
return_token_logprobs,
)
else:
# Single-item scoring: process each result separately
return self._process_single_item_scoring_results(
results, label_token_ids, apply_softmax, return_pooled_hidden_states
results,
label_token_ids,
apply_softmax,
return_pooled_hidden_states,
temperature,
return_token_logprobs,
)
def _convert_logprobs_to_scores(
@@ -645,13 +716,20 @@ class TokenizerManagerScoreMixin:
logprobs: Dict[int, float],
label_token_ids: List[int],
apply_softmax: bool,
temperature: float = 1.0,
) -> List[float]:
score_list = [
logprobs.get(token_id, float("-inf")) for token_id in label_token_ids
]
if apply_softmax:
score_list = torch.softmax(torch.tensor(score_list), dim=0).tolist()
# center before scaling so small temperatures do not overflow
maximum = max(score_list)
weights = [
math.exp((score - maximum) / temperature) for score in score_list
]
denominator = sum(weights)
score_list = [weight / denominator for weight in weights]
else:
# Convert logprobs to probabilities if not using softmax
score_list = [