[feature] add per-item candidate token scoring and calibration (#40826)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
@@ -30,13 +30,15 @@ class EngineScoreMixin:
|
||||
self,
|
||||
query: Optional[Union[str, List[int]]] = None,
|
||||
items: Optional[Union[str, List[str], List[List[int]]]] = None,
|
||||
label_token_ids: Optional[List[int]] = None,
|
||||
label_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
|
||||
apply_softmax: bool = False,
|
||||
item_first: bool = False,
|
||||
embed_override_token_id: Optional[int] = None,
|
||||
query_embed_overrides: Optional[List[torch.Tensor]] = None,
|
||||
item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None,
|
||||
return_pooled_hidden_states: bool = False,
|
||||
temperature: float = 1.0,
|
||||
return_token_logprobs: bool = False,
|
||||
) -> ScoreResult:
|
||||
"""
|
||||
Score items against a query using the loaded model.
|
||||
@@ -54,8 +56,8 @@ class EngineScoreMixin:
|
||||
Args:
|
||||
query: The query text or pre-tokenized token IDs.
|
||||
items: The item text(s) or pre-tokenized token IDs.
|
||||
label_token_ids: Token IDs to score (required for CausalLM; ignored for
|
||||
SequenceClassification).
|
||||
label_token_ids: Shared token IDs or one list per item, preserving candidate
|
||||
order (required for CausalLM; ignored for SequenceClassification).
|
||||
apply_softmax: Whether to normalize scores using softmax.
|
||||
item_first: If True, prepend items before query (single-item mode only).
|
||||
embed_override_token_id: Placeholder token ID used to locate override positions.
|
||||
@@ -64,6 +66,10 @@ class EngineScoreMixin:
|
||||
return_pooled_hidden_states: Whether to include raw pooled transformer
|
||||
hidden states (before the task head) in the result. Only supported
|
||||
for non-generation models (SequenceClassification, RewardModel).
|
||||
temperature: Positive calibration temperature, applied before candidate/class
|
||||
softmax. Requires apply_softmax=True when different from 1.
|
||||
return_token_logprobs: Include uncalibrated full-vocabulary logprobs for
|
||||
each item's candidates. Only supported for CausalLM, not raw logits.
|
||||
|
||||
Returns:
|
||||
ScoreResult with scores (one list per item), prompt token count, and
|
||||
@@ -81,6 +87,8 @@ class EngineScoreMixin:
|
||||
item_embed_overrides=item_embed_overrides,
|
||||
request=None,
|
||||
return_pooled_hidden_states=return_pooled_hidden_states,
|
||||
temperature=temperature,
|
||||
return_token_logprobs=return_token_logprobs,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -88,13 +96,15 @@ class EngineScoreMixin:
|
||||
self,
|
||||
query: Optional[Union[str, List[int]]] = None,
|
||||
items: Optional[Union[str, List[str], List[List[int]]]] = None,
|
||||
label_token_ids: Optional[List[int]] = None,
|
||||
label_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
|
||||
apply_softmax: bool = False,
|
||||
item_first: bool = False,
|
||||
embed_override_token_id: Optional[int] = None,
|
||||
query_embed_overrides: Optional[List[torch.Tensor]] = None,
|
||||
item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None,
|
||||
return_pooled_hidden_states: bool = False,
|
||||
temperature: float = 1.0,
|
||||
return_token_logprobs: bool = False,
|
||||
) -> ScoreResult:
|
||||
"""Asynchronous version of score(). See score() for full documentation."""
|
||||
return await self.tokenizer_manager.score_request(
|
||||
@@ -108,4 +118,6 @@ class EngineScoreMixin:
|
||||
item_embed_overrides=item_embed_overrides,
|
||||
request=None,
|
||||
return_pooled_hidden_states=return_pooled_hidden_states,
|
||||
temperature=temperature,
|
||||
return_token_logprobs=return_token_logprobs,
|
||||
)
|
||||
|
||||
@@ -1398,10 +1398,12 @@ class ScoringRequest(BaseModel):
|
||||
item_embed_overrides: Optional[List[Optional[List[List[float]]]]] = (
|
||||
None # [num_items][num_item_embed_overrides][hidden_size]
|
||||
)
|
||||
label_token_ids: Optional[List[int]] = (
|
||||
None # Token IDs to compute probabilities for
|
||||
label_token_ids: Optional[Union[List[int], List[List[int]]]] = (
|
||||
None # shared candidates or one candidate list per item
|
||||
)
|
||||
apply_softmax: bool = False
|
||||
temperature: float = Field(default=1.0, gt=0, allow_inf_nan=False)
|
||||
return_token_logprobs: bool = False
|
||||
item_first: bool = False
|
||||
return_pooled_hidden_states: bool = False
|
||||
model: str = DEFAULT_MODEL_NAME
|
||||
@@ -1412,6 +1414,7 @@ class ScoringResponse(BaseModel):
|
||||
List[float]
|
||||
] # List of lists of probabilities, each in the order of label_token_ids
|
||||
pooled_hidden_states: Optional[List[Optional[List[float]]]] = None
|
||||
token_logprobs: Optional[List[List[float]]] = None
|
||||
model: str
|
||||
usage: Optional[UsageInfo] = None
|
||||
object: str = "scoring"
|
||||
|
||||
@@ -78,6 +78,8 @@ class OpenAIServingScore(OpenAIServingBase):
|
||||
item_embed_overrides=item_embed_overrides,
|
||||
request=raw_request,
|
||||
return_pooled_hidden_states=request.return_pooled_hidden_states,
|
||||
temperature=request.temperature,
|
||||
return_token_logprobs=request.return_token_logprobs,
|
||||
)
|
||||
|
||||
phs_as_lists = None
|
||||
@@ -90,6 +92,7 @@ class OpenAIServingScore(OpenAIServingBase):
|
||||
response = ScoringResponse(
|
||||
scores=result.scores,
|
||||
pooled_hidden_states=phs_as_lists,
|
||||
token_logprobs=result.token_logprobs,
|
||||
model=request.model,
|
||||
usage=UsageInfo(
|
||||
prompt_tokens=result.prompt_tokens,
|
||||
|
||||
@@ -24,15 +24,19 @@ class ScoreResult:
|
||||
# The HTTP path converts to lists in serving_score.py before JSON serialization.
|
||||
# Same layout as scores: one tensor per item (not a single packed 2D tensor).
|
||||
pooled_hidden_states: Optional[List[Optional[torch.Tensor]]] = None
|
||||
# uncalibrated full-vocabulary logprobs, in each item's candidate order
|
||||
token_logprobs: Optional[List[List[float]]] = None
|
||||
|
||||
|
||||
class TokenizerManagerScoreMixin:
|
||||
async def score_prompts(
|
||||
self,
|
||||
prompts: Union[str, List[str], List[List[int]]],
|
||||
label_token_ids: List[int],
|
||||
label_token_ids: Union[List[int], List[List[int]]],
|
||||
apply_softmax: bool = False,
|
||||
request: Optional[Any] = None,
|
||||
temperature: float = 1.0,
|
||||
return_token_logprobs: bool = False,
|
||||
) -> ScoreResult:
|
||||
"""
|
||||
Score probabilities of specified token IDs after each *full prompt*.
|
||||
@@ -51,6 +55,8 @@ class TokenizerManagerScoreMixin:
|
||||
apply_softmax=apply_softmax,
|
||||
item_first=False,
|
||||
request=request,
|
||||
temperature=temperature,
|
||||
return_token_logprobs=return_token_logprobs,
|
||||
)
|
||||
|
||||
# Tokenized prompts
|
||||
@@ -62,6 +68,8 @@ class TokenizerManagerScoreMixin:
|
||||
apply_softmax=apply_softmax,
|
||||
item_first=False,
|
||||
request=request,
|
||||
temperature=temperature,
|
||||
return_token_logprobs=return_token_logprobs,
|
||||
)
|
||||
|
||||
raise ValueError("Invalid prompts type for score_prompts.")
|
||||
@@ -102,7 +110,8 @@ class TokenizerManagerScoreMixin:
|
||||
items_ids = []
|
||||
for item in items_list:
|
||||
if isinstance(item, str):
|
||||
items_ids.append(self.tokenizer.encode(item))
|
||||
# items continue the query, so only the query gets special tokens
|
||||
items_ids.append(self.tokenizer.encode(item, add_special_tokens=False))
|
||||
else:
|
||||
items_ids.append(list(item))
|
||||
|
||||
@@ -112,10 +121,12 @@ class TokenizerManagerScoreMixin:
|
||||
self,
|
||||
results: Any,
|
||||
items: List,
|
||||
label_token_ids: Optional[List[int]],
|
||||
label_token_ids: Optional[List[List[int]]],
|
||||
apply_softmax: bool,
|
||||
batch_request=None,
|
||||
return_pooled_hidden_states: bool = False,
|
||||
temperature: float = 1.0,
|
||||
return_token_logprobs: bool = False,
|
||||
) -> ScoreResult:
|
||||
"""
|
||||
Process results from multi-item scoring request.
|
||||
@@ -135,27 +146,31 @@ class TokenizerManagerScoreMixin:
|
||||
# Extract per-delimiter scores from whichever field has them
|
||||
input_logprobs = meta_info.get("input_token_ids_logprobs", [])
|
||||
embedding = single_result.get("embedding")
|
||||
token_logprobs = [] if return_token_logprobs else None
|
||||
|
||||
if input_logprobs:
|
||||
# Generation model: extract label-token logprobs at each delimiter
|
||||
per_delimiter_scores = []
|
||||
for logprobs_data in input_logprobs:
|
||||
logprobs = self._extract_logprobs_for_tokens(
|
||||
logprobs_data, label_token_ids
|
||||
if len(input_logprobs) != expected_count:
|
||||
raise RuntimeError(
|
||||
f"Expected {expected_count} delimiter entries, got {len(input_logprobs)}"
|
||||
)
|
||||
per_delimiter_scores = [[]] # the query boundary is not a decision
|
||||
for logprobs_data, labels in zip(input_logprobs[1:], label_token_ids):
|
||||
logprobs = self._extract_logprobs_for_tokens(logprobs_data, labels)
|
||||
score_list = self._convert_logprobs_to_scores(
|
||||
logprobs, label_token_ids, apply_softmax
|
||||
logprobs, labels, apply_softmax, temperature
|
||||
)
|
||||
per_delimiter_scores.append(score_list)
|
||||
if return_token_logprobs:
|
||||
token_logprobs.append([logprobs[token] for token in labels])
|
||||
elif embedding is not None:
|
||||
# Classification model: scores are directly in 2D embedding.
|
||||
if apply_softmax:
|
||||
scores_tensor = (
|
||||
torch.tensor(embedding)
|
||||
if isinstance(embedding, list)
|
||||
else embedding
|
||||
scores_tensor = torch.as_tensor(embedding, dtype=torch.float64)
|
||||
scores_tensor = scores_tensor - scores_tensor.amax(dim=-1, keepdim=True)
|
||||
scores_tensor = torch.nn.functional.softmax(
|
||||
scores_tensor / temperature, dim=-1
|
||||
)
|
||||
scores_tensor = torch.nn.functional.softmax(scores_tensor, dim=-1)
|
||||
per_delimiter_scores = scores_tensor.tolist()
|
||||
else:
|
||||
per_delimiter_scores = (
|
||||
@@ -188,14 +203,17 @@ class TokenizerManagerScoreMixin:
|
||||
scores=scores,
|
||||
prompt_tokens=prompt_tokens,
|
||||
pooled_hidden_states=phs_list,
|
||||
token_logprobs=token_logprobs,
|
||||
)
|
||||
|
||||
def _process_single_item_scoring_results(
|
||||
self,
|
||||
results: Any,
|
||||
label_token_ids: Optional[List[int]],
|
||||
label_token_ids: Optional[List[List[int]]],
|
||||
apply_softmax: bool,
|
||||
return_pooled_hidden_states: bool = False,
|
||||
temperature: float = 1.0,
|
||||
return_token_logprobs: bool = False,
|
||||
) -> ScoreResult:
|
||||
"""
|
||||
Process results from single-item scoring request.
|
||||
@@ -208,10 +226,11 @@ class TokenizerManagerScoreMixin:
|
||||
phs_list = []
|
||||
has_phs = False
|
||||
prompt_tokens = 0
|
||||
token_logprobs = [] if return_token_logprobs else None
|
||||
|
||||
is_generation = self.is_generation
|
||||
if is_generation:
|
||||
for result in results:
|
||||
for result, labels in zip(results, label_token_ids):
|
||||
# For single-item scoring, logprobs are in output_token_ids_logprobs
|
||||
output_logprobs = result["meta_info"].get(
|
||||
"output_token_ids_logprobs", []
|
||||
@@ -225,13 +244,13 @@ class TokenizerManagerScoreMixin:
|
||||
)
|
||||
|
||||
# Extract logprobs for the first (and only) position
|
||||
logprobs = self._extract_logprobs_for_tokens(
|
||||
output_logprobs[0], label_token_ids
|
||||
)
|
||||
logprobs = self._extract_logprobs_for_tokens(output_logprobs[0], labels)
|
||||
score_list = self._convert_logprobs_to_scores(
|
||||
logprobs, label_token_ids, apply_softmax
|
||||
logprobs, labels, apply_softmax, temperature
|
||||
)
|
||||
scores.append(score_list)
|
||||
if return_token_logprobs:
|
||||
token_logprobs.append([logprobs[token] for token in labels])
|
||||
else:
|
||||
for result in results:
|
||||
embedding = result.get("embedding", None)
|
||||
@@ -241,8 +260,9 @@ class TokenizerManagerScoreMixin:
|
||||
prompt_tokens += result.get("meta_info", {}).get("prompt_tokens", 0)
|
||||
|
||||
if apply_softmax:
|
||||
scores_tensor = torch.as_tensor(embedding, dtype=torch.float64)
|
||||
embedding = torch.softmax(
|
||||
torch.as_tensor(embedding), dim=-1
|
||||
(scores_tensor - scores_tensor.max()) / temperature, dim=-1
|
||||
).tolist()
|
||||
|
||||
# The classification head produces per-token logits, which the pooler reduces
|
||||
@@ -262,6 +282,7 @@ class TokenizerManagerScoreMixin:
|
||||
scores=scores,
|
||||
prompt_tokens=prompt_tokens,
|
||||
pooled_hidden_states=phs_list if has_phs else None,
|
||||
token_logprobs=token_logprobs,
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
@@ -445,7 +466,7 @@ class TokenizerManagerScoreMixin:
|
||||
self,
|
||||
query: Optional[Union[str, List[int]]] = None,
|
||||
items: Optional[Union[str, List[str], List[List[int]]]] = None,
|
||||
label_token_ids: Optional[List[int]] = None,
|
||||
label_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
|
||||
apply_softmax: bool = False,
|
||||
item_first: bool = False,
|
||||
embed_override_token_id: Optional[int] = None,
|
||||
@@ -453,6 +474,8 @@ class TokenizerManagerScoreMixin:
|
||||
item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None,
|
||||
request: Optional[Any] = None,
|
||||
return_pooled_hidden_states: bool = False,
|
||||
temperature: float = 1.0,
|
||||
return_token_logprobs: bool = False,
|
||||
) -> ScoreResult:
|
||||
"""
|
||||
Score the probability of specified token IDs appearing after the given (query + item) pair.
|
||||
@@ -477,6 +500,15 @@ class TokenizerManagerScoreMixin:
|
||||
"""
|
||||
is_generation = self.is_generation
|
||||
|
||||
if not math.isfinite(temperature) or temperature <= 0:
|
||||
raise ValueError("temperature must be finite and greater than zero")
|
||||
if temperature != 1.0 and not apply_softmax:
|
||||
raise ValueError("temperature requires apply_softmax=True")
|
||||
if return_token_logprobs and not is_generation:
|
||||
raise ValueError(
|
||||
"return_token_logprobs is only supported for CausalLM models"
|
||||
)
|
||||
|
||||
if is_generation and label_token_ids is None:
|
||||
raise ValueError(
|
||||
"label_token_ids is required for generation (CausalLM) models."
|
||||
@@ -484,7 +516,37 @@ class TokenizerManagerScoreMixin:
|
||||
if items is None:
|
||||
raise ValueError("items must be provided")
|
||||
if not items:
|
||||
return ScoreResult(scores=[], prompt_tokens=0)
|
||||
return ScoreResult(
|
||||
scores=[], token_logprobs=[] if return_token_logprobs else None
|
||||
)
|
||||
|
||||
num_items = 1 if isinstance(items, str) else len(items)
|
||||
if is_generation:
|
||||
if not isinstance(label_token_ids, list) or not label_token_ids:
|
||||
raise ValueError("label_token_ids must be a nonempty list")
|
||||
if isinstance(label_token_ids[0], list):
|
||||
if len(label_token_ids) != num_items:
|
||||
raise ValueError("label_token_ids must have one list per item")
|
||||
else:
|
||||
label_token_ids = [label_token_ids] * num_items
|
||||
for labels in label_token_ids:
|
||||
if not isinstance(labels, list) or not labels:
|
||||
raise ValueError("each item must have a nonempty candidate list")
|
||||
if any(type(token) is not int or token < 0 for token in labels):
|
||||
raise ValueError(
|
||||
"label_token_ids must contain nonnegative integers"
|
||||
)
|
||||
if len(set(labels)) != len(labels):
|
||||
raise ValueError(
|
||||
"label_token_ids must not contain duplicate tokens"
|
||||
)
|
||||
if self.tokenizer is not None:
|
||||
vocab_size = self.tokenizer.vocab_size
|
||||
for token_id in labels:
|
||||
if token_id >= vocab_size:
|
||||
raise ValueError(
|
||||
f"Token ID {token_id} is out of vocabulary (vocab size: {vocab_size})"
|
||||
)
|
||||
|
||||
has_embeds = (
|
||||
query_embed_overrides is not None or item_embed_overrides is not None
|
||||
@@ -501,14 +563,6 @@ class TokenizerManagerScoreMixin:
|
||||
f"item_embed_overrides length ({len(item_embed_overrides)}) "
|
||||
f"must match items length ({len(items)})."
|
||||
)
|
||||
if self.tokenizer is not None and label_token_ids is not None:
|
||||
vocab_size = self.tokenizer.vocab_size
|
||||
for token_id in label_token_ids:
|
||||
if token_id >= vocab_size:
|
||||
raise ValueError(
|
||||
f"Token ID {token_id} is out of vocabulary (vocab size: {vocab_size})"
|
||||
)
|
||||
|
||||
# Check if multi-item scoring is enabled
|
||||
use_multi_item_scoring = get_exec().features.enable_mis
|
||||
|
||||
@@ -601,10 +655,20 @@ class TokenizerManagerScoreMixin:
|
||||
# Create the appropriate request type
|
||||
mis_delimiter_indices = [delimiter_indices] if use_multi_item_scoring else None
|
||||
if is_generation:
|
||||
# packed MIS requests gather the union, then select per item
|
||||
request_labels = (
|
||||
list(
|
||||
dict.fromkeys(
|
||||
token for labels in label_token_ids for token in labels
|
||||
)
|
||||
)
|
||||
if use_multi_item_scoring
|
||||
else label_token_ids
|
||||
)
|
||||
batch_request = GenerateReqInput(
|
||||
text=text_prompts,
|
||||
input_ids=input_ids,
|
||||
token_ids_logprob=label_token_ids,
|
||||
token_ids_logprob=request_labels,
|
||||
return_logprob=True,
|
||||
# Set logprob_start_len=0 for multi-item scoring since we want logprobs at all delimiter positions
|
||||
logprob_start_len=0 if use_multi_item_scoring else -1,
|
||||
@@ -633,11 +697,18 @@ class TokenizerManagerScoreMixin:
|
||||
apply_softmax,
|
||||
batch_request,
|
||||
return_pooled_hidden_states,
|
||||
temperature,
|
||||
return_token_logprobs,
|
||||
)
|
||||
else:
|
||||
# Single-item scoring: process each result separately
|
||||
return self._process_single_item_scoring_results(
|
||||
results, label_token_ids, apply_softmax, return_pooled_hidden_states
|
||||
results,
|
||||
label_token_ids,
|
||||
apply_softmax,
|
||||
return_pooled_hidden_states,
|
||||
temperature,
|
||||
return_token_logprobs,
|
||||
)
|
||||
|
||||
def _convert_logprobs_to_scores(
|
||||
@@ -645,13 +716,20 @@ class TokenizerManagerScoreMixin:
|
||||
logprobs: Dict[int, float],
|
||||
label_token_ids: List[int],
|
||||
apply_softmax: bool,
|
||||
temperature: float = 1.0,
|
||||
) -> List[float]:
|
||||
score_list = [
|
||||
logprobs.get(token_id, float("-inf")) for token_id in label_token_ids
|
||||
]
|
||||
|
||||
if apply_softmax:
|
||||
score_list = torch.softmax(torch.tensor(score_list), dim=0).tolist()
|
||||
# center before scaling so small temperatures do not overflow
|
||||
maximum = max(score_list)
|
||||
weights = [
|
||||
math.exp((score - maximum) / temperature) for score in score_list
|
||||
]
|
||||
denominator = sum(weights)
|
||||
score_list = [weight / denominator for weight in weights]
|
||||
else:
|
||||
# Convert logprobs to probabilities if not using softmax
|
||||
score_list = [
|
||||
|
||||
Reference in New Issue
Block a user