[feature] add per-item candidate token scoring and calibration (#40826)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,72 @@
|
||||
"""Score SemIf JSONL decisions using its original prompts and an SGLang server."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
from semif_phase1.direct import PROMPT_VERSION, encode_prompt
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:30000")
|
||||
parser.add_argument("--model", default="Qwen/Qwen3.5-4B")
|
||||
parser.add_argument("--revision", required=True)
|
||||
parser.add_argument("--input", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--batch-size", type=int, default=32)
|
||||
parser.add_argument("--max-tokens", type=int, default=4096)
|
||||
parser.add_argument("--temperature", type=float, default=1.0)
|
||||
args = parser.parse_args()
|
||||
if args.batch_size < 1:
|
||||
parser.error("--batch-size must be positive")
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(args.model, revision=args.revision)
|
||||
rows = [
|
||||
json.loads(line) for line in args.input.read_text().splitlines() if line.strip()
|
||||
]
|
||||
with args.output.open("x") as output, requests.Session() as session:
|
||||
for start in range(0, len(rows), args.batch_size):
|
||||
batch = rows[start : start + args.batch_size]
|
||||
encoded = [encode_prompt(tokenizer, row, args.max_tokens) for row in batch]
|
||||
response = session.post(
|
||||
args.base_url.rstrip("/") + "/v1/score",
|
||||
json={
|
||||
"model": args.model,
|
||||
"query": [],
|
||||
"items": [ids for ids, _, _ in encoded],
|
||||
"label_token_ids": [slots for _, slots, _ in encoded],
|
||||
"apply_softmax": True,
|
||||
"temperature": args.temperature,
|
||||
"return_token_logprobs": True,
|
||||
},
|
||||
timeout=300,
|
||||
)
|
||||
response.raise_for_status()
|
||||
result = response.json()
|
||||
for row, (ids, _, prompt_hash), probabilities, logprobs in zip(
|
||||
batch, encoded, result["scores"], result["token_logprobs"], strict=True
|
||||
):
|
||||
output.write(
|
||||
json.dumps(
|
||||
{
|
||||
"id": row["id"],
|
||||
"option_ids": [option["id"] for option in row["options"]],
|
||||
"probabilities": probabilities,
|
||||
"token_logprobs": logprobs,
|
||||
"input_tokens": len(ids),
|
||||
"prompt_sha256": prompt_hash,
|
||||
"prompt_version": PROMPT_VERSION,
|
||||
"temperature": args.temperature,
|
||||
"readout": "sglang candidate token logprobs",
|
||||
},
|
||||
allow_nan=False,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -30,13 +30,15 @@ class EngineScoreMixin:
|
||||
self,
|
||||
query: Optional[Union[str, List[int]]] = None,
|
||||
items: Optional[Union[str, List[str], List[List[int]]]] = None,
|
||||
label_token_ids: Optional[List[int]] = None,
|
||||
label_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
|
||||
apply_softmax: bool = False,
|
||||
item_first: bool = False,
|
||||
embed_override_token_id: Optional[int] = None,
|
||||
query_embed_overrides: Optional[List[torch.Tensor]] = None,
|
||||
item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None,
|
||||
return_pooled_hidden_states: bool = False,
|
||||
temperature: float = 1.0,
|
||||
return_token_logprobs: bool = False,
|
||||
) -> ScoreResult:
|
||||
"""
|
||||
Score items against a query using the loaded model.
|
||||
@@ -54,8 +56,8 @@ class EngineScoreMixin:
|
||||
Args:
|
||||
query: The query text or pre-tokenized token IDs.
|
||||
items: The item text(s) or pre-tokenized token IDs.
|
||||
label_token_ids: Token IDs to score (required for CausalLM; ignored for
|
||||
SequenceClassification).
|
||||
label_token_ids: Shared token IDs or one list per item, preserving candidate
|
||||
order (required for CausalLM; ignored for SequenceClassification).
|
||||
apply_softmax: Whether to normalize scores using softmax.
|
||||
item_first: If True, prepend items before query (single-item mode only).
|
||||
embed_override_token_id: Placeholder token ID used to locate override positions.
|
||||
@@ -64,6 +66,10 @@ class EngineScoreMixin:
|
||||
return_pooled_hidden_states: Whether to include raw pooled transformer
|
||||
hidden states (before the task head) in the result. Only supported
|
||||
for non-generation models (SequenceClassification, RewardModel).
|
||||
temperature: Positive calibration temperature, applied before candidate/class
|
||||
softmax. Requires apply_softmax=True when different from 1.
|
||||
return_token_logprobs: Include uncalibrated full-vocabulary logprobs for
|
||||
each item's candidates. Only supported for CausalLM, not raw logits.
|
||||
|
||||
Returns:
|
||||
ScoreResult with scores (one list per item), prompt token count, and
|
||||
@@ -81,6 +87,8 @@ class EngineScoreMixin:
|
||||
item_embed_overrides=item_embed_overrides,
|
||||
request=None,
|
||||
return_pooled_hidden_states=return_pooled_hidden_states,
|
||||
temperature=temperature,
|
||||
return_token_logprobs=return_token_logprobs,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -88,13 +96,15 @@ class EngineScoreMixin:
|
||||
self,
|
||||
query: Optional[Union[str, List[int]]] = None,
|
||||
items: Optional[Union[str, List[str], List[List[int]]]] = None,
|
||||
label_token_ids: Optional[List[int]] = None,
|
||||
label_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
|
||||
apply_softmax: bool = False,
|
||||
item_first: bool = False,
|
||||
embed_override_token_id: Optional[int] = None,
|
||||
query_embed_overrides: Optional[List[torch.Tensor]] = None,
|
||||
item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None,
|
||||
return_pooled_hidden_states: bool = False,
|
||||
temperature: float = 1.0,
|
||||
return_token_logprobs: bool = False,
|
||||
) -> ScoreResult:
|
||||
"""Asynchronous version of score(). See score() for full documentation."""
|
||||
return await self.tokenizer_manager.score_request(
|
||||
@@ -108,4 +118,6 @@ class EngineScoreMixin:
|
||||
item_embed_overrides=item_embed_overrides,
|
||||
request=None,
|
||||
return_pooled_hidden_states=return_pooled_hidden_states,
|
||||
temperature=temperature,
|
||||
return_token_logprobs=return_token_logprobs,
|
||||
)
|
||||
|
||||
@@ -1398,10 +1398,12 @@ class ScoringRequest(BaseModel):
|
||||
item_embed_overrides: Optional[List[Optional[List[List[float]]]]] = (
|
||||
None # [num_items][num_item_embed_overrides][hidden_size]
|
||||
)
|
||||
label_token_ids: Optional[List[int]] = (
|
||||
None # Token IDs to compute probabilities for
|
||||
label_token_ids: Optional[Union[List[int], List[List[int]]]] = (
|
||||
None # shared candidates or one candidate list per item
|
||||
)
|
||||
apply_softmax: bool = False
|
||||
temperature: float = Field(default=1.0, gt=0, allow_inf_nan=False)
|
||||
return_token_logprobs: bool = False
|
||||
item_first: bool = False
|
||||
return_pooled_hidden_states: bool = False
|
||||
model: str = DEFAULT_MODEL_NAME
|
||||
@@ -1412,6 +1414,7 @@ class ScoringResponse(BaseModel):
|
||||
List[float]
|
||||
] # List of lists of probabilities, each in the order of label_token_ids
|
||||
pooled_hidden_states: Optional[List[Optional[List[float]]]] = None
|
||||
token_logprobs: Optional[List[List[float]]] = None
|
||||
model: str
|
||||
usage: Optional[UsageInfo] = None
|
||||
object: str = "scoring"
|
||||
|
||||
@@ -78,6 +78,8 @@ class OpenAIServingScore(OpenAIServingBase):
|
||||
item_embed_overrides=item_embed_overrides,
|
||||
request=raw_request,
|
||||
return_pooled_hidden_states=request.return_pooled_hidden_states,
|
||||
temperature=request.temperature,
|
||||
return_token_logprobs=request.return_token_logprobs,
|
||||
)
|
||||
|
||||
phs_as_lists = None
|
||||
@@ -90,6 +92,7 @@ class OpenAIServingScore(OpenAIServingBase):
|
||||
response = ScoringResponse(
|
||||
scores=result.scores,
|
||||
pooled_hidden_states=phs_as_lists,
|
||||
token_logprobs=result.token_logprobs,
|
||||
model=request.model,
|
||||
usage=UsageInfo(
|
||||
prompt_tokens=result.prompt_tokens,
|
||||
|
||||
@@ -24,15 +24,19 @@ class ScoreResult:
|
||||
# The HTTP path converts to lists in serving_score.py before JSON serialization.
|
||||
# Same layout as scores: one tensor per item (not a single packed 2D tensor).
|
||||
pooled_hidden_states: Optional[List[Optional[torch.Tensor]]] = None
|
||||
# uncalibrated full-vocabulary logprobs, in each item's candidate order
|
||||
token_logprobs: Optional[List[List[float]]] = None
|
||||
|
||||
|
||||
class TokenizerManagerScoreMixin:
|
||||
async def score_prompts(
|
||||
self,
|
||||
prompts: Union[str, List[str], List[List[int]]],
|
||||
label_token_ids: List[int],
|
||||
label_token_ids: Union[List[int], List[List[int]]],
|
||||
apply_softmax: bool = False,
|
||||
request: Optional[Any] = None,
|
||||
temperature: float = 1.0,
|
||||
return_token_logprobs: bool = False,
|
||||
) -> ScoreResult:
|
||||
"""
|
||||
Score probabilities of specified token IDs after each *full prompt*.
|
||||
@@ -51,6 +55,8 @@ class TokenizerManagerScoreMixin:
|
||||
apply_softmax=apply_softmax,
|
||||
item_first=False,
|
||||
request=request,
|
||||
temperature=temperature,
|
||||
return_token_logprobs=return_token_logprobs,
|
||||
)
|
||||
|
||||
# Tokenized prompts
|
||||
@@ -62,6 +68,8 @@ class TokenizerManagerScoreMixin:
|
||||
apply_softmax=apply_softmax,
|
||||
item_first=False,
|
||||
request=request,
|
||||
temperature=temperature,
|
||||
return_token_logprobs=return_token_logprobs,
|
||||
)
|
||||
|
||||
raise ValueError("Invalid prompts type for score_prompts.")
|
||||
@@ -102,7 +110,8 @@ class TokenizerManagerScoreMixin:
|
||||
items_ids = []
|
||||
for item in items_list:
|
||||
if isinstance(item, str):
|
||||
items_ids.append(self.tokenizer.encode(item))
|
||||
# items continue the query, so only the query gets special tokens
|
||||
items_ids.append(self.tokenizer.encode(item, add_special_tokens=False))
|
||||
else:
|
||||
items_ids.append(list(item))
|
||||
|
||||
@@ -112,10 +121,12 @@ class TokenizerManagerScoreMixin:
|
||||
self,
|
||||
results: Any,
|
||||
items: List,
|
||||
label_token_ids: Optional[List[int]],
|
||||
label_token_ids: Optional[List[List[int]]],
|
||||
apply_softmax: bool,
|
||||
batch_request=None,
|
||||
return_pooled_hidden_states: bool = False,
|
||||
temperature: float = 1.0,
|
||||
return_token_logprobs: bool = False,
|
||||
) -> ScoreResult:
|
||||
"""
|
||||
Process results from multi-item scoring request.
|
||||
@@ -135,27 +146,31 @@ class TokenizerManagerScoreMixin:
|
||||
# Extract per-delimiter scores from whichever field has them
|
||||
input_logprobs = meta_info.get("input_token_ids_logprobs", [])
|
||||
embedding = single_result.get("embedding")
|
||||
token_logprobs = [] if return_token_logprobs else None
|
||||
|
||||
if input_logprobs:
|
||||
# Generation model: extract label-token logprobs at each delimiter
|
||||
per_delimiter_scores = []
|
||||
for logprobs_data in input_logprobs:
|
||||
logprobs = self._extract_logprobs_for_tokens(
|
||||
logprobs_data, label_token_ids
|
||||
if len(input_logprobs) != expected_count:
|
||||
raise RuntimeError(
|
||||
f"Expected {expected_count} delimiter entries, got {len(input_logprobs)}"
|
||||
)
|
||||
per_delimiter_scores = [[]] # the query boundary is not a decision
|
||||
for logprobs_data, labels in zip(input_logprobs[1:], label_token_ids):
|
||||
logprobs = self._extract_logprobs_for_tokens(logprobs_data, labels)
|
||||
score_list = self._convert_logprobs_to_scores(
|
||||
logprobs, label_token_ids, apply_softmax
|
||||
logprobs, labels, apply_softmax, temperature
|
||||
)
|
||||
per_delimiter_scores.append(score_list)
|
||||
if return_token_logprobs:
|
||||
token_logprobs.append([logprobs[token] for token in labels])
|
||||
elif embedding is not None:
|
||||
# Classification model: scores are directly in 2D embedding.
|
||||
if apply_softmax:
|
||||
scores_tensor = (
|
||||
torch.tensor(embedding)
|
||||
if isinstance(embedding, list)
|
||||
else embedding
|
||||
scores_tensor = torch.as_tensor(embedding, dtype=torch.float64)
|
||||
scores_tensor = scores_tensor - scores_tensor.amax(dim=-1, keepdim=True)
|
||||
scores_tensor = torch.nn.functional.softmax(
|
||||
scores_tensor / temperature, dim=-1
|
||||
)
|
||||
scores_tensor = torch.nn.functional.softmax(scores_tensor, dim=-1)
|
||||
per_delimiter_scores = scores_tensor.tolist()
|
||||
else:
|
||||
per_delimiter_scores = (
|
||||
@@ -188,14 +203,17 @@ class TokenizerManagerScoreMixin:
|
||||
scores=scores,
|
||||
prompt_tokens=prompt_tokens,
|
||||
pooled_hidden_states=phs_list,
|
||||
token_logprobs=token_logprobs,
|
||||
)
|
||||
|
||||
def _process_single_item_scoring_results(
|
||||
self,
|
||||
results: Any,
|
||||
label_token_ids: Optional[List[int]],
|
||||
label_token_ids: Optional[List[List[int]]],
|
||||
apply_softmax: bool,
|
||||
return_pooled_hidden_states: bool = False,
|
||||
temperature: float = 1.0,
|
||||
return_token_logprobs: bool = False,
|
||||
) -> ScoreResult:
|
||||
"""
|
||||
Process results from single-item scoring request.
|
||||
@@ -208,10 +226,11 @@ class TokenizerManagerScoreMixin:
|
||||
phs_list = []
|
||||
has_phs = False
|
||||
prompt_tokens = 0
|
||||
token_logprobs = [] if return_token_logprobs else None
|
||||
|
||||
is_generation = self.is_generation
|
||||
if is_generation:
|
||||
for result in results:
|
||||
for result, labels in zip(results, label_token_ids):
|
||||
# For single-item scoring, logprobs are in output_token_ids_logprobs
|
||||
output_logprobs = result["meta_info"].get(
|
||||
"output_token_ids_logprobs", []
|
||||
@@ -225,13 +244,13 @@ class TokenizerManagerScoreMixin:
|
||||
)
|
||||
|
||||
# Extract logprobs for the first (and only) position
|
||||
logprobs = self._extract_logprobs_for_tokens(
|
||||
output_logprobs[0], label_token_ids
|
||||
)
|
||||
logprobs = self._extract_logprobs_for_tokens(output_logprobs[0], labels)
|
||||
score_list = self._convert_logprobs_to_scores(
|
||||
logprobs, label_token_ids, apply_softmax
|
||||
logprobs, labels, apply_softmax, temperature
|
||||
)
|
||||
scores.append(score_list)
|
||||
if return_token_logprobs:
|
||||
token_logprobs.append([logprobs[token] for token in labels])
|
||||
else:
|
||||
for result in results:
|
||||
embedding = result.get("embedding", None)
|
||||
@@ -241,8 +260,9 @@ class TokenizerManagerScoreMixin:
|
||||
prompt_tokens += result.get("meta_info", {}).get("prompt_tokens", 0)
|
||||
|
||||
if apply_softmax:
|
||||
scores_tensor = torch.as_tensor(embedding, dtype=torch.float64)
|
||||
embedding = torch.softmax(
|
||||
torch.as_tensor(embedding), dim=-1
|
||||
(scores_tensor - scores_tensor.max()) / temperature, dim=-1
|
||||
).tolist()
|
||||
|
||||
# The classification head produces per-token logits, which the pooler reduces
|
||||
@@ -262,6 +282,7 @@ class TokenizerManagerScoreMixin:
|
||||
scores=scores,
|
||||
prompt_tokens=prompt_tokens,
|
||||
pooled_hidden_states=phs_list if has_phs else None,
|
||||
token_logprobs=token_logprobs,
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
@@ -445,7 +466,7 @@ class TokenizerManagerScoreMixin:
|
||||
self,
|
||||
query: Optional[Union[str, List[int]]] = None,
|
||||
items: Optional[Union[str, List[str], List[List[int]]]] = None,
|
||||
label_token_ids: Optional[List[int]] = None,
|
||||
label_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
|
||||
apply_softmax: bool = False,
|
||||
item_first: bool = False,
|
||||
embed_override_token_id: Optional[int] = None,
|
||||
@@ -453,6 +474,8 @@ class TokenizerManagerScoreMixin:
|
||||
item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None,
|
||||
request: Optional[Any] = None,
|
||||
return_pooled_hidden_states: bool = False,
|
||||
temperature: float = 1.0,
|
||||
return_token_logprobs: bool = False,
|
||||
) -> ScoreResult:
|
||||
"""
|
||||
Score the probability of specified token IDs appearing after the given (query + item) pair.
|
||||
@@ -477,6 +500,15 @@ class TokenizerManagerScoreMixin:
|
||||
"""
|
||||
is_generation = self.is_generation
|
||||
|
||||
if not math.isfinite(temperature) or temperature <= 0:
|
||||
raise ValueError("temperature must be finite and greater than zero")
|
||||
if temperature != 1.0 and not apply_softmax:
|
||||
raise ValueError("temperature requires apply_softmax=True")
|
||||
if return_token_logprobs and not is_generation:
|
||||
raise ValueError(
|
||||
"return_token_logprobs is only supported for CausalLM models"
|
||||
)
|
||||
|
||||
if is_generation and label_token_ids is None:
|
||||
raise ValueError(
|
||||
"label_token_ids is required for generation (CausalLM) models."
|
||||
@@ -484,7 +516,37 @@ class TokenizerManagerScoreMixin:
|
||||
if items is None:
|
||||
raise ValueError("items must be provided")
|
||||
if not items:
|
||||
return ScoreResult(scores=[], prompt_tokens=0)
|
||||
return ScoreResult(
|
||||
scores=[], token_logprobs=[] if return_token_logprobs else None
|
||||
)
|
||||
|
||||
num_items = 1 if isinstance(items, str) else len(items)
|
||||
if is_generation:
|
||||
if not isinstance(label_token_ids, list) or not label_token_ids:
|
||||
raise ValueError("label_token_ids must be a nonempty list")
|
||||
if isinstance(label_token_ids[0], list):
|
||||
if len(label_token_ids) != num_items:
|
||||
raise ValueError("label_token_ids must have one list per item")
|
||||
else:
|
||||
label_token_ids = [label_token_ids] * num_items
|
||||
for labels in label_token_ids:
|
||||
if not isinstance(labels, list) or not labels:
|
||||
raise ValueError("each item must have a nonempty candidate list")
|
||||
if any(type(token) is not int or token < 0 for token in labels):
|
||||
raise ValueError(
|
||||
"label_token_ids must contain nonnegative integers"
|
||||
)
|
||||
if len(set(labels)) != len(labels):
|
||||
raise ValueError(
|
||||
"label_token_ids must not contain duplicate tokens"
|
||||
)
|
||||
if self.tokenizer is not None:
|
||||
vocab_size = self.tokenizer.vocab_size
|
||||
for token_id in labels:
|
||||
if token_id >= vocab_size:
|
||||
raise ValueError(
|
||||
f"Token ID {token_id} is out of vocabulary (vocab size: {vocab_size})"
|
||||
)
|
||||
|
||||
has_embeds = (
|
||||
query_embed_overrides is not None or item_embed_overrides is not None
|
||||
@@ -501,14 +563,6 @@ class TokenizerManagerScoreMixin:
|
||||
f"item_embed_overrides length ({len(item_embed_overrides)}) "
|
||||
f"must match items length ({len(items)})."
|
||||
)
|
||||
if self.tokenizer is not None and label_token_ids is not None:
|
||||
vocab_size = self.tokenizer.vocab_size
|
||||
for token_id in label_token_ids:
|
||||
if token_id >= vocab_size:
|
||||
raise ValueError(
|
||||
f"Token ID {token_id} is out of vocabulary (vocab size: {vocab_size})"
|
||||
)
|
||||
|
||||
# Check if multi-item scoring is enabled
|
||||
use_multi_item_scoring = get_exec().features.enable_mis
|
||||
|
||||
@@ -601,10 +655,20 @@ class TokenizerManagerScoreMixin:
|
||||
# Create the appropriate request type
|
||||
mis_delimiter_indices = [delimiter_indices] if use_multi_item_scoring else None
|
||||
if is_generation:
|
||||
# packed MIS requests gather the union, then select per item
|
||||
request_labels = (
|
||||
list(
|
||||
dict.fromkeys(
|
||||
token for labels in label_token_ids for token in labels
|
||||
)
|
||||
)
|
||||
if use_multi_item_scoring
|
||||
else label_token_ids
|
||||
)
|
||||
batch_request = GenerateReqInput(
|
||||
text=text_prompts,
|
||||
input_ids=input_ids,
|
||||
token_ids_logprob=label_token_ids,
|
||||
token_ids_logprob=request_labels,
|
||||
return_logprob=True,
|
||||
# Set logprob_start_len=0 for multi-item scoring since we want logprobs at all delimiter positions
|
||||
logprob_start_len=0 if use_multi_item_scoring else -1,
|
||||
@@ -633,11 +697,18 @@ class TokenizerManagerScoreMixin:
|
||||
apply_softmax,
|
||||
batch_request,
|
||||
return_pooled_hidden_states,
|
||||
temperature,
|
||||
return_token_logprobs,
|
||||
)
|
||||
else:
|
||||
# Single-item scoring: process each result separately
|
||||
return self._process_single_item_scoring_results(
|
||||
results, label_token_ids, apply_softmax, return_pooled_hidden_states
|
||||
results,
|
||||
label_token_ids,
|
||||
apply_softmax,
|
||||
return_pooled_hidden_states,
|
||||
temperature,
|
||||
return_token_logprobs,
|
||||
)
|
||||
|
||||
def _convert_logprobs_to_scores(
|
||||
@@ -645,13 +716,20 @@ class TokenizerManagerScoreMixin:
|
||||
logprobs: Dict[int, float],
|
||||
label_token_ids: List[int],
|
||||
apply_softmax: bool,
|
||||
temperature: float = 1.0,
|
||||
) -> List[float]:
|
||||
score_list = [
|
||||
logprobs.get(token_id, float("-inf")) for token_id in label_token_ids
|
||||
]
|
||||
|
||||
if apply_softmax:
|
||||
score_list = torch.softmax(torch.tensor(score_list), dim=0).tolist()
|
||||
# center before scaling so small temperatures do not overflow
|
||||
maximum = max(score_list)
|
||||
weights = [
|
||||
math.exp((score - maximum) / temperature) for score in score_list
|
||||
]
|
||||
denominator = sum(weights)
|
||||
score_list = [weight / denominator for weight in weights]
|
||||
else:
|
||||
# Convert logprobs to probabilities if not using softmax
|
||||
score_list = [
|
||||
|
||||
@@ -433,7 +433,12 @@ class TestMultiItemScoringParity(CustomTestCase):
|
||||
tokenizer = AutoTokenizer.from_pretrained(TEST_MODEL_NAME)
|
||||
|
||||
def label_ids(labels):
|
||||
return [tokenizer.encode(lb, add_special_tokens=False)[0] for lb in labels]
|
||||
token_ids = []
|
||||
for label in labels:
|
||||
(token_id,) = tokenizer.encode(label, add_special_tokens=False)
|
||||
token_ids.append(token_id)
|
||||
assert len(set(token_ids)) == len(token_ids), token_ids
|
||||
return token_ids
|
||||
|
||||
return {
|
||||
"basic": dict(
|
||||
@@ -460,7 +465,7 @@ class TestMultiItemScoringParity(CustomTestCase):
|
||||
"many_items": dict(
|
||||
query="Rate this option from 1 to 5:",
|
||||
items=[f" Option {i}" for i in range(10)],
|
||||
label_token_ids=label_ids([" 1", " 2", " 3", " 4", " 5"]),
|
||||
label_token_ids=label_ids(["1", "2", "3", "4", "5"]),
|
||||
apply_softmax=True,
|
||||
),
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ test_score_engine.py. These tests focus on the HTTP integration seam:
|
||||
Pydantic schema defaults, FastAPI routing, and server argument wiring.
|
||||
"""
|
||||
|
||||
import math
|
||||
import os
|
||||
import unittest
|
||||
|
||||
@@ -136,6 +137,44 @@ class TestCausalLMScoringHTTP(CustomTestCase):
|
||||
with self.subTest(payload=list(payload.keys())):
|
||||
self.assertGreaterEqual(self._post(payload).status_code, 400)
|
||||
|
||||
def test_calibrated_decisions(self):
|
||||
response = self._post(
|
||||
{
|
||||
"query": [],
|
||||
"items": [[1, 2, 3], [4, 5]],
|
||||
"label_token_ids": [[32, 33], [34, 32, 33]],
|
||||
"apply_softmax": True,
|
||||
"temperature": 2.0,
|
||||
"return_token_logprobs": True,
|
||||
}
|
||||
)
|
||||
self.assertEqual(response.status_code, 200, response.text)
|
||||
body = response.json()
|
||||
self.assertEqual([len(row) for row in body["scores"]], [2, 3])
|
||||
for scores, logprobs in zip(body["scores"], body["token_logprobs"]):
|
||||
weights = [math.exp((x - max(logprobs)) / 2) for x in logprobs]
|
||||
for actual, weight in zip(scores, weights):
|
||||
self.assertAlmostEqual(actual, weight / sum(weights), places=6)
|
||||
self.assertEqual(body["usage"]["completion_tokens"], 0)
|
||||
|
||||
def test_invalid_decision_options(self):
|
||||
for extra in (
|
||||
{"label_token_ids": [[1, 2]]},
|
||||
{"label_token_ids": [[1], []]},
|
||||
{"label_token_ids": [1, 1]},
|
||||
{"temperature": 0},
|
||||
{"temperature": 2.0},
|
||||
):
|
||||
response = self._post(
|
||||
{
|
||||
"query": "",
|
||||
"items": ["A", "B"],
|
||||
"label_token_ids": [1, 2],
|
||||
**extra,
|
||||
}
|
||||
)
|
||||
self.assertIn(response.status_code, (400, 422), response.text)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MIS scoring (with --enable-mis)
|
||||
@@ -203,6 +242,27 @@ class TestCausalLMMISScoringHTTP(CustomTestCase):
|
||||
result = self._score("Test query", [], [1, 2])
|
||||
self.assertEqual(len(result["scores"]), 0)
|
||||
|
||||
def test_per_item_candidates(self):
|
||||
labels = [[32, 33], [34, 32, 33]]
|
||||
response = requests.post(
|
||||
self.base_url + "/v1/score",
|
||||
json={
|
||||
"query": "Select an option:",
|
||||
"items": ["A or B?", "A, B or C?"],
|
||||
"label_token_ids": labels,
|
||||
"apply_softmax": True,
|
||||
"temperature": 2.0,
|
||||
"return_token_logprobs": True,
|
||||
},
|
||||
)
|
||||
self.assertEqual(response.status_code, 200, response.text)
|
||||
body = response.json()
|
||||
self.assertEqual([len(row) for row in body["scores"]], [2, 3])
|
||||
for scores, logprobs in zip(body["scores"], body["token_logprobs"]):
|
||||
weights = [math.exp((x - max(logprobs)) / 2) for x in logprobs]
|
||||
for actual, weight in zip(scores, weights):
|
||||
self.assertAlmostEqual(actual, weight / sum(weights), places=6)
|
||||
|
||||
def test_varying_item_counts(self):
|
||||
"""1, 2, 4, and 6 items all return the correct number of score vectors."""
|
||||
label_token_ids = [1, 2, 3, 4, 5]
|
||||
|
||||
@@ -249,6 +249,36 @@ class TestCausalLMScoring(CustomTestCase):
|
||||
query="Q", items=None, label_token_ids=[1, 2], apply_softmax=True
|
||||
)
|
||||
|
||||
def test_decision_scoring(self):
|
||||
tokenizer = AutoTokenizer.from_pretrained(_CAUSAL_LM_MODEL)
|
||||
prompts = ["Answer A or B: Is Paris in France?", "Choose A, B or C: 2 + 2 ="]
|
||||
inputs = [tokenizer.encode(prompt) for prompt in prompts]
|
||||
labels = [[33, 32], [32, 34, 33]]
|
||||
kwargs = dict(apply_softmax=True, temperature=1.7, return_token_logprobs=True)
|
||||
batch = self.engine.score(
|
||||
query=[], items=inputs, label_token_ids=labels, **kwargs
|
||||
)
|
||||
self.assertEqual(len(batch.scores), 2)
|
||||
for i, (ids, candidates) in enumerate(zip(inputs, labels)):
|
||||
single = self.engine.score(
|
||||
query=[], items=[ids], label_token_ids=candidates, **kwargs
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
torch.tensor(batch.scores[i]),
|
||||
torch.tensor(single.scores[0]),
|
||||
atol=1e-3,
|
||||
rtol=1e-2,
|
||||
)
|
||||
expected = torch.softmax(torch.tensor(batch.token_logprobs[i]) / 1.7, 0)
|
||||
torch.testing.assert_close(torch.tensor(batch.scores[i]), expected)
|
||||
repeated = self.engine.score(
|
||||
query=[], items=inputs, label_token_ids=labels, **kwargs
|
||||
)
|
||||
for cold, warm in zip(batch.scores, repeated.scores):
|
||||
torch.testing.assert_close(
|
||||
torch.tensor(cold), torch.tensor(warm), atol=1e-3, rtol=1e-2
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SequenceClassification — single-item mode
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
import asyncio
|
||||
import json
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
from tokenizers import Tokenizer
|
||||
from tokenizers.models import WordLevel
|
||||
from tokenizers.pre_tokenizers import Whitespace
|
||||
from tokenizers.processors import TemplateProcessing
|
||||
from transformers import PreTrainedTokenizerFast
|
||||
|
||||
from sglang.srt.entrypoints.engine_score_mixin import EngineScoreMixin
|
||||
from sglang.srt.entrypoints.openai.protocol import ScoringRequest
|
||||
from sglang.srt.entrypoints.openai.serving_score import OpenAIServingScore
|
||||
from sglang.srt.managers.tokenizer_manager_score_mixin import TokenizerManagerScoreMixin
|
||||
from sglang.srt.runtime_context import publish, restore_context, snapshot_context
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class ScoringManager(TokenizerManagerScoreMixin):
|
||||
"""Replace only model execution; keep request construction and score extraction real."""
|
||||
|
||||
def __init__(self, enable_mis=False, generation=True):
|
||||
self.server_args = ServerArgs(model_path="dummy", enable_mis=enable_mis)
|
||||
publish(self.server_args, role="test")
|
||||
self.is_generation = generation
|
||||
self.tokenizer = SimpleNamespace(vocab_size=8)
|
||||
self.logits = torch.tensor([-1000.0, -999.0, -997.0, -996.0, 0, 1, 2, 3])
|
||||
self.requests = []
|
||||
|
||||
async def generate_request(self, request, raw_request):
|
||||
self.requests.append(request)
|
||||
request.normalize_batch_and_arguments()
|
||||
results = []
|
||||
for index, ids in enumerate(request.input_ids):
|
||||
meta = {"prompt_tokens": len(ids)}
|
||||
if self.is_generation:
|
||||
assert request.sampling_params[index]["max_new_tokens"] == 0
|
||||
labels = request.token_ids_logprob[index]
|
||||
values = torch.log_softmax(self.logits, dim=0)
|
||||
logprobs = [(values[token].item(), token, None) for token in labels]
|
||||
if self.server_args.enable_mis:
|
||||
count = len(request.multi_item_delimiter_indices[index])
|
||||
meta["input_token_ids_logprobs"] = [logprobs] * count
|
||||
else:
|
||||
meta["output_token_ids_logprobs"] = [logprobs]
|
||||
results.append({"meta_info": meta})
|
||||
else:
|
||||
embedding = [1.0, 3.0]
|
||||
if self.server_args.enable_mis:
|
||||
count = len(request.multi_item_delimiter_indices[index])
|
||||
embedding = [embedding] * count
|
||||
results.append({"meta_info": meta, "embedding": embedding})
|
||||
yield results
|
||||
|
||||
|
||||
class TestTokenScoring(unittest.IsolatedAsyncioTestCase):
|
||||
def setUp(self):
|
||||
self.addCleanup(restore_context, snapshot_context())
|
||||
|
||||
async def test_http_to_manager_ragged_candidates(self):
|
||||
for enable_mis in (False, True):
|
||||
with self.subTest(enable_mis=enable_mis):
|
||||
manager = ScoringManager(enable_mis=enable_mis)
|
||||
request = ScoringRequest(
|
||||
query=[4],
|
||||
items=[[5], [6, 7]],
|
||||
label_token_ids=[[3, 1], [0, 2, 1]],
|
||||
apply_softmax=True,
|
||||
temperature=2.0,
|
||||
return_token_logprobs=True,
|
||||
)
|
||||
handler = OpenAIServingScore(manager)
|
||||
response = await handler._handle_non_streaming_request(
|
||||
request, request, None
|
||||
)
|
||||
body = json.loads(response.body)
|
||||
for labels, scores, logprobs in zip(
|
||||
request.label_token_ids, body["scores"], body["token_logprobs"]
|
||||
):
|
||||
expected = torch.softmax(manager.logits[labels] / 2, dim=0)
|
||||
torch.testing.assert_close(torch.tensor(scores), expected)
|
||||
torch.testing.assert_close(
|
||||
torch.tensor(logprobs),
|
||||
torch.log_softmax(manager.logits, dim=0)[labels],
|
||||
)
|
||||
self.assertEqual(body["usage"]["completion_tokens"], 0)
|
||||
|
||||
async def test_default_scores_are_vocabulary_probabilities(self):
|
||||
manager = ScoringManager()
|
||||
result = await manager.score_request(
|
||||
query=[], items=[[4], [5]], label_token_ids=[7, 4]
|
||||
)
|
||||
expected = torch.softmax(manager.logits, dim=0)[[7, 4]]
|
||||
for scores in result.scores:
|
||||
torch.testing.assert_close(torch.tensor(scores), expected)
|
||||
self.assertIsNone(result.token_logprobs)
|
||||
|
||||
async def test_mis_text_items_do_not_add_special_tokens(self):
|
||||
vocab = {"[UNK]": 0, "[BOS]": 1, "Rate": 2, ":": 3, "Option": 4, "A": 5, "B": 6}
|
||||
tokenizer = Tokenizer(WordLevel(vocab, unk_token="[UNK]"))
|
||||
tokenizer.pre_tokenizer = Whitespace()
|
||||
tokenizer.post_processor = TemplateProcessing(
|
||||
single="[BOS] $A", special_tokens=[("[BOS]", vocab["[BOS]"])]
|
||||
)
|
||||
manager = ScoringManager(enable_mis=True)
|
||||
manager.tokenizer = PreTrainedTokenizerFast(
|
||||
tokenizer_object=tokenizer, bos_token="[BOS]", unk_token="[UNK]"
|
||||
)
|
||||
items = [" Option A", " Option B"]
|
||||
for query in ("Rate:", ""):
|
||||
with self.subTest(query=query):
|
||||
await manager.score_request(
|
||||
query=query, items=items, label_token_ids=[5, 6]
|
||||
)
|
||||
request = manager.requests[-1]
|
||||
packed = request.input_ids[0]
|
||||
delimiters = request.multi_item_delimiter_indices[0]
|
||||
prefix = packed[: delimiters[0]]
|
||||
for item, start, end in zip(items, delimiters, delimiters[1:]):
|
||||
self.assertEqual(
|
||||
prefix + packed[start + 1 : end],
|
||||
manager.tokenizer.encode(query + item),
|
||||
)
|
||||
|
||||
async def test_async_engine_and_full_prompts(self):
|
||||
manager = ScoringManager()
|
||||
engine = EngineScoreMixin()
|
||||
engine.tokenizer_manager = manager
|
||||
kwargs = dict(
|
||||
label_token_ids=[[1, 3], [2]],
|
||||
apply_softmax=True,
|
||||
temperature=0.5,
|
||||
return_token_logprobs=True,
|
||||
)
|
||||
actual = await engine.async_score(query=[], items=[[4], [5]], **kwargs)
|
||||
expected = await manager.score_prompts([[4], [5]], **kwargs)
|
||||
self.assertEqual(actual, expected)
|
||||
self.assertEqual(actual.scores[1], [1.0])
|
||||
|
||||
async def test_classification_temperature(self):
|
||||
for enable_mis in (False, True):
|
||||
manager = ScoringManager(enable_mis=enable_mis, generation=False)
|
||||
result = await manager.score_request(
|
||||
query=[4], items=[[5], [6]], apply_softmax=True, temperature=2.0
|
||||
)
|
||||
for scores in result.scores:
|
||||
torch.testing.assert_close(
|
||||
torch.tensor(scores), torch.softmax(torch.tensor([0.5, 1.5]), 0)
|
||||
)
|
||||
with self.assertRaisesRegex(ValueError, "only supported for CausalLM"):
|
||||
await manager.score_request(
|
||||
query=[], items=[[4]], return_token_logprobs=True
|
||||
)
|
||||
|
||||
async def test_invalid_candidates_and_temperature(self):
|
||||
manager = ScoringManager()
|
||||
for labels in ([], [[]], [[1]], [[1], []], [1, 1], [-1], [8], [True], "bad"):
|
||||
with self.subTest(labels=labels), self.assertRaises(ValueError):
|
||||
await manager.score_request(
|
||||
query=[], items=[[4], [5]], label_token_ids=labels
|
||||
)
|
||||
for temperature in (0, -1, float("nan"), float("inf")):
|
||||
with self.subTest(temperature=temperature), self.assertRaises(ValueError):
|
||||
await manager.score_request(
|
||||
query=[],
|
||||
items=[[4]],
|
||||
label_token_ids=[1],
|
||||
temperature=temperature,
|
||||
apply_softmax=True,
|
||||
)
|
||||
with self.assertRaisesRegex(ValueError, "requires apply_softmax"):
|
||||
await manager.score_request(
|
||||
query=[], items=[[4]], label_token_ids=[1], temperature=2.0
|
||||
)
|
||||
self.assertEqual(manager.requests, [])
|
||||
|
||||
async def test_http_out_of_vocabulary_error(self):
|
||||
for enable_mis in (False, True):
|
||||
for token_id in (8, 999999):
|
||||
for labels in ([7, token_id], [[7], [1, token_id]]):
|
||||
with self.subTest(enable_mis=enable_mis, labels=labels):
|
||||
manager = ScoringManager(enable_mis=enable_mis)
|
||||
request = ScoringRequest(
|
||||
query=[],
|
||||
items=[[4], [5]],
|
||||
label_token_ids=labels,
|
||||
apply_softmax=True,
|
||||
)
|
||||
response = await OpenAIServingScore(
|
||||
manager
|
||||
)._handle_non_streaming_request(request, request, None)
|
||||
self.assertEqual(response.status_code, 400)
|
||||
body = json.loads(response.body)
|
||||
self.assertEqual(body["type"], "BadRequestError")
|
||||
self.assertEqual(
|
||||
body["message"],
|
||||
f"Token ID {token_id} is out of vocabulary (vocab size: 8)",
|
||||
)
|
||||
self.assertEqual(manager.requests, [])
|
||||
|
||||
async def test_small_temperature_is_finite(self):
|
||||
for enable_mis in (False, True):
|
||||
for generation in (False, True):
|
||||
manager = ScoringManager(enable_mis=enable_mis, generation=generation)
|
||||
result = await manager.score_request(
|
||||
query=[4],
|
||||
items=[[5]],
|
||||
label_token_ids=[1, 3],
|
||||
apply_softmax=True,
|
||||
temperature=1e-300,
|
||||
)
|
||||
self.assertEqual(result.scores, [[0.0, 1.0]])
|
||||
|
||||
async def test_empty_batch(self):
|
||||
manager = ScoringManager()
|
||||
result = await manager.score_request(
|
||||
query=[], items=[], label_token_ids=[], return_token_logprobs=True
|
||||
)
|
||||
self.assertEqual(result.scores, [])
|
||||
self.assertEqual(result.token_logprobs, [])
|
||||
self.assertEqual(manager.requests, [])
|
||||
|
||||
|
||||
class TestSyncTokenScoring(unittest.TestCase):
|
||||
def test_engine_score(self):
|
||||
self.addCleanup(restore_context, snapshot_context())
|
||||
engine = EngineScoreMixin()
|
||||
engine.tokenizer_manager = ScoringManager()
|
||||
engine.loop = asyncio.new_event_loop()
|
||||
try:
|
||||
result = engine.score(
|
||||
query=[],
|
||||
items=[[4]],
|
||||
label_token_ids=[[1, 3]],
|
||||
apply_softmax=True,
|
||||
temperature=2.0,
|
||||
return_token_logprobs=True,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
torch.tensor(result.scores[0]),
|
||||
torch.softmax(engine.tokenizer_manager.logits[[1, 3]] / 2, 0),
|
||||
)
|
||||
finally:
|
||||
engine.loop.run_until_complete(engine.loop.shutdown_asyncgens())
|
||||
engine.loop.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user