[feature] add per-item candidate token scoring and calibration (#40826)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Mick
2026-09-24 17:00:13 +08:00
committed by GitHub
co-authored by Mick Qian
parent 41812afe43
commit 174a5f37b9
9 changed files with 558 additions and 41 deletions
@@ -0,0 +1,72 @@
"""Score SemIf JSONL decisions using its original prompts and an SGLang server."""
import argparse
import json
from pathlib import Path
import requests
from semif_phase1.direct import PROMPT_VERSION, encode_prompt
from transformers import AutoTokenizer
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:30000")
parser.add_argument("--model", default="Qwen/Qwen3.5-4B")
parser.add_argument("--revision", required=True)
parser.add_argument("--input", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--batch-size", type=int, default=32)
parser.add_argument("--max-tokens", type=int, default=4096)
parser.add_argument("--temperature", type=float, default=1.0)
args = parser.parse_args()
if args.batch_size < 1:
parser.error("--batch-size must be positive")
tokenizer = AutoTokenizer.from_pretrained(args.model, revision=args.revision)
rows = [
json.loads(line) for line in args.input.read_text().splitlines() if line.strip()
]
with args.output.open("x") as output, requests.Session() as session:
for start in range(0, len(rows), args.batch_size):
batch = rows[start : start + args.batch_size]
encoded = [encode_prompt(tokenizer, row, args.max_tokens) for row in batch]
response = session.post(
args.base_url.rstrip("/") + "/v1/score",
json={
"model": args.model,
"query": [],
"items": [ids for ids, _, _ in encoded],
"label_token_ids": [slots for _, slots, _ in encoded],
"apply_softmax": True,
"temperature": args.temperature,
"return_token_logprobs": True,
},
timeout=300,
)
response.raise_for_status()
result = response.json()
for row, (ids, _, prompt_hash), probabilities, logprobs in zip(
batch, encoded, result["scores"], result["token_logprobs"], strict=True
):
output.write(
json.dumps(
{
"id": row["id"],
"option_ids": [option["id"] for option in row["options"]],
"probabilities": probabilities,
"token_logprobs": logprobs,
"input_tokens": len(ids),
"prompt_sha256": prompt_hash,
"prompt_version": PROMPT_VERSION,
"temperature": args.temperature,
"readout": "sglang candidate token logprobs",
},
allow_nan=False,
)
+ "\n"
)
if __name__ == "__main__":
main()
@@ -30,13 +30,15 @@ class EngineScoreMixin:
self,
query: Optional[Union[str, List[int]]] = None,
items: Optional[Union[str, List[str], List[List[int]]]] = None,
label_token_ids: Optional[List[int]] = None,
label_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
apply_softmax: bool = False,
item_first: bool = False,
embed_override_token_id: Optional[int] = None,
query_embed_overrides: Optional[List[torch.Tensor]] = None,
item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None,
return_pooled_hidden_states: bool = False,
temperature: float = 1.0,
return_token_logprobs: bool = False,
) -> ScoreResult:
"""
Score items against a query using the loaded model.
@@ -54,8 +56,8 @@ class EngineScoreMixin:
Args:
query: The query text or pre-tokenized token IDs.
items: The item text(s) or pre-tokenized token IDs.
label_token_ids: Token IDs to score (required for CausalLM; ignored for
SequenceClassification).
label_token_ids: Shared token IDs or one list per item, preserving candidate
order (required for CausalLM; ignored for SequenceClassification).
apply_softmax: Whether to normalize scores using softmax.
item_first: If True, prepend items before query (single-item mode only).
embed_override_token_id: Placeholder token ID used to locate override positions.
@@ -64,6 +66,10 @@ class EngineScoreMixin:
return_pooled_hidden_states: Whether to include raw pooled transformer
hidden states (before the task head) in the result. Only supported
for non-generation models (SequenceClassification, RewardModel).
temperature: Positive calibration temperature, applied before candidate/class
softmax. Requires apply_softmax=True when different from 1.
return_token_logprobs: Include uncalibrated full-vocabulary logprobs for
each item's candidates. Only supported for CausalLM, not raw logits.
Returns:
ScoreResult with scores (one list per item), prompt token count, and
@@ -81,6 +87,8 @@ class EngineScoreMixin:
item_embed_overrides=item_embed_overrides,
request=None,
return_pooled_hidden_states=return_pooled_hidden_states,
temperature=temperature,
return_token_logprobs=return_token_logprobs,
)
)
@@ -88,13 +96,15 @@ class EngineScoreMixin:
self,
query: Optional[Union[str, List[int]]] = None,
items: Optional[Union[str, List[str], List[List[int]]]] = None,
label_token_ids: Optional[List[int]] = None,
label_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
apply_softmax: bool = False,
item_first: bool = False,
embed_override_token_id: Optional[int] = None,
query_embed_overrides: Optional[List[torch.Tensor]] = None,
item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None,
return_pooled_hidden_states: bool = False,
temperature: float = 1.0,
return_token_logprobs: bool = False,
) -> ScoreResult:
"""Asynchronous version of score(). See score() for full documentation."""
return await self.tokenizer_manager.score_request(
@@ -108,4 +118,6 @@ class EngineScoreMixin:
item_embed_overrides=item_embed_overrides,
request=None,
return_pooled_hidden_states=return_pooled_hidden_states,
temperature=temperature,
return_token_logprobs=return_token_logprobs,
)
@@ -1398,10 +1398,12 @@ class ScoringRequest(BaseModel):
item_embed_overrides: Optional[List[Optional[List[List[float]]]]] = (
None # [num_items][num_item_embed_overrides][hidden_size]
)
label_token_ids: Optional[List[int]] = (
None # Token IDs to compute probabilities for
label_token_ids: Optional[Union[List[int], List[List[int]]]] = (
None # shared candidates or one candidate list per item
)
apply_softmax: bool = False
temperature: float = Field(default=1.0, gt=0, allow_inf_nan=False)
return_token_logprobs: bool = False
item_first: bool = False
return_pooled_hidden_states: bool = False
model: str = DEFAULT_MODEL_NAME
@@ -1412,6 +1414,7 @@ class ScoringResponse(BaseModel):
List[float]
] # List of lists of probabilities, each in the order of label_token_ids
pooled_hidden_states: Optional[List[Optional[List[float]]]] = None
token_logprobs: Optional[List[List[float]]] = None
model: str
usage: Optional[UsageInfo] = None
object: str = "scoring"
@@ -78,6 +78,8 @@ class OpenAIServingScore(OpenAIServingBase):
item_embed_overrides=item_embed_overrides,
request=raw_request,
return_pooled_hidden_states=request.return_pooled_hidden_states,
temperature=request.temperature,
return_token_logprobs=request.return_token_logprobs,
)
phs_as_lists = None
@@ -90,6 +92,7 @@ class OpenAIServingScore(OpenAIServingBase):
response = ScoringResponse(
scores=result.scores,
pooled_hidden_states=phs_as_lists,
token_logprobs=result.token_logprobs,
model=request.model,
usage=UsageInfo(
prompt_tokens=result.prompt_tokens,
@@ -24,15 +24,19 @@ class ScoreResult:
# The HTTP path converts to lists in serving_score.py before JSON serialization.
# Same layout as scores: one tensor per item (not a single packed 2D tensor).
pooled_hidden_states: Optional[List[Optional[torch.Tensor]]] = None
# uncalibrated full-vocabulary logprobs, in each item's candidate order
token_logprobs: Optional[List[List[float]]] = None
class TokenizerManagerScoreMixin:
async def score_prompts(
self,
prompts: Union[str, List[str], List[List[int]]],
label_token_ids: List[int],
label_token_ids: Union[List[int], List[List[int]]],
apply_softmax: bool = False,
request: Optional[Any] = None,
temperature: float = 1.0,
return_token_logprobs: bool = False,
) -> ScoreResult:
"""
Score probabilities of specified token IDs after each *full prompt*.
@@ -51,6 +55,8 @@ class TokenizerManagerScoreMixin:
apply_softmax=apply_softmax,
item_first=False,
request=request,
temperature=temperature,
return_token_logprobs=return_token_logprobs,
)
# Tokenized prompts
@@ -62,6 +68,8 @@ class TokenizerManagerScoreMixin:
apply_softmax=apply_softmax,
item_first=False,
request=request,
temperature=temperature,
return_token_logprobs=return_token_logprobs,
)
raise ValueError("Invalid prompts type for score_prompts.")
@@ -102,7 +110,8 @@ class TokenizerManagerScoreMixin:
items_ids = []
for item in items_list:
if isinstance(item, str):
items_ids.append(self.tokenizer.encode(item))
# items continue the query, so only the query gets special tokens
items_ids.append(self.tokenizer.encode(item, add_special_tokens=False))
else:
items_ids.append(list(item))
@@ -112,10 +121,12 @@ class TokenizerManagerScoreMixin:
self,
results: Any,
items: List,
label_token_ids: Optional[List[int]],
label_token_ids: Optional[List[List[int]]],
apply_softmax: bool,
batch_request=None,
return_pooled_hidden_states: bool = False,
temperature: float = 1.0,
return_token_logprobs: bool = False,
) -> ScoreResult:
"""
Process results from multi-item scoring request.
@@ -135,27 +146,31 @@ class TokenizerManagerScoreMixin:
# Extract per-delimiter scores from whichever field has them
input_logprobs = meta_info.get("input_token_ids_logprobs", [])
embedding = single_result.get("embedding")
token_logprobs = [] if return_token_logprobs else None
if input_logprobs:
# Generation model: extract label-token logprobs at each delimiter
per_delimiter_scores = []
for logprobs_data in input_logprobs:
logprobs = self._extract_logprobs_for_tokens(
logprobs_data, label_token_ids
if len(input_logprobs) != expected_count:
raise RuntimeError(
f"Expected {expected_count} delimiter entries, got {len(input_logprobs)}"
)
per_delimiter_scores = [[]] # the query boundary is not a decision
for logprobs_data, labels in zip(input_logprobs[1:], label_token_ids):
logprobs = self._extract_logprobs_for_tokens(logprobs_data, labels)
score_list = self._convert_logprobs_to_scores(
logprobs, label_token_ids, apply_softmax
logprobs, labels, apply_softmax, temperature
)
per_delimiter_scores.append(score_list)
if return_token_logprobs:
token_logprobs.append([logprobs[token] for token in labels])
elif embedding is not None:
# Classification model: scores are directly in 2D embedding.
if apply_softmax:
scores_tensor = (
torch.tensor(embedding)
if isinstance(embedding, list)
else embedding
scores_tensor = torch.as_tensor(embedding, dtype=torch.float64)
scores_tensor = scores_tensor - scores_tensor.amax(dim=-1, keepdim=True)
scores_tensor = torch.nn.functional.softmax(
scores_tensor / temperature, dim=-1
)
scores_tensor = torch.nn.functional.softmax(scores_tensor, dim=-1)
per_delimiter_scores = scores_tensor.tolist()
else:
per_delimiter_scores = (
@@ -188,14 +203,17 @@ class TokenizerManagerScoreMixin:
scores=scores,
prompt_tokens=prompt_tokens,
pooled_hidden_states=phs_list,
token_logprobs=token_logprobs,
)
def _process_single_item_scoring_results(
self,
results: Any,
label_token_ids: Optional[List[int]],
label_token_ids: Optional[List[List[int]]],
apply_softmax: bool,
return_pooled_hidden_states: bool = False,
temperature: float = 1.0,
return_token_logprobs: bool = False,
) -> ScoreResult:
"""
Process results from single-item scoring request.
@@ -208,10 +226,11 @@ class TokenizerManagerScoreMixin:
phs_list = []
has_phs = False
prompt_tokens = 0
token_logprobs = [] if return_token_logprobs else None
is_generation = self.is_generation
if is_generation:
for result in results:
for result, labels in zip(results, label_token_ids):
# For single-item scoring, logprobs are in output_token_ids_logprobs
output_logprobs = result["meta_info"].get(
"output_token_ids_logprobs", []
@@ -225,13 +244,13 @@ class TokenizerManagerScoreMixin:
)
# Extract logprobs for the first (and only) position
logprobs = self._extract_logprobs_for_tokens(
output_logprobs[0], label_token_ids
)
logprobs = self._extract_logprobs_for_tokens(output_logprobs[0], labels)
score_list = self._convert_logprobs_to_scores(
logprobs, label_token_ids, apply_softmax
logprobs, labels, apply_softmax, temperature
)
scores.append(score_list)
if return_token_logprobs:
token_logprobs.append([logprobs[token] for token in labels])
else:
for result in results:
embedding = result.get("embedding", None)
@@ -241,8 +260,9 @@ class TokenizerManagerScoreMixin:
prompt_tokens += result.get("meta_info", {}).get("prompt_tokens", 0)
if apply_softmax:
scores_tensor = torch.as_tensor(embedding, dtype=torch.float64)
embedding = torch.softmax(
torch.as_tensor(embedding), dim=-1
(scores_tensor - scores_tensor.max()) / temperature, dim=-1
).tolist()
# The classification head produces per-token logits, which the pooler reduces
@@ -262,6 +282,7 @@ class TokenizerManagerScoreMixin:
scores=scores,
prompt_tokens=prompt_tokens,
pooled_hidden_states=phs_list if has_phs else None,
token_logprobs=token_logprobs,
)
# ------------------------------------------------------------------
@@ -445,7 +466,7 @@ class TokenizerManagerScoreMixin:
self,
query: Optional[Union[str, List[int]]] = None,
items: Optional[Union[str, List[str], List[List[int]]]] = None,
label_token_ids: Optional[List[int]] = None,
label_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
apply_softmax: bool = False,
item_first: bool = False,
embed_override_token_id: Optional[int] = None,
@@ -453,6 +474,8 @@ class TokenizerManagerScoreMixin:
item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None,
request: Optional[Any] = None,
return_pooled_hidden_states: bool = False,
temperature: float = 1.0,
return_token_logprobs: bool = False,
) -> ScoreResult:
"""
Score the probability of specified token IDs appearing after the given (query + item) pair.
@@ -477,6 +500,15 @@ class TokenizerManagerScoreMixin:
"""
is_generation = self.is_generation
if not math.isfinite(temperature) or temperature <= 0:
raise ValueError("temperature must be finite and greater than zero")
if temperature != 1.0 and not apply_softmax:
raise ValueError("temperature requires apply_softmax=True")
if return_token_logprobs and not is_generation:
raise ValueError(
"return_token_logprobs is only supported for CausalLM models"
)
if is_generation and label_token_ids is None:
raise ValueError(
"label_token_ids is required for generation (CausalLM) models."
@@ -484,7 +516,37 @@ class TokenizerManagerScoreMixin:
if items is None:
raise ValueError("items must be provided")
if not items:
return ScoreResult(scores=[], prompt_tokens=0)
return ScoreResult(
scores=[], token_logprobs=[] if return_token_logprobs else None
)
num_items = 1 if isinstance(items, str) else len(items)
if is_generation:
if not isinstance(label_token_ids, list) or not label_token_ids:
raise ValueError("label_token_ids must be a nonempty list")
if isinstance(label_token_ids[0], list):
if len(label_token_ids) != num_items:
raise ValueError("label_token_ids must have one list per item")
else:
label_token_ids = [label_token_ids] * num_items
for labels in label_token_ids:
if not isinstance(labels, list) or not labels:
raise ValueError("each item must have a nonempty candidate list")
if any(type(token) is not int or token < 0 for token in labels):
raise ValueError(
"label_token_ids must contain nonnegative integers"
)
if len(set(labels)) != len(labels):
raise ValueError(
"label_token_ids must not contain duplicate tokens"
)
if self.tokenizer is not None:
vocab_size = self.tokenizer.vocab_size
for token_id in labels:
if token_id >= vocab_size:
raise ValueError(
f"Token ID {token_id} is out of vocabulary (vocab size: {vocab_size})"
)
has_embeds = (
query_embed_overrides is not None or item_embed_overrides is not None
@@ -501,14 +563,6 @@ class TokenizerManagerScoreMixin:
f"item_embed_overrides length ({len(item_embed_overrides)}) "
f"must match items length ({len(items)})."
)
if self.tokenizer is not None and label_token_ids is not None:
vocab_size = self.tokenizer.vocab_size
for token_id in label_token_ids:
if token_id >= vocab_size:
raise ValueError(
f"Token ID {token_id} is out of vocabulary (vocab size: {vocab_size})"
)
# Check if multi-item scoring is enabled
use_multi_item_scoring = get_exec().features.enable_mis
@@ -601,10 +655,20 @@ class TokenizerManagerScoreMixin:
# Create the appropriate request type
mis_delimiter_indices = [delimiter_indices] if use_multi_item_scoring else None
if is_generation:
# packed MIS requests gather the union, then select per item
request_labels = (
list(
dict.fromkeys(
token for labels in label_token_ids for token in labels
)
)
if use_multi_item_scoring
else label_token_ids
)
batch_request = GenerateReqInput(
text=text_prompts,
input_ids=input_ids,
token_ids_logprob=label_token_ids,
token_ids_logprob=request_labels,
return_logprob=True,
# Set logprob_start_len=0 for multi-item scoring since we want logprobs at all delimiter positions
logprob_start_len=0 if use_multi_item_scoring else -1,
@@ -633,11 +697,18 @@ class TokenizerManagerScoreMixin:
apply_softmax,
batch_request,
return_pooled_hidden_states,
temperature,
return_token_logprobs,
)
else:
# Single-item scoring: process each result separately
return self._process_single_item_scoring_results(
results, label_token_ids, apply_softmax, return_pooled_hidden_states
results,
label_token_ids,
apply_softmax,
return_pooled_hidden_states,
temperature,
return_token_logprobs,
)
def _convert_logprobs_to_scores(
@@ -645,13 +716,20 @@ class TokenizerManagerScoreMixin:
logprobs: Dict[int, float],
label_token_ids: List[int],
apply_softmax: bool,
temperature: float = 1.0,
) -> List[float]:
score_list = [
logprobs.get(token_id, float("-inf")) for token_id in label_token_ids
]
if apply_softmax:
score_list = torch.softmax(torch.tensor(score_list), dim=0).tolist()
# center before scaling so small temperatures do not overflow
maximum = max(score_list)
weights = [
math.exp((score - maximum) / temperature) for score in score_list
]
denominator = sum(weights)
score_list = [weight / denominator for weight in weights]
else:
# Convert logprobs to probabilities if not using softmax
score_list = [
@@ -433,7 +433,12 @@ class TestMultiItemScoringParity(CustomTestCase):
tokenizer = AutoTokenizer.from_pretrained(TEST_MODEL_NAME)
def label_ids(labels):
return [tokenizer.encode(lb, add_special_tokens=False)[0] for lb in labels]
token_ids = []
for label in labels:
(token_id,) = tokenizer.encode(label, add_special_tokens=False)
token_ids.append(token_id)
assert len(set(token_ids)) == len(token_ids), token_ids
return token_ids
return {
"basic": dict(
@@ -460,7 +465,7 @@ class TestMultiItemScoringParity(CustomTestCase):
"many_items": dict(
query="Rate this option from 1 to 5:",
items=[f" Option {i}" for i in range(10)],
label_token_ids=label_ids([" 1", " 2", " 3", " 4", " 5"]),
label_token_ids=label_ids(["1", "2", "3", "4", "5"]),
apply_softmax=True,
),
}
@@ -12,6 +12,7 @@ test_score_engine.py. These tests focus on the HTTP integration seam:
Pydantic schema defaults, FastAPI routing, and server argument wiring.
"""
import math
import os
import unittest
@@ -136,6 +137,44 @@ class TestCausalLMScoringHTTP(CustomTestCase):
with self.subTest(payload=list(payload.keys())):
self.assertGreaterEqual(self._post(payload).status_code, 400)
def test_calibrated_decisions(self):
response = self._post(
{
"query": [],
"items": [[1, 2, 3], [4, 5]],
"label_token_ids": [[32, 33], [34, 32, 33]],
"apply_softmax": True,
"temperature": 2.0,
"return_token_logprobs": True,
}
)
self.assertEqual(response.status_code, 200, response.text)
body = response.json()
self.assertEqual([len(row) for row in body["scores"]], [2, 3])
for scores, logprobs in zip(body["scores"], body["token_logprobs"]):
weights = [math.exp((x - max(logprobs)) / 2) for x in logprobs]
for actual, weight in zip(scores, weights):
self.assertAlmostEqual(actual, weight / sum(weights), places=6)
self.assertEqual(body["usage"]["completion_tokens"], 0)
def test_invalid_decision_options(self):
for extra in (
{"label_token_ids": [[1, 2]]},
{"label_token_ids": [[1], []]},
{"label_token_ids": [1, 1]},
{"temperature": 0},
{"temperature": 2.0},
):
response = self._post(
{
"query": "",
"items": ["A", "B"],
"label_token_ids": [1, 2],
**extra,
}
)
self.assertIn(response.status_code, (400, 422), response.text)
# ---------------------------------------------------------------------------
# MIS scoring (with --enable-mis)
@@ -203,6 +242,27 @@ class TestCausalLMMISScoringHTTP(CustomTestCase):
result = self._score("Test query", [], [1, 2])
self.assertEqual(len(result["scores"]), 0)
def test_per_item_candidates(self):
labels = [[32, 33], [34, 32, 33]]
response = requests.post(
self.base_url + "/v1/score",
json={
"query": "Select an option:",
"items": ["A or B?", "A, B or C?"],
"label_token_ids": labels,
"apply_softmax": True,
"temperature": 2.0,
"return_token_logprobs": True,
},
)
self.assertEqual(response.status_code, 200, response.text)
body = response.json()
self.assertEqual([len(row) for row in body["scores"]], [2, 3])
for scores, logprobs in zip(body["scores"], body["token_logprobs"]):
weights = [math.exp((x - max(logprobs)) / 2) for x in logprobs]
for actual, weight in zip(scores, weights):
self.assertAlmostEqual(actual, weight / sum(weights), places=6)
def test_varying_item_counts(self):
"""1, 2, 4, and 6 items all return the correct number of score vectors."""
label_token_ids = [1, 2, 3, 4, 5]
@@ -249,6 +249,36 @@ class TestCausalLMScoring(CustomTestCase):
query="Q", items=None, label_token_ids=[1, 2], apply_softmax=True
)
def test_decision_scoring(self):
tokenizer = AutoTokenizer.from_pretrained(_CAUSAL_LM_MODEL)
prompts = ["Answer A or B: Is Paris in France?", "Choose A, B or C: 2 + 2 ="]
inputs = [tokenizer.encode(prompt) for prompt in prompts]
labels = [[33, 32], [32, 34, 33]]
kwargs = dict(apply_softmax=True, temperature=1.7, return_token_logprobs=True)
batch = self.engine.score(
query=[], items=inputs, label_token_ids=labels, **kwargs
)
self.assertEqual(len(batch.scores), 2)
for i, (ids, candidates) in enumerate(zip(inputs, labels)):
single = self.engine.score(
query=[], items=[ids], label_token_ids=candidates, **kwargs
)
torch.testing.assert_close(
torch.tensor(batch.scores[i]),
torch.tensor(single.scores[0]),
atol=1e-3,
rtol=1e-2,
)
expected = torch.softmax(torch.tensor(batch.token_logprobs[i]) / 1.7, 0)
torch.testing.assert_close(torch.tensor(batch.scores[i]), expected)
repeated = self.engine.score(
query=[], items=inputs, label_token_ids=labels, **kwargs
)
for cold, warm in zip(batch.scores, repeated.scores):
torch.testing.assert_close(
torch.tensor(cold), torch.tensor(warm), atol=1e-3, rtol=1e-2
)
# ---------------------------------------------------------------------------
# SequenceClassification — single-item mode
+254
View File
@@ -0,0 +1,254 @@
import asyncio
import json
import unittest
from types import SimpleNamespace
import torch
from tokenizers import Tokenizer
from tokenizers.models import WordLevel
from tokenizers.pre_tokenizers import Whitespace
from tokenizers.processors import TemplateProcessing
from transformers import PreTrainedTokenizerFast
from sglang.srt.entrypoints.engine_score_mixin import EngineScoreMixin
from sglang.srt.entrypoints.openai.protocol import ScoringRequest
from sglang.srt.entrypoints.openai.serving_score import OpenAIServingScore
from sglang.srt.managers.tokenizer_manager_score_mixin import TokenizerManagerScoreMixin
from sglang.srt.runtime_context import publish, restore_context, snapshot_context
from sglang.srt.server_args import ServerArgs
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
class ScoringManager(TokenizerManagerScoreMixin):
"""Replace only model execution; keep request construction and score extraction real."""
def __init__(self, enable_mis=False, generation=True):
self.server_args = ServerArgs(model_path="dummy", enable_mis=enable_mis)
publish(self.server_args, role="test")
self.is_generation = generation
self.tokenizer = SimpleNamespace(vocab_size=8)
self.logits = torch.tensor([-1000.0, -999.0, -997.0, -996.0, 0, 1, 2, 3])
self.requests = []
async def generate_request(self, request, raw_request):
self.requests.append(request)
request.normalize_batch_and_arguments()
results = []
for index, ids in enumerate(request.input_ids):
meta = {"prompt_tokens": len(ids)}
if self.is_generation:
assert request.sampling_params[index]["max_new_tokens"] == 0
labels = request.token_ids_logprob[index]
values = torch.log_softmax(self.logits, dim=0)
logprobs = [(values[token].item(), token, None) for token in labels]
if self.server_args.enable_mis:
count = len(request.multi_item_delimiter_indices[index])
meta["input_token_ids_logprobs"] = [logprobs] * count
else:
meta["output_token_ids_logprobs"] = [logprobs]
results.append({"meta_info": meta})
else:
embedding = [1.0, 3.0]
if self.server_args.enable_mis:
count = len(request.multi_item_delimiter_indices[index])
embedding = [embedding] * count
results.append({"meta_info": meta, "embedding": embedding})
yield results
class TestTokenScoring(unittest.IsolatedAsyncioTestCase):
def setUp(self):
self.addCleanup(restore_context, snapshot_context())
async def test_http_to_manager_ragged_candidates(self):
for enable_mis in (False, True):
with self.subTest(enable_mis=enable_mis):
manager = ScoringManager(enable_mis=enable_mis)
request = ScoringRequest(
query=[4],
items=[[5], [6, 7]],
label_token_ids=[[3, 1], [0, 2, 1]],
apply_softmax=True,
temperature=2.0,
return_token_logprobs=True,
)
handler = OpenAIServingScore(manager)
response = await handler._handle_non_streaming_request(
request, request, None
)
body = json.loads(response.body)
for labels, scores, logprobs in zip(
request.label_token_ids, body["scores"], body["token_logprobs"]
):
expected = torch.softmax(manager.logits[labels] / 2, dim=0)
torch.testing.assert_close(torch.tensor(scores), expected)
torch.testing.assert_close(
torch.tensor(logprobs),
torch.log_softmax(manager.logits, dim=0)[labels],
)
self.assertEqual(body["usage"]["completion_tokens"], 0)
async def test_default_scores_are_vocabulary_probabilities(self):
manager = ScoringManager()
result = await manager.score_request(
query=[], items=[[4], [5]], label_token_ids=[7, 4]
)
expected = torch.softmax(manager.logits, dim=0)[[7, 4]]
for scores in result.scores:
torch.testing.assert_close(torch.tensor(scores), expected)
self.assertIsNone(result.token_logprobs)
async def test_mis_text_items_do_not_add_special_tokens(self):
vocab = {"[UNK]": 0, "[BOS]": 1, "Rate": 2, ":": 3, "Option": 4, "A": 5, "B": 6}
tokenizer = Tokenizer(WordLevel(vocab, unk_token="[UNK]"))
tokenizer.pre_tokenizer = Whitespace()
tokenizer.post_processor = TemplateProcessing(
single="[BOS] $A", special_tokens=[("[BOS]", vocab["[BOS]"])]
)
manager = ScoringManager(enable_mis=True)
manager.tokenizer = PreTrainedTokenizerFast(
tokenizer_object=tokenizer, bos_token="[BOS]", unk_token="[UNK]"
)
items = [" Option A", " Option B"]
for query in ("Rate:", ""):
with self.subTest(query=query):
await manager.score_request(
query=query, items=items, label_token_ids=[5, 6]
)
request = manager.requests[-1]
packed = request.input_ids[0]
delimiters = request.multi_item_delimiter_indices[0]
prefix = packed[: delimiters[0]]
for item, start, end in zip(items, delimiters, delimiters[1:]):
self.assertEqual(
prefix + packed[start + 1 : end],
manager.tokenizer.encode(query + item),
)
async def test_async_engine_and_full_prompts(self):
manager = ScoringManager()
engine = EngineScoreMixin()
engine.tokenizer_manager = manager
kwargs = dict(
label_token_ids=[[1, 3], [2]],
apply_softmax=True,
temperature=0.5,
return_token_logprobs=True,
)
actual = await engine.async_score(query=[], items=[[4], [5]], **kwargs)
expected = await manager.score_prompts([[4], [5]], **kwargs)
self.assertEqual(actual, expected)
self.assertEqual(actual.scores[1], [1.0])
async def test_classification_temperature(self):
for enable_mis in (False, True):
manager = ScoringManager(enable_mis=enable_mis, generation=False)
result = await manager.score_request(
query=[4], items=[[5], [6]], apply_softmax=True, temperature=2.0
)
for scores in result.scores:
torch.testing.assert_close(
torch.tensor(scores), torch.softmax(torch.tensor([0.5, 1.5]), 0)
)
with self.assertRaisesRegex(ValueError, "only supported for CausalLM"):
await manager.score_request(
query=[], items=[[4]], return_token_logprobs=True
)
async def test_invalid_candidates_and_temperature(self):
manager = ScoringManager()
for labels in ([], [[]], [[1]], [[1], []], [1, 1], [-1], [8], [True], "bad"):
with self.subTest(labels=labels), self.assertRaises(ValueError):
await manager.score_request(
query=[], items=[[4], [5]], label_token_ids=labels
)
for temperature in (0, -1, float("nan"), float("inf")):
with self.subTest(temperature=temperature), self.assertRaises(ValueError):
await manager.score_request(
query=[],
items=[[4]],
label_token_ids=[1],
temperature=temperature,
apply_softmax=True,
)
with self.assertRaisesRegex(ValueError, "requires apply_softmax"):
await manager.score_request(
query=[], items=[[4]], label_token_ids=[1], temperature=2.0
)
self.assertEqual(manager.requests, [])
async def test_http_out_of_vocabulary_error(self):
for enable_mis in (False, True):
for token_id in (8, 999999):
for labels in ([7, token_id], [[7], [1, token_id]]):
with self.subTest(enable_mis=enable_mis, labels=labels):
manager = ScoringManager(enable_mis=enable_mis)
request = ScoringRequest(
query=[],
items=[[4], [5]],
label_token_ids=labels,
apply_softmax=True,
)
response = await OpenAIServingScore(
manager
)._handle_non_streaming_request(request, request, None)
self.assertEqual(response.status_code, 400)
body = json.loads(response.body)
self.assertEqual(body["type"], "BadRequestError")
self.assertEqual(
body["message"],
f"Token ID {token_id} is out of vocabulary (vocab size: 8)",
)
self.assertEqual(manager.requests, [])
async def test_small_temperature_is_finite(self):
for enable_mis in (False, True):
for generation in (False, True):
manager = ScoringManager(enable_mis=enable_mis, generation=generation)
result = await manager.score_request(
query=[4],
items=[[5]],
label_token_ids=[1, 3],
apply_softmax=True,
temperature=1e-300,
)
self.assertEqual(result.scores, [[0.0, 1.0]])
async def test_empty_batch(self):
manager = ScoringManager()
result = await manager.score_request(
query=[], items=[], label_token_ids=[], return_token_logprobs=True
)
self.assertEqual(result.scores, [])
self.assertEqual(result.token_logprobs, [])
self.assertEqual(manager.requests, [])
class TestSyncTokenScoring(unittest.TestCase):
def test_engine_score(self):
self.addCleanup(restore_context, snapshot_context())
engine = EngineScoreMixin()
engine.tokenizer_manager = ScoringManager()
engine.loop = asyncio.new_event_loop()
try:
result = engine.score(
query=[],
items=[[4]],
label_token_ids=[[1, 3]],
apply_softmax=True,
temperature=2.0,
return_token_logprobs=True,
)
torch.testing.assert_close(
torch.tensor(result.scores[0]),
torch.softmax(engine.tokenizer_manager.logits[[1, 3]] / 2, 0),
)
finally:
engine.loop.run_until_complete(engine.loop.shutdown_asyncgens())
engine.loop.close()
if __name__ == "__main__":
unittest.main()