From 174a5f37b99d9027f355c954b6d52f4feff87c29 Mon Sep 17 00:00:00 2001 From: Mick Date: Thu, 24 Sep 2026 17:00:13 +0800 Subject: [PATCH] [feature] add per-item candidate token scoring and calibration (#40826) Co-authored-by: Mick Qian --- examples/runtime/semantic_scoring/semif.py | 72 +++++ .../srt/entrypoints/engine_score_mixin.py | 20 +- .../sglang/srt/entrypoints/openai/protocol.py | 7 +- .../srt/entrypoints/openai/serving_score.py | 3 + .../managers/tokenizer_manager_score_mixin.py | 144 +++++++--- .../prefill_only/test_multi_item_scoring.py | 9 +- .../registered/prefill_only/test_score_api.py | 60 +++++ .../prefill_only/test_score_engine.py | 30 +++ test/registered/unit/test_token_scoring.py | 254 ++++++++++++++++++ 9 files changed, 558 insertions(+), 41 deletions(-) create mode 100644 examples/runtime/semantic_scoring/semif.py create mode 100644 test/registered/unit/test_token_scoring.py diff --git a/examples/runtime/semantic_scoring/semif.py b/examples/runtime/semantic_scoring/semif.py new file mode 100644 index 0000000000..1c465e3f91 --- /dev/null +++ b/examples/runtime/semantic_scoring/semif.py @@ -0,0 +1,72 @@ +"""Score SemIf JSONL decisions using its original prompts and an SGLang server.""" + +import argparse +import json +from pathlib import Path + +import requests +from semif_phase1.direct import PROMPT_VERSION, encode_prompt +from transformers import AutoTokenizer + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default="http://127.0.0.1:30000") + parser.add_argument("--model", default="Qwen/Qwen3.5-4B") + parser.add_argument("--revision", required=True) + parser.add_argument("--input", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--batch-size", type=int, default=32) + parser.add_argument("--max-tokens", type=int, default=4096) + parser.add_argument("--temperature", type=float, default=1.0) + args = parser.parse_args() + if args.batch_size < 1: + parser.error("--batch-size must be positive") + + tokenizer = AutoTokenizer.from_pretrained(args.model, revision=args.revision) + rows = [ + json.loads(line) for line in args.input.read_text().splitlines() if line.strip() + ] + with args.output.open("x") as output, requests.Session() as session: + for start in range(0, len(rows), args.batch_size): + batch = rows[start : start + args.batch_size] + encoded = [encode_prompt(tokenizer, row, args.max_tokens) for row in batch] + response = session.post( + args.base_url.rstrip("/") + "/v1/score", + json={ + "model": args.model, + "query": [], + "items": [ids for ids, _, _ in encoded], + "label_token_ids": [slots for _, slots, _ in encoded], + "apply_softmax": True, + "temperature": args.temperature, + "return_token_logprobs": True, + }, + timeout=300, + ) + response.raise_for_status() + result = response.json() + for row, (ids, _, prompt_hash), probabilities, logprobs in zip( + batch, encoded, result["scores"], result["token_logprobs"], strict=True + ): + output.write( + json.dumps( + { + "id": row["id"], + "option_ids": [option["id"] for option in row["options"]], + "probabilities": probabilities, + "token_logprobs": logprobs, + "input_tokens": len(ids), + "prompt_sha256": prompt_hash, + "prompt_version": PROMPT_VERSION, + "temperature": args.temperature, + "readout": "sglang candidate token logprobs", + }, + allow_nan=False, + ) + + "\n" + ) + + +if __name__ == "__main__": + main() diff --git a/python/sglang/srt/entrypoints/engine_score_mixin.py b/python/sglang/srt/entrypoints/engine_score_mixin.py index 085e006ce3..3f96419ae7 100644 --- a/python/sglang/srt/entrypoints/engine_score_mixin.py +++ b/python/sglang/srt/entrypoints/engine_score_mixin.py @@ -30,13 +30,15 @@ class EngineScoreMixin: self, query: Optional[Union[str, List[int]]] = None, items: Optional[Union[str, List[str], List[List[int]]]] = None, - label_token_ids: Optional[List[int]] = None, + label_token_ids: Optional[Union[List[int], List[List[int]]]] = None, apply_softmax: bool = False, item_first: bool = False, embed_override_token_id: Optional[int] = None, query_embed_overrides: Optional[List[torch.Tensor]] = None, item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None, return_pooled_hidden_states: bool = False, + temperature: float = 1.0, + return_token_logprobs: bool = False, ) -> ScoreResult: """ Score items against a query using the loaded model. @@ -54,8 +56,8 @@ class EngineScoreMixin: Args: query: The query text or pre-tokenized token IDs. items: The item text(s) or pre-tokenized token IDs. - label_token_ids: Token IDs to score (required for CausalLM; ignored for - SequenceClassification). + label_token_ids: Shared token IDs or one list per item, preserving candidate + order (required for CausalLM; ignored for SequenceClassification). apply_softmax: Whether to normalize scores using softmax. item_first: If True, prepend items before query (single-item mode only). embed_override_token_id: Placeholder token ID used to locate override positions. @@ -64,6 +66,10 @@ class EngineScoreMixin: return_pooled_hidden_states: Whether to include raw pooled transformer hidden states (before the task head) in the result. Only supported for non-generation models (SequenceClassification, RewardModel). + temperature: Positive calibration temperature, applied before candidate/class + softmax. Requires apply_softmax=True when different from 1. + return_token_logprobs: Include uncalibrated full-vocabulary logprobs for + each item's candidates. Only supported for CausalLM, not raw logits. Returns: ScoreResult with scores (one list per item), prompt token count, and @@ -81,6 +87,8 @@ class EngineScoreMixin: item_embed_overrides=item_embed_overrides, request=None, return_pooled_hidden_states=return_pooled_hidden_states, + temperature=temperature, + return_token_logprobs=return_token_logprobs, ) ) @@ -88,13 +96,15 @@ class EngineScoreMixin: self, query: Optional[Union[str, List[int]]] = None, items: Optional[Union[str, List[str], List[List[int]]]] = None, - label_token_ids: Optional[List[int]] = None, + label_token_ids: Optional[Union[List[int], List[List[int]]]] = None, apply_softmax: bool = False, item_first: bool = False, embed_override_token_id: Optional[int] = None, query_embed_overrides: Optional[List[torch.Tensor]] = None, item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None, return_pooled_hidden_states: bool = False, + temperature: float = 1.0, + return_token_logprobs: bool = False, ) -> ScoreResult: """Asynchronous version of score(). See score() for full documentation.""" return await self.tokenizer_manager.score_request( @@ -108,4 +118,6 @@ class EngineScoreMixin: item_embed_overrides=item_embed_overrides, request=None, return_pooled_hidden_states=return_pooled_hidden_states, + temperature=temperature, + return_token_logprobs=return_token_logprobs, ) diff --git a/python/sglang/srt/entrypoints/openai/protocol.py b/python/sglang/srt/entrypoints/openai/protocol.py index 88b33753a8..243b40e6dd 100644 --- a/python/sglang/srt/entrypoints/openai/protocol.py +++ b/python/sglang/srt/entrypoints/openai/protocol.py @@ -1398,10 +1398,12 @@ class ScoringRequest(BaseModel): item_embed_overrides: Optional[List[Optional[List[List[float]]]]] = ( None # [num_items][num_item_embed_overrides][hidden_size] ) - label_token_ids: Optional[List[int]] = ( - None # Token IDs to compute probabilities for + label_token_ids: Optional[Union[List[int], List[List[int]]]] = ( + None # shared candidates or one candidate list per item ) apply_softmax: bool = False + temperature: float = Field(default=1.0, gt=0, allow_inf_nan=False) + return_token_logprobs: bool = False item_first: bool = False return_pooled_hidden_states: bool = False model: str = DEFAULT_MODEL_NAME @@ -1412,6 +1414,7 @@ class ScoringResponse(BaseModel): List[float] ] # List of lists of probabilities, each in the order of label_token_ids pooled_hidden_states: Optional[List[Optional[List[float]]]] = None + token_logprobs: Optional[List[List[float]]] = None model: str usage: Optional[UsageInfo] = None object: str = "scoring" diff --git a/python/sglang/srt/entrypoints/openai/serving_score.py b/python/sglang/srt/entrypoints/openai/serving_score.py index 9a81716f39..79cb15b9a4 100644 --- a/python/sglang/srt/entrypoints/openai/serving_score.py +++ b/python/sglang/srt/entrypoints/openai/serving_score.py @@ -78,6 +78,8 @@ class OpenAIServingScore(OpenAIServingBase): item_embed_overrides=item_embed_overrides, request=raw_request, return_pooled_hidden_states=request.return_pooled_hidden_states, + temperature=request.temperature, + return_token_logprobs=request.return_token_logprobs, ) phs_as_lists = None @@ -90,6 +92,7 @@ class OpenAIServingScore(OpenAIServingBase): response = ScoringResponse( scores=result.scores, pooled_hidden_states=phs_as_lists, + token_logprobs=result.token_logprobs, model=request.model, usage=UsageInfo( prompt_tokens=result.prompt_tokens, diff --git a/python/sglang/srt/managers/tokenizer_manager_score_mixin.py b/python/sglang/srt/managers/tokenizer_manager_score_mixin.py index 367fa35d42..589ea1dda7 100644 --- a/python/sglang/srt/managers/tokenizer_manager_score_mixin.py +++ b/python/sglang/srt/managers/tokenizer_manager_score_mixin.py @@ -24,15 +24,19 @@ class ScoreResult: # The HTTP path converts to lists in serving_score.py before JSON serialization. # Same layout as scores: one tensor per item (not a single packed 2D tensor). pooled_hidden_states: Optional[List[Optional[torch.Tensor]]] = None + # uncalibrated full-vocabulary logprobs, in each item's candidate order + token_logprobs: Optional[List[List[float]]] = None class TokenizerManagerScoreMixin: async def score_prompts( self, prompts: Union[str, List[str], List[List[int]]], - label_token_ids: List[int], + label_token_ids: Union[List[int], List[List[int]]], apply_softmax: bool = False, request: Optional[Any] = None, + temperature: float = 1.0, + return_token_logprobs: bool = False, ) -> ScoreResult: """ Score probabilities of specified token IDs after each *full prompt*. @@ -51,6 +55,8 @@ class TokenizerManagerScoreMixin: apply_softmax=apply_softmax, item_first=False, request=request, + temperature=temperature, + return_token_logprobs=return_token_logprobs, ) # Tokenized prompts @@ -62,6 +68,8 @@ class TokenizerManagerScoreMixin: apply_softmax=apply_softmax, item_first=False, request=request, + temperature=temperature, + return_token_logprobs=return_token_logprobs, ) raise ValueError("Invalid prompts type for score_prompts.") @@ -102,7 +110,8 @@ class TokenizerManagerScoreMixin: items_ids = [] for item in items_list: if isinstance(item, str): - items_ids.append(self.tokenizer.encode(item)) + # items continue the query, so only the query gets special tokens + items_ids.append(self.tokenizer.encode(item, add_special_tokens=False)) else: items_ids.append(list(item)) @@ -112,10 +121,12 @@ class TokenizerManagerScoreMixin: self, results: Any, items: List, - label_token_ids: Optional[List[int]], + label_token_ids: Optional[List[List[int]]], apply_softmax: bool, batch_request=None, return_pooled_hidden_states: bool = False, + temperature: float = 1.0, + return_token_logprobs: bool = False, ) -> ScoreResult: """ Process results from multi-item scoring request. @@ -135,27 +146,31 @@ class TokenizerManagerScoreMixin: # Extract per-delimiter scores from whichever field has them input_logprobs = meta_info.get("input_token_ids_logprobs", []) embedding = single_result.get("embedding") + token_logprobs = [] if return_token_logprobs else None if input_logprobs: # Generation model: extract label-token logprobs at each delimiter - per_delimiter_scores = [] - for logprobs_data in input_logprobs: - logprobs = self._extract_logprobs_for_tokens( - logprobs_data, label_token_ids + if len(input_logprobs) != expected_count: + raise RuntimeError( + f"Expected {expected_count} delimiter entries, got {len(input_logprobs)}" ) + per_delimiter_scores = [[]] # the query boundary is not a decision + for logprobs_data, labels in zip(input_logprobs[1:], label_token_ids): + logprobs = self._extract_logprobs_for_tokens(logprobs_data, labels) score_list = self._convert_logprobs_to_scores( - logprobs, label_token_ids, apply_softmax + logprobs, labels, apply_softmax, temperature ) per_delimiter_scores.append(score_list) + if return_token_logprobs: + token_logprobs.append([logprobs[token] for token in labels]) elif embedding is not None: # Classification model: scores are directly in 2D embedding. if apply_softmax: - scores_tensor = ( - torch.tensor(embedding) - if isinstance(embedding, list) - else embedding + scores_tensor = torch.as_tensor(embedding, dtype=torch.float64) + scores_tensor = scores_tensor - scores_tensor.amax(dim=-1, keepdim=True) + scores_tensor = torch.nn.functional.softmax( + scores_tensor / temperature, dim=-1 ) - scores_tensor = torch.nn.functional.softmax(scores_tensor, dim=-1) per_delimiter_scores = scores_tensor.tolist() else: per_delimiter_scores = ( @@ -188,14 +203,17 @@ class TokenizerManagerScoreMixin: scores=scores, prompt_tokens=prompt_tokens, pooled_hidden_states=phs_list, + token_logprobs=token_logprobs, ) def _process_single_item_scoring_results( self, results: Any, - label_token_ids: Optional[List[int]], + label_token_ids: Optional[List[List[int]]], apply_softmax: bool, return_pooled_hidden_states: bool = False, + temperature: float = 1.0, + return_token_logprobs: bool = False, ) -> ScoreResult: """ Process results from single-item scoring request. @@ -208,10 +226,11 @@ class TokenizerManagerScoreMixin: phs_list = [] has_phs = False prompt_tokens = 0 + token_logprobs = [] if return_token_logprobs else None is_generation = self.is_generation if is_generation: - for result in results: + for result, labels in zip(results, label_token_ids): # For single-item scoring, logprobs are in output_token_ids_logprobs output_logprobs = result["meta_info"].get( "output_token_ids_logprobs", [] @@ -225,13 +244,13 @@ class TokenizerManagerScoreMixin: ) # Extract logprobs for the first (and only) position - logprobs = self._extract_logprobs_for_tokens( - output_logprobs[0], label_token_ids - ) + logprobs = self._extract_logprobs_for_tokens(output_logprobs[0], labels) score_list = self._convert_logprobs_to_scores( - logprobs, label_token_ids, apply_softmax + logprobs, labels, apply_softmax, temperature ) scores.append(score_list) + if return_token_logprobs: + token_logprobs.append([logprobs[token] for token in labels]) else: for result in results: embedding = result.get("embedding", None) @@ -241,8 +260,9 @@ class TokenizerManagerScoreMixin: prompt_tokens += result.get("meta_info", {}).get("prompt_tokens", 0) if apply_softmax: + scores_tensor = torch.as_tensor(embedding, dtype=torch.float64) embedding = torch.softmax( - torch.as_tensor(embedding), dim=-1 + (scores_tensor - scores_tensor.max()) / temperature, dim=-1 ).tolist() # The classification head produces per-token logits, which the pooler reduces @@ -262,6 +282,7 @@ class TokenizerManagerScoreMixin: scores=scores, prompt_tokens=prompt_tokens, pooled_hidden_states=phs_list if has_phs else None, + token_logprobs=token_logprobs, ) # ------------------------------------------------------------------ @@ -445,7 +466,7 @@ class TokenizerManagerScoreMixin: self, query: Optional[Union[str, List[int]]] = None, items: Optional[Union[str, List[str], List[List[int]]]] = None, - label_token_ids: Optional[List[int]] = None, + label_token_ids: Optional[Union[List[int], List[List[int]]]] = None, apply_softmax: bool = False, item_first: bool = False, embed_override_token_id: Optional[int] = None, @@ -453,6 +474,8 @@ class TokenizerManagerScoreMixin: item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None, request: Optional[Any] = None, return_pooled_hidden_states: bool = False, + temperature: float = 1.0, + return_token_logprobs: bool = False, ) -> ScoreResult: """ Score the probability of specified token IDs appearing after the given (query + item) pair. @@ -477,6 +500,15 @@ class TokenizerManagerScoreMixin: """ is_generation = self.is_generation + if not math.isfinite(temperature) or temperature <= 0: + raise ValueError("temperature must be finite and greater than zero") + if temperature != 1.0 and not apply_softmax: + raise ValueError("temperature requires apply_softmax=True") + if return_token_logprobs and not is_generation: + raise ValueError( + "return_token_logprobs is only supported for CausalLM models" + ) + if is_generation and label_token_ids is None: raise ValueError( "label_token_ids is required for generation (CausalLM) models." @@ -484,7 +516,37 @@ class TokenizerManagerScoreMixin: if items is None: raise ValueError("items must be provided") if not items: - return ScoreResult(scores=[], prompt_tokens=0) + return ScoreResult( + scores=[], token_logprobs=[] if return_token_logprobs else None + ) + + num_items = 1 if isinstance(items, str) else len(items) + if is_generation: + if not isinstance(label_token_ids, list) or not label_token_ids: + raise ValueError("label_token_ids must be a nonempty list") + if isinstance(label_token_ids[0], list): + if len(label_token_ids) != num_items: + raise ValueError("label_token_ids must have one list per item") + else: + label_token_ids = [label_token_ids] * num_items + for labels in label_token_ids: + if not isinstance(labels, list) or not labels: + raise ValueError("each item must have a nonempty candidate list") + if any(type(token) is not int or token < 0 for token in labels): + raise ValueError( + "label_token_ids must contain nonnegative integers" + ) + if len(set(labels)) != len(labels): + raise ValueError( + "label_token_ids must not contain duplicate tokens" + ) + if self.tokenizer is not None: + vocab_size = self.tokenizer.vocab_size + for token_id in labels: + if token_id >= vocab_size: + raise ValueError( + f"Token ID {token_id} is out of vocabulary (vocab size: {vocab_size})" + ) has_embeds = ( query_embed_overrides is not None or item_embed_overrides is not None @@ -501,14 +563,6 @@ class TokenizerManagerScoreMixin: f"item_embed_overrides length ({len(item_embed_overrides)}) " f"must match items length ({len(items)})." ) - if self.tokenizer is not None and label_token_ids is not None: - vocab_size = self.tokenizer.vocab_size - for token_id in label_token_ids: - if token_id >= vocab_size: - raise ValueError( - f"Token ID {token_id} is out of vocabulary (vocab size: {vocab_size})" - ) - # Check if multi-item scoring is enabled use_multi_item_scoring = get_exec().features.enable_mis @@ -601,10 +655,20 @@ class TokenizerManagerScoreMixin: # Create the appropriate request type mis_delimiter_indices = [delimiter_indices] if use_multi_item_scoring else None if is_generation: + # packed MIS requests gather the union, then select per item + request_labels = ( + list( + dict.fromkeys( + token for labels in label_token_ids for token in labels + ) + ) + if use_multi_item_scoring + else label_token_ids + ) batch_request = GenerateReqInput( text=text_prompts, input_ids=input_ids, - token_ids_logprob=label_token_ids, + token_ids_logprob=request_labels, return_logprob=True, # Set logprob_start_len=0 for multi-item scoring since we want logprobs at all delimiter positions logprob_start_len=0 if use_multi_item_scoring else -1, @@ -633,11 +697,18 @@ class TokenizerManagerScoreMixin: apply_softmax, batch_request, return_pooled_hidden_states, + temperature, + return_token_logprobs, ) else: # Single-item scoring: process each result separately return self._process_single_item_scoring_results( - results, label_token_ids, apply_softmax, return_pooled_hidden_states + results, + label_token_ids, + apply_softmax, + return_pooled_hidden_states, + temperature, + return_token_logprobs, ) def _convert_logprobs_to_scores( @@ -645,13 +716,20 @@ class TokenizerManagerScoreMixin: logprobs: Dict[int, float], label_token_ids: List[int], apply_softmax: bool, + temperature: float = 1.0, ) -> List[float]: score_list = [ logprobs.get(token_id, float("-inf")) for token_id in label_token_ids ] if apply_softmax: - score_list = torch.softmax(torch.tensor(score_list), dim=0).tolist() + # center before scaling so small temperatures do not overflow + maximum = max(score_list) + weights = [ + math.exp((score - maximum) / temperature) for score in score_list + ] + denominator = sum(weights) + score_list = [weight / denominator for weight in weights] else: # Convert logprobs to probabilities if not using softmax score_list = [ diff --git a/test/registered/prefill_only/test_multi_item_scoring.py b/test/registered/prefill_only/test_multi_item_scoring.py index e943464084..73a10e506f 100644 --- a/test/registered/prefill_only/test_multi_item_scoring.py +++ b/test/registered/prefill_only/test_multi_item_scoring.py @@ -433,7 +433,12 @@ class TestMultiItemScoringParity(CustomTestCase): tokenizer = AutoTokenizer.from_pretrained(TEST_MODEL_NAME) def label_ids(labels): - return [tokenizer.encode(lb, add_special_tokens=False)[0] for lb in labels] + token_ids = [] + for label in labels: + (token_id,) = tokenizer.encode(label, add_special_tokens=False) + token_ids.append(token_id) + assert len(set(token_ids)) == len(token_ids), token_ids + return token_ids return { "basic": dict( @@ -460,7 +465,7 @@ class TestMultiItemScoringParity(CustomTestCase): "many_items": dict( query="Rate this option from 1 to 5:", items=[f" Option {i}" for i in range(10)], - label_token_ids=label_ids([" 1", " 2", " 3", " 4", " 5"]), + label_token_ids=label_ids(["1", "2", "3", "4", "5"]), apply_softmax=True, ), } diff --git a/test/registered/prefill_only/test_score_api.py b/test/registered/prefill_only/test_score_api.py index 76d8f9873e..19b1bff177 100644 --- a/test/registered/prefill_only/test_score_api.py +++ b/test/registered/prefill_only/test_score_api.py @@ -12,6 +12,7 @@ test_score_engine.py. These tests focus on the HTTP integration seam: Pydantic schema defaults, FastAPI routing, and server argument wiring. """ +import math import os import unittest @@ -136,6 +137,44 @@ class TestCausalLMScoringHTTP(CustomTestCase): with self.subTest(payload=list(payload.keys())): self.assertGreaterEqual(self._post(payload).status_code, 400) + def test_calibrated_decisions(self): + response = self._post( + { + "query": [], + "items": [[1, 2, 3], [4, 5]], + "label_token_ids": [[32, 33], [34, 32, 33]], + "apply_softmax": True, + "temperature": 2.0, + "return_token_logprobs": True, + } + ) + self.assertEqual(response.status_code, 200, response.text) + body = response.json() + self.assertEqual([len(row) for row in body["scores"]], [2, 3]) + for scores, logprobs in zip(body["scores"], body["token_logprobs"]): + weights = [math.exp((x - max(logprobs)) / 2) for x in logprobs] + for actual, weight in zip(scores, weights): + self.assertAlmostEqual(actual, weight / sum(weights), places=6) + self.assertEqual(body["usage"]["completion_tokens"], 0) + + def test_invalid_decision_options(self): + for extra in ( + {"label_token_ids": [[1, 2]]}, + {"label_token_ids": [[1], []]}, + {"label_token_ids": [1, 1]}, + {"temperature": 0}, + {"temperature": 2.0}, + ): + response = self._post( + { + "query": "", + "items": ["A", "B"], + "label_token_ids": [1, 2], + **extra, + } + ) + self.assertIn(response.status_code, (400, 422), response.text) + # --------------------------------------------------------------------------- # MIS scoring (with --enable-mis) @@ -203,6 +242,27 @@ class TestCausalLMMISScoringHTTP(CustomTestCase): result = self._score("Test query", [], [1, 2]) self.assertEqual(len(result["scores"]), 0) + def test_per_item_candidates(self): + labels = [[32, 33], [34, 32, 33]] + response = requests.post( + self.base_url + "/v1/score", + json={ + "query": "Select an option:", + "items": ["A or B?", "A, B or C?"], + "label_token_ids": labels, + "apply_softmax": True, + "temperature": 2.0, + "return_token_logprobs": True, + }, + ) + self.assertEqual(response.status_code, 200, response.text) + body = response.json() + self.assertEqual([len(row) for row in body["scores"]], [2, 3]) + for scores, logprobs in zip(body["scores"], body["token_logprobs"]): + weights = [math.exp((x - max(logprobs)) / 2) for x in logprobs] + for actual, weight in zip(scores, weights): + self.assertAlmostEqual(actual, weight / sum(weights), places=6) + def test_varying_item_counts(self): """1, 2, 4, and 6 items all return the correct number of score vectors.""" label_token_ids = [1, 2, 3, 4, 5] diff --git a/test/registered/prefill_only/test_score_engine.py b/test/registered/prefill_only/test_score_engine.py index cab0f8e1c5..df45a4375d 100644 --- a/test/registered/prefill_only/test_score_engine.py +++ b/test/registered/prefill_only/test_score_engine.py @@ -249,6 +249,36 @@ class TestCausalLMScoring(CustomTestCase): query="Q", items=None, label_token_ids=[1, 2], apply_softmax=True ) + def test_decision_scoring(self): + tokenizer = AutoTokenizer.from_pretrained(_CAUSAL_LM_MODEL) + prompts = ["Answer A or B: Is Paris in France?", "Choose A, B or C: 2 + 2 ="] + inputs = [tokenizer.encode(prompt) for prompt in prompts] + labels = [[33, 32], [32, 34, 33]] + kwargs = dict(apply_softmax=True, temperature=1.7, return_token_logprobs=True) + batch = self.engine.score( + query=[], items=inputs, label_token_ids=labels, **kwargs + ) + self.assertEqual(len(batch.scores), 2) + for i, (ids, candidates) in enumerate(zip(inputs, labels)): + single = self.engine.score( + query=[], items=[ids], label_token_ids=candidates, **kwargs + ) + torch.testing.assert_close( + torch.tensor(batch.scores[i]), + torch.tensor(single.scores[0]), + atol=1e-3, + rtol=1e-2, + ) + expected = torch.softmax(torch.tensor(batch.token_logprobs[i]) / 1.7, 0) + torch.testing.assert_close(torch.tensor(batch.scores[i]), expected) + repeated = self.engine.score( + query=[], items=inputs, label_token_ids=labels, **kwargs + ) + for cold, warm in zip(batch.scores, repeated.scores): + torch.testing.assert_close( + torch.tensor(cold), torch.tensor(warm), atol=1e-3, rtol=1e-2 + ) + # --------------------------------------------------------------------------- # SequenceClassification — single-item mode diff --git a/test/registered/unit/test_token_scoring.py b/test/registered/unit/test_token_scoring.py new file mode 100644 index 0000000000..6c8e0994b7 --- /dev/null +++ b/test/registered/unit/test_token_scoring.py @@ -0,0 +1,254 @@ +import asyncio +import json +import unittest +from types import SimpleNamespace + +import torch +from tokenizers import Tokenizer +from tokenizers.models import WordLevel +from tokenizers.pre_tokenizers import Whitespace +from tokenizers.processors import TemplateProcessing +from transformers import PreTrainedTokenizerFast + +from sglang.srt.entrypoints.engine_score_mixin import EngineScoreMixin +from sglang.srt.entrypoints.openai.protocol import ScoringRequest +from sglang.srt.entrypoints.openai.serving_score import OpenAIServingScore +from sglang.srt.managers.tokenizer_manager_score_mixin import TokenizerManagerScoreMixin +from sglang.srt.runtime_context import publish, restore_context, snapshot_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class ScoringManager(TokenizerManagerScoreMixin): + """Replace only model execution; keep request construction and score extraction real.""" + + def __init__(self, enable_mis=False, generation=True): + self.server_args = ServerArgs(model_path="dummy", enable_mis=enable_mis) + publish(self.server_args, role="test") + self.is_generation = generation + self.tokenizer = SimpleNamespace(vocab_size=8) + self.logits = torch.tensor([-1000.0, -999.0, -997.0, -996.0, 0, 1, 2, 3]) + self.requests = [] + + async def generate_request(self, request, raw_request): + self.requests.append(request) + request.normalize_batch_and_arguments() + results = [] + for index, ids in enumerate(request.input_ids): + meta = {"prompt_tokens": len(ids)} + if self.is_generation: + assert request.sampling_params[index]["max_new_tokens"] == 0 + labels = request.token_ids_logprob[index] + values = torch.log_softmax(self.logits, dim=0) + logprobs = [(values[token].item(), token, None) for token in labels] + if self.server_args.enable_mis: + count = len(request.multi_item_delimiter_indices[index]) + meta["input_token_ids_logprobs"] = [logprobs] * count + else: + meta["output_token_ids_logprobs"] = [logprobs] + results.append({"meta_info": meta}) + else: + embedding = [1.0, 3.0] + if self.server_args.enable_mis: + count = len(request.multi_item_delimiter_indices[index]) + embedding = [embedding] * count + results.append({"meta_info": meta, "embedding": embedding}) + yield results + + +class TestTokenScoring(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.addCleanup(restore_context, snapshot_context()) + + async def test_http_to_manager_ragged_candidates(self): + for enable_mis in (False, True): + with self.subTest(enable_mis=enable_mis): + manager = ScoringManager(enable_mis=enable_mis) + request = ScoringRequest( + query=[4], + items=[[5], [6, 7]], + label_token_ids=[[3, 1], [0, 2, 1]], + apply_softmax=True, + temperature=2.0, + return_token_logprobs=True, + ) + handler = OpenAIServingScore(manager) + response = await handler._handle_non_streaming_request( + request, request, None + ) + body = json.loads(response.body) + for labels, scores, logprobs in zip( + request.label_token_ids, body["scores"], body["token_logprobs"] + ): + expected = torch.softmax(manager.logits[labels] / 2, dim=0) + torch.testing.assert_close(torch.tensor(scores), expected) + torch.testing.assert_close( + torch.tensor(logprobs), + torch.log_softmax(manager.logits, dim=0)[labels], + ) + self.assertEqual(body["usage"]["completion_tokens"], 0) + + async def test_default_scores_are_vocabulary_probabilities(self): + manager = ScoringManager() + result = await manager.score_request( + query=[], items=[[4], [5]], label_token_ids=[7, 4] + ) + expected = torch.softmax(manager.logits, dim=0)[[7, 4]] + for scores in result.scores: + torch.testing.assert_close(torch.tensor(scores), expected) + self.assertIsNone(result.token_logprobs) + + async def test_mis_text_items_do_not_add_special_tokens(self): + vocab = {"[UNK]": 0, "[BOS]": 1, "Rate": 2, ":": 3, "Option": 4, "A": 5, "B": 6} + tokenizer = Tokenizer(WordLevel(vocab, unk_token="[UNK]")) + tokenizer.pre_tokenizer = Whitespace() + tokenizer.post_processor = TemplateProcessing( + single="[BOS] $A", special_tokens=[("[BOS]", vocab["[BOS]"])] + ) + manager = ScoringManager(enable_mis=True) + manager.tokenizer = PreTrainedTokenizerFast( + tokenizer_object=tokenizer, bos_token="[BOS]", unk_token="[UNK]" + ) + items = [" Option A", " Option B"] + for query in ("Rate:", ""): + with self.subTest(query=query): + await manager.score_request( + query=query, items=items, label_token_ids=[5, 6] + ) + request = manager.requests[-1] + packed = request.input_ids[0] + delimiters = request.multi_item_delimiter_indices[0] + prefix = packed[: delimiters[0]] + for item, start, end in zip(items, delimiters, delimiters[1:]): + self.assertEqual( + prefix + packed[start + 1 : end], + manager.tokenizer.encode(query + item), + ) + + async def test_async_engine_and_full_prompts(self): + manager = ScoringManager() + engine = EngineScoreMixin() + engine.tokenizer_manager = manager + kwargs = dict( + label_token_ids=[[1, 3], [2]], + apply_softmax=True, + temperature=0.5, + return_token_logprobs=True, + ) + actual = await engine.async_score(query=[], items=[[4], [5]], **kwargs) + expected = await manager.score_prompts([[4], [5]], **kwargs) + self.assertEqual(actual, expected) + self.assertEqual(actual.scores[1], [1.0]) + + async def test_classification_temperature(self): + for enable_mis in (False, True): + manager = ScoringManager(enable_mis=enable_mis, generation=False) + result = await manager.score_request( + query=[4], items=[[5], [6]], apply_softmax=True, temperature=2.0 + ) + for scores in result.scores: + torch.testing.assert_close( + torch.tensor(scores), torch.softmax(torch.tensor([0.5, 1.5]), 0) + ) + with self.assertRaisesRegex(ValueError, "only supported for CausalLM"): + await manager.score_request( + query=[], items=[[4]], return_token_logprobs=True + ) + + async def test_invalid_candidates_and_temperature(self): + manager = ScoringManager() + for labels in ([], [[]], [[1]], [[1], []], [1, 1], [-1], [8], [True], "bad"): + with self.subTest(labels=labels), self.assertRaises(ValueError): + await manager.score_request( + query=[], items=[[4], [5]], label_token_ids=labels + ) + for temperature in (0, -1, float("nan"), float("inf")): + with self.subTest(temperature=temperature), self.assertRaises(ValueError): + await manager.score_request( + query=[], + items=[[4]], + label_token_ids=[1], + temperature=temperature, + apply_softmax=True, + ) + with self.assertRaisesRegex(ValueError, "requires apply_softmax"): + await manager.score_request( + query=[], items=[[4]], label_token_ids=[1], temperature=2.0 + ) + self.assertEqual(manager.requests, []) + + async def test_http_out_of_vocabulary_error(self): + for enable_mis in (False, True): + for token_id in (8, 999999): + for labels in ([7, token_id], [[7], [1, token_id]]): + with self.subTest(enable_mis=enable_mis, labels=labels): + manager = ScoringManager(enable_mis=enable_mis) + request = ScoringRequest( + query=[], + items=[[4], [5]], + label_token_ids=labels, + apply_softmax=True, + ) + response = await OpenAIServingScore( + manager + )._handle_non_streaming_request(request, request, None) + self.assertEqual(response.status_code, 400) + body = json.loads(response.body) + self.assertEqual(body["type"], "BadRequestError") + self.assertEqual( + body["message"], + f"Token ID {token_id} is out of vocabulary (vocab size: 8)", + ) + self.assertEqual(manager.requests, []) + + async def test_small_temperature_is_finite(self): + for enable_mis in (False, True): + for generation in (False, True): + manager = ScoringManager(enable_mis=enable_mis, generation=generation) + result = await manager.score_request( + query=[4], + items=[[5]], + label_token_ids=[1, 3], + apply_softmax=True, + temperature=1e-300, + ) + self.assertEqual(result.scores, [[0.0, 1.0]]) + + async def test_empty_batch(self): + manager = ScoringManager() + result = await manager.score_request( + query=[], items=[], label_token_ids=[], return_token_logprobs=True + ) + self.assertEqual(result.scores, []) + self.assertEqual(result.token_logprobs, []) + self.assertEqual(manager.requests, []) + + +class TestSyncTokenScoring(unittest.TestCase): + def test_engine_score(self): + self.addCleanup(restore_context, snapshot_context()) + engine = EngineScoreMixin() + engine.tokenizer_manager = ScoringManager() + engine.loop = asyncio.new_event_loop() + try: + result = engine.score( + query=[], + items=[[4]], + label_token_ids=[[1, 3]], + apply_softmax=True, + temperature=2.0, + return_token_logprobs=True, + ) + torch.testing.assert_close( + torch.tensor(result.scores[0]), + torch.softmax(engine.tokenizer_manager.logits[[1, 3]] / 2, 0), + ) + finally: + engine.loop.run_until_complete(engine.loop.shutdown_asyncgens()) + engine.loop.close() + + +if __name__ == "__main__": + unittest.main()