[Score API] Setwise Scoring Support (#38965)

This commit is contained in:
Sundara Raman Ramachandran
2026-09-24 16:50:58 -07:00
committed by GitHub
parent 7c5bdb940c
commit 2f2f9d12f8
18 changed files with 1882 additions and 51 deletions
@@ -0,0 +1,271 @@
"""Engine benchmark for setwise (multi-position) scoring.
Benchmarks `Engine.async_score(..., score_extraction_token_id=...)` — the setwise
path that pools the classification head at each candidate's anchor token and
returns an ``[N, num_labels]`` matrix in a single forward pass.
Unlike the general ``bench_score.py`` (label-token / MIS scoring), this is a
focused, self-contained harness: it sweeps the **set size** (number of
candidates packed into one item) and reports latency percentiles and throughput
(requests/s and candidates/s) so you can see how a single-forward setwise pass
scales with N.
It talks to the Engine directly (no HTTP) and requires the setwise deployment
constraints (``--disable-radix-cache``, ``--chunked-prefill-size -1``), which it
sets automatically.
Usage:
# Default sweep over set sizes on the target model
python bench_setwise_score.py --model-path /path/to/seqcls_model
# Custom sweep + concurrency + per-candidate length
python bench_setwise_score.py --model-path /path/to/model \
--set-sizes 8 16 32 64 --concurrency 16 --num-requests 100 \
--item-tokens 180 --query-tokens 64
# Include pooled hidden states in the response (per-anchor pre-head vectors)
python bench_setwise_score.py --model-path /path/to/model --return-pooled-hidden-states
"""
import argparse
import asyncio
import csv
import random
import statistics
import time
from typing import List, Optional
from transformers import AutoTokenizer
def parse_args():
parser = argparse.ArgumentParser(
description="Benchmark setwise scoring via Engine.async_score",
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument(
"--model-path",
type=str,
required=True,
help="Path to a SequenceClassification model (e.g. Qwen3ForSequenceClassification).",
)
parser.add_argument(
"--score-extraction-token",
type=str,
default="<|object_ref_start|>",
help="Anchor token pooled at each candidate (default: <|object_ref_start|>).",
)
parser.add_argument(
"--set-sizes",
type=int,
nargs="+",
default=[8, 16, 32, 64],
help="Candidates per set (each is one anchor in one item). Swept in order.",
)
parser.add_argument(
"--item-tokens",
type=int,
default=180,
help="Content tokens per candidate before its anchor (default: 180).",
)
parser.add_argument(
"--query-tokens",
type=int,
default=0,
help="Shared query-prefix tokens (default: 0).",
)
parser.add_argument(
"--num-requests",
type=int,
default=50,
help="Timed requests per set size (default: 50).",
)
parser.add_argument(
"--concurrency",
type=int,
default=8,
help="Max in-flight requests (default: 8).",
)
parser.add_argument(
"--warmup",
type=int,
default=3,
help="Warmup requests per set size, not timed (default: 3).",
)
parser.add_argument("--apply-softmax", action="store_true")
parser.add_argument("--return-pooled-hidden-states", action="store_true")
# Engine configuration
parser.add_argument("--dtype", type=str, default="float16")
parser.add_argument("--mem-fraction-static", type=float, default=0.5)
parser.add_argument("--max-prefill-tokens", type=int, default=60000)
parser.add_argument("--attention-backend", type=str, default="flashinfer")
parser.add_argument("--seed", type=int, default=0)
parser.add_argument(
"--output", "-o", type=str, default=None, help="Optional CSV output path."
)
return parser.parse_args()
def _random_content(num_tokens: int, anchor_id: int, lo: int = 10, hi: int = 30000):
"""Random content token ids that never collide with the anchor id."""
ids = [random.randint(lo, hi) for _ in range(num_tokens)]
return [t + 1 if t == anchor_id else t for t in ids]
def _build_setwise_request(args, anchor_id: int, n_candidates: int):
"""One setwise request: shared query prefix + ONE item with N anchors."""
query_ids = (
_random_content(args.query_tokens, anchor_id) if args.query_tokens else []
)
item_ids: List[int] = []
for _ in range(n_candidates):
item_ids.extend(_random_content(args.item_tokens, anchor_id))
item_ids.append(anchor_id)
return query_ids, [item_ids]
def _percentile(sorted_vals: List[float], pct: float) -> float:
if not sorted_vals:
return 0.0
k = max(0, min(len(sorted_vals) - 1, round((pct / 100.0) * (len(sorted_vals) - 1))))
return sorted_vals[k]
async def _run_set_size(engine, args, anchor_id: int, n: int) -> Optional[dict]:
# Pre-build requests so generation cost is outside the timed section.
requests = [
_build_setwise_request(args, anchor_id, n)
for _ in range(args.num_requests + args.warmup)
]
prompt_tokens = len(requests[0][0]) + len(requests[0][1][0])
async def _one(query_ids, items):
t0 = time.perf_counter()
res = await engine.async_score(
query=query_ids,
items=items,
score_extraction_token_id=anchor_id,
apply_softmax=args.apply_softmax,
return_pooled_hidden_states=args.return_pooled_hidden_states,
)
# Setwise scores are nested (one [Ni x num_labels] matrix per item) and
# this harness sends exactly one item, so the per-candidate row count is
# len(res.scores[0]), not len(res.scores) (which is always 1 = num items).
return time.perf_counter() - t0, len(res.scores[0]) if res.scores else 0
# Warmup (also validates the row count == N).
for q, it in requests[: args.warmup]:
_, rows = await _one(q, it)
if rows != n:
print(f"[warn] set_size={n}: expected {n} score rows, got {rows}")
sem = asyncio.Semaphore(args.concurrency)
async def _guarded(q, it):
async with sem:
return await _one(q, it)
timed = requests[args.warmup :]
wall_start = time.perf_counter()
results = await asyncio.gather(*(_guarded(q, it) for q, it in timed))
wall = time.perf_counter() - wall_start
latencies_ms = sorted(1000.0 * lat for lat, _ in results)
n_req = len(results)
return {
"set_size": n,
"prompt_tokens": prompt_tokens,
"requests": n_req,
"concurrency": args.concurrency,
"avg_ms": statistics.mean(latencies_ms),
"p50_ms": _percentile(latencies_ms, 50),
"p95_ms": _percentile(latencies_ms, 95),
"p99_ms": _percentile(latencies_ms, 99),
"req_per_s": n_req / wall if wall > 0 else 0.0,
"cand_per_s": (n_req * n) / wall if wall > 0 else 0.0,
}
def _print_table(rows: List[dict]):
headers = [
"set_size",
"prompt_tokens",
"requests",
"avg_ms",
"p50_ms",
"p95_ms",
"p99_ms",
"req_per_s",
"cand_per_s",
]
print("\n" + " | ".join(f"{h:>13}" for h in headers))
print("-" * (16 * len(headers)))
for r in rows:
cells = []
for h in headers:
v = r[h]
cells.append(f"{v:>13.2f}" if isinstance(v, float) else f"{v:>13}")
print(" | ".join(cells))
async def main():
args = parse_args()
random.seed(args.seed)
tokenizer = AutoTokenizer.from_pretrained(args.model_path, trust_remote_code=True)
anchor_id = tokenizer.convert_tokens_to_ids(args.score_extraction_token)
unk_id = tokenizer.unk_token_id
if anchor_id is None or (unk_id is not None and anchor_id == unk_id):
raise ValueError(
f"score_extraction_token {args.score_extraction_token!r} did not resolve "
f"to a dedicated token id."
)
from sglang.srt.entrypoints.engine import Engine
# Setwise deployment constraints: positions are full-prompt coordinates, so
# radix reuse / chunked prefill must be off (the engine also validates this).
engine = Engine(
model_path=args.model_path,
dtype=args.dtype,
mem_fraction_static=args.mem_fraction_static,
max_prefill_tokens=args.max_prefill_tokens,
attention_backend=args.attention_backend,
disable_radix_cache=True,
chunked_prefill_size=-1,
)
print(
f"Setwise scoring benchmark | model={args.model_path}\n"
f"anchor={args.score_extraction_token!r} (id={anchor_id}) | "
f"item_tokens={args.item_tokens} query_tokens={args.query_tokens} | "
f"concurrency={args.concurrency} num_requests={args.num_requests}"
)
rows = []
try:
for n in args.set_sizes:
row = await _run_set_size(engine, args, anchor_id, n)
if row is not None:
rows.append(row)
print(
f"set_size={n:>4}: avg={row['avg_ms']:.1f}ms "
f"p95={row['p95_ms']:.1f}ms "
f"{row['req_per_s']:.1f} req/s {row['cand_per_s']:.0f} cand/s"
)
finally:
engine.shutdown()
_print_table(rows)
if args.output and rows:
with open(args.output, "w", newline="") as f:
writer = csv.DictWriter(f, fieldnames=list(rows[0].keys()))
writer.writeheader()
writer.writerows(rows)
print(f"\nWrote {len(rows)} rows to {args.output}")
if __name__ == "__main__":
asyncio.run(main())
+14
View File
@@ -2398,6 +2398,20 @@ def is_cross_encoding_pooler_model(model_architectures: List[str]) -> bool:
return any(arch in _cross_encoding_pooler_archs for arch in model_architectures)
# SequenceClassification models whose forward routes the head through
# score_and_pool (per-position pooling); only these support setwise readout
# (token_indices_to_pool). Keep in sync with callers of layers.pooler.score_and_pool.
_score_and_pool_archs = [
"LlamaForSequenceClassification",
"Qwen2ForSequenceClassification",
"Qwen3ForSequenceClassification",
]
def is_score_and_pool_model(model_architectures: List[str]) -> bool:
return any(arch in _score_and_pool_archs for arch in model_architectures)
def yarn_get_mscale(scale: float = 1, mscale: float = 1) -> float:
if scale <= 1:
return 1.0
@@ -37,6 +37,7 @@ class EngineScoreMixin:
query_embed_overrides: Optional[List[torch.Tensor]] = None,
item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None,
return_pooled_hidden_states: bool = False,
score_extraction_token_id: Optional[int] = None,
temperature: float = 1.0,
return_token_logprobs: bool = False,
) -> ScoreResult:
@@ -53,6 +54,11 @@ class EngineScoreMixin:
For SequenceClassification models, returns the pooled class logits directly from
the classification head. label_token_ids is optional and ignored.
Setwise scoring is expressed via ``score_extraction_token_id``: pass a single
item containing the whole candidate block with one extraction token per
candidate, and the head is pooled AT those positions, so ``scores`` becomes
the ``[N x num_labels]`` per-candidate matrix.
Args:
query: The query text or pre-tokenized token IDs.
items: The item text(s) or pre-tokenized token IDs.
@@ -63,6 +69,11 @@ class EngineScoreMixin:
embed_override_token_id: Placeholder token ID used to locate override positions.
query_embed_overrides: Embedding vectors replacing placeholder tokens in query.
item_embed_overrides: Per-item embedding vectors replacing placeholder tokens in items.
score_extraction_token_id: SequenceClassification-only. When set, pool the
head at every occurrence of this token per sequence instead of the last
token; ``scores`` becomes nested — one ``[Ni x num_labels]`` matrix per
item (``len(scores) == len(items)``), where ``Ni`` is the number of
extraction tokens (candidates) in item ``i``.
return_pooled_hidden_states: Whether to include raw pooled transformer
hidden states (before the task head) in the result. Only supported
for non-generation models (SequenceClassification, RewardModel).
@@ -72,8 +83,10 @@ class EngineScoreMixin:
each item's candidates. Only supported for CausalLM, not raw logits.
Returns:
ScoreResult with scores (one list per item), prompt token count, and
optional pooled_hidden_states tensors.
ScoreResult with scores (flat ``[num_items x num_labels]`` for pointwise
scoring, or nested ``[num_items][Ni x num_labels]`` when
score_extraction_token_id is set), prompt token count, and optional
pooled_hidden_states tensors.
"""
return self.loop.run_until_complete(
self.tokenizer_manager.score_request(
@@ -85,6 +98,7 @@ class EngineScoreMixin:
embed_override_token_id=embed_override_token_id,
query_embed_overrides=query_embed_overrides,
item_embed_overrides=item_embed_overrides,
score_extraction_token_id=score_extraction_token_id,
request=None,
return_pooled_hidden_states=return_pooled_hidden_states,
temperature=temperature,
@@ -103,6 +117,7 @@ class EngineScoreMixin:
query_embed_overrides: Optional[List[torch.Tensor]] = None,
item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None,
return_pooled_hidden_states: bool = False,
score_extraction_token_id: Optional[int] = None,
temperature: float = 1.0,
return_token_logprobs: bool = False,
) -> ScoreResult:
@@ -116,6 +131,7 @@ class EngineScoreMixin:
embed_override_token_id=embed_override_token_id,
query_embed_overrides=query_embed_overrides,
item_embed_overrides=item_embed_overrides,
score_extraction_token_id=score_extraction_token_id,
request=None,
return_pooled_hidden_states=return_pooled_hidden_states,
temperature=temperature,
@@ -1406,14 +1406,32 @@ class ScoringRequest(BaseModel):
return_token_logprobs: bool = False
item_first: bool = False
return_pooled_hidden_states: bool = False
# Setwise readout (SequenceClassification-only): when set, the head is pooled
# AT every occurrence of this token in each `query + item` sequence instead of
# the last token, and `scores` is returned nested (one `[Nᵢ x num_labels]`
# matrix per item). --enable-mis fuses items; otherwise each is scored alone.
score_extraction_token: Optional[str] = None
model: str = DEFAULT_MODEL_NAME
class ScoringResponse(BaseModel):
scores: List[
List[float]
] # List of lists of probabilities, each in the order of label_token_ids
pooled_hidden_states: Optional[List[Optional[List[float]]]] = None
# Pointwise (no `score_extraction_token`): a single [num_rows x num_labels]
# matrix. Setwise: nested [num_items][Nᵢ x num_labels]. The nesting depth
# disambiguates the two shapes.
scores: Union[
List[List[float]], # pointwise: [num_rows x num_labels]
List[List[List[float]]], # setwise: [num_items][Nᵢ x num_labels]
]
# Parallel to `scores`: flat [num_rows x hidden] (pointwise) or one such
# matrix per item (setwise).
pooled_hidden_states: Optional[
Union[
List[Optional[List[float]]], # pointwise: [num_rows x hidden]
List[List[Optional[List[float]]]], # setwise: [num_items][Nᵢ x hidden]
]
] = None
token_logprobs: Optional[List[List[float]]] = None
model: str
usage: Optional[UsageInfo] = None
@@ -67,6 +67,16 @@ class OpenAIServingScore(OpenAIServingBase):
else None
)
# Resolve the extraction token to an id here (this process owns the
# tokenizer); score_request then scans for it and pools the head there.
score_extraction_token_id = None
if request.score_extraction_token is not None:
score_extraction_token_id = (
self.tokenizer_manager._resolve_score_extraction_token_id(
request.score_extraction_token
)
)
result = await self.tokenizer_manager.score_request(
query=request.query,
items=request.items,
@@ -76,18 +86,23 @@ class OpenAIServingScore(OpenAIServingBase):
embed_override_token_id=request.embed_override_token_id,
query_embed_overrides=query_embed_overrides,
item_embed_overrides=item_embed_overrides,
score_extraction_token_id=score_extraction_token_id,
request=raw_request,
return_pooled_hidden_states=request.return_pooled_hidden_states,
temperature=request.temperature,
return_token_logprobs=request.return_token_logprobs,
)
phs_as_lists = None
if result.pooled_hidden_states is not None:
phs_as_lists = [
t.tolist() if t is not None else None
for t in result.pooled_hidden_states
]
# pooled_hidden_states is flat (pointwise / single-set setwise) or
# nested per item (multi-item setwise); convert tensors at any depth.
def _tensors_to_lists(value):
if value is None:
return None
if isinstance(value, list):
return [_tensors_to_lists(v) for v in value]
return value.tolist()
phs_as_lists = _tensors_to_lists(result.pooled_hidden_states)
response = ScoringResponse(
scores=result.scores,
@@ -988,8 +988,12 @@ class OutputLogprobProcessor:
# Handle token_ids logprobs if requested
if needs_token_ids_logprobs:
# no_copy_to_cpu avoids the .item()/.tolist() CUDA sync that regresses
# prefill-only Score throughput at high concurrency.
(
result.token_ids_logprobs_val,
result.token_ids_logprobs_idx,
) = get_token_ids_logprobs_batch_optimized(logprobs, token_ids_logprobs)
) = get_token_ids_logprobs(
logprobs, token_ids_logprobs, no_copy_to_cpu=True
)
return result
+68 -1
View File
@@ -35,7 +35,8 @@ class EmbeddingPoolerOutput:
pooled_hidden_states: Raw transformer hidden states *before* the
task-specific head, present only when
``forward_batch.return_pooled_hidden_states`` is True. Tensor
(standard path) or list of tensors (MIS path, one per delimiter).
(standard path) or list of tensors (one per position — MIS: one per
delimiter; setwise / multi-position readout: one per pooled position).
"""
# Pooler can return list[tensor] instead of tensor if the dimension of each tensor in the batch is different
@@ -114,6 +115,45 @@ def pool_at_delimiter_positions(
return list(data[index_tensor].split(delim_counts))
def pool_at_positions(
data: torch.Tensor,
per_request_indices: List[torch.Tensor],
forward_batch: ForwardBatch,
device: torch.device,
) -> List[torch.Tensor]:
"""Pool a tensor exactly AT the given token positions for every request.
Unlike ``pool_at_delimiter_positions`` (which pools at delimiter - 1), this
pools at the position itself with no offset shift and no discarded entry.
Args:
data: 2-D tensor [total_tokens, dim] — hidden states or logits.
per_request_indices: Per-request anchor positions (CPU tensors), each
relative to the start of that request's sequence.
forward_batch: Forward batch with extend_seq_lens_cpu populated.
device: Device for the index tensor.
Returns:
One tensor per request, shaped [num_anchors, dim].
"""
all_index_tensors: List[torch.Tensor] = []
counts: List[int] = []
offset = 0
for req_idx, req_seq_len in enumerate(forward_batch.extend_seq_lens_cpu):
indices_tensor = per_request_indices[req_idx]
n = len(indices_tensor)
if n > 0:
all_index_tensors.append(indices_tensor + offset)
counts.append(n)
offset += req_seq_len
if all_index_tensors:
index_tensor = torch.cat(all_index_tensors).to(device, non_blocking=True)
else:
index_tensor = torch.empty(0, dtype=torch.long, device=device)
return list(data[index_tensor].split(counts))
def score_and_pool(
score_head: nn.Module,
pooler: Pooler,
@@ -123,6 +163,10 @@ def score_and_pool(
) -> EmbeddingPoolerOutput:
"""Apply a classification/score head with MIS and pooled-hidden-states support.
Multi-position readout path (``token_indices_to_pool`` on forward_batch, e.g.
setwise scoring): pool the head AT each caller-specified position (no offset,
no discarded row), then split per-request.
MIS path (pre-computed delimiter indices on forward_batch): extract hidden
states at positions just before each delimiter, apply the score head, then
split per-request.
@@ -132,6 +176,29 @@ def score_and_pool(
When ``forward_batch.return_pooled_hidden_states`` is True, the raw pooled
hidden states (before the score head) are included in the output.
"""
if (
forward_batch.token_indices_to_pool is not None
and forward_batch.is_prefill_only
):
# Setwise readout: pool the head AT each requested position (no -1 shift,
# no discarded boundary row). Concat to call the head once, then split.
per_request_phs = pool_at_positions(
hidden_states,
forward_batch.token_indices_to_pool,
forward_batch,
input_ids.device,
)
phs_flat = torch.cat(per_request_phs, dim=0)
scores_flat = score_head(phs_flat)
pos_counts = [t.shape[0] for t in per_request_phs]
per_request_scores = list(scores_flat.split(pos_counts))
return EmbeddingPoolerOutput(
embeddings=per_request_scores,
pooled_hidden_states=(
per_request_phs if forward_batch.return_pooled_hidden_states else None
),
)
if (
forward_batch.multi_item_delimiter_indices is not None
and forward_batch.is_prefill_only
+18 -1
View File
@@ -1233,6 +1233,12 @@ class EmbeddingReqInput:
# Batch-level: List[List[int]] (one per request). After __getitem__: List[int].
multi_item_delimiter_indices: Optional[Union[List[List[int]], List[int]]] = None
# Token positions the pooler reads out AT (head / hidden states), instead of
# the default LAST/CLS pooling. Generic multi-position primitive (setwise
# scoring today). Unlike MIS, pooling is AT the position (no delimiter - 1).
# Batch-level: List[List[int]] (one per request). After __getitem__: List[int].
token_indices_to_pool: Optional[Union[List[List[int]], List[int]]] = None
def regenerate_rid(self):
"""Generate a new request ID and return it."""
if isinstance(self.rid, list):
@@ -1359,6 +1365,11 @@ class EmbeddingReqInput:
if self.multi_item_delimiter_indices is not None
else None
),
token_indices_to_pool=(
self.token_indices_to_pool[i]
if self.token_indices_to_pool is not None
else None
),
)
else:
sub = EmbeddingReqInput(
@@ -1390,6 +1401,11 @@ class EmbeddingReqInput:
if self.multi_item_delimiter_indices is not None
else None
),
token_indices_to_pool=(
self.token_indices_to_pool[i]
if self.token_indices_to_pool is not None
else None
),
)
cache[i] = sub
return sub
@@ -1419,7 +1435,8 @@ class TokenizedEmbeddingReqInput(BaseReq, kw_only=True):
return_pooled_hidden_states: bool = False
# Pre-computed delimiter indices for multi-item scoring
multi_item_delimiter_indices: Optional[List[int]] = None
# Token positions for setwise pooling readout
token_indices_to_pool: Optional[List[int]] = None
# For observability
# Pickled Optional[Union[APIServerReqTimeStats, DPControllerReqTimeStats]]
time_stats: Optional[PickleWrapper] = None
@@ -1025,6 +1025,7 @@ class Req(ReqDllmMixin):
] = None,
return_pooled_hidden_states: bool = False,
multi_item_delimiter_indices: Optional[List[int]] = None,
token_indices_to_pool: Optional[List[int]] = None,
session_id: Optional[str] = None,
cache_salt: Optional[str] = None,
):
@@ -1056,6 +1057,7 @@ class Req(ReqDllmMixin):
self.input_embeds = input_embeds
self.positional_embed_overrides = positional_embed_overrides
self.multi_item_delimiter_indices = multi_item_delimiter_indices
self.token_indices_to_pool = token_indices_to_pool
# For req-level memory management
self.kv = ReqKvInfo()
+10
View File
@@ -3431,6 +3431,7 @@ class Scheduler(
time_stats=recv_req.time_stats,
return_pooled_hidden_states=recv_req.return_pooled_hidden_states,
multi_item_delimiter_indices=recv_req.multi_item_delimiter_indices,
token_indices_to_pool=recv_req.token_indices_to_pool,
)
req.tokenizer = self.tokenizer
self._maybe_namespace_elastic_radix_cache(req)
@@ -3934,6 +3935,15 @@ class Scheduler(
if self.enable_lora and not self.can_schedule_lora_req(req, running_loras):
continue
# A forward batch runs one pooling mode, so setwise readout requests
# (token_indices_to_pool) cannot share a batch with last-token ones.
# Admit only the first request's mode; the other stays queued.
if adder.can_run_list and (
(req.token_indices_to_pool is not None)
!= (adder.can_run_list[0].token_indices_to_pool is not None)
):
continue
running_bs = len(running_batch.reqs)
candidate_beam_width = (
req.beam_group.beam_width if req.beam_group is not None else None
@@ -1561,6 +1561,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
http_worker_ipc=obj.http_worker_ipc,
return_pooled_hidden_states=obj.return_pooled_hidden_states,
multi_item_delimiter_indices=obj.multi_item_delimiter_indices,
token_indices_to_pool=obj.token_indices_to_pool,
)
tokenized_obj.time_stats = self.rid_to_state[obj.rid].time_stats
@@ -1,29 +1,41 @@
import logging
import math
from dataclasses import dataclass
from typing import Any, Dict, List, Optional, Tuple, Union
from typing import Any, Dict, List, Optional, Tuple, TypeAlias, Union
import torch
from sglang.srt.configs.model_config import is_cross_encoding_pooler_model
from sglang.srt.configs.model_config import (
is_cross_encoding_pooler_model,
is_score_and_pool_model,
)
from sglang.srt.constants import MIS_DELIMITER_TOKEN_ID
from sglang.srt.managers.embed_types import PositionalEmbeds
from sglang.srt.managers.io_struct import EmbeddingReqInput, GenerateReqInput
from sglang.srt.runtime_context import get_exec
from sglang.srt.runtime_context import get_exec, get_memory, get_schedule, get_serving
logger = logging.getLogger(__name__)
ScoreRow: TypeAlias = List[float]
ScoreMatrix: TypeAlias = List[ScoreRow]
# Pointwise: [num_items, num_labels]. Setwise (multi-position): one
# [N_i, num_labels] matrix per item.
ScoreOutput: TypeAlias = Union[ScoreMatrix, List[ScoreMatrix]]
PooledHiddenStateRows: TypeAlias = List[Optional[torch.Tensor]]
PooledHiddenStateOutput: TypeAlias = Union[
PooledHiddenStateRows, List[Optional[List[torch.Tensor]]]
]
@dataclass(frozen=True, slots=True)
class ScoreResult:
scores: List[List[float]]
scores: ScoreOutput
prompt_tokens: int = 0
# Per-item pooled hidden states (pre-head transformer output).
# CPU tensors when return_pooled_hidden_states=True; kept as tensors so
# in-process consumers (gRPC, engine API) avoid a .tolist() round-trip.
# The HTTP path converts to lists in serving_score.py before JSON serialization.
# Same layout as scores: one tensor per item (not a single packed 2D tensor).
pooled_hidden_states: Optional[List[Optional[torch.Tensor]]] = None
# Pre-head CPU tensors parallel to scores; kept as tensors so in-process
# consumers avoid a .tolist() round-trip (the HTTP path converts before JSON).
pooled_hidden_states: Optional[PooledHiddenStateOutput] = None
# uncalibrated full-vocabulary logprobs, in each item's candidate order
token_logprobs: Optional[List[List[float]]] = None
@@ -197,7 +209,7 @@ class TokenizerManagerScoreMixin:
if return_pooled_hidden_states:
raw_phs = single_result.get("pooled_hidden_state")
if raw_phs is not None and len(raw_phs) == expected_count:
phs_list = raw_phs[1:]
phs_list = list(raw_phs[1:])
return ScoreResult(
scores=scores,
@@ -206,21 +218,110 @@ class TokenizerManagerScoreMixin:
token_logprobs=token_logprobs,
)
# ------------------------------------------------------------------
# Multi-position pooling result helpers (setwise scoring): the head is read
# at every score-extraction token instead of the last token. Shared by the
# batched and fused (--enable-mis) result processors.
# ------------------------------------------------------------------
def _multi_position_score_rows(
self, embedding: Any, apply_softmax: bool
) -> List[List[float]]:
"""Validate a 2-D multi-position result embedding and return its per-token rows."""
embedding_tensor = torch.as_tensor(embedding)
if embedding_tensor.ndim != 2:
# A 1-D result means the model pooled a single vector (reward/embedding
# head) instead of per-position scores; ValueError -> clean 400.
raise ValueError(
"Multi-position scoring expected a 2-D "
"[num_positions, num_labels] result, but got shape "
f"{tuple(embedding_tensor.shape)}. This model may not support "
"score_extraction_token_id (only SequenceClassification heads that "
"pool per position do)."
)
if apply_softmax:
return torch.softmax(embedding_tensor, dim=-1).tolist()
return embedding if isinstance(embedding, list) else embedding_tensor.tolist()
def _multi_position_phs_matrix(self, phs: Any, expected_rows: int) -> torch.Tensor:
"""Validate multi-position pooled hidden states are 2-D with one row per position."""
phs_tensor = torch.as_tensor(phs)
if phs_tensor.ndim != 2 or phs_tensor.shape[0] != expected_rows:
raise ValueError(
"Multi-position pooled hidden states must be a 2-D tensor with "
"one row per score position."
)
return phs_tensor
def _process_multi_item_extraction_results(
self,
results: Any,
per_item_anchor_counts: List[int],
apply_softmax: bool,
return_pooled_hidden_states: bool = False,
) -> ScoreResult:
"""Process a fused multi-item score-extraction request (``--enable-mis``).
The fused sequence returns one ``[ΣNᵢ, num_labels]`` matrix of every
anchor's logits in item order; split it back into one ``[Nᵢ, num_labels]``
matrix per item using per_item_anchor_counts.
"""
single_result = results[0] if isinstance(results, list) else results
meta_info = single_result.get("meta_info", {})
request_id = meta_info.get("id", "<unknown>")
prompt_tokens = meta_info.get("prompt_tokens", 0)
embedding = single_result.get("embedding")
if embedding is None:
raise ValueError("Embedding not found in the result.")
rows = self._multi_position_score_rows(embedding, apply_softmax)
total_anchors = sum(per_item_anchor_counts)
if len(rows) != total_anchors:
raise RuntimeError(
f"Expected {total_anchors} anchor rows across "
f"{len(per_item_anchor_counts)} items, but got {len(rows)}. "
f"Request ID: {request_id}"
)
scores = []
offset = 0
for count in per_item_anchor_counts:
scores.append(rows[offset : offset + count])
offset += count
phs_grouped = None
if return_pooled_hidden_states:
raw_phs = single_result.get("pooled_hidden_state")
if raw_phs is not None:
phs_matrix = self._multi_position_phs_matrix(raw_phs, total_anchors)
phs_grouped = [
list(group) for group in phs_matrix.split(per_item_anchor_counts)
]
return ScoreResult(
scores=scores,
prompt_tokens=prompt_tokens,
pooled_hidden_states=phs_grouped,
)
def _process_single_item_scoring_results(
self,
results: Any,
label_token_ids: Optional[List[List[int]]],
apply_softmax: bool,
return_pooled_hidden_states: bool = False,
per_item_matrix: bool = False,
temperature: float = 1.0,
return_token_logprobs: bool = False,
) -> ScoreResult:
"""
Process results from single-item scoring request.
For generation (CausalLM) models: reads output_token_ids_logprobs.
For non-generation (SequenceClassification) models: reads the embedding field
which contains pooled class logits from the classification head.
CausalLM models read output_token_ids_logprobs; SequenceClassification
models read the embedding field (pooled class logits from the head).
When per_item_matrix (setwise), each result's 2-D matrix becomes one
item's entry so ``scores`` is nested; otherwise each result is one row.
"""
scores = []
phs_list = []
@@ -259,24 +360,29 @@ class TokenizerManagerScoreMixin:
prompt_tokens += result.get("meta_info", {}).get("prompt_tokens", 0)
if apply_softmax:
scores_tensor = torch.as_tensor(embedding, dtype=torch.float64)
embedding = torch.softmax(
(scores_tensor - scores_tensor.max()) / temperature, dim=-1
).tolist()
# The classification head produces per-token logits, which the pooler reduces
# into a single vector per input. That vector is returned in the `.embeddings`
# field — not as semantic embeddings, but as pooled classification logits.
# The field name is reused for compatibility with the existing
# EmbeddingPoolerOutput API.
scores.append(embedding)
if per_item_matrix:
rows = self._multi_position_score_rows(embedding, apply_softmax)
scores.append(rows)
else:
if apply_softmax:
scores_tensor = torch.as_tensor(embedding, dtype=torch.float64)
embedding = torch.softmax(
(scores_tensor - scores_tensor.max()) / temperature, dim=-1
).tolist()
# Pooled classification logits, reusing the EmbeddingPoolerOutput field.
scores.append(embedding)
if return_pooled_hidden_states:
phs = result.get("pooled_hidden_state")
phs_list.append(phs)
if phs is not None:
if per_item_matrix and phs is not None:
phs_list.append(
list(self._multi_position_phs_matrix(phs, len(rows)))
)
has_phs = True
else:
phs_list.append(phs)
if phs is not None:
has_phs = True
return ScoreResult(
scores=scores,
@@ -458,6 +564,190 @@ class TokenizerManagerScoreMixin:
None,
)
# ------------------------------------------------------------------
# Multi-position pooling readout (setwise scoring)
# ------------------------------------------------------------------
def _resolve_score_extraction_token_id(self, token: str) -> int:
"""Resolve the score-extraction token string to a dedicated token id.
Runs in the tokenizer-manager process; used by the HTTP score handler to
turn a client token string into the id ``score_request`` scans for.
"""
if self.tokenizer is None:
raise ValueError(
"A tokenizer is required to resolve score_extraction_token."
)
token_id = self.tokenizer.convert_tokens_to_ids(token)
unk_id = self.tokenizer.unk_token_id
if token_id is None or (unk_id is not None and token_id == unk_id):
raise ValueError(
f"score_extraction_token {token!r} did not resolve to a dedicated "
f"token id."
)
return token_id
def _validate_score_extraction(
self,
is_generation: bool,
item_first: bool,
) -> None:
"""Validate that multi-position pooling readout is applicable.
Supported only by SequenceClassification models whose forward routes the
head through ``score_and_pool`` (per-position pooling, in
``is_score_and_pool_model``). Generation, cross-encoder, reward, and
embedding models pool a single vector and are rejected here before
inference. Requires radix cache and chunked prefill off and no auto-truncate,
since pooling positions are full-prompt coordinates.
"""
if is_generation:
raise ValueError(
"score_extraction_token_id is only supported for "
"SequenceClassification models, not generation (CausalLM) models."
)
architectures = self.model_config.hf_config.architectures or []
if not is_score_and_pool_model(architectures):
raise ValueError(
"score_extraction_token_id is only supported for "
"SequenceClassification models that pool the head per position "
"(score_and_pool); model architecture(s) "
f"{architectures} do not (cross-encoder, reward, and embedding "
"models pool a single vector and ignore the readout positions)."
)
if item_first:
raise ValueError(
"item_first is not supported with score_extraction_token_id."
)
if not get_memory().disable_radix_cache:
raise ValueError(
"score_extraction_token_id requires --disable-radix-cache because "
"pooling positions are relative to the full prompt."
)
if get_schedule().chunked_prefill_size != -1:
raise ValueError(
"score_extraction_token_id requires --chunked-prefill-size -1 "
"because all pooling positions must be processed in one prefill."
)
if get_serving().allow_auto_truncate:
raise ValueError(
"score_extraction_token_id is not supported with "
"--allow-auto-truncate: pooling positions are computed from the "
"full prompt, and truncating the sequence would leave them "
"pointing past the truncated length."
)
def _reject_query_prefix_anchor(
self,
query_token_ids: List[int],
score_extraction_token_id: int,
) -> None:
"""Reject a shared query prefix that itself contains the extraction token.
The non-MIS path scans the whole ``query + item`` sequence, so a query-side
hit would emit an extra, misaligned score row instead of one per candidate.
"""
if score_extraction_token_id in query_token_ids:
raise ValueError(
"score-extraction token found in the shared query prefix; it must "
"appear only within items, one per candidate."
)
def _build_score_extraction_text_inputs(
self,
query: str,
items: Union[str, List[str]],
score_extraction_token_id: int,
) -> List[List[int]]:
"""Tokenize each complete query-item prompt once for extraction scanning."""
items_list = [items] if isinstance(items, str) else items
if not all(isinstance(item, str) for item in items_list):
raise ValueError(
"query and items must both be text or both be token IDs when "
"score_extraction_token_id is set."
)
if query:
self._reject_query_prefix_anchor(
self.tokenizer.encode(query), score_extraction_token_id
)
return [self.tokenizer.encode(f"{query}{item}") for item in items_list]
def _resolve_score_extraction_indices(
self,
input_ids: List[List[int]],
score_extraction_token_id: int,
) -> List[List[int]]:
"""Scan each built sequence for the score-extraction token.
The head is later pooled AT these positions (via ``token_indices_to_pool``)
instead of the last token. Raises ValueError if any sequence has no token,
which would silently drop that sequence's scores.
"""
token_indices_to_pool = [
[i for i, t in enumerate(seq) if t == score_extraction_token_id]
for seq in input_ids
]
if any(not seq_indices for seq_indices in token_indices_to_pool):
raise ValueError(
f"No score-extraction token (id={score_extraction_token_id}) "
"found in one or more input(s); the client must render one per "
"candidate."
)
return token_indices_to_pool
def _anchor_counts_per_item(
self,
delimiter_indices: List[int],
anchor_positions: List[int],
) -> List[int]:
"""Count extraction-token anchors falling inside each fused item's block.
``--enable-mis`` fuses items as ``query <delim> item0 <delim> item1 …``, so
item ``i`` spans ``(delimiter_indices[i], delimiter_indices[i + 1])``.
Raises ValueError (clean 400 before inference) if an item has no anchor or
an anchor falls outside every item block.
"""
counts = [
sum(1 for p in anchor_positions if lo < p < hi)
for lo, hi in zip(delimiter_indices, delimiter_indices[1:])
]
# An anchor in the shared query prefix is not attributable to any item and
# would leave the pooled forward with more rows than sum(counts); reject it.
if sum(counts) != len(anchor_positions):
raise ValueError(
"score-extraction token found outside the candidate items (e.g. in "
"the shared query prefix) under --enable-mis multi-item scoring; it "
"must appear only within items, one per candidate."
)
if any(c == 0 for c in counts):
raise ValueError(
"Each item (candidate set) must contain at least one "
"score-extraction token under --enable-mis multi-item scoring."
)
return counts
def _resolve_multi_position_pooling(
self,
input_ids: List[List[int]],
score_extraction_token_id: int,
use_multi_item_scoring: bool,
delimiter_indices: Optional[List[int]],
) -> Tuple[List[List[int]], Optional[List[int]]]:
"""Resolve where the head is pooled for a score-extraction request.
Returns ``(token_indices_to_pool, per_item_anchor_counts)``; the latter is
None outside ``--enable-mis`` and otherwise buckets anchors back per item.
"""
token_indices_to_pool = self._resolve_score_extraction_indices(
input_ids, score_extraction_token_id
)
per_item_anchor_counts = None
if use_multi_item_scoring:
per_item_anchor_counts = self._anchor_counts_per_item(
delimiter_indices, token_indices_to_pool[0]
)
return token_indices_to_pool, per_item_anchor_counts
# ------------------------------------------------------------------
# Main entry point
# ------------------------------------------------------------------
@@ -474,6 +764,7 @@ class TokenizerManagerScoreMixin:
item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None,
request: Optional[Any] = None,
return_pooled_hidden_states: bool = False,
score_extraction_token_id: Optional[int] = None,
temperature: float = 1.0,
return_token_logprobs: bool = False,
) -> ScoreResult:
@@ -495,6 +786,13 @@ class TokenizerManagerScoreMixin:
- Generation (CausalLM): Requires label_token_ids; returns logprob-based scores.
- SequenceClassification: label_token_ids is optional; returns pooled class logits.
Setwise scoring (SequenceClassification-only) is expressed via
score_extraction_token_id: when set, the head is pooled AT every occurrence
of this token in each ``query + item`` sequence instead of the last token,
and ``scores`` is returned nested (one ``[Nᵢ x num_labels]`` matrix per
item). With ``--enable-mis`` the items are fused into one multi-item
sequence; otherwise each item is scored independently in one batch.
return_pooled_hidden_states is only supported for non-generation models
(SequenceClassification, RewardModel); raises ValueError for CausalLM.
"""
@@ -548,6 +846,12 @@ class TokenizerManagerScoreMixin:
f"Token ID {token_id} is out of vocabulary (vocab size: {vocab_size})"
)
# Normalize an omitted query by item type so path selection sees a concrete
# prefix: [] for pre-tokenized items (token-ID path), "" for text items.
if query is None:
items_are_token_ids = isinstance(items, list) and isinstance(items[0], list)
query = [] if items_are_token_ids else ""
has_embeds = (
query_embed_overrides is not None or item_embed_overrides is not None
)
@@ -566,12 +870,19 @@ class TokenizerManagerScoreMixin:
# Check if multi-item scoring is enabled
use_multi_item_scoring = get_exec().features.enable_mis
# Setwise readout: pool the head at every score_extraction_token_id.
use_score_extraction = score_extraction_token_id is not None
if use_score_extraction:
self._validate_score_extraction(is_generation, item_first)
input_ids = None
text_prompts = None
positional_embed_overrides = None
delimiter_indices = None
use_text_prompts = isinstance(query, str) and not has_embeds
use_text_prompts = (
isinstance(query, str) and not has_embeds and not use_score_extraction
)
if use_text_prompts:
# Both query and items are text
@@ -605,6 +916,10 @@ class TokenizerManagerScoreMixin:
):
# Both query and items are token IDs — tokenize text inputs if needed for embed overrides
query_ids, items_ids = query, items
if use_score_extraction and not use_multi_item_scoring:
# Non-MIS concatenates query + item, so a query-prefix anchor would
# emit a misaligned row; reject it (MIS confines anchors per item).
self._reject_query_prefix_anchor(query_ids, score_extraction_token_id)
_, input_ids, positional_embed_overrides, delimiter_indices = (
self._build_token_id_inputs(
query_ids,
@@ -616,9 +931,25 @@ class TokenizerManagerScoreMixin:
item_embed_overrides,
)
)
elif has_embeds:
# Text inputs with embed overrides — need to tokenize first to resolve positions
query_ids, items_ids = self._batch_tokenize_query_and_items(query, items)
elif (
use_score_extraction
and not use_multi_item_scoring
and not has_embeds
and (query is None or isinstance(query, str))
):
# Single-set setwise: text (or omitted) query + text items, one
# sequence per item. --enable-mis instead fuses the items below.
input_ids = self._build_score_extraction_text_inputs(
query or "", items, score_extraction_token_id
)
elif has_embeds or use_score_extraction:
# Tokenize text inputs to token IDs, then build via the token-id path
# (embed overrides need positions; score extraction needs token IDs).
query_ids, items_ids = self._batch_tokenize_query_and_items(
query or "" if use_multi_item_scoring else query, items
)
if use_score_extraction and not use_multi_item_scoring:
self._reject_query_prefix_anchor(query_ids, score_extraction_token_id)
_, input_ids, positional_embed_overrides, delimiter_indices = (
self._build_token_id_inputs(
query_ids,
@@ -635,6 +966,21 @@ class TokenizerManagerScoreMixin:
"Invalid combination of query/items types for score_request."
)
# Setwise readout: scan each sequence for the extraction token (positions
# the pooler reads the head at); under --enable-mis also return per-item
# anchor counts to split the fused score matrix back per item.
token_indices_to_pool = None
per_item_anchor_counts = None
if use_score_extraction:
token_indices_to_pool, per_item_anchor_counts = (
self._resolve_multi_position_pooling(
input_ids,
score_extraction_token_id,
use_multi_item_scoring,
delimiter_indices,
)
)
if return_pooled_hidden_states:
if is_generation:
raise ValueError(
@@ -684,11 +1030,22 @@ class TokenizerManagerScoreMixin:
positional_embed_overrides=positional_embed_overrides,
return_pooled_hidden_states=return_pooled_hidden_states,
multi_item_delimiter_indices=mis_delimiter_indices,
token_indices_to_pool=token_indices_to_pool,
)
results = await self.generate_request(batch_request, request).__anext__()
if use_multi_item_scoring:
if use_multi_item_scoring and use_score_extraction:
# Multi-item setwise: the items were fused into one sequence and the
# head pooled at every anchor. Split the flat [ΣNᵢ, num_labels] result
# back into one matrix per item (nested scores).
return self._process_multi_item_extraction_results(
results,
per_item_anchor_counts,
apply_softmax,
return_pooled_hidden_states,
)
elif use_multi_item_scoring:
# Multi-item scoring: extract scores from input_token_ids_logprobs or embedding
return self._process_multi_item_scoring_results(
results,
@@ -701,14 +1058,18 @@ class TokenizerManagerScoreMixin:
return_token_logprobs,
)
else:
# Single-item scoring: process each result separately
# Single-item scoring: process each result separately. With
# score_extraction_token_id, each item is scored as an independent
# query+item sequence and its 2-D embedding ([positions x num_labels])
# is kept as one score matrix per item (nested scores).
return self._process_single_item_scoring_results(
results,
label_token_ids,
apply_softmax,
return_pooled_hidden_states,
temperature,
return_token_logprobs,
per_item_matrix=use_score_extraction,
temperature=temperature,
return_token_logprobs=return_token_logprobs,
)
def _convert_logprobs_to_scores(
@@ -571,6 +571,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
# Pre-computed delimiter indices for multi-item scoring (CPU tensors, one per request)
multi_item_delimiter_indices: Optional[List[torch.Tensor]] = None
# Setwise pooling readout positions (CPU tensors, one per request)
token_indices_to_pool: Optional[List[torch.Tensor]] = None
# === Borrowed from ScheduleBatch: compound (carry their own device tensors) ===
# Sampling info
sampling_info: SamplingBatchInfo = None
@@ -1158,6 +1161,18 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
for r in batch.reqs
]
# Setwise readout: pool the head AT token_indices_to_pool instead of
# the last token. The scheduler keeps readout batches homogeneous
# (get_new_batch_prefill), so build only when every request carries
# the field and otherwise fall back to standard pooling.
if batch.reqs and all(
r.token_indices_to_pool is not None for r in batch.reqs
):
self.token_indices_to_pool = [
torch.tensor(r.token_indices_to_pool, dtype=torch.int64)
for r in batch.reqs
]
token_type_ids = [
r.token_type_ids for r in batch.reqs if r.token_type_ids is not None
]
@@ -1900,6 +1900,7 @@ class ModelRunner:
and not isinstance(self.prefill_cuda_graph_runner, EagerRunner)
and self.prefill_cuda_graph_runner is not None
and self.prefill_cuda_graph_runner.can_run_graph(forward_batch)
and forward_batch.token_indices_to_pool is None
and _prefill_cuda_graph_allows_context_parallel(
self.prefill_cuda_graph_runner, forward_batch
)
@@ -0,0 +1,401 @@
"""HuggingFace parity tests for setwise (multi-position) scoring.
Setwise scoring pools the SequenceClassification head at every occurrence of a
``score_extraction_token`` (one per candidate) instead of the last token. These
tests verify the *numerical* correctness of that readout against a HuggingFace
reference (run the checkpoint through HF, gather the final hidden state at the
anchor positions, apply the model's own classification head), so a wrong pooling
position or head application changes the numbers, not just the shape.
The engine runs ``float16`` on ``flashinfer`` (its scoring backend of record);
the HF golden is ``float32`` and compared with an fp16-appropriate tolerance.
The anchor token ``<|object_ref_start|>`` is a Qwen3 special token (single
dedicated id). All model/dtype/tolerance knobs are overridable via env.
"""
import asyncio
import os
import unittest
import torch
from transformers import AutoModelForSequenceClassification, AutoTokenizer
from sglang.srt.entrypoints.engine import Engine
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=200, stage="base-b", runner_config="1-gpu-small")
_SEQCLS_MODEL = os.environ.get(
"TEST_CLASSIFICATION_BASE_MODEL",
"tomaarsen/Qwen3-Reranker-0.6B-seq-cls",
)
_ANCHOR_TOKEN = os.environ.get("TEST_SCORE_EXTRACTION_TOKEN", "<|object_ref_start|>")
# float16 on flashinfer (no float32 prefill kernel); HF golden is float32 and the
# tolerance below covers the fp16-vs-fp32 gap. All overridable via env.
_DTYPE = os.environ.get("TEST_SEQCLS_DTYPE", "float16")
_ATOL = float(os.environ.get("TEST_SETWISE_ATOL", "0.2"))
_RTOL = float(os.environ.get("TEST_SETWISE_RTOL", "0.05"))
class TestSetwiseScoringHFParity(CustomTestCase):
"""Setwise engine scores must match an HF head-at-anchor reference."""
@classmethod
def setUpClass(cls):
cls.tokenizer = AutoTokenizer.from_pretrained(_SEQCLS_MODEL)
cls.anchor_id = cls.tokenizer.convert_tokens_to_ids(_ANCHOR_TOKEN)
assert (
cls.anchor_id is not None and cls.anchor_id != cls.tokenizer.unk_token_id
), f"{_ANCHOR_TOKEN!r} did not resolve to a dedicated token id"
cls.engine = Engine(
model_path=_SEQCLS_MODEL,
disable_radix_cache=True,
chunked_prefill_size=-1,
attention_backend="flashinfer",
dtype=_DTYPE,
mem_fraction_static=0.15,
)
@classmethod
def tearDownClass(cls):
if getattr(cls, "engine", None) is not None:
cls.engine.shutdown()
torch.cuda.empty_cache()
# ------------------------------------------------------------------
# HF reference
# ------------------------------------------------------------------
def _hf_setwise_reference(self, prompt: str):
"""Reference: HF hidden state at each anchor -> classification head.
Returns (ref_scores [N, num_labels] as list-of-lists, anchor_positions).
Tokenized exactly like the engine (``tokenizer.encode`` with default
special tokens) so the anchor positions line up.
"""
input_ids = self.tokenizer.encode(prompt)
anchor_positions = [i for i, t in enumerate(input_ids) if t == self.anchor_id]
self.assertGreater(len(anchor_positions), 0, "prompt has no anchor tokens")
model = AutoModelForSequenceClassification.from_pretrained(
_SEQCLS_MODEL, torch_dtype=torch.float32
).eval()
try:
ids = torch.tensor([input_ids], dtype=torch.long)
with torch.no_grad():
outputs = model(input_ids=ids, output_hidden_states=True)
last_hidden = outputs.hidden_states[-1][0] # [seq, hidden]
# The classification head is `score` on Qwen/Llama SeqCls models.
head = getattr(model, "score", None) or getattr(model, "classifier")
anchor_hidden = last_hidden[anchor_positions].to(head.weight.dtype)
ref = head(anchor_hidden) # [N, num_labels]
return ref.tolist(), anchor_positions
finally:
model.cpu()
del model
torch.cuda.empty_cache()
def _hf_anchor_hidden_states(self, prompt: str):
"""Reference: HF final hidden state at each anchor position (pre-head).
Returns (hidden [N, hidden] as list-of-lists, anchor_positions).
"""
input_ids = self.tokenizer.encode(prompt)
anchor_positions = [i for i, t in enumerate(input_ids) if t == self.anchor_id]
self.assertGreater(len(anchor_positions), 0, "prompt has no anchor tokens")
model = AutoModelForSequenceClassification.from_pretrained(
_SEQCLS_MODEL, torch_dtype=torch.float32
).eval()
try:
ids = torch.tensor([input_ids], dtype=torch.long)
with torch.no_grad():
outputs = model(input_ids=ids, output_hidden_states=True)
last_hidden = outputs.hidden_states[-1][0] # [seq, hidden]
return last_hidden[anchor_positions].tolist(), anchor_positions
finally:
model.cpu()
del model
torch.cuda.empty_cache()
def _assert_close(self, ref, sgl, atol=_ATOL, rtol=_RTOL):
self.assertEqual(len(ref), len(sgl), "row count mismatch")
for i, (rrow, srow) in enumerate(zip(ref, sgl)):
self.assertEqual(len(rrow), len(srow), f"row {i} width mismatch")
for j, (r, s) in enumerate(zip(rrow, srow)):
self.assertLessEqual(
abs(r - s),
atol + rtol * abs(r),
f"row {i} label {j}: HF={r:.6f} SGLang={s:.6f}",
)
# ------------------------------------------------------------------
# Tests
# ------------------------------------------------------------------
def _build_prompt(self, n_anchors: int) -> str:
candidates = " ".join(f"Candidate {i}." for i in range(n_anchors))
return f"Rank the candidates. {candidates} Scores:" + (
_ANCHOR_TOKEN * n_anchors
)
def test_setwise_scores_match_hf_reference(self):
"""Engine setwise logits == HF head applied at each anchor position."""
prompt = self._build_prompt(3)
ref, anchor_positions = self._hf_setwise_reference(prompt)
# `scores` is nested per item (one [N x num_labels] matrix per item); a
# single item here yields one matrix with one row per anchor.
sgl = self.engine.score(
query="",
items=[prompt],
apply_softmax=False,
score_extraction_token_id=self.anchor_id,
).scores
self.assertEqual(len(sgl), 1) # one item
item = sgl[0]
self.assertEqual(len(item), len(anchor_positions)) # one row per anchor
self._assert_close(ref, item)
def test_setwise_multiple_items_match_hf(self):
"""Multiple items -> one HF-matching matrix per item (independent, no MIS)."""
prompt0 = self._build_prompt(3)
prompt1 = self._build_prompt(2)
ref0, anchors0 = self._hf_setwise_reference(prompt0)
ref1, anchors1 = self._hf_setwise_reference(prompt1)
sgl = self.engine.score(
query="",
items=[prompt0, prompt1],
apply_softmax=False,
score_extraction_token_id=self.anchor_id,
).scores
self.assertEqual(len(sgl), 2) # one matrix per item
self.assertEqual(len(sgl[0]), len(anchors0))
self.assertEqual(len(sgl[1]), len(anchors1))
self._assert_close(ref0, sgl[0])
self._assert_close(ref1, sgl[1])
def test_setwise_single_anchor_matches_hf(self):
"""A single anchor is the degenerate case and must still match HF."""
prompt = self._build_prompt(1)
ref, _ = self._hf_setwise_reference(prompt)
sgl = self.engine.score(
query="",
items=[prompt],
apply_softmax=False,
score_extraction_token_id=self.anchor_id,
).scores
self.assertEqual(len(sgl), 1) # one item
self.assertEqual(len(sgl[0]), 1) # one anchor row
self._assert_close(ref, sgl[0])
def test_setwise_candidate_ranking_matches_hf(self):
"""Ranking (argsort over label 0) agrees with HF — robust to fp noise."""
prompt = self._build_prompt(5)
ref, _ = self._hf_setwise_reference(prompt)
# Single item -> unwrap its [N x num_labels] matrix.
sgl = self.engine.score(
query="",
items=[prompt],
apply_softmax=False,
score_extraction_token_id=self.anchor_id,
).scores[0]
ref_order = sorted(range(len(ref)), key=lambda i: ref[i][0])
sgl_order = sorted(range(len(sgl)), key=lambda i: sgl[i][0])
self.assertEqual(ref_order, sgl_order)
def test_setwise_pooled_hidden_states_match_hf(self):
"""return_pooled_hidden_states -> per-anchor pre-head vectors == HF."""
prompt = self._build_prompt(3)
ref_hidden, anchor_positions = self._hf_anchor_hidden_states(prompt)
result = self.engine.score(
query="",
items=[prompt],
apply_softmax=False,
score_extraction_token_id=self.anchor_id,
return_pooled_hidden_states=True,
)
phs = result.pooled_hidden_states
self.assertIsNotNone(phs)
# Nested per item, parallel to scores: a single item -> one group of
# pre-head vectors, one per anchor, aligned 1:1 with the score rows.
self.assertEqual(len(phs), 1) # one item
self.assertEqual(len(result.scores), 1)
item_phs = phs[0]
self.assertEqual(len(item_phs), len(anchor_positions))
self.assertEqual(len(item_phs), len(result.scores[0]))
sgl_hidden = [vec.tolist() for vec in item_phs]
self._assert_close(ref_hidden, sgl_hidden)
def test_setwise_tokenized_items_omitted_query_match_hf(self):
"""Token-ID items with the query omitted take the token-ID path (query -> [])."""
prompt = self._build_prompt(3)
ref, anchor_positions = self._hf_setwise_reference(prompt)
token_ids = self.tokenizer.encode(prompt)
sgl = self.engine.score(
items=[token_ids], # query omitted -> normalized to []
apply_softmax=False,
score_extraction_token_id=self.anchor_id,
).scores
self.assertEqual(len(sgl), 1)
self.assertEqual(len(sgl[0]), len(anchor_positions))
self._assert_close(ref, sgl[0])
def test_mixed_setwise_and_pointwise_concurrent_batch(self):
"""Setwise and pointwise requests batched together must both succeed.
Both share the ``/v1/score`` queue; the scheduler partitions prefill
batches by pooling mode, so mixed traffic must not abort the setwise
requests (pre-fix a mixed batch returned HTTP 400 for them).
"""
setwise_prompt = self._build_prompt(3)
async def _run():
coros = []
for _ in range(6):
coros.append(
self.engine.async_score(
query="",
items=[setwise_prompt],
apply_softmax=False,
score_extraction_token_id=self.anchor_id,
)
)
coros.append(
self.engine.async_score(
query="",
items=["Candidate A.", "Candidate B."],
apply_softmax=False,
)
)
return await asyncio.gather(*coros, return_exceptions=True)
results = self.engine.loop.run_until_complete(_run())
errors = [r for r in results if isinstance(r, BaseException)]
self.assertEqual(errors, [], f"mixed batch produced errors: {errors}")
for i, r in enumerate(results):
if i % 2 == 0: # setwise: one item, 3 anchor rows (nested)
self.assertEqual(len(r.scores), 1)
self.assertEqual(len(r.scores[0]), 3)
else: # pointwise: one score row per item (flat)
self.assertEqual(len(r.scores), 2)
class TestSetwiseMultiItemMISScoring(CustomTestCase):
"""Multi-item setwise scoring under ``--enable-mis`` (fused sequence).
The fused path carries the standard MIS delimiter tokens, so it is not
bit-identical to independent HF scoring (inherent to MIS). These tests assert
the nested per-item shape and block-diagonal set isolation rather than HF
parity, which ``TestSetwiseScoringHFParity`` covers for the batched path.
"""
@classmethod
def setUpClass(cls):
cls.tokenizer = AutoTokenizer.from_pretrained(_SEQCLS_MODEL)
cls.anchor_id = cls.tokenizer.convert_tokens_to_ids(_ANCHOR_TOKEN)
cls.engine = Engine(
model_path=_SEQCLS_MODEL,
disable_radix_cache=True,
chunked_prefill_size=-1,
enable_mis=True,
attention_backend="flashinfer",
dtype=_DTYPE,
mem_fraction_static=0.15,
)
@classmethod
def tearDownClass(cls):
if getattr(cls, "engine", None) is not None:
cls.engine.shutdown()
torch.cuda.empty_cache()
@staticmethod
def _set_prompt(n_anchors: int) -> str:
candidates = " ".join(f"Candidate {i}." for i in range(n_anchors))
return f"Rank the candidates. {candidates} Scores:" + (
_ANCHOR_TOKEN * n_anchors
)
# atol=5e-2 is a floor set by fp16, not slack: one ULP on these logits is
# ~0.002-0.004, and fusing set0 with another set changes reduction order and
# stacks several such steps (observed drift ~0.012). 5e-2 clears that while
# still catching real cross-set leakage, which would diverge far more.
def _assert_matrix_close(self, a, b, atol=5e-2):
self.assertEqual(len(a), len(b), "row count mismatch")
for i, (ra, rb) in enumerate(zip(a, b)):
self.assertEqual(len(ra), len(rb), f"row {i} width mismatch")
for x, y in zip(ra, rb):
self.assertLessEqual(abs(x - y), atol, f"{x} vs {y}")
def test_mis_multi_item_nested_shape(self):
"""Fused multi-item -> one [Nᵢ x num_labels] matrix per item."""
scores = self.engine.score(
query="Rank:",
items=[self._set_prompt(3), self._set_prompt(2)],
apply_softmax=False,
score_extraction_token_id=self.anchor_id,
).scores
self.assertEqual(len(scores), 2) # one matrix per item
self.assertEqual(len(scores[0]), 3) # 3 candidates
self.assertEqual(len(scores[1]), 2) # 2 candidates
num_labels = len(scores[0][0])
self.assertTrue(
all(len(row) == num_labels for matrix in scores for row in matrix)
)
def test_mis_set_isolation(self):
"""Block-diagonal mask: a set's scores are unchanged by other sets.
The first set is scored alone and again fused with a second set; the
block-diagonal mask means the first set attends only over the shared query
prefix and itself, so its matrix must be identical either way.
"""
set0 = self._set_prompt(3)
alone = self.engine.score(
query="Rank:",
items=[set0],
apply_softmax=False,
score_extraction_token_id=self.anchor_id,
).scores
fused = self.engine.score(
query="Rank:",
items=[set0, self._set_prompt(2)],
apply_softmax=False,
score_extraction_token_id=self.anchor_id,
).scores
self.assertEqual(len(alone), 1)
self.assertEqual(len(fused), 2)
# set0 is the first item in both requests -> identical matrix.
self._assert_matrix_close(alone[0], fused[0])
def test_mis_rejects_item_without_anchor(self):
"""An item carrying no extraction token is rejected under --enable-mis."""
with self.assertRaisesRegex(ValueError, "at least one"):
self.engine.score(
query="Rank:",
items=[self._set_prompt(2), "no anchors here"],
apply_softmax=False,
score_extraction_token_id=self.anchor_id,
)
if __name__ == "__main__":
unittest.main()
@@ -25,6 +25,7 @@ register_cpu_ci(est_time=10, suite="base-a-test-cpu")
def _make_forward_batch(
extend_seq_lens,
multi_item_delimiter_indices=None,
token_indices_to_pool=None,
return_pooled_hidden_states=False,
is_prefill_only=True,
):
@@ -33,6 +34,7 @@ def _make_forward_batch(
extend_seq_lens=torch.tensor(extend_seq_lens, dtype=torch.long),
extend_seq_lens_cpu=extend_seq_lens,
multi_item_delimiter_indices=multi_item_delimiter_indices,
token_indices_to_pool=token_indices_to_pool,
dimensions=None,
return_pooled_hidden_states=return_pooled_hidden_states,
is_prefill_only=is_prefill_only,
@@ -186,6 +188,85 @@ class TestScoreAndPool(CustomTestCase):
self.assertEqual(len(out.embeddings), 1)
self.assertEqual(out.embeddings[0].shape, (0, self.num_labels))
def test_setwise_returns_per_request_list(self):
"""Anchor positions provided -> one [num_anchors, num_labels] tensor per request."""
# Sequence: query + items + suffix; anchors at positions 8, 10, 12.
input_ids = torch.arange(14)
hidden = torch.randn(len(input_ids), self.hidden_dim)
fb = _make_forward_batch(
extend_seq_lens=[len(input_ids)],
token_indices_to_pool=[torch.tensor([8, 10, 12])],
)
out = score_and_pool(self.score_head, self.pooler, hidden, fb, input_ids)
self.assertIsInstance(out.embeddings, list)
self.assertEqual(len(out.embeddings), 1)
# 3 anchors -> 3 score rows (no query-boundary row, unlike MIS).
self.assertEqual(out.embeddings[0].shape, (3, self.num_labels))
def test_setwise_extracts_at_anchor_positions(self):
"""Setwise pools AT the anchor index (no delimiter-1 shift)."""
input_ids = torch.tensor([0, 1, 2, 3, 4, 5])
hidden = (
torch.arange(len(input_ids))
.unsqueeze(1)
.float()
.expand(-1, self.hidden_dim)
.clone()
)
fb = _make_forward_batch(
extend_seq_lens=[len(input_ids)],
token_indices_to_pool=[torch.tensor([2, 5])],
)
identity_head = nn.Linear(self.hidden_dim, self.hidden_dim, bias=False)
nn.init.eye_(identity_head.weight)
out = score_and_pool(identity_head, self.pooler, hidden, fb, input_ids)
scores = out.embeddings[0]
# Pooled exactly at indices 2 and 5 (not 1 and 4).
torch.testing.assert_close(scores[0], hidden[2])
torch.testing.assert_close(scores[1], hidden[5])
def test_setwise_batched_splits_per_request(self):
"""Two batched setwise requests -> a list of length 2."""
req1 = list(range(10, 17)) # 7 tokens, anchors at 5, 6
req2 = list(range(20, 24)) # 4 tokens, anchor at 3
input_ids = torch.tensor(req1 + req2)
hidden = torch.randn(len(input_ids), self.hidden_dim)
fb = _make_forward_batch(
extend_seq_lens=[len(req1), len(req2)],
token_indices_to_pool=[
torch.tensor([5, 6]),
torch.tensor([3]),
],
)
out = score_and_pool(self.score_head, self.pooler, hidden, fb, input_ids)
self.assertIsInstance(out.embeddings, list)
self.assertEqual(len(out.embeddings), 2)
self.assertEqual(out.embeddings[0].shape, (2, self.num_labels))
self.assertEqual(out.embeddings[1].shape, (1, self.num_labels))
def test_setwise_returns_pooled_hidden_states(self):
"""return_pooled_hidden_states -> per-anchor pre-head hidden states."""
input_ids = torch.arange(6)
hidden = torch.randn(len(input_ids), self.hidden_dim)
fb = _make_forward_batch(
extend_seq_lens=[len(input_ids)],
token_indices_to_pool=[torch.tensor([3, 5])],
return_pooled_hidden_states=True,
)
out = score_and_pool(self.score_head, self.pooler, hidden, fb, input_ids)
self.assertIsNotNone(out.pooled_hidden_states)
self.assertEqual(len(out.pooled_hidden_states), 1)
self.assertEqual(out.pooled_hidden_states[0].shape, (2, self.hidden_dim))
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,536 @@
"""Unit tests for setwise result grouping on
``TokenizerManagerScoreMixin._process_single_item_scoring_results`` and
``_process_multi_item_extraction_results``.
Setwise scoring pools the SequenceClassification head at every occurrence of a
``score_extraction_token_id``, so each item carries a 2-D
``[num_positions x num_labels]`` embedding kept grouped (``per_item_matrix=True``)
as nested ``scores`` ``[num_items][Nᵢ x num_labels]``. These tests cover the
grouping, validation, anchor bucketing, and the serialization round-trip on CPU.
"""
import unittest
from types import SimpleNamespace
import torch
from sglang.srt.managers.tokenizer_manager_score_mixin import (
ScoreResult,
TokenizerManagerScoreMixin,
)
from sglang.srt.runtime_context import get_context
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
# Config that satisfies _validate_score_extraction (dedicated setwise endpoint).
_VALID_SETWISE_CONFIG = dict(
disable_radix_cache=True,
chunked_prefill_size=-1,
allow_auto_truncate=False,
)
class _ClsHarness(TokenizerManagerScoreMixin):
"""Bare SequenceClassification harness for the pure result parser."""
is_generation = False
def __init__(self, architecture="Qwen3ForSequenceClassification"):
self.model_config = SimpleNamespace(
hf_config=SimpleNamespace(architectures=[architecture])
)
class TestSingleItemScoringResults(CustomTestCase):
@staticmethod
def _result(embedding, phs=None, prompt_tokens=7):
r = {
"meta_info": {"id": "rid-1", "prompt_tokens": prompt_tokens},
"embedding": embedding,
}
if phs is not None:
r["pooled_hidden_state"] = phs
return r
def setUp(self):
super().setUp()
self.h = _ClsHarness()
# _validate_score_extraction reads resolved config bags, so publish a
# valid setwise config; validation tests re-publish with one field flipped.
self._saved_server_args = get_context()._server_args
self.addCleanup(self._restore_server_args)
self._publish(**_VALID_SETWISE_CONFIG)
def _restore_server_args(self):
if self._saved_server_args is not None:
get_context().set_server_args(self._saved_server_args)
def _publish(self, **fields):
override = get_context().override_server_args(**fields)
override.install()
self.addCleanup(override.restore)
# ---- default (per-item) CLS behavior -------------------
def test_pointwise_one_row_per_item(self):
results = [
self._result([0.1, 0.2]),
self._result([0.3, 0.4]),
]
res = self.h._process_single_item_scoring_results(
results, label_token_ids=None, apply_softmax=False
)
self.assertIsInstance(res, ScoreResult)
self.assertEqual(res.scores, [[0.1, 0.2], [0.3, 0.4]])
self.assertEqual(res.prompt_tokens, 14)
def test_mis_pooled_hidden_states_are_list_of_item_tensors(self):
# Standard MIS returns one fused result with a row at each delimiter.
# The first row is the query boundary and is discarded; the public
# ScoreResult contract is one CPU tensor per remaining item.
phs = torch.arange(12).reshape(3, 4)
result = self._result([[0.0, 0.0], [1.0, 0.0], [0.0, 1.0]], phs=phs)
res = self.h._process_multi_item_scoring_results(
[result],
items=["item0", "item1"],
label_token_ids=None,
apply_softmax=False,
return_pooled_hidden_states=True,
)
self.assertEqual(res.scores, [[1.0, 0.0], [0.0, 1.0]])
self.assertIsInstance(res.pooled_hidden_states, list)
self.assertEqual(len(res.pooled_hidden_states), 2)
torch.testing.assert_close(res.pooled_hidden_states[0], phs[1])
torch.testing.assert_close(res.pooled_hidden_states[1], phs[2])
# ---- setwise: per_item_matrix=True ----------------------------------
def test_setwise_one_matrix_per_item(self):
# One item (single big candidate block), 3 extraction positions -> one
# [3 x num_labels] matrix, nested under the item.
emb = [[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]]
res = self.h._process_single_item_scoring_results(
[self._result(emb)],
label_token_ids=None,
apply_softmax=False,
per_item_matrix=True,
)
self.assertEqual(res.scores, [emb]) # [num_items][N x num_labels]
self.assertEqual(len(res.scores), 1)
self.assertEqual(len(res.scores[0]), 3)
def test_setwise_apply_softmax_over_labels(self):
emb = [[0.0, 0.0], [2.0, 2.0]]
res = self.h._process_single_item_scoring_results(
[self._result(emb)],
label_token_ids=None,
apply_softmax=True,
per_item_matrix=True,
)
for row in res.scores[0]:
self.assertAlmostEqual(sum(row), 1.0, places=5)
self.assertAlmostEqual(row[0], 0.5, places=5)
def test_setwise_multiple_items_grouped_per_item(self):
# Two candidate sets scored independently -> one matrix per item (nested).
res = self.h._process_single_item_scoring_results(
[self._result([[1.0, 0.0], [0.0, 1.0]]), self._result([[0.5, 0.5]])],
label_token_ids=None,
apply_softmax=False,
per_item_matrix=True,
)
self.assertEqual(res.scores, [[[1.0, 0.0], [0.0, 1.0]], [[0.5, 0.5]]])
def test_setwise_groups_pooled_hidden_states_per_item(self):
emb = [[0.1, 0.2], [0.3, 0.4]]
phs = torch.randn(2, 4) # one pre-head vector per position
res = self.h._process_single_item_scoring_results(
[self._result(emb, phs=phs)],
label_token_ids=None,
apply_softmax=False,
return_pooled_hidden_states=True,
per_item_matrix=True,
)
self.assertIsNotNone(res.pooled_hidden_states)
# One item -> one matrix of 2 vectors.
self.assertEqual(len(res.pooled_hidden_states), 1)
self.assertEqual(len(res.pooled_hidden_states[0]), 2)
for v in res.pooled_hidden_states[0]:
self.assertEqual(tuple(v.shape), (4,))
def test_setwise_no_phs_when_not_requested(self):
emb = [[0.1, 0.2]]
phs = torch.randn(1, 4)
res = self.h._process_single_item_scoring_results(
[self._result(emb, phs=phs)],
label_token_ids=None,
apply_softmax=False,
return_pooled_hidden_states=False,
per_item_matrix=True,
)
self.assertIsNone(res.pooled_hidden_states)
def test_setwise_rejects_non_matrix_scores(self):
with self.assertRaisesRegex(ValueError, "expected a 2-D"):
self.h._process_single_item_scoring_results(
[self._result([0.1, 0.2])],
label_token_ids=None,
apply_softmax=False,
per_item_matrix=True,
)
def test_setwise_rejects_misaligned_pooled_hidden_states(self):
with self.assertRaisesRegex(ValueError, "one row per score position"):
self.h._process_single_item_scoring_results(
[self._result([[0.1], [0.2]], phs=torch.randn(1, 4))],
label_token_ids=None,
apply_softmax=False,
return_pooled_hidden_states=True,
per_item_matrix=True,
)
def test_validation_rejects_radix_cache(self):
self._publish(disable_radix_cache=False, chunked_prefill_size=-1)
with self.assertRaisesRegex(ValueError, "--disable-radix-cache"):
self.h._validate_score_extraction(False, False)
def test_validation_rejects_chunked_prefill(self):
self._publish(disable_radix_cache=True, chunked_prefill_size=2048)
with self.assertRaisesRegex(ValueError, "--chunked-prefill-size -1"):
self.h._validate_score_extraction(False, False)
def test_validation_rejects_cross_encoder(self):
harness = _ClsHarness(architecture="BertForSequenceClassification")
with self.assertRaisesRegex(ValueError, "cross-encoder"):
harness._validate_score_extraction(False, False)
def test_validation_rejects_reward_model(self):
# Reward models (last-token pooling, not score_and_pool) ignore the readout
# positions -> rejected up front, not after a wasted forward.
harness = _ClsHarness(architecture="InternLM2ForRewardModel")
with self.assertRaisesRegex(ValueError, "pool the head per position"):
harness._validate_score_extraction(False, False)
def test_validation_rejects_embedding_model(self):
# A base embedding model likewise does not route through score_and_pool.
harness = _ClsHarness(architecture="Qwen3Model")
with self.assertRaisesRegex(ValueError, "pool the head per position"):
harness._validate_score_extraction(False, False)
def test_validation_rejects_auto_truncate(self):
self._publish(**{**_VALID_SETWISE_CONFIG, "allow_auto_truncate": True})
with self.assertRaisesRegex(ValueError, "--allow-auto-truncate"):
self.h._validate_score_extraction(False, False)
def test_validation_allows_multi_item(self):
# Multiple items (candidate sets) are allowed with or without --enable-mis;
# both return one score matrix per item.
self.h._validate_score_extraction(False, False)
def test_anchor_counts_per_item_buckets_by_delimiter(self):
# Fused: q <d0> item0(2 anchors) <d1> item1(1 anchor) <d2>.
# delimiter_indices point at the delimiter tokens.
delimiter_indices = [3, 8, 12]
anchor_positions = [5, 7, 10] # 5,7 in item0 (3<p<8); 10 in item1 (8<p<12)
counts = self.h._anchor_counts_per_item(delimiter_indices, anchor_positions)
self.assertEqual(counts, [2, 1])
def test_anchor_counts_per_item_rejects_item_without_anchor(self):
delimiter_indices = [3, 8, 12]
anchor_positions = [5, 7] # nothing in item1
with self.assertRaisesRegex(ValueError, "at least one"):
self.h._anchor_counts_per_item(delimiter_indices, anchor_positions)
def test_anchor_counts_per_item_rejects_anchor_outside_items(self):
# An anchor before the first delimiter (shared query prefix) is not
# attributable to any item and must be rejected up front.
delimiter_indices = [3, 8, 12]
anchor_positions = [1, 5, 7, 10] # position 1 is in the query region
with self.assertRaisesRegex(ValueError, "outside the candidate items"):
self.h._anchor_counts_per_item(delimiter_indices, anchor_positions)
def test_multi_item_extraction_groups_scores_per_item(self):
# Fused request returns one flat [ΣNᵢ, num_labels] embedding; split back
# into one matrix per item using the per-item anchor counts.
res = self.h._process_multi_item_extraction_results(
[self._result([[1.0, 0.0], [0.0, 1.0], [0.5, 0.5]])],
per_item_anchor_counts=[2, 1],
apply_softmax=False,
)
self.assertEqual(res.scores, [[[1.0, 0.0], [0.0, 1.0]], [[0.5, 0.5]]])
self.assertEqual(res.prompt_tokens, 7)
def test_multi_item_extraction_softmax_over_labels(self):
res = self.h._process_multi_item_extraction_results(
[self._result([[0.0, 0.0], [2.0, 2.0]])],
per_item_anchor_counts=[1, 1],
apply_softmax=True,
)
for item in res.scores:
for row in item:
self.assertAlmostEqual(sum(row), 1.0, places=5)
def test_multi_item_extraction_groups_pooled_hidden_states(self):
res = self.h._process_multi_item_extraction_results(
[self._result([[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]], phs=torch.randn(3, 4))],
per_item_anchor_counts=[2, 1],
apply_softmax=False,
return_pooled_hidden_states=True,
)
self.assertIsNotNone(res.pooled_hidden_states)
self.assertEqual(len(res.pooled_hidden_states), 2)
self.assertEqual(len(res.pooled_hidden_states[0]), 2)
self.assertEqual(len(res.pooled_hidden_states[1]), 1)
self.assertEqual(tuple(res.pooled_hidden_states[0][0].shape), (4,))
def test_multi_item_extraction_rejects_row_count_mismatch(self):
with self.assertRaisesRegex(RuntimeError, "anchor rows"):
self.h._process_multi_item_extraction_results(
[self._result([[1.0, 0.0], [0.0, 1.0]])],
per_item_anchor_counts=[2, 1],
apply_softmax=False,
)
def test_text_extraction_tokenizes_combined_prompt_once(self):
class _RecordingTokenizer:
def __init__(self):
self.inputs = []
def encode(self, text):
self.inputs.append(text)
return [len(text)]
self.h.tokenizer = _RecordingTokenizer()
# anchor id 999 never appears (encode returns [len(text)]), so the
# query-prefix guard passes; the query is scanned once and each item is
# combined-encoded once.
input_ids = self.h._build_score_extraction_text_inputs("query", ["a", "b"], 999)
self.assertEqual(self.h.tokenizer.inputs, ["query", "querya", "queryb"])
self.assertEqual(input_ids, [[6], [6]])
def test_text_extraction_rejects_extraction_token_in_query(self):
# A query-side anchor would emit an extra, misaligned row. anchor id 5 ==
# len("query"), so encode("query") -> [5] contains it and must reject.
class _LenTokenizer:
def encode(self, text):
return [len(text)]
self.h.tokenizer = _LenTokenizer()
with self.assertRaisesRegex(ValueError, "query prefix"):
self.h._build_score_extraction_text_inputs("query", ["a"], 5)
def test_reject_query_prefix_anchor_rejects_pretokenized_query(self):
# A pre-tokenized query whose ids contain the extraction token is rejected:
# the non-MIS path scans the whole query + item sequence.
with self.assertRaisesRegex(ValueError, "query prefix"):
self.h._reject_query_prefix_anchor([1, 7, 2], score_extraction_token_id=7)
def test_reject_query_prefix_anchor_allows_clean_query(self):
# A query prefix without the extraction token passes.
self.h._reject_query_prefix_anchor([1, 2, 3], score_extraction_token_id=7)
def test_resolve_indices_returns_positions_per_sequence(self):
# anchor id = 7; positions are per-sequence, in order.
indices = self.h._resolve_score_extraction_indices(
[[1, 7, 2, 7], [7, 3]], score_extraction_token_id=7
)
self.assertEqual(indices, [[1, 3], [0]])
def test_resolve_indices_rejects_any_sequence_without_anchor(self):
# Second sequence has no anchor -> must fail (not silently drop its rows),
# even though another sequence does contain the anchor.
with self.assertRaisesRegex(ValueError, "one or more"):
self.h._resolve_score_extraction_indices(
[[1, 7, 2], [3, 4]], score_extraction_token_id=7
)
# ---- _resolve_multi_position_pooling (extraction-token pooling) -------
def test_resolve_multi_position_pooling_batched_returns_no_counts(self):
# No MIS: one sequence per item; anchors resolved per sequence and no
# per-item anchor counts (the batched path splits by result, not counts).
token_indices, counts = self.h._resolve_multi_position_pooling(
[[1, 7, 2, 7], [7, 3]],
score_extraction_token_id=7,
use_multi_item_scoring=False,
delimiter_indices=None,
)
self.assertEqual(token_indices, [[1, 3], [0]])
self.assertIsNone(counts)
def test_resolve_multi_position_pooling_mis_buckets_anchors_by_item(self):
# MIS: items fused into one sequence with delimiters; anchors are scanned
# over the fused sequence and bucketed back to items via the delimiters.
# Fused (D=0, anchor=7): q q <D> 7 9 7 <D> 9 7 <D>
# positions: 0 1 2 3 4 5 6 7 8 9
# delimiters at 2,6,9; anchors at 3,5,8 -> item0 has 2, item1 has 1.
fused = [1, 1, 0, 7, 9, 7, 0, 9, 7, 0]
token_indices, counts = self.h._resolve_multi_position_pooling(
[fused],
score_extraction_token_id=7,
use_multi_item_scoring=True,
delimiter_indices=[2, 6, 9],
)
self.assertEqual(token_indices, [[3, 5, 8]])
self.assertEqual(counts, [2, 1])
def test_resolve_multi_position_pooling_mis_rejects_item_without_anchor(self):
# An item with no anchor inside its fused block is rejected.
# Fused (D=0, anchor=7): q <D> 7 <D> 9 <D> -> item1 (positions 3<p<5) empty.
fused = [1, 0, 7, 0, 9, 0]
with self.assertRaisesRegex(ValueError, "at least one"):
self.h._resolve_multi_position_pooling(
[fused],
score_extraction_token_id=7,
use_multi_item_scoring=True,
delimiter_indices=[1, 3, 5],
)
def test_resolve_multi_position_pooling_mis_rejects_query_side_anchor(self):
# The extraction token also appears in the query prefix -> rejected before
# inference.
# Fused (D=0, anchor=7): q 7 <D> 7 <D> 7 <D> (query anchor at position 1)
# positions: 0 1 2 3 4 5 6
# delimiters at 2,4,6; in-item anchors at 3,5 but the scan also finds 1.
fused = [1, 7, 0, 7, 0, 7, 0]
with self.assertRaisesRegex(ValueError, "outside the candidate items"):
self.h._resolve_multi_position_pooling(
[fused],
score_extraction_token_id=7,
use_multi_item_scoring=True,
delimiter_indices=[2, 4, 6],
)
# ---- _multi_position_score_rows / _multi_position_phs_matrix helpers --
def test_multi_position_score_rows_preserves_list_values_without_softmax(self):
# No softmax returns the original list as-is (no float round-trip).
emb = [[0.1, 0.2], [0.3, 0.4]]
rows = self.h._multi_position_score_rows(emb, apply_softmax=False)
self.assertIs(rows, emb)
def test_multi_position_score_rows_softmax_over_labels(self):
rows = self.h._multi_position_score_rows(
[[0.0, 0.0], [2.0, 2.0]], apply_softmax=True
)
for row in rows:
self.assertAlmostEqual(sum(row), 1.0, places=5)
self.assertAlmostEqual(row[0], 0.5, places=5)
def test_multi_position_score_rows_rejects_non_matrix(self):
with self.assertRaisesRegex(ValueError, "expected a 2-D"):
self.h._multi_position_score_rows([0.1, 0.2], apply_softmax=False)
def test_multi_position_phs_matrix_rejects_row_count_mismatch(self):
with self.assertRaisesRegex(ValueError, "one row per score position"):
self.h._multi_position_phs_matrix(torch.randn(2, 4), expected_rows=3)
def test_multi_position_phs_matrix_rejects_non_matrix(self):
with self.assertRaisesRegex(ValueError, "one row per score position"):
self.h._multi_position_phs_matrix(torch.randn(4), expected_rows=4)
class TestSetwisePooledHiddenStatesRoundTrip(CustomTestCase):
"""Validate per-position pooled hidden states survive scheduler serialization.
The pooler's ``[num_positions, hidden]`` tensor is packed by
``stream_output_embedding`` (stacked when shapes match, list when they differ)
and reconstructed on the receiver by index. These tests exercise that real
transmission path, then confirm the parser groups it as one item's vectors.
"""
@staticmethod
def _run_stream_output_embedding(phs_per_req):
# Lazy import so the heavier scheduler module loads only when this runs.
from sglang.srt.managers.scheduler_components.output_streamer import (
SchedulerOutputStreamer,
)
captured = []
reqs = []
for idx, phs in enumerate(phs_per_req):
reqs.append(
SimpleNamespace(
finished=lambda: True,
rid=f"rid-{idx}",
http_worker_ipc=None,
finished_reason=SimpleNamespace(to_json=lambda: {"type": "stop"}),
embedding=[[0.0, 0.0] for _ in range(phs.shape[0])],
origin_input_ids=[1, 2, 3],
cached_tokens=0,
time_stats=SimpleNamespace(),
retraction_count=0,
pooled_hidden_state=phs,
)
)
fake_self = SimpleNamespace(
get_cached_tokens_details=lambda req: None,
send_to_detokenizer=SimpleNamespace(
send_output=lambda obj: captured.append(obj)
),
)
SchedulerOutputStreamer._stream_output_embedding(fake_self, reqs)
out = captured[0]
# Mirror the receiver-side reconstruction in tokenizer_manager.py: a
# single stacked ``[stacked(N, ...)]`` (len 1, N > 1) is unwrapped back to
# per-request indexing regardless of wire format.
phs = out.pooled_hidden_states
if phs is not None and len(phs) == 1 and len(reqs) > 1:
out.pooled_hidden_states = phs[0]
return out
def test_uniform_anchor_counts_stack_and_reconstruct(self):
# Two requests, each with 2 anchors -> stackable; receiver [i] must recover
# each request's [num_positions, hidden] matrix intact.
phs0 = torch.arange(8).float().reshape(2, 4)
phs1 = (torch.arange(8).float() + 100).reshape(2, 4)
out = self._run_stream_output_embedding([phs0, phs1])
torch.testing.assert_close(torch.as_tensor(out.pooled_hidden_states[0]), phs0)
torch.testing.assert_close(torch.as_tensor(out.pooled_hidden_states[1]), phs1)
def test_ragged_anchor_counts_keep_list(self):
# Different anchor counts cannot be stacked; the packer must fall back to a
# list so per-request indexing stays correct.
phs0 = torch.randn(3, 4)
phs1 = torch.randn(1, 4)
out = self._run_stream_output_embedding([phs0, phs1])
self.assertEqual(torch.as_tensor(out.pooled_hidden_states[0]).shape, (3, 4))
self.assertEqual(torch.as_tensor(out.pooled_hidden_states[1]).shape, (1, 4))
torch.testing.assert_close(torch.as_tensor(out.pooled_hidden_states[0]), phs0)
torch.testing.assert_close(torch.as_tensor(out.pooled_hidden_states[1]), phs1)
def test_full_roundtrip_into_parser_groups_per_item(self):
# pooler 2-D PHS -> stream_output_embedding -> receiver [i] -> parser:
# the ScoreResult must hold the item's vectors grouped as one item.
phs = torch.arange(6).float().reshape(3, 2) # 3 positions, hidden=2
out = self._run_stream_output_embedding([phs])
received = out.pooled_hidden_states[0] # receiver-side indexing
parser = _ClsHarness()
result = {
"meta_info": {"id": "rid-0", "prompt_tokens": 5},
"embedding": [[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]],
"pooled_hidden_state": received,
}
res = parser._process_single_item_scoring_results(
[result],
label_token_ids=None,
apply_softmax=False,
return_pooled_hidden_states=True,
per_item_matrix=True,
)
self.assertIsNotNone(res.pooled_hidden_states)
# One item -> one group of 3 position vectors.
self.assertEqual(len(res.pooled_hidden_states), 1)
self.assertEqual(len(res.pooled_hidden_states[0]), 3)
for i in range(3):
torch.testing.assert_close(res.pooled_hidden_states[0][i], phs[i])
if __name__ == "__main__":
unittest.main()
@@ -1044,6 +1044,7 @@ class TestSchedulerMmTransportBoundary(CustomTestCase):
time_stats=None,
return_pooled_hidden_states=False,
multi_item_delimiter_indices=None,
token_indices_to_pool=None,
mm_inputs=object(),
)