[Score API] Setwise Scoring Support (#38965)

This commit is contained in:
Sundara Raman Ramachandran
2026-09-24 16:50:58 -07:00
committed by GitHub
parent 7c5bdb940c
commit 2f2f9d12f8
18 changed files with 1882 additions and 51 deletions
+14
View File
@@ -2398,6 +2398,20 @@ def is_cross_encoding_pooler_model(model_architectures: List[str]) -> bool:
return any(arch in _cross_encoding_pooler_archs for arch in model_architectures)
# SequenceClassification models whose forward routes the head through
# score_and_pool (per-position pooling); only these support setwise readout
# (token_indices_to_pool). Keep in sync with callers of layers.pooler.score_and_pool.
_score_and_pool_archs = [
"LlamaForSequenceClassification",
"Qwen2ForSequenceClassification",
"Qwen3ForSequenceClassification",
]
def is_score_and_pool_model(model_architectures: List[str]) -> bool:
return any(arch in _score_and_pool_archs for arch in model_architectures)
def yarn_get_mscale(scale: float = 1, mscale: float = 1) -> float:
if scale <= 1:
return 1.0
@@ -37,6 +37,7 @@ class EngineScoreMixin:
query_embed_overrides: Optional[List[torch.Tensor]] = None,
item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None,
return_pooled_hidden_states: bool = False,
score_extraction_token_id: Optional[int] = None,
temperature: float = 1.0,
return_token_logprobs: bool = False,
) -> ScoreResult:
@@ -53,6 +54,11 @@ class EngineScoreMixin:
For SequenceClassification models, returns the pooled class logits directly from
the classification head. label_token_ids is optional and ignored.
Setwise scoring is expressed via ``score_extraction_token_id``: pass a single
item containing the whole candidate block with one extraction token per
candidate, and the head is pooled AT those positions, so ``scores`` becomes
the ``[N x num_labels]`` per-candidate matrix.
Args:
query: The query text or pre-tokenized token IDs.
items: The item text(s) or pre-tokenized token IDs.
@@ -63,6 +69,11 @@ class EngineScoreMixin:
embed_override_token_id: Placeholder token ID used to locate override positions.
query_embed_overrides: Embedding vectors replacing placeholder tokens in query.
item_embed_overrides: Per-item embedding vectors replacing placeholder tokens in items.
score_extraction_token_id: SequenceClassification-only. When set, pool the
head at every occurrence of this token per sequence instead of the last
token; ``scores`` becomes nested — one ``[Ni x num_labels]`` matrix per
item (``len(scores) == len(items)``), where ``Ni`` is the number of
extraction tokens (candidates) in item ``i``.
return_pooled_hidden_states: Whether to include raw pooled transformer
hidden states (before the task head) in the result. Only supported
for non-generation models (SequenceClassification, RewardModel).
@@ -72,8 +83,10 @@ class EngineScoreMixin:
each item's candidates. Only supported for CausalLM, not raw logits.
Returns:
ScoreResult with scores (one list per item), prompt token count, and
optional pooled_hidden_states tensors.
ScoreResult with scores (flat ``[num_items x num_labels]`` for pointwise
scoring, or nested ``[num_items][Ni x num_labels]`` when
score_extraction_token_id is set), prompt token count, and optional
pooled_hidden_states tensors.
"""
return self.loop.run_until_complete(
self.tokenizer_manager.score_request(
@@ -85,6 +98,7 @@ class EngineScoreMixin:
embed_override_token_id=embed_override_token_id,
query_embed_overrides=query_embed_overrides,
item_embed_overrides=item_embed_overrides,
score_extraction_token_id=score_extraction_token_id,
request=None,
return_pooled_hidden_states=return_pooled_hidden_states,
temperature=temperature,
@@ -103,6 +117,7 @@ class EngineScoreMixin:
query_embed_overrides: Optional[List[torch.Tensor]] = None,
item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None,
return_pooled_hidden_states: bool = False,
score_extraction_token_id: Optional[int] = None,
temperature: float = 1.0,
return_token_logprobs: bool = False,
) -> ScoreResult:
@@ -116,6 +131,7 @@ class EngineScoreMixin:
embed_override_token_id=embed_override_token_id,
query_embed_overrides=query_embed_overrides,
item_embed_overrides=item_embed_overrides,
score_extraction_token_id=score_extraction_token_id,
request=None,
return_pooled_hidden_states=return_pooled_hidden_states,
temperature=temperature,
@@ -1406,14 +1406,32 @@ class ScoringRequest(BaseModel):
return_token_logprobs: bool = False
item_first: bool = False
return_pooled_hidden_states: bool = False
# Setwise readout (SequenceClassification-only): when set, the head is pooled
# AT every occurrence of this token in each `query + item` sequence instead of
# the last token, and `scores` is returned nested (one `[Nᵢ x num_labels]`
# matrix per item). --enable-mis fuses items; otherwise each is scored alone.
score_extraction_token: Optional[str] = None
model: str = DEFAULT_MODEL_NAME
class ScoringResponse(BaseModel):
scores: List[
List[float]
] # List of lists of probabilities, each in the order of label_token_ids
pooled_hidden_states: Optional[List[Optional[List[float]]]] = None
# Pointwise (no `score_extraction_token`): a single [num_rows x num_labels]
# matrix. Setwise: nested [num_items][Nᵢ x num_labels]. The nesting depth
# disambiguates the two shapes.
scores: Union[
List[List[float]], # pointwise: [num_rows x num_labels]
List[List[List[float]]], # setwise: [num_items][Nᵢ x num_labels]
]
# Parallel to `scores`: flat [num_rows x hidden] (pointwise) or one such
# matrix per item (setwise).
pooled_hidden_states: Optional[
Union[
List[Optional[List[float]]], # pointwise: [num_rows x hidden]
List[List[Optional[List[float]]]], # setwise: [num_items][Nᵢ x hidden]
]
] = None
token_logprobs: Optional[List[List[float]]] = None
model: str
usage: Optional[UsageInfo] = None
@@ -67,6 +67,16 @@ class OpenAIServingScore(OpenAIServingBase):
else None
)
# Resolve the extraction token to an id here (this process owns the
# tokenizer); score_request then scans for it and pools the head there.
score_extraction_token_id = None
if request.score_extraction_token is not None:
score_extraction_token_id = (
self.tokenizer_manager._resolve_score_extraction_token_id(
request.score_extraction_token
)
)
result = await self.tokenizer_manager.score_request(
query=request.query,
items=request.items,
@@ -76,18 +86,23 @@ class OpenAIServingScore(OpenAIServingBase):
embed_override_token_id=request.embed_override_token_id,
query_embed_overrides=query_embed_overrides,
item_embed_overrides=item_embed_overrides,
score_extraction_token_id=score_extraction_token_id,
request=raw_request,
return_pooled_hidden_states=request.return_pooled_hidden_states,
temperature=request.temperature,
return_token_logprobs=request.return_token_logprobs,
)
phs_as_lists = None
if result.pooled_hidden_states is not None:
phs_as_lists = [
t.tolist() if t is not None else None
for t in result.pooled_hidden_states
]
# pooled_hidden_states is flat (pointwise / single-set setwise) or
# nested per item (multi-item setwise); convert tensors at any depth.
def _tensors_to_lists(value):
if value is None:
return None
if isinstance(value, list):
return [_tensors_to_lists(v) for v in value]
return value.tolist()
phs_as_lists = _tensors_to_lists(result.pooled_hidden_states)
response = ScoringResponse(
scores=result.scores,
@@ -988,8 +988,12 @@ class OutputLogprobProcessor:
# Handle token_ids logprobs if requested
if needs_token_ids_logprobs:
# no_copy_to_cpu avoids the .item()/.tolist() CUDA sync that regresses
# prefill-only Score throughput at high concurrency.
(
result.token_ids_logprobs_val,
result.token_ids_logprobs_idx,
) = get_token_ids_logprobs_batch_optimized(logprobs, token_ids_logprobs)
) = get_token_ids_logprobs(
logprobs, token_ids_logprobs, no_copy_to_cpu=True
)
return result
+68 -1
View File
@@ -35,7 +35,8 @@ class EmbeddingPoolerOutput:
pooled_hidden_states: Raw transformer hidden states *before* the
task-specific head, present only when
``forward_batch.return_pooled_hidden_states`` is True. Tensor
(standard path) or list of tensors (MIS path, one per delimiter).
(standard path) or list of tensors (one per position — MIS: one per
delimiter; setwise / multi-position readout: one per pooled position).
"""
# Pooler can return list[tensor] instead of tensor if the dimension of each tensor in the batch is different
@@ -114,6 +115,45 @@ def pool_at_delimiter_positions(
return list(data[index_tensor].split(delim_counts))
def pool_at_positions(
data: torch.Tensor,
per_request_indices: List[torch.Tensor],
forward_batch: ForwardBatch,
device: torch.device,
) -> List[torch.Tensor]:
"""Pool a tensor exactly AT the given token positions for every request.
Unlike ``pool_at_delimiter_positions`` (which pools at delimiter - 1), this
pools at the position itself with no offset shift and no discarded entry.
Args:
data: 2-D tensor [total_tokens, dim] — hidden states or logits.
per_request_indices: Per-request anchor positions (CPU tensors), each
relative to the start of that request's sequence.
forward_batch: Forward batch with extend_seq_lens_cpu populated.
device: Device for the index tensor.
Returns:
One tensor per request, shaped [num_anchors, dim].
"""
all_index_tensors: List[torch.Tensor] = []
counts: List[int] = []
offset = 0
for req_idx, req_seq_len in enumerate(forward_batch.extend_seq_lens_cpu):
indices_tensor = per_request_indices[req_idx]
n = len(indices_tensor)
if n > 0:
all_index_tensors.append(indices_tensor + offset)
counts.append(n)
offset += req_seq_len
if all_index_tensors:
index_tensor = torch.cat(all_index_tensors).to(device, non_blocking=True)
else:
index_tensor = torch.empty(0, dtype=torch.long, device=device)
return list(data[index_tensor].split(counts))
def score_and_pool(
score_head: nn.Module,
pooler: Pooler,
@@ -123,6 +163,10 @@ def score_and_pool(
) -> EmbeddingPoolerOutput:
"""Apply a classification/score head with MIS and pooled-hidden-states support.
Multi-position readout path (``token_indices_to_pool`` on forward_batch, e.g.
setwise scoring): pool the head AT each caller-specified position (no offset,
no discarded row), then split per-request.
MIS path (pre-computed delimiter indices on forward_batch): extract hidden
states at positions just before each delimiter, apply the score head, then
split per-request.
@@ -132,6 +176,29 @@ def score_and_pool(
When ``forward_batch.return_pooled_hidden_states`` is True, the raw pooled
hidden states (before the score head) are included in the output.
"""
if (
forward_batch.token_indices_to_pool is not None
and forward_batch.is_prefill_only
):
# Setwise readout: pool the head AT each requested position (no -1 shift,
# no discarded boundary row). Concat to call the head once, then split.
per_request_phs = pool_at_positions(
hidden_states,
forward_batch.token_indices_to_pool,
forward_batch,
input_ids.device,
)
phs_flat = torch.cat(per_request_phs, dim=0)
scores_flat = score_head(phs_flat)
pos_counts = [t.shape[0] for t in per_request_phs]
per_request_scores = list(scores_flat.split(pos_counts))
return EmbeddingPoolerOutput(
embeddings=per_request_scores,
pooled_hidden_states=(
per_request_phs if forward_batch.return_pooled_hidden_states else None
),
)
if (
forward_batch.multi_item_delimiter_indices is not None
and forward_batch.is_prefill_only
+18 -1
View File
@@ -1233,6 +1233,12 @@ class EmbeddingReqInput:
# Batch-level: List[List[int]] (one per request). After __getitem__: List[int].
multi_item_delimiter_indices: Optional[Union[List[List[int]], List[int]]] = None
# Token positions the pooler reads out AT (head / hidden states), instead of
# the default LAST/CLS pooling. Generic multi-position primitive (setwise
# scoring today). Unlike MIS, pooling is AT the position (no delimiter - 1).
# Batch-level: List[List[int]] (one per request). After __getitem__: List[int].
token_indices_to_pool: Optional[Union[List[List[int]], List[int]]] = None
def regenerate_rid(self):
"""Generate a new request ID and return it."""
if isinstance(self.rid, list):
@@ -1359,6 +1365,11 @@ class EmbeddingReqInput:
if self.multi_item_delimiter_indices is not None
else None
),
token_indices_to_pool=(
self.token_indices_to_pool[i]
if self.token_indices_to_pool is not None
else None
),
)
else:
sub = EmbeddingReqInput(
@@ -1390,6 +1401,11 @@ class EmbeddingReqInput:
if self.multi_item_delimiter_indices is not None
else None
),
token_indices_to_pool=(
self.token_indices_to_pool[i]
if self.token_indices_to_pool is not None
else None
),
)
cache[i] = sub
return sub
@@ -1419,7 +1435,8 @@ class TokenizedEmbeddingReqInput(BaseReq, kw_only=True):
return_pooled_hidden_states: bool = False
# Pre-computed delimiter indices for multi-item scoring
multi_item_delimiter_indices: Optional[List[int]] = None
# Token positions for setwise pooling readout
token_indices_to_pool: Optional[List[int]] = None
# For observability
# Pickled Optional[Union[APIServerReqTimeStats, DPControllerReqTimeStats]]
time_stats: Optional[PickleWrapper] = None
@@ -1025,6 +1025,7 @@ class Req(ReqDllmMixin):
] = None,
return_pooled_hidden_states: bool = False,
multi_item_delimiter_indices: Optional[List[int]] = None,
token_indices_to_pool: Optional[List[int]] = None,
session_id: Optional[str] = None,
cache_salt: Optional[str] = None,
):
@@ -1056,6 +1057,7 @@ class Req(ReqDllmMixin):
self.input_embeds = input_embeds
self.positional_embed_overrides = positional_embed_overrides
self.multi_item_delimiter_indices = multi_item_delimiter_indices
self.token_indices_to_pool = token_indices_to_pool
# For req-level memory management
self.kv = ReqKvInfo()
+10
View File
@@ -3431,6 +3431,7 @@ class Scheduler(
time_stats=recv_req.time_stats,
return_pooled_hidden_states=recv_req.return_pooled_hidden_states,
multi_item_delimiter_indices=recv_req.multi_item_delimiter_indices,
token_indices_to_pool=recv_req.token_indices_to_pool,
)
req.tokenizer = self.tokenizer
self._maybe_namespace_elastic_radix_cache(req)
@@ -3934,6 +3935,15 @@ class Scheduler(
if self.enable_lora and not self.can_schedule_lora_req(req, running_loras):
continue
# A forward batch runs one pooling mode, so setwise readout requests
# (token_indices_to_pool) cannot share a batch with last-token ones.
# Admit only the first request's mode; the other stays queued.
if adder.can_run_list and (
(req.token_indices_to_pool is not None)
!= (adder.can_run_list[0].token_indices_to_pool is not None)
):
continue
running_bs = len(running_batch.reqs)
candidate_beam_width = (
req.beam_group.beam_width if req.beam_group is not None else None
@@ -1561,6 +1561,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
http_worker_ipc=obj.http_worker_ipc,
return_pooled_hidden_states=obj.return_pooled_hidden_states,
multi_item_delimiter_indices=obj.multi_item_delimiter_indices,
token_indices_to_pool=obj.token_indices_to_pool,
)
tokenized_obj.time_stats = self.rid_to_state[obj.rid].time_stats
@@ -1,29 +1,41 @@
import logging
import math
from dataclasses import dataclass
from typing import Any, Dict, List, Optional, Tuple, Union
from typing import Any, Dict, List, Optional, Tuple, TypeAlias, Union
import torch
from sglang.srt.configs.model_config import is_cross_encoding_pooler_model
from sglang.srt.configs.model_config import (
is_cross_encoding_pooler_model,
is_score_and_pool_model,
)
from sglang.srt.constants import MIS_DELIMITER_TOKEN_ID
from sglang.srt.managers.embed_types import PositionalEmbeds
from sglang.srt.managers.io_struct import EmbeddingReqInput, GenerateReqInput
from sglang.srt.runtime_context import get_exec
from sglang.srt.runtime_context import get_exec, get_memory, get_schedule, get_serving
logger = logging.getLogger(__name__)
ScoreRow: TypeAlias = List[float]
ScoreMatrix: TypeAlias = List[ScoreRow]
# Pointwise: [num_items, num_labels]. Setwise (multi-position): one
# [N_i, num_labels] matrix per item.
ScoreOutput: TypeAlias = Union[ScoreMatrix, List[ScoreMatrix]]
PooledHiddenStateRows: TypeAlias = List[Optional[torch.Tensor]]
PooledHiddenStateOutput: TypeAlias = Union[
PooledHiddenStateRows, List[Optional[List[torch.Tensor]]]
]
@dataclass(frozen=True, slots=True)
class ScoreResult:
scores: List[List[float]]
scores: ScoreOutput
prompt_tokens: int = 0
# Per-item pooled hidden states (pre-head transformer output).
# CPU tensors when return_pooled_hidden_states=True; kept as tensors so
# in-process consumers (gRPC, engine API) avoid a .tolist() round-trip.
# The HTTP path converts to lists in serving_score.py before JSON serialization.
# Same layout as scores: one tensor per item (not a single packed 2D tensor).
pooled_hidden_states: Optional[List[Optional[torch.Tensor]]] = None
# Pre-head CPU tensors parallel to scores; kept as tensors so in-process
# consumers avoid a .tolist() round-trip (the HTTP path converts before JSON).
pooled_hidden_states: Optional[PooledHiddenStateOutput] = None
# uncalibrated full-vocabulary logprobs, in each item's candidate order
token_logprobs: Optional[List[List[float]]] = None
@@ -197,7 +209,7 @@ class TokenizerManagerScoreMixin:
if return_pooled_hidden_states:
raw_phs = single_result.get("pooled_hidden_state")
if raw_phs is not None and len(raw_phs) == expected_count:
phs_list = raw_phs[1:]
phs_list = list(raw_phs[1:])
return ScoreResult(
scores=scores,
@@ -206,21 +218,110 @@ class TokenizerManagerScoreMixin:
token_logprobs=token_logprobs,
)
# ------------------------------------------------------------------
# Multi-position pooling result helpers (setwise scoring): the head is read
# at every score-extraction token instead of the last token. Shared by the
# batched and fused (--enable-mis) result processors.
# ------------------------------------------------------------------
def _multi_position_score_rows(
self, embedding: Any, apply_softmax: bool
) -> List[List[float]]:
"""Validate a 2-D multi-position result embedding and return its per-token rows."""
embedding_tensor = torch.as_tensor(embedding)
if embedding_tensor.ndim != 2:
# A 1-D result means the model pooled a single vector (reward/embedding
# head) instead of per-position scores; ValueError -> clean 400.
raise ValueError(
"Multi-position scoring expected a 2-D "
"[num_positions, num_labels] result, but got shape "
f"{tuple(embedding_tensor.shape)}. This model may not support "
"score_extraction_token_id (only SequenceClassification heads that "
"pool per position do)."
)
if apply_softmax:
return torch.softmax(embedding_tensor, dim=-1).tolist()
return embedding if isinstance(embedding, list) else embedding_tensor.tolist()
def _multi_position_phs_matrix(self, phs: Any, expected_rows: int) -> torch.Tensor:
"""Validate multi-position pooled hidden states are 2-D with one row per position."""
phs_tensor = torch.as_tensor(phs)
if phs_tensor.ndim != 2 or phs_tensor.shape[0] != expected_rows:
raise ValueError(
"Multi-position pooled hidden states must be a 2-D tensor with "
"one row per score position."
)
return phs_tensor
def _process_multi_item_extraction_results(
self,
results: Any,
per_item_anchor_counts: List[int],
apply_softmax: bool,
return_pooled_hidden_states: bool = False,
) -> ScoreResult:
"""Process a fused multi-item score-extraction request (``--enable-mis``).
The fused sequence returns one ``[ΣNᵢ, num_labels]`` matrix of every
anchor's logits in item order; split it back into one ``[Nᵢ, num_labels]``
matrix per item using per_item_anchor_counts.
"""
single_result = results[0] if isinstance(results, list) else results
meta_info = single_result.get("meta_info", {})
request_id = meta_info.get("id", "<unknown>")
prompt_tokens = meta_info.get("prompt_tokens", 0)
embedding = single_result.get("embedding")
if embedding is None:
raise ValueError("Embedding not found in the result.")
rows = self._multi_position_score_rows(embedding, apply_softmax)
total_anchors = sum(per_item_anchor_counts)
if len(rows) != total_anchors:
raise RuntimeError(
f"Expected {total_anchors} anchor rows across "
f"{len(per_item_anchor_counts)} items, but got {len(rows)}. "
f"Request ID: {request_id}"
)
scores = []
offset = 0
for count in per_item_anchor_counts:
scores.append(rows[offset : offset + count])
offset += count
phs_grouped = None
if return_pooled_hidden_states:
raw_phs = single_result.get("pooled_hidden_state")
if raw_phs is not None:
phs_matrix = self._multi_position_phs_matrix(raw_phs, total_anchors)
phs_grouped = [
list(group) for group in phs_matrix.split(per_item_anchor_counts)
]
return ScoreResult(
scores=scores,
prompt_tokens=prompt_tokens,
pooled_hidden_states=phs_grouped,
)
def _process_single_item_scoring_results(
self,
results: Any,
label_token_ids: Optional[List[List[int]]],
apply_softmax: bool,
return_pooled_hidden_states: bool = False,
per_item_matrix: bool = False,
temperature: float = 1.0,
return_token_logprobs: bool = False,
) -> ScoreResult:
"""
Process results from single-item scoring request.
For generation (CausalLM) models: reads output_token_ids_logprobs.
For non-generation (SequenceClassification) models: reads the embedding field
which contains pooled class logits from the classification head.
CausalLM models read output_token_ids_logprobs; SequenceClassification
models read the embedding field (pooled class logits from the head).
When per_item_matrix (setwise), each result's 2-D matrix becomes one
item's entry so ``scores`` is nested; otherwise each result is one row.
"""
scores = []
phs_list = []
@@ -259,24 +360,29 @@ class TokenizerManagerScoreMixin:
prompt_tokens += result.get("meta_info", {}).get("prompt_tokens", 0)
if apply_softmax:
scores_tensor = torch.as_tensor(embedding, dtype=torch.float64)
embedding = torch.softmax(
(scores_tensor - scores_tensor.max()) / temperature, dim=-1
).tolist()
# The classification head produces per-token logits, which the pooler reduces
# into a single vector per input. That vector is returned in the `.embeddings`
# field — not as semantic embeddings, but as pooled classification logits.
# The field name is reused for compatibility with the existing
# EmbeddingPoolerOutput API.
scores.append(embedding)
if per_item_matrix:
rows = self._multi_position_score_rows(embedding, apply_softmax)
scores.append(rows)
else:
if apply_softmax:
scores_tensor = torch.as_tensor(embedding, dtype=torch.float64)
embedding = torch.softmax(
(scores_tensor - scores_tensor.max()) / temperature, dim=-1
).tolist()
# Pooled classification logits, reusing the EmbeddingPoolerOutput field.
scores.append(embedding)
if return_pooled_hidden_states:
phs = result.get("pooled_hidden_state")
phs_list.append(phs)
if phs is not None:
if per_item_matrix and phs is not None:
phs_list.append(
list(self._multi_position_phs_matrix(phs, len(rows)))
)
has_phs = True
else:
phs_list.append(phs)
if phs is not None:
has_phs = True
return ScoreResult(
scores=scores,
@@ -458,6 +564,190 @@ class TokenizerManagerScoreMixin:
None,
)
# ------------------------------------------------------------------
# Multi-position pooling readout (setwise scoring)
# ------------------------------------------------------------------
def _resolve_score_extraction_token_id(self, token: str) -> int:
"""Resolve the score-extraction token string to a dedicated token id.
Runs in the tokenizer-manager process; used by the HTTP score handler to
turn a client token string into the id ``score_request`` scans for.
"""
if self.tokenizer is None:
raise ValueError(
"A tokenizer is required to resolve score_extraction_token."
)
token_id = self.tokenizer.convert_tokens_to_ids(token)
unk_id = self.tokenizer.unk_token_id
if token_id is None or (unk_id is not None and token_id == unk_id):
raise ValueError(
f"score_extraction_token {token!r} did not resolve to a dedicated "
f"token id."
)
return token_id
def _validate_score_extraction(
self,
is_generation: bool,
item_first: bool,
) -> None:
"""Validate that multi-position pooling readout is applicable.
Supported only by SequenceClassification models whose forward routes the
head through ``score_and_pool`` (per-position pooling, in
``is_score_and_pool_model``). Generation, cross-encoder, reward, and
embedding models pool a single vector and are rejected here before
inference. Requires radix cache and chunked prefill off and no auto-truncate,
since pooling positions are full-prompt coordinates.
"""
if is_generation:
raise ValueError(
"score_extraction_token_id is only supported for "
"SequenceClassification models, not generation (CausalLM) models."
)
architectures = self.model_config.hf_config.architectures or []
if not is_score_and_pool_model(architectures):
raise ValueError(
"score_extraction_token_id is only supported for "
"SequenceClassification models that pool the head per position "
"(score_and_pool); model architecture(s) "
f"{architectures} do not (cross-encoder, reward, and embedding "
"models pool a single vector and ignore the readout positions)."
)
if item_first:
raise ValueError(
"item_first is not supported with score_extraction_token_id."
)
if not get_memory().disable_radix_cache:
raise ValueError(
"score_extraction_token_id requires --disable-radix-cache because "
"pooling positions are relative to the full prompt."
)
if get_schedule().chunked_prefill_size != -1:
raise ValueError(
"score_extraction_token_id requires --chunked-prefill-size -1 "
"because all pooling positions must be processed in one prefill."
)
if get_serving().allow_auto_truncate:
raise ValueError(
"score_extraction_token_id is not supported with "
"--allow-auto-truncate: pooling positions are computed from the "
"full prompt, and truncating the sequence would leave them "
"pointing past the truncated length."
)
def _reject_query_prefix_anchor(
self,
query_token_ids: List[int],
score_extraction_token_id: int,
) -> None:
"""Reject a shared query prefix that itself contains the extraction token.
The non-MIS path scans the whole ``query + item`` sequence, so a query-side
hit would emit an extra, misaligned score row instead of one per candidate.
"""
if score_extraction_token_id in query_token_ids:
raise ValueError(
"score-extraction token found in the shared query prefix; it must "
"appear only within items, one per candidate."
)
def _build_score_extraction_text_inputs(
self,
query: str,
items: Union[str, List[str]],
score_extraction_token_id: int,
) -> List[List[int]]:
"""Tokenize each complete query-item prompt once for extraction scanning."""
items_list = [items] if isinstance(items, str) else items
if not all(isinstance(item, str) for item in items_list):
raise ValueError(
"query and items must both be text or both be token IDs when "
"score_extraction_token_id is set."
)
if query:
self._reject_query_prefix_anchor(
self.tokenizer.encode(query), score_extraction_token_id
)
return [self.tokenizer.encode(f"{query}{item}") for item in items_list]
def _resolve_score_extraction_indices(
self,
input_ids: List[List[int]],
score_extraction_token_id: int,
) -> List[List[int]]:
"""Scan each built sequence for the score-extraction token.
The head is later pooled AT these positions (via ``token_indices_to_pool``)
instead of the last token. Raises ValueError if any sequence has no token,
which would silently drop that sequence's scores.
"""
token_indices_to_pool = [
[i for i, t in enumerate(seq) if t == score_extraction_token_id]
for seq in input_ids
]
if any(not seq_indices for seq_indices in token_indices_to_pool):
raise ValueError(
f"No score-extraction token (id={score_extraction_token_id}) "
"found in one or more input(s); the client must render one per "
"candidate."
)
return token_indices_to_pool
def _anchor_counts_per_item(
self,
delimiter_indices: List[int],
anchor_positions: List[int],
) -> List[int]:
"""Count extraction-token anchors falling inside each fused item's block.
``--enable-mis`` fuses items as ``query <delim> item0 <delim> item1 …``, so
item ``i`` spans ``(delimiter_indices[i], delimiter_indices[i + 1])``.
Raises ValueError (clean 400 before inference) if an item has no anchor or
an anchor falls outside every item block.
"""
counts = [
sum(1 for p in anchor_positions if lo < p < hi)
for lo, hi in zip(delimiter_indices, delimiter_indices[1:])
]
# An anchor in the shared query prefix is not attributable to any item and
# would leave the pooled forward with more rows than sum(counts); reject it.
if sum(counts) != len(anchor_positions):
raise ValueError(
"score-extraction token found outside the candidate items (e.g. in "
"the shared query prefix) under --enable-mis multi-item scoring; it "
"must appear only within items, one per candidate."
)
if any(c == 0 for c in counts):
raise ValueError(
"Each item (candidate set) must contain at least one "
"score-extraction token under --enable-mis multi-item scoring."
)
return counts
def _resolve_multi_position_pooling(
self,
input_ids: List[List[int]],
score_extraction_token_id: int,
use_multi_item_scoring: bool,
delimiter_indices: Optional[List[int]],
) -> Tuple[List[List[int]], Optional[List[int]]]:
"""Resolve where the head is pooled for a score-extraction request.
Returns ``(token_indices_to_pool, per_item_anchor_counts)``; the latter is
None outside ``--enable-mis`` and otherwise buckets anchors back per item.
"""
token_indices_to_pool = self._resolve_score_extraction_indices(
input_ids, score_extraction_token_id
)
per_item_anchor_counts = None
if use_multi_item_scoring:
per_item_anchor_counts = self._anchor_counts_per_item(
delimiter_indices, token_indices_to_pool[0]
)
return token_indices_to_pool, per_item_anchor_counts
# ------------------------------------------------------------------
# Main entry point
# ------------------------------------------------------------------
@@ -474,6 +764,7 @@ class TokenizerManagerScoreMixin:
item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None,
request: Optional[Any] = None,
return_pooled_hidden_states: bool = False,
score_extraction_token_id: Optional[int] = None,
temperature: float = 1.0,
return_token_logprobs: bool = False,
) -> ScoreResult:
@@ -495,6 +786,13 @@ class TokenizerManagerScoreMixin:
- Generation (CausalLM): Requires label_token_ids; returns logprob-based scores.
- SequenceClassification: label_token_ids is optional; returns pooled class logits.
Setwise scoring (SequenceClassification-only) is expressed via
score_extraction_token_id: when set, the head is pooled AT every occurrence
of this token in each ``query + item`` sequence instead of the last token,
and ``scores`` is returned nested (one ``[Nᵢ x num_labels]`` matrix per
item). With ``--enable-mis`` the items are fused into one multi-item
sequence; otherwise each item is scored independently in one batch.
return_pooled_hidden_states is only supported for non-generation models
(SequenceClassification, RewardModel); raises ValueError for CausalLM.
"""
@@ -548,6 +846,12 @@ class TokenizerManagerScoreMixin:
f"Token ID {token_id} is out of vocabulary (vocab size: {vocab_size})"
)
# Normalize an omitted query by item type so path selection sees a concrete
# prefix: [] for pre-tokenized items (token-ID path), "" for text items.
if query is None:
items_are_token_ids = isinstance(items, list) and isinstance(items[0], list)
query = [] if items_are_token_ids else ""
has_embeds = (
query_embed_overrides is not None or item_embed_overrides is not None
)
@@ -566,12 +870,19 @@ class TokenizerManagerScoreMixin:
# Check if multi-item scoring is enabled
use_multi_item_scoring = get_exec().features.enable_mis
# Setwise readout: pool the head at every score_extraction_token_id.
use_score_extraction = score_extraction_token_id is not None
if use_score_extraction:
self._validate_score_extraction(is_generation, item_first)
input_ids = None
text_prompts = None
positional_embed_overrides = None
delimiter_indices = None
use_text_prompts = isinstance(query, str) and not has_embeds
use_text_prompts = (
isinstance(query, str) and not has_embeds and not use_score_extraction
)
if use_text_prompts:
# Both query and items are text
@@ -605,6 +916,10 @@ class TokenizerManagerScoreMixin:
):
# Both query and items are token IDs — tokenize text inputs if needed for embed overrides
query_ids, items_ids = query, items
if use_score_extraction and not use_multi_item_scoring:
# Non-MIS concatenates query + item, so a query-prefix anchor would
# emit a misaligned row; reject it (MIS confines anchors per item).
self._reject_query_prefix_anchor(query_ids, score_extraction_token_id)
_, input_ids, positional_embed_overrides, delimiter_indices = (
self._build_token_id_inputs(
query_ids,
@@ -616,9 +931,25 @@ class TokenizerManagerScoreMixin:
item_embed_overrides,
)
)
elif has_embeds:
# Text inputs with embed overrides — need to tokenize first to resolve positions
query_ids, items_ids = self._batch_tokenize_query_and_items(query, items)
elif (
use_score_extraction
and not use_multi_item_scoring
and not has_embeds
and (query is None or isinstance(query, str))
):
# Single-set setwise: text (or omitted) query + text items, one
# sequence per item. --enable-mis instead fuses the items below.
input_ids = self._build_score_extraction_text_inputs(
query or "", items, score_extraction_token_id
)
elif has_embeds or use_score_extraction:
# Tokenize text inputs to token IDs, then build via the token-id path
# (embed overrides need positions; score extraction needs token IDs).
query_ids, items_ids = self._batch_tokenize_query_and_items(
query or "" if use_multi_item_scoring else query, items
)
if use_score_extraction and not use_multi_item_scoring:
self._reject_query_prefix_anchor(query_ids, score_extraction_token_id)
_, input_ids, positional_embed_overrides, delimiter_indices = (
self._build_token_id_inputs(
query_ids,
@@ -635,6 +966,21 @@ class TokenizerManagerScoreMixin:
"Invalid combination of query/items types for score_request."
)
# Setwise readout: scan each sequence for the extraction token (positions
# the pooler reads the head at); under --enable-mis also return per-item
# anchor counts to split the fused score matrix back per item.
token_indices_to_pool = None
per_item_anchor_counts = None
if use_score_extraction:
token_indices_to_pool, per_item_anchor_counts = (
self._resolve_multi_position_pooling(
input_ids,
score_extraction_token_id,
use_multi_item_scoring,
delimiter_indices,
)
)
if return_pooled_hidden_states:
if is_generation:
raise ValueError(
@@ -684,11 +1030,22 @@ class TokenizerManagerScoreMixin:
positional_embed_overrides=positional_embed_overrides,
return_pooled_hidden_states=return_pooled_hidden_states,
multi_item_delimiter_indices=mis_delimiter_indices,
token_indices_to_pool=token_indices_to_pool,
)
results = await self.generate_request(batch_request, request).__anext__()
if use_multi_item_scoring:
if use_multi_item_scoring and use_score_extraction:
# Multi-item setwise: the items were fused into one sequence and the
# head pooled at every anchor. Split the flat [ΣNᵢ, num_labels] result
# back into one matrix per item (nested scores).
return self._process_multi_item_extraction_results(
results,
per_item_anchor_counts,
apply_softmax,
return_pooled_hidden_states,
)
elif use_multi_item_scoring:
# Multi-item scoring: extract scores from input_token_ids_logprobs or embedding
return self._process_multi_item_scoring_results(
results,
@@ -701,14 +1058,18 @@ class TokenizerManagerScoreMixin:
return_token_logprobs,
)
else:
# Single-item scoring: process each result separately
# Single-item scoring: process each result separately. With
# score_extraction_token_id, each item is scored as an independent
# query+item sequence and its 2-D embedding ([positions x num_labels])
# is kept as one score matrix per item (nested scores).
return self._process_single_item_scoring_results(
results,
label_token_ids,
apply_softmax,
return_pooled_hidden_states,
temperature,
return_token_logprobs,
per_item_matrix=use_score_extraction,
temperature=temperature,
return_token_logprobs=return_token_logprobs,
)
def _convert_logprobs_to_scores(
@@ -571,6 +571,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
# Pre-computed delimiter indices for multi-item scoring (CPU tensors, one per request)
multi_item_delimiter_indices: Optional[List[torch.Tensor]] = None
# Setwise pooling readout positions (CPU tensors, one per request)
token_indices_to_pool: Optional[List[torch.Tensor]] = None
# === Borrowed from ScheduleBatch: compound (carry their own device tensors) ===
# Sampling info
sampling_info: SamplingBatchInfo = None
@@ -1158,6 +1161,18 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
for r in batch.reqs
]
# Setwise readout: pool the head AT token_indices_to_pool instead of
# the last token. The scheduler keeps readout batches homogeneous
# (get_new_batch_prefill), so build only when every request carries
# the field and otherwise fall back to standard pooling.
if batch.reqs and all(
r.token_indices_to_pool is not None for r in batch.reqs
):
self.token_indices_to_pool = [
torch.tensor(r.token_indices_to_pool, dtype=torch.int64)
for r in batch.reqs
]
token_type_ids = [
r.token_type_ids for r in batch.reqs if r.token_type_ids is not None
]
@@ -1900,6 +1900,7 @@ class ModelRunner:
and not isinstance(self.prefill_cuda_graph_runner, EagerRunner)
and self.prefill_cuda_graph_runner is not None
and self.prefill_cuda_graph_runner.can_run_graph(forward_batch)
and forward_batch.token_indices_to_pool is None
and _prefill_cuda_graph_allows_context_parallel(
self.prefill_cuda_graph_runner, forward_batch
)