[Score API] Setwise scoring: CausalLM support (batched + --enable-mis) (#41188)
This commit is contained in:
@@ -56,8 +56,10 @@ class EngineScoreMixin:
|
||||
|
||||
Setwise scoring is expressed via ``score_extraction_token_id``: pass a single
|
||||
item containing the whole candidate block with one extraction token per
|
||||
candidate, and the head is pooled AT those positions, so ``scores`` becomes
|
||||
the ``[N x num_labels]`` per-candidate matrix.
|
||||
candidate, and the readout is taken AT those positions, so ``scores``
|
||||
becomes the ``[N x num_labels]`` per-candidate matrix. SequenceClassification
|
||||
pools the head there; CausalLM reads label-token logprobs there (batched
|
||||
path only).
|
||||
|
||||
Args:
|
||||
query: The query text or pre-tokenized token IDs.
|
||||
@@ -69,10 +71,11 @@ class EngineScoreMixin:
|
||||
embed_override_token_id: Placeholder token ID used to locate override positions.
|
||||
query_embed_overrides: Embedding vectors replacing placeholder tokens in query.
|
||||
item_embed_overrides: Per-item embedding vectors replacing placeholder tokens in items.
|
||||
score_extraction_token_id: SequenceClassification-only. When set, pool the
|
||||
head at every occurrence of this token per sequence instead of the last
|
||||
token; ``scores`` becomes nested — one ``[Ni x num_labels]`` matrix per
|
||||
item (``len(scores) == len(items)``), where ``Ni`` is the number of
|
||||
score_extraction_token_id: When set, read the score head (SeqCls) or
|
||||
label-token logprobs (CausalLM) at every occurrence of
|
||||
this token per sequence instead of the last token; ``scores`` becomes
|
||||
nested — one ``[Ni x num_labels]`` matrix per item
|
||||
(``len(scores) == len(items)``), where ``Ni`` is the number of
|
||||
extraction tokens (candidates) in item ``i``.
|
||||
return_pooled_hidden_states: Whether to include raw pooled transformer
|
||||
hidden states (before the task head) in the result. Only supported
|
||||
|
||||
@@ -1408,10 +1408,11 @@ class ScoringRequest(BaseModel):
|
||||
item_first: bool = False
|
||||
return_pooled_hidden_states: bool = False
|
||||
|
||||
# Setwise readout (SequenceClassification-only): when set, the head is pooled
|
||||
# AT every occurrence of this token in each `query + item` sequence instead of
|
||||
# the last token, and `scores` is returned nested (one `[Nᵢ x num_labels]`
|
||||
# matrix per item). --enable-mis fuses items; otherwise each is scored alone.
|
||||
# Setwise readout: when set, the readout is taken AT every occurrence of this
|
||||
# token in each `query + item` sequence instead of the last token, and `scores`
|
||||
# is returned nested (one `[Nᵢ x num_labels]` matrix per item). SeqCls pools the
|
||||
# head there; CausalLM reads label-token logprobs there. Both support batched
|
||||
# and --enable-mis.
|
||||
score_extraction_token: Optional[str] = None
|
||||
|
||||
model: str = DEFAULT_MODEL_NAME
|
||||
|
||||
@@ -510,10 +510,12 @@ class LogitsProcessor(nn.Module):
|
||||
aux_hidden_states: Optional[AuxHiddenStates] = None,
|
||||
hidden_states_before_norm: Optional[torch.Tensor] = None,
|
||||
) -> LogitsProcessorOutput:
|
||||
# Extract MIS indices before ForwardBatch → LogitsMetadata conversion
|
||||
# Extract MIS / setwise indices before ForwardBatch → LogitsMetadata conversion
|
||||
multi_item_delimiter_indices = None
|
||||
token_indices_to_pool = None
|
||||
if isinstance(logits_metadata, ForwardBatch):
|
||||
multi_item_delimiter_indices = logits_metadata.multi_item_delimiter_indices
|
||||
token_indices_to_pool = logits_metadata.token_indices_to_pool
|
||||
logits_metadata = LogitsMetadata.from_forward_batch(logits_metadata)
|
||||
|
||||
# Autotune dummy run discards this output. `is False` not `not`: None
|
||||
@@ -531,6 +533,18 @@ class LogitsProcessor(nn.Module):
|
||||
forward_mode=logits_metadata.forward_mode,
|
||||
)
|
||||
|
||||
# Setwise scoring (CausalLM): read label-token logprobs AT each anchor
|
||||
# position instead of the last token. Takes precedence over the MIS
|
||||
# delimiter path; the two are mutually exclusive for generation.
|
||||
if token_indices_to_pool is not None and logits_metadata.is_prefill_only:
|
||||
return self.compute_logprobs_at_positions(
|
||||
input_ids,
|
||||
hidden_states,
|
||||
lm_head,
|
||||
logits_metadata,
|
||||
token_indices_to_pool,
|
||||
)
|
||||
|
||||
# Multi-item scoring only for prefill-only requests with pre-computed indices.
|
||||
if multi_item_delimiter_indices is not None and logits_metadata.is_prefill_only:
|
||||
return self.compute_logprobs_for_multi_item_scoring(
|
||||
@@ -1263,6 +1277,89 @@ class LogitsProcessor(nn.Module):
|
||||
mm_input_embeds=logits_metadata.mm_input_embeds,
|
||||
)
|
||||
|
||||
def compute_logprobs_at_positions(
|
||||
self,
|
||||
input_ids,
|
||||
hidden_states,
|
||||
lm_head: VocabParallelEmbedding,
|
||||
logits_metadata: Union[LogitsMetadata, ForwardBatch],
|
||||
token_indices_to_pool: List[torch.Tensor],
|
||||
):
|
||||
"""Compute label-token logprobs AT each requested position (setwise, CausalLM).
|
||||
|
||||
Mirrors ``compute_logprobs_for_multi_item_scoring`` but reads the LM head
|
||||
AT ``token_indices_to_pool`` (no delimiter - 1 shift, no discarded row):
|
||||
each anchor's logprobs are P(next token | prefix up to and including the
|
||||
anchor), one row per anchor.
|
||||
"""
|
||||
device = input_ids.device
|
||||
all_tensors = []
|
||||
if logits_metadata.extend_seq_lens_cpu is not None:
|
||||
offset = 0
|
||||
for req_seq_len, indices_tensor in zip(
|
||||
logits_metadata.extend_seq_lens_cpu, token_indices_to_pool
|
||||
):
|
||||
if len(indices_tensor) > 0:
|
||||
all_tensors.append(indices_tensor + offset)
|
||||
offset += req_seq_len
|
||||
else:
|
||||
all_tensors.append(token_indices_to_pool[0])
|
||||
pooled_indices = torch.cat(all_tensors).to(device, non_blocking=True)
|
||||
|
||||
sliced_hidden = hidden_states[pooled_indices]
|
||||
sliced_logits = self._get_logits(sliced_hidden, lm_head, logits_metadata)
|
||||
sliced_logprobs = torch.nn.functional.log_softmax(sliced_logits, dim=-1)
|
||||
|
||||
input_token_ids_logprobs_val = []
|
||||
input_token_ids_logprobs_idx = []
|
||||
input_top_logprobs_val = None
|
||||
input_top_logprobs_idx = None
|
||||
|
||||
if (
|
||||
logits_metadata.token_ids_logprobs
|
||||
or logits_metadata.extend_return_top_logprob
|
||||
):
|
||||
logits_metadata.extend_logprob_pruned_lens_cpu = [
|
||||
len(t) for t in token_indices_to_pool
|
||||
]
|
||||
|
||||
if logits_metadata.extend_token_ids_logprob:
|
||||
(
|
||||
input_token_ids_logprobs_val,
|
||||
input_token_ids_logprobs_idx,
|
||||
) = get_token_ids_logprobs_raw(
|
||||
sliced_logprobs,
|
||||
logits_metadata.token_ids_logprobs,
|
||||
stage=LogprobStage.PREFILL,
|
||||
extend_logprob_pruned_lens_cpu=logits_metadata.extend_logprob_pruned_lens_cpu,
|
||||
no_copy_to_cpu=True,
|
||||
)
|
||||
|
||||
if logits_metadata.extend_return_top_logprob:
|
||||
(
|
||||
input_top_logprobs_val,
|
||||
input_top_logprobs_idx,
|
||||
) = get_top_logprobs_raw(
|
||||
sliced_logprobs,
|
||||
logits_metadata.top_logprobs_nums,
|
||||
stage=LogprobStage.PREFILL,
|
||||
extend_logprob_pruned_lens_cpu=logits_metadata.extend_logprob_pruned_lens_cpu,
|
||||
)
|
||||
|
||||
# Zeros to satisfy the shared logprob pipeline's non-None / length asserts;
|
||||
# score_request() reads only input_token_ids_logprobs_val (see the MIS path).
|
||||
input_token_logprobs = torch.zeros(pooled_indices.shape[0], device=device)
|
||||
|
||||
return LogitsProcessorOutput(
|
||||
next_token_logits=None,
|
||||
input_token_logprobs=input_token_logprobs,
|
||||
input_top_logprobs_val=input_top_logprobs_val,
|
||||
input_top_logprobs_idx=input_top_logprobs_idx,
|
||||
input_token_ids_logprobs_val=input_token_ids_logprobs_val,
|
||||
input_token_ids_logprobs_idx=input_token_ids_logprobs_idx,
|
||||
mm_input_embeds=logits_metadata.mm_input_embeds,
|
||||
)
|
||||
|
||||
|
||||
def _reassemble_tp_lm_head_all_to_all_output(
|
||||
all_to_all_output: torch.Tensor,
|
||||
|
||||
@@ -357,6 +357,11 @@ class GenerateReqInput:
|
||||
# Batch-level: List[List[int]] (one per request). After __getitem__: List[int].
|
||||
multi_item_delimiter_indices: Optional[Union[List[List[int]], List[int]]] = None
|
||||
|
||||
# Token positions for setwise pooling readout (CausalLM: label-token logprobs
|
||||
# are read AT these positions instead of the last token).
|
||||
# Batch-level: List[List[int]] (one per request). After __getitem__: List[int].
|
||||
token_indices_to_pool: Optional[Union[List[List[int]], List[int]]] = None
|
||||
|
||||
# Cache namespace used to isolate otherwise-identical prefixes.
|
||||
cache_salt: Optional[Union[List[str], str]] = None
|
||||
|
||||
@@ -1023,6 +1028,11 @@ class GenerateReqInput:
|
||||
if self.multi_item_delimiter_indices is not None
|
||||
else None
|
||||
),
|
||||
token_indices_to_pool=(
|
||||
self.token_indices_to_pool[i]
|
||||
if self.token_indices_to_pool is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
cache[i] = sub
|
||||
return sub
|
||||
@@ -1120,6 +1130,9 @@ class TokenizedGenerateReqInput(BaseReq, kw_only=True):
|
||||
# Pre-computed delimiter indices for multi-item scoring
|
||||
multi_item_delimiter_indices: Optional[List[int]] = None
|
||||
|
||||
# Token positions for setwise pooling readout (CausalLM)
|
||||
token_indices_to_pool: Optional[List[int]] = None
|
||||
|
||||
# For observability
|
||||
# Pickled Optional[Union[APIServerReqTimeStats, DPControllerReqTimeStats]]
|
||||
time_stats: Optional[PickleWrapper] = None
|
||||
|
||||
@@ -2801,6 +2801,7 @@ class Scheduler(
|
||||
dllm_config=self.dllm_config,
|
||||
time_stats=recv_req.time_stats,
|
||||
multi_item_delimiter_indices=recv_req.multi_item_delimiter_indices,
|
||||
token_indices_to_pool=recv_req.token_indices_to_pool,
|
||||
)
|
||||
req.tokenizer = self.tokenizer
|
||||
|
||||
|
||||
@@ -27,24 +27,24 @@ class SchedulerLogprobResultProcessor:
|
||||
self, req: Req, input_token_logprobs: List
|
||||
) -> None:
|
||||
"""Process input token logprobs values and indices."""
|
||||
is_multi_item_scoring = self._is_multi_item_scoring(req)
|
||||
uses_scoring_positions = self._uses_scoring_positions(req)
|
||||
|
||||
# Process logprob values - handle multi-item scoring vs regular requests
|
||||
if is_multi_item_scoring:
|
||||
# Multi-item scoring: use all logprobs as-is
|
||||
# Process logprob values - handle position-based scoring vs regular requests
|
||||
if uses_scoring_positions:
|
||||
# Position-based scoring: use all logprobs as-is
|
||||
req.logprob.input_token_logprobs_val = input_token_logprobs
|
||||
else:
|
||||
# Regular request: add None at start, remove last (sampling token)
|
||||
req.logprob.input_token_logprobs_val = [None] + input_token_logprobs[:-1]
|
||||
|
||||
# Process logprob indices based on scoring type
|
||||
if is_multi_item_scoring:
|
||||
# MIS scores come from input_token_ids_logprobs, not input_token_logprobs.
|
||||
if uses_scoring_positions:
|
||||
# Position-based scores come from input_token_ids_logprobs, not input_token_logprobs.
|
||||
# But the shared pipeline requires input_token_logprobs_idx to be the same
|
||||
# length as input_token_logprobs_val (validated at line 816). We fill with
|
||||
# MIS_DELIMITER_TOKEN_ID as a dummy — score_request() ignores this field.
|
||||
delimiter_count = len(req.multi_item_delimiter_indices)
|
||||
input_token_logprobs_idx = [MIS_DELIMITER_TOKEN_ID] * delimiter_count
|
||||
position_count = len(self._scoring_positions(req))
|
||||
input_token_logprobs_idx = [MIS_DELIMITER_TOKEN_ID] * position_count
|
||||
else:
|
||||
# Regular request: include all tokens from logprob_start_len onwards
|
||||
input_token_logprobs_idx = req.origin_input_ids[req.logprob_start_len :]
|
||||
@@ -60,11 +60,11 @@ class SchedulerLogprobResultProcessor:
|
||||
if req.logprob.top_logprobs_num <= 0:
|
||||
return
|
||||
|
||||
is_multi_item_scoring = self._is_multi_item_scoring(req)
|
||||
uses_scoring_positions = self._uses_scoring_positions(req)
|
||||
|
||||
# Initialize arrays - multi-item scoring starts empty, others start with None
|
||||
req.logprob.input_top_logprobs_val = [] if is_multi_item_scoring else [None]
|
||||
req.logprob.input_top_logprobs_idx = [] if is_multi_item_scoring else [None]
|
||||
# Initialize arrays - position-based scoring starts empty, others start with None
|
||||
req.logprob.input_top_logprobs_val = [] if uses_scoring_positions else [None]
|
||||
req.logprob.input_top_logprobs_idx = [] if uses_scoring_positions else [None]
|
||||
|
||||
# Extend arrays with temp values
|
||||
for val, idx in zip(
|
||||
@@ -75,8 +75,8 @@ class SchedulerLogprobResultProcessor:
|
||||
req.logprob.input_top_logprobs_val.extend(val)
|
||||
req.logprob.input_top_logprobs_idx.extend(idx)
|
||||
|
||||
# Remove last token (sampling token) for non multi-item scoring requests
|
||||
if not is_multi_item_scoring:
|
||||
# Remove last token (sampling token) for non position-based scoring requests
|
||||
if not uses_scoring_positions:
|
||||
req.logprob.input_top_logprobs_val.pop()
|
||||
req.logprob.input_top_logprobs_idx.pop()
|
||||
|
||||
@@ -118,14 +118,14 @@ class SchedulerLogprobResultProcessor:
|
||||
if req.logprob.token_ids_logprob is None:
|
||||
return
|
||||
|
||||
is_multi_item_scoring = self._is_multi_item_scoring(req)
|
||||
uses_scoring_positions = self._uses_scoring_positions(req)
|
||||
|
||||
# Initialize arrays - multi-item scoring starts empty, others start with None
|
||||
# Initialize arrays - position-based scoring starts empty, others start with None
|
||||
req.logprob.input_token_ids_logprobs_val = (
|
||||
[] if is_multi_item_scoring else [None]
|
||||
[] if uses_scoring_positions else [None]
|
||||
)
|
||||
req.logprob.input_token_ids_logprobs_idx = (
|
||||
[] if is_multi_item_scoring else [None]
|
||||
[] if uses_scoring_positions else [None]
|
||||
)
|
||||
|
||||
# Process temp values - convert tensors to lists and extend arrays
|
||||
@@ -140,8 +140,8 @@ class SchedulerLogprobResultProcessor:
|
||||
)
|
||||
req.logprob.input_token_ids_logprobs_idx.extend(idx)
|
||||
|
||||
# Remove last token (sampling token) for non multi-item scoring requests
|
||||
if not is_multi_item_scoring:
|
||||
# Remove last token (sampling token) for non position-based scoring requests
|
||||
if not uses_scoring_positions:
|
||||
req.logprob.input_token_ids_logprobs_val.pop()
|
||||
req.logprob.input_token_ids_logprobs_idx.pop()
|
||||
|
||||
@@ -150,15 +150,16 @@ class SchedulerLogprobResultProcessor:
|
||||
req.temp_input_token_ids_logprobs_val = None
|
||||
|
||||
def _calculate_relevant_tokens_len(self, req: Req) -> int:
|
||||
"""Calculate the expected length of logprob arrays based on whether multi-item scoring is enabled.
|
||||
"""Calculate the expected length of logprob arrays based on whether position-based scoring is used.
|
||||
|
||||
For multi-item scoring, only delimiter positions have logprobs.
|
||||
For regular requests, all positions from logprob_start_len onwards have logprobs.
|
||||
For position-based scoring (MIS delimiters or setwise anchors), only those
|
||||
positions have logprobs. For regular requests, all positions from
|
||||
logprob_start_len onwards have logprobs.
|
||||
"""
|
||||
is_multi_item_scoring = self._is_multi_item_scoring(req)
|
||||
positions = self._scoring_positions(req)
|
||||
|
||||
if is_multi_item_scoring:
|
||||
return len(req.multi_item_delimiter_indices)
|
||||
if positions is not None:
|
||||
return len(positions)
|
||||
else:
|
||||
return len(req.origin_input_ids[req.logprob_start_len :])
|
||||
|
||||
@@ -168,36 +169,53 @@ class SchedulerLogprobResultProcessor:
|
||||
extend_input_len: int,
|
||||
extend_logprob_start_len: int,
|
||||
) -> int:
|
||||
"""Calculate the number of input logprobs based on whether multi-item scoring is enabled.
|
||||
"""Calculate the number of input logprobs based on whether position-based scoring is used.
|
||||
|
||||
For multi-item scoring, only delimiter positions have logprobs.
|
||||
For regular requests, all positions in the range have logprobs.
|
||||
For position-based scoring (MIS delimiters or setwise anchors), only those
|
||||
positions have logprobs. For regular requests, all positions in the range
|
||||
have logprobs.
|
||||
"""
|
||||
is_multi_item_scoring = self._is_multi_item_scoring(req)
|
||||
positions = self._scoring_positions(req)
|
||||
|
||||
if is_multi_item_scoring:
|
||||
# Count pre-computed delimiter indices within the extend range
|
||||
if positions is not None:
|
||||
# Count pre-computed scoring positions within the extend range
|
||||
return sum(
|
||||
1
|
||||
for idx in req.multi_item_delimiter_indices
|
||||
for idx in positions
|
||||
if extend_logprob_start_len <= idx < extend_input_len
|
||||
)
|
||||
else:
|
||||
# Regular request: all tokens in the range
|
||||
return extend_input_len - extend_logprob_start_len
|
||||
|
||||
def _is_multi_item_scoring(self, req: Req) -> bool:
|
||||
"""Check if request uses multi-item scoring.
|
||||
def _scoring_positions(self, req: Req):
|
||||
"""Position-based scoring readout indices for a prefill-only request, else None.
|
||||
|
||||
Multi-item scoring applies to prefill-only requests when a delimiter
|
||||
token is configured. In this mode, only positions containing the
|
||||
delimiter token receive logprobs.
|
||||
Both MIS (delimiter positions, --enable-mis) and setwise scoring
|
||||
(token_indices_to_pool, per-anchor readout) compute label-token logprobs
|
||||
at a fixed set of positions instead of at every input token; the logprob
|
||||
bookkeeping below is identical for the two.
|
||||
"""
|
||||
return (
|
||||
if not req.is_prefill_only:
|
||||
return None
|
||||
# Setwise anchors take precedence over MIS delimiters: a fused setwise
|
||||
# request carries both, and its logprobs are read at the anchors.
|
||||
if req.token_indices_to_pool is not None:
|
||||
return req.token_indices_to_pool
|
||||
if (
|
||||
get_exec().features.enable_mis
|
||||
and req.is_prefill_only
|
||||
and req.multi_item_delimiter_indices is not None
|
||||
)
|
||||
):
|
||||
return req.multi_item_delimiter_indices
|
||||
return None
|
||||
|
||||
def _uses_scoring_positions(self, req: Req) -> bool:
|
||||
"""Whether the request uses position-based scoring (MIS or setwise).
|
||||
|
||||
In this mode only the scoring positions receive logprobs, so the shared
|
||||
logprob pipeline skips the None prefix and last-token pop.
|
||||
"""
|
||||
return self._scoring_positions(req) is not None
|
||||
|
||||
def add_input_logprob_return_values(
|
||||
self,
|
||||
|
||||
@@ -1535,6 +1535,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
need_wait_for_mm_inputs=obj.need_wait_for_mm_inputs,
|
||||
num_items_assigned=obj.num_items_assigned,
|
||||
multi_item_delimiter_indices=obj.multi_item_delimiter_indices,
|
||||
token_indices_to_pool=obj.token_indices_to_pool,
|
||||
encoder_urls=obj.encoder_urls,
|
||||
)
|
||||
elif isinstance(obj, EmbeddingReqInput):
|
||||
|
||||
@@ -258,25 +258,61 @@ class TokenizerManagerScoreMixin:
|
||||
results: Any,
|
||||
per_item_anchor_counts: List[int],
|
||||
apply_softmax: bool,
|
||||
label_token_ids: Optional[List[List[int]]] = None,
|
||||
temperature: float = 1.0,
|
||||
return_pooled_hidden_states: bool = False,
|
||||
) -> ScoreResult:
|
||||
"""Process a fused multi-item score-extraction request (``--enable-mis``).
|
||||
|
||||
The fused sequence returns one ``[ΣNᵢ, num_labels]`` matrix of every
|
||||
anchor's logits in item order; split it back into one ``[Nᵢ, num_labels]``
|
||||
matrix per item using per_item_anchor_counts.
|
||||
The fused sequence yields every anchor's scores in item order (a
|
||||
``[ΣNᵢ, num_labels]`` embedding for SequenceClassification, or per-anchor
|
||||
label-token logprobs for CausalLM); split it back into one
|
||||
``[Nᵢ, num_labels]`` matrix per item using per_item_anchor_counts.
|
||||
"""
|
||||
single_result = results[0] if isinstance(results, list) else results
|
||||
meta_info = single_result.get("meta_info", {})
|
||||
request_id = meta_info.get("id", "<unknown>")
|
||||
prompt_tokens = meta_info.get("prompt_tokens", 0)
|
||||
total_anchors = sum(per_item_anchor_counts)
|
||||
|
||||
if self.is_generation:
|
||||
# CausalLM: per-anchor label-token logprobs in item order; item i's
|
||||
# anchors use item i's candidate labels (label_token_ids[i]).
|
||||
anchor_logprobs = meta_info.get("input_token_ids_logprobs", [])
|
||||
if not anchor_logprobs:
|
||||
raise ValueError(
|
||||
f"input_token_ids_logprobs not found in the result for "
|
||||
f"request {request_id}."
|
||||
)
|
||||
if len(anchor_logprobs) != total_anchors:
|
||||
raise RuntimeError(
|
||||
f"Expected {total_anchors} anchor rows across "
|
||||
f"{len(per_item_anchor_counts)} items, but got "
|
||||
f"{len(anchor_logprobs)}. Request ID: {request_id}"
|
||||
)
|
||||
scores = []
|
||||
offset = 0
|
||||
for labels, count in zip(label_token_ids, per_item_anchor_counts):
|
||||
scores.append(
|
||||
[
|
||||
self._convert_logprobs_to_scores(
|
||||
self._extract_logprobs_for_tokens(
|
||||
anchor_logprobs[offset + j], labels
|
||||
),
|
||||
labels,
|
||||
apply_softmax,
|
||||
temperature,
|
||||
)
|
||||
for j in range(count)
|
||||
]
|
||||
)
|
||||
offset += count
|
||||
return ScoreResult(scores=scores, prompt_tokens=prompt_tokens)
|
||||
|
||||
embedding = single_result.get("embedding")
|
||||
if embedding is None:
|
||||
raise ValueError("Embedding not found in the result.")
|
||||
|
||||
rows = self._multi_position_score_rows(embedding, apply_softmax)
|
||||
total_anchors = sum(per_item_anchor_counts)
|
||||
if len(rows) != total_anchors:
|
||||
raise RuntimeError(
|
||||
f"Expected {total_anchors} anchor rows across "
|
||||
@@ -332,19 +368,38 @@ class TokenizerManagerScoreMixin:
|
||||
is_generation = self.is_generation
|
||||
if is_generation:
|
||||
for result, labels in zip(results, label_token_ids):
|
||||
# For single-item scoring, logprobs are in output_token_ids_logprobs
|
||||
output_logprobs = result["meta_info"].get(
|
||||
"output_token_ids_logprobs", []
|
||||
)
|
||||
prompt_tokens += result["meta_info"].get("prompt_tokens", 0)
|
||||
meta = result["meta_info"]
|
||||
prompt_tokens += meta.get("prompt_tokens", 0)
|
||||
|
||||
if per_item_matrix:
|
||||
# Setwise: label-token logprobs were read AT each anchor
|
||||
# position; one row per anchor, grouped as one item's matrix.
|
||||
anchor_logprobs = meta.get("input_token_ids_logprobs", [])
|
||||
if not anchor_logprobs:
|
||||
raise RuntimeError(
|
||||
f"input_token_ids_logprobs is empty for request "
|
||||
f"{meta.get('id', '<unknown>')}."
|
||||
)
|
||||
item_matrix = [
|
||||
self._convert_logprobs_to_scores(
|
||||
self._extract_logprobs_for_tokens(pos_logprobs, labels),
|
||||
labels,
|
||||
apply_softmax,
|
||||
temperature,
|
||||
)
|
||||
for pos_logprobs in anchor_logprobs
|
||||
]
|
||||
scores.append(item_matrix)
|
||||
continue
|
||||
|
||||
# Pointwise: logprobs are in output_token_ids_logprobs (last position)
|
||||
output_logprobs = meta.get("output_token_ids_logprobs", [])
|
||||
if not output_logprobs or len(output_logprobs) == 0:
|
||||
raise RuntimeError(
|
||||
f"output_logprobs is empty for request "
|
||||
f"{result['meta_info'].get('id', '<unknown>')}."
|
||||
f"{meta.get('id', '<unknown>')}."
|
||||
)
|
||||
|
||||
# Extract logprobs for the first (and only) position
|
||||
logprobs = self._extract_logprobs_for_tokens(output_logprobs[0], labels)
|
||||
score_list = self._convert_logprobs_to_scores(
|
||||
logprobs, labels, apply_softmax, temperature
|
||||
@@ -592,29 +647,26 @@ class TokenizerManagerScoreMixin:
|
||||
is_generation: bool,
|
||||
item_first: bool,
|
||||
) -> None:
|
||||
"""Validate that multi-position pooling readout is applicable.
|
||||
"""Validate that setwise pooling readout is applicable.
|
||||
|
||||
Supported only by SequenceClassification models whose forward routes the
|
||||
head through ``score_and_pool`` (per-position pooling, in
|
||||
``is_score_and_pool_model``). Generation, cross-encoder, reward, and
|
||||
embedding models pool a single vector and are rejected here before
|
||||
inference. Requires radix cache and chunked prefill off and no auto-truncate,
|
||||
since pooling positions are full-prompt coordinates.
|
||||
SequenceClassification models must route the head through
|
||||
``score_and_pool`` (per-position pooling, in ``is_score_and_pool_model``);
|
||||
cross-encoder, reward, and embedding models pool a single vector and are
|
||||
rejected. CausalLM (generation) models read label-token logprobs at each
|
||||
anchor via the LM head and are supported in both the batched and fused
|
||||
(``--enable-mis``) paths. Requires radix cache and chunked prefill off and
|
||||
no auto-truncate, since pooling positions are full-prompt coordinates.
|
||||
"""
|
||||
if is_generation:
|
||||
raise ValueError(
|
||||
"score_extraction_token_id is only supported for "
|
||||
"SequenceClassification models, not generation (CausalLM) models."
|
||||
)
|
||||
architectures = self.model_config.hf_config.architectures or []
|
||||
if not is_score_and_pool_model(architectures):
|
||||
raise ValueError(
|
||||
"score_extraction_token_id is only supported for "
|
||||
"SequenceClassification models that pool the head per position "
|
||||
"(score_and_pool); model architecture(s) "
|
||||
f"{architectures} do not (cross-encoder, reward, and embedding "
|
||||
"models pool a single vector and ignore the readout positions)."
|
||||
)
|
||||
if not is_generation:
|
||||
architectures = self.model_config.hf_config.architectures or []
|
||||
if not is_score_and_pool_model(architectures):
|
||||
raise ValueError(
|
||||
"score_extraction_token_id is only supported for "
|
||||
"SequenceClassification models that pool the head per position "
|
||||
"(score_and_pool); model architecture(s) "
|
||||
f"{architectures} do not (cross-encoder, reward, and embedding "
|
||||
"models pool a single vector and ignore the readout positions)."
|
||||
)
|
||||
if item_first:
|
||||
raise ValueError(
|
||||
"item_first is not supported with score_extraction_token_id."
|
||||
@@ -786,12 +838,12 @@ class TokenizerManagerScoreMixin:
|
||||
- Generation (CausalLM): Requires label_token_ids; returns logprob-based scores.
|
||||
- SequenceClassification: label_token_ids is optional; returns pooled class logits.
|
||||
|
||||
Setwise scoring (SequenceClassification-only) is expressed via
|
||||
score_extraction_token_id: when set, the head is pooled AT every occurrence
|
||||
of this token in each ``query + item`` sequence instead of the last token,
|
||||
and ``scores`` is returned nested (one ``[Nᵢ x num_labels]`` matrix per
|
||||
item). With ``--enable-mis`` the items are fused into one multi-item
|
||||
sequence; otherwise each item is scored independently in one batch.
|
||||
Setwise scoring is expressed via score_extraction_token_id: when set, the
|
||||
readout is taken AT every occurrence of this token in each ``query + item``
|
||||
sequence instead of the last token, and ``scores`` is returned nested (one
|
||||
``[Nᵢ x num_labels]`` matrix per item). SequenceClassification pools the
|
||||
head at those positions; CausalLM reads label-token logprobs there. Both
|
||||
support the batched and fused (``--enable-mis``) paths.
|
||||
|
||||
return_pooled_hidden_states is only supported for non-generation models
|
||||
(SequenceClassification, RewardModel); raises ValueError for CausalLM.
|
||||
@@ -1016,12 +1068,16 @@ class TokenizerManagerScoreMixin:
|
||||
input_ids=input_ids,
|
||||
token_ids_logprob=request_labels,
|
||||
return_logprob=True,
|
||||
# Set logprob_start_len=0 for multi-item scoring since we want logprobs at all delimiter positions
|
||||
logprob_start_len=0 if use_multi_item_scoring else -1,
|
||||
# logprob_start_len=0 so input-position logprobs are computed:
|
||||
# multi-item scoring reads them at delimiters, setwise at anchors.
|
||||
logprob_start_len=(
|
||||
0 if (use_multi_item_scoring or use_score_extraction) else -1
|
||||
),
|
||||
stream=False,
|
||||
sampling_params={"max_new_tokens": 0},
|
||||
positional_embed_overrides=positional_embed_overrides,
|
||||
multi_item_delimiter_indices=mis_delimiter_indices,
|
||||
token_indices_to_pool=token_indices_to_pool,
|
||||
)
|
||||
else:
|
||||
batch_request = EmbeddingReqInput(
|
||||
@@ -1037,13 +1093,15 @@ class TokenizerManagerScoreMixin:
|
||||
|
||||
if use_multi_item_scoring and use_score_extraction:
|
||||
# Multi-item setwise: the items were fused into one sequence and the
|
||||
# head pooled at every anchor. Split the flat [ΣNᵢ, num_labels] result
|
||||
# back into one matrix per item (nested scores).
|
||||
# readout taken at every anchor. Split the flat [ΣNᵢ, num_labels]
|
||||
# result back into one matrix per item (nested scores).
|
||||
return self._process_multi_item_extraction_results(
|
||||
results,
|
||||
per_item_anchor_counts,
|
||||
apply_softmax,
|
||||
return_pooled_hidden_states,
|
||||
label_token_ids=label_token_ids,
|
||||
temperature=temperature,
|
||||
return_pooled_hidden_states=return_pooled_hidden_states,
|
||||
)
|
||||
elif use_multi_item_scoring:
|
||||
# Multi-item scoring: extract scores from input_token_ids_logprobs or embedding
|
||||
|
||||
Reference in New Issue
Block a user