[Score API] Setwise scoring: CausalLM support (batched + --enable-mis) (#41188)

This commit is contained in:
Sundara Raman Ramachandran
2026-09-25 23:02:44 -07:00
committed by GitHub
parent 4bb645b8e2
commit 3ed56a341b
10 changed files with 637 additions and 97 deletions
@@ -56,8 +56,10 @@ class EngineScoreMixin:
Setwise scoring is expressed via ``score_extraction_token_id``: pass a single
item containing the whole candidate block with one extraction token per
candidate, and the head is pooled AT those positions, so ``scores`` becomes
the ``[N x num_labels]`` per-candidate matrix.
candidate, and the readout is taken AT those positions, so ``scores``
becomes the ``[N x num_labels]`` per-candidate matrix. SequenceClassification
pools the head there; CausalLM reads label-token logprobs there (batched
path only).
Args:
query: The query text or pre-tokenized token IDs.
@@ -69,10 +71,11 @@ class EngineScoreMixin:
embed_override_token_id: Placeholder token ID used to locate override positions.
query_embed_overrides: Embedding vectors replacing placeholder tokens in query.
item_embed_overrides: Per-item embedding vectors replacing placeholder tokens in items.
score_extraction_token_id: SequenceClassification-only. When set, pool the
head at every occurrence of this token per sequence instead of the last
token; ``scores`` becomes nested — one ``[Ni x num_labels]`` matrix per
item (``len(scores) == len(items)``), where ``Ni`` is the number of
score_extraction_token_id: When set, read the score head (SeqCls) or
label-token logprobs (CausalLM) at every occurrence of
this token per sequence instead of the last token; ``scores`` becomes
nested — one ``[Ni x num_labels]`` matrix per item
(``len(scores) == len(items)``), where ``Ni`` is the number of
extraction tokens (candidates) in item ``i``.
return_pooled_hidden_states: Whether to include raw pooled transformer
hidden states (before the task head) in the result. Only supported
@@ -1408,10 +1408,11 @@ class ScoringRequest(BaseModel):
item_first: bool = False
return_pooled_hidden_states: bool = False
# Setwise readout (SequenceClassification-only): when set, the head is pooled
# AT every occurrence of this token in each `query + item` sequence instead of
# the last token, and `scores` is returned nested (one `[Nᵢ x num_labels]`
# matrix per item). --enable-mis fuses items; otherwise each is scored alone.
# Setwise readout: when set, the readout is taken AT every occurrence of this
# token in each `query + item` sequence instead of the last token, and `scores`
# is returned nested (one `[Nᵢ x num_labels]` matrix per item). SeqCls pools the
# head there; CausalLM reads label-token logprobs there. Both support batched
# and --enable-mis.
score_extraction_token: Optional[str] = None
model: str = DEFAULT_MODEL_NAME
+98 -1
View File
@@ -510,10 +510,12 @@ class LogitsProcessor(nn.Module):
aux_hidden_states: Optional[AuxHiddenStates] = None,
hidden_states_before_norm: Optional[torch.Tensor] = None,
) -> LogitsProcessorOutput:
# Extract MIS indices before ForwardBatch → LogitsMetadata conversion
# Extract MIS / setwise indices before ForwardBatch → LogitsMetadata conversion
multi_item_delimiter_indices = None
token_indices_to_pool = None
if isinstance(logits_metadata, ForwardBatch):
multi_item_delimiter_indices = logits_metadata.multi_item_delimiter_indices
token_indices_to_pool = logits_metadata.token_indices_to_pool
logits_metadata = LogitsMetadata.from_forward_batch(logits_metadata)
# Autotune dummy run discards this output. `is False` not `not`: None
@@ -531,6 +533,18 @@ class LogitsProcessor(nn.Module):
forward_mode=logits_metadata.forward_mode,
)
# Setwise scoring (CausalLM): read label-token logprobs AT each anchor
# position instead of the last token. Takes precedence over the MIS
# delimiter path; the two are mutually exclusive for generation.
if token_indices_to_pool is not None and logits_metadata.is_prefill_only:
return self.compute_logprobs_at_positions(
input_ids,
hidden_states,
lm_head,
logits_metadata,
token_indices_to_pool,
)
# Multi-item scoring only for prefill-only requests with pre-computed indices.
if multi_item_delimiter_indices is not None and logits_metadata.is_prefill_only:
return self.compute_logprobs_for_multi_item_scoring(
@@ -1263,6 +1277,89 @@ class LogitsProcessor(nn.Module):
mm_input_embeds=logits_metadata.mm_input_embeds,
)
def compute_logprobs_at_positions(
self,
input_ids,
hidden_states,
lm_head: VocabParallelEmbedding,
logits_metadata: Union[LogitsMetadata, ForwardBatch],
token_indices_to_pool: List[torch.Tensor],
):
"""Compute label-token logprobs AT each requested position (setwise, CausalLM).
Mirrors ``compute_logprobs_for_multi_item_scoring`` but reads the LM head
AT ``token_indices_to_pool`` (no delimiter - 1 shift, no discarded row):
each anchor's logprobs are P(next token | prefix up to and including the
anchor), one row per anchor.
"""
device = input_ids.device
all_tensors = []
if logits_metadata.extend_seq_lens_cpu is not None:
offset = 0
for req_seq_len, indices_tensor in zip(
logits_metadata.extend_seq_lens_cpu, token_indices_to_pool
):
if len(indices_tensor) > 0:
all_tensors.append(indices_tensor + offset)
offset += req_seq_len
else:
all_tensors.append(token_indices_to_pool[0])
pooled_indices = torch.cat(all_tensors).to(device, non_blocking=True)
sliced_hidden = hidden_states[pooled_indices]
sliced_logits = self._get_logits(sliced_hidden, lm_head, logits_metadata)
sliced_logprobs = torch.nn.functional.log_softmax(sliced_logits, dim=-1)
input_token_ids_logprobs_val = []
input_token_ids_logprobs_idx = []
input_top_logprobs_val = None
input_top_logprobs_idx = None
if (
logits_metadata.token_ids_logprobs
or logits_metadata.extend_return_top_logprob
):
logits_metadata.extend_logprob_pruned_lens_cpu = [
len(t) for t in token_indices_to_pool
]
if logits_metadata.extend_token_ids_logprob:
(
input_token_ids_logprobs_val,
input_token_ids_logprobs_idx,
) = get_token_ids_logprobs_raw(
sliced_logprobs,
logits_metadata.token_ids_logprobs,
stage=LogprobStage.PREFILL,
extend_logprob_pruned_lens_cpu=logits_metadata.extend_logprob_pruned_lens_cpu,
no_copy_to_cpu=True,
)
if logits_metadata.extend_return_top_logprob:
(
input_top_logprobs_val,
input_top_logprobs_idx,
) = get_top_logprobs_raw(
sliced_logprobs,
logits_metadata.top_logprobs_nums,
stage=LogprobStage.PREFILL,
extend_logprob_pruned_lens_cpu=logits_metadata.extend_logprob_pruned_lens_cpu,
)
# Zeros to satisfy the shared logprob pipeline's non-None / length asserts;
# score_request() reads only input_token_ids_logprobs_val (see the MIS path).
input_token_logprobs = torch.zeros(pooled_indices.shape[0], device=device)
return LogitsProcessorOutput(
next_token_logits=None,
input_token_logprobs=input_token_logprobs,
input_top_logprobs_val=input_top_logprobs_val,
input_top_logprobs_idx=input_top_logprobs_idx,
input_token_ids_logprobs_val=input_token_ids_logprobs_val,
input_token_ids_logprobs_idx=input_token_ids_logprobs_idx,
mm_input_embeds=logits_metadata.mm_input_embeds,
)
def _reassemble_tp_lm_head_all_to_all_output(
all_to_all_output: torch.Tensor,
+13
View File
@@ -357,6 +357,11 @@ class GenerateReqInput:
# Batch-level: List[List[int]] (one per request). After __getitem__: List[int].
multi_item_delimiter_indices: Optional[Union[List[List[int]], List[int]]] = None
# Token positions for setwise pooling readout (CausalLM: label-token logprobs
# are read AT these positions instead of the last token).
# Batch-level: List[List[int]] (one per request). After __getitem__: List[int].
token_indices_to_pool: Optional[Union[List[List[int]], List[int]]] = None
# Cache namespace used to isolate otherwise-identical prefixes.
cache_salt: Optional[Union[List[str], str]] = None
@@ -1023,6 +1028,11 @@ class GenerateReqInput:
if self.multi_item_delimiter_indices is not None
else None
),
token_indices_to_pool=(
self.token_indices_to_pool[i]
if self.token_indices_to_pool is not None
else None
),
)
cache[i] = sub
return sub
@@ -1120,6 +1130,9 @@ class TokenizedGenerateReqInput(BaseReq, kw_only=True):
# Pre-computed delimiter indices for multi-item scoring
multi_item_delimiter_indices: Optional[List[int]] = None
# Token positions for setwise pooling readout (CausalLM)
token_indices_to_pool: Optional[List[int]] = None
# For observability
# Pickled Optional[Union[APIServerReqTimeStats, DPControllerReqTimeStats]]
time_stats: Optional[PickleWrapper] = None
+1
View File
@@ -2801,6 +2801,7 @@ class Scheduler(
dllm_config=self.dllm_config,
time_stats=recv_req.time_stats,
multi_item_delimiter_indices=recv_req.multi_item_delimiter_indices,
token_indices_to_pool=recv_req.token_indices_to_pool,
)
req.tokenizer = self.tokenizer
@@ -27,24 +27,24 @@ class SchedulerLogprobResultProcessor:
self, req: Req, input_token_logprobs: List
) -> None:
"""Process input token logprobs values and indices."""
is_multi_item_scoring = self._is_multi_item_scoring(req)
uses_scoring_positions = self._uses_scoring_positions(req)
# Process logprob values - handle multi-item scoring vs regular requests
if is_multi_item_scoring:
# Multi-item scoring: use all logprobs as-is
# Process logprob values - handle position-based scoring vs regular requests
if uses_scoring_positions:
# Position-based scoring: use all logprobs as-is
req.logprob.input_token_logprobs_val = input_token_logprobs
else:
# Regular request: add None at start, remove last (sampling token)
req.logprob.input_token_logprobs_val = [None] + input_token_logprobs[:-1]
# Process logprob indices based on scoring type
if is_multi_item_scoring:
# MIS scores come from input_token_ids_logprobs, not input_token_logprobs.
if uses_scoring_positions:
# Position-based scores come from input_token_ids_logprobs, not input_token_logprobs.
# But the shared pipeline requires input_token_logprobs_idx to be the same
# length as input_token_logprobs_val (validated at line 816). We fill with
# MIS_DELIMITER_TOKEN_ID as a dummy — score_request() ignores this field.
delimiter_count = len(req.multi_item_delimiter_indices)
input_token_logprobs_idx = [MIS_DELIMITER_TOKEN_ID] * delimiter_count
position_count = len(self._scoring_positions(req))
input_token_logprobs_idx = [MIS_DELIMITER_TOKEN_ID] * position_count
else:
# Regular request: include all tokens from logprob_start_len onwards
input_token_logprobs_idx = req.origin_input_ids[req.logprob_start_len :]
@@ -60,11 +60,11 @@ class SchedulerLogprobResultProcessor:
if req.logprob.top_logprobs_num <= 0:
return
is_multi_item_scoring = self._is_multi_item_scoring(req)
uses_scoring_positions = self._uses_scoring_positions(req)
# Initialize arrays - multi-item scoring starts empty, others start with None
req.logprob.input_top_logprobs_val = [] if is_multi_item_scoring else [None]
req.logprob.input_top_logprobs_idx = [] if is_multi_item_scoring else [None]
# Initialize arrays - position-based scoring starts empty, others start with None
req.logprob.input_top_logprobs_val = [] if uses_scoring_positions else [None]
req.logprob.input_top_logprobs_idx = [] if uses_scoring_positions else [None]
# Extend arrays with temp values
for val, idx in zip(
@@ -75,8 +75,8 @@ class SchedulerLogprobResultProcessor:
req.logprob.input_top_logprobs_val.extend(val)
req.logprob.input_top_logprobs_idx.extend(idx)
# Remove last token (sampling token) for non multi-item scoring requests
if not is_multi_item_scoring:
# Remove last token (sampling token) for non position-based scoring requests
if not uses_scoring_positions:
req.logprob.input_top_logprobs_val.pop()
req.logprob.input_top_logprobs_idx.pop()
@@ -118,14 +118,14 @@ class SchedulerLogprobResultProcessor:
if req.logprob.token_ids_logprob is None:
return
is_multi_item_scoring = self._is_multi_item_scoring(req)
uses_scoring_positions = self._uses_scoring_positions(req)
# Initialize arrays - multi-item scoring starts empty, others start with None
# Initialize arrays - position-based scoring starts empty, others start with None
req.logprob.input_token_ids_logprobs_val = (
[] if is_multi_item_scoring else [None]
[] if uses_scoring_positions else [None]
)
req.logprob.input_token_ids_logprobs_idx = (
[] if is_multi_item_scoring else [None]
[] if uses_scoring_positions else [None]
)
# Process temp values - convert tensors to lists and extend arrays
@@ -140,8 +140,8 @@ class SchedulerLogprobResultProcessor:
)
req.logprob.input_token_ids_logprobs_idx.extend(idx)
# Remove last token (sampling token) for non multi-item scoring requests
if not is_multi_item_scoring:
# Remove last token (sampling token) for non position-based scoring requests
if not uses_scoring_positions:
req.logprob.input_token_ids_logprobs_val.pop()
req.logprob.input_token_ids_logprobs_idx.pop()
@@ -150,15 +150,16 @@ class SchedulerLogprobResultProcessor:
req.temp_input_token_ids_logprobs_val = None
def _calculate_relevant_tokens_len(self, req: Req) -> int:
"""Calculate the expected length of logprob arrays based on whether multi-item scoring is enabled.
"""Calculate the expected length of logprob arrays based on whether position-based scoring is used.
For multi-item scoring, only delimiter positions have logprobs.
For regular requests, all positions from logprob_start_len onwards have logprobs.
For position-based scoring (MIS delimiters or setwise anchors), only those
positions have logprobs. For regular requests, all positions from
logprob_start_len onwards have logprobs.
"""
is_multi_item_scoring = self._is_multi_item_scoring(req)
positions = self._scoring_positions(req)
if is_multi_item_scoring:
return len(req.multi_item_delimiter_indices)
if positions is not None:
return len(positions)
else:
return len(req.origin_input_ids[req.logprob_start_len :])
@@ -168,36 +169,53 @@ class SchedulerLogprobResultProcessor:
extend_input_len: int,
extend_logprob_start_len: int,
) -> int:
"""Calculate the number of input logprobs based on whether multi-item scoring is enabled.
"""Calculate the number of input logprobs based on whether position-based scoring is used.
For multi-item scoring, only delimiter positions have logprobs.
For regular requests, all positions in the range have logprobs.
For position-based scoring (MIS delimiters or setwise anchors), only those
positions have logprobs. For regular requests, all positions in the range
have logprobs.
"""
is_multi_item_scoring = self._is_multi_item_scoring(req)
positions = self._scoring_positions(req)
if is_multi_item_scoring:
# Count pre-computed delimiter indices within the extend range
if positions is not None:
# Count pre-computed scoring positions within the extend range
return sum(
1
for idx in req.multi_item_delimiter_indices
for idx in positions
if extend_logprob_start_len <= idx < extend_input_len
)
else:
# Regular request: all tokens in the range
return extend_input_len - extend_logprob_start_len
def _is_multi_item_scoring(self, req: Req) -> bool:
"""Check if request uses multi-item scoring.
def _scoring_positions(self, req: Req):
"""Position-based scoring readout indices for a prefill-only request, else None.
Multi-item scoring applies to prefill-only requests when a delimiter
token is configured. In this mode, only positions containing the
delimiter token receive logprobs.
Both MIS (delimiter positions, --enable-mis) and setwise scoring
(token_indices_to_pool, per-anchor readout) compute label-token logprobs
at a fixed set of positions instead of at every input token; the logprob
bookkeeping below is identical for the two.
"""
return (
if not req.is_prefill_only:
return None
# Setwise anchors take precedence over MIS delimiters: a fused setwise
# request carries both, and its logprobs are read at the anchors.
if req.token_indices_to_pool is not None:
return req.token_indices_to_pool
if (
get_exec().features.enable_mis
and req.is_prefill_only
and req.multi_item_delimiter_indices is not None
)
):
return req.multi_item_delimiter_indices
return None
def _uses_scoring_positions(self, req: Req) -> bool:
"""Whether the request uses position-based scoring (MIS or setwise).
In this mode only the scoring positions receive logprobs, so the shared
logprob pipeline skips the None prefix and last-token pop.
"""
return self._scoring_positions(req) is not None
def add_input_logprob_return_values(
self,
@@ -1535,6 +1535,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
need_wait_for_mm_inputs=obj.need_wait_for_mm_inputs,
num_items_assigned=obj.num_items_assigned,
multi_item_delimiter_indices=obj.multi_item_delimiter_indices,
token_indices_to_pool=obj.token_indices_to_pool,
encoder_urls=obj.encoder_urls,
)
elif isinstance(obj, EmbeddingReqInput):
@@ -258,25 +258,61 @@ class TokenizerManagerScoreMixin:
results: Any,
per_item_anchor_counts: List[int],
apply_softmax: bool,
label_token_ids: Optional[List[List[int]]] = None,
temperature: float = 1.0,
return_pooled_hidden_states: bool = False,
) -> ScoreResult:
"""Process a fused multi-item score-extraction request (``--enable-mis``).
The fused sequence returns one ``[ΣNᵢ, num_labels]`` matrix of every
anchor's logits in item order; split it back into one ``[Nᵢ, num_labels]``
matrix per item using per_item_anchor_counts.
The fused sequence yields every anchor's scores in item order (a
``[ΣNᵢ, num_labels]`` embedding for SequenceClassification, or per-anchor
label-token logprobs for CausalLM); split it back into one
``[Nᵢ, num_labels]`` matrix per item using per_item_anchor_counts.
"""
single_result = results[0] if isinstance(results, list) else results
meta_info = single_result.get("meta_info", {})
request_id = meta_info.get("id", "<unknown>")
prompt_tokens = meta_info.get("prompt_tokens", 0)
total_anchors = sum(per_item_anchor_counts)
if self.is_generation:
# CausalLM: per-anchor label-token logprobs in item order; item i's
# anchors use item i's candidate labels (label_token_ids[i]).
anchor_logprobs = meta_info.get("input_token_ids_logprobs", [])
if not anchor_logprobs:
raise ValueError(
f"input_token_ids_logprobs not found in the result for "
f"request {request_id}."
)
if len(anchor_logprobs) != total_anchors:
raise RuntimeError(
f"Expected {total_anchors} anchor rows across "
f"{len(per_item_anchor_counts)} items, but got "
f"{len(anchor_logprobs)}. Request ID: {request_id}"
)
scores = []
offset = 0
for labels, count in zip(label_token_ids, per_item_anchor_counts):
scores.append(
[
self._convert_logprobs_to_scores(
self._extract_logprobs_for_tokens(
anchor_logprobs[offset + j], labels
),
labels,
apply_softmax,
temperature,
)
for j in range(count)
]
)
offset += count
return ScoreResult(scores=scores, prompt_tokens=prompt_tokens)
embedding = single_result.get("embedding")
if embedding is None:
raise ValueError("Embedding not found in the result.")
rows = self._multi_position_score_rows(embedding, apply_softmax)
total_anchors = sum(per_item_anchor_counts)
if len(rows) != total_anchors:
raise RuntimeError(
f"Expected {total_anchors} anchor rows across "
@@ -332,19 +368,38 @@ class TokenizerManagerScoreMixin:
is_generation = self.is_generation
if is_generation:
for result, labels in zip(results, label_token_ids):
# For single-item scoring, logprobs are in output_token_ids_logprobs
output_logprobs = result["meta_info"].get(
"output_token_ids_logprobs", []
)
prompt_tokens += result["meta_info"].get("prompt_tokens", 0)
meta = result["meta_info"]
prompt_tokens += meta.get("prompt_tokens", 0)
if per_item_matrix:
# Setwise: label-token logprobs were read AT each anchor
# position; one row per anchor, grouped as one item's matrix.
anchor_logprobs = meta.get("input_token_ids_logprobs", [])
if not anchor_logprobs:
raise RuntimeError(
f"input_token_ids_logprobs is empty for request "
f"{meta.get('id', '<unknown>')}."
)
item_matrix = [
self._convert_logprobs_to_scores(
self._extract_logprobs_for_tokens(pos_logprobs, labels),
labels,
apply_softmax,
temperature,
)
for pos_logprobs in anchor_logprobs
]
scores.append(item_matrix)
continue
# Pointwise: logprobs are in output_token_ids_logprobs (last position)
output_logprobs = meta.get("output_token_ids_logprobs", [])
if not output_logprobs or len(output_logprobs) == 0:
raise RuntimeError(
f"output_logprobs is empty for request "
f"{result['meta_info'].get('id', '<unknown>')}."
f"{meta.get('id', '<unknown>')}."
)
# Extract logprobs for the first (and only) position
logprobs = self._extract_logprobs_for_tokens(output_logprobs[0], labels)
score_list = self._convert_logprobs_to_scores(
logprobs, labels, apply_softmax, temperature
@@ -592,29 +647,26 @@ class TokenizerManagerScoreMixin:
is_generation: bool,
item_first: bool,
) -> None:
"""Validate that multi-position pooling readout is applicable.
"""Validate that setwise pooling readout is applicable.
Supported only by SequenceClassification models whose forward routes the
head through ``score_and_pool`` (per-position pooling, in
``is_score_and_pool_model``). Generation, cross-encoder, reward, and
embedding models pool a single vector and are rejected here before
inference. Requires radix cache and chunked prefill off and no auto-truncate,
since pooling positions are full-prompt coordinates.
SequenceClassification models must route the head through
``score_and_pool`` (per-position pooling, in ``is_score_and_pool_model``);
cross-encoder, reward, and embedding models pool a single vector and are
rejected. CausalLM (generation) models read label-token logprobs at each
anchor via the LM head and are supported in both the batched and fused
(``--enable-mis``) paths. Requires radix cache and chunked prefill off and
no auto-truncate, since pooling positions are full-prompt coordinates.
"""
if is_generation:
raise ValueError(
"score_extraction_token_id is only supported for "
"SequenceClassification models, not generation (CausalLM) models."
)
architectures = self.model_config.hf_config.architectures or []
if not is_score_and_pool_model(architectures):
raise ValueError(
"score_extraction_token_id is only supported for "
"SequenceClassification models that pool the head per position "
"(score_and_pool); model architecture(s) "
f"{architectures} do not (cross-encoder, reward, and embedding "
"models pool a single vector and ignore the readout positions)."
)
if not is_generation:
architectures = self.model_config.hf_config.architectures or []
if not is_score_and_pool_model(architectures):
raise ValueError(
"score_extraction_token_id is only supported for "
"SequenceClassification models that pool the head per position "
"(score_and_pool); model architecture(s) "
f"{architectures} do not (cross-encoder, reward, and embedding "
"models pool a single vector and ignore the readout positions)."
)
if item_first:
raise ValueError(
"item_first is not supported with score_extraction_token_id."
@@ -786,12 +838,12 @@ class TokenizerManagerScoreMixin:
- Generation (CausalLM): Requires label_token_ids; returns logprob-based scores.
- SequenceClassification: label_token_ids is optional; returns pooled class logits.
Setwise scoring (SequenceClassification-only) is expressed via
score_extraction_token_id: when set, the head is pooled AT every occurrence
of this token in each ``query + item`` sequence instead of the last token,
and ``scores`` is returned nested (one ``[Nᵢ x num_labels]`` matrix per
item). With ``--enable-mis`` the items are fused into one multi-item
sequence; otherwise each item is scored independently in one batch.
Setwise scoring is expressed via score_extraction_token_id: when set, the
readout is taken AT every occurrence of this token in each ``query + item``
sequence instead of the last token, and ``scores`` is returned nested (one
``[Nᵢ x num_labels]`` matrix per item). SequenceClassification pools the
head at those positions; CausalLM reads label-token logprobs there. Both
support the batched and fused (``--enable-mis``) paths.
return_pooled_hidden_states is only supported for non-generation models
(SequenceClassification, RewardModel); raises ValueError for CausalLM.
@@ -1016,12 +1068,16 @@ class TokenizerManagerScoreMixin:
input_ids=input_ids,
token_ids_logprob=request_labels,
return_logprob=True,
# Set logprob_start_len=0 for multi-item scoring since we want logprobs at all delimiter positions
logprob_start_len=0 if use_multi_item_scoring else -1,
# logprob_start_len=0 so input-position logprobs are computed:
# multi-item scoring reads them at delimiters, setwise at anchors.
logprob_start_len=(
0 if (use_multi_item_scoring or use_score_extraction) else -1
),
stream=False,
sampling_params={"max_new_tokens": 0},
positional_embed_overrides=positional_embed_overrides,
multi_item_delimiter_indices=mis_delimiter_indices,
token_indices_to_pool=token_indices_to_pool,
)
else:
batch_request = EmbeddingReqInput(
@@ -1037,13 +1093,15 @@ class TokenizerManagerScoreMixin:
if use_multi_item_scoring and use_score_extraction:
# Multi-item setwise: the items were fused into one sequence and the
# head pooled at every anchor. Split the flat [ΣNᵢ, num_labels] result
# back into one matrix per item (nested scores).
# readout taken at every anchor. Split the flat [ΣNᵢ, num_labels]
# result back into one matrix per item (nested scores).
return self._process_multi_item_extraction_results(
results,
per_item_anchor_counts,
apply_softmax,
return_pooled_hidden_states,
label_token_ids=label_token_ids,
temperature=temperature,
return_pooled_hidden_states=return_pooled_hidden_states,
)
elif use_multi_item_scoring:
# Multi-item scoring: extract scores from input_token_ids_logprobs or embedding