[Score API] Setwise Scoring Support (#38965)
This commit is contained in:
@@ -2398,6 +2398,20 @@ def is_cross_encoding_pooler_model(model_architectures: List[str]) -> bool:
|
||||
return any(arch in _cross_encoding_pooler_archs for arch in model_architectures)
|
||||
|
||||
|
||||
# SequenceClassification models whose forward routes the head through
|
||||
# score_and_pool (per-position pooling); only these support setwise readout
|
||||
# (token_indices_to_pool). Keep in sync with callers of layers.pooler.score_and_pool.
|
||||
_score_and_pool_archs = [
|
||||
"LlamaForSequenceClassification",
|
||||
"Qwen2ForSequenceClassification",
|
||||
"Qwen3ForSequenceClassification",
|
||||
]
|
||||
|
||||
|
||||
def is_score_and_pool_model(model_architectures: List[str]) -> bool:
|
||||
return any(arch in _score_and_pool_archs for arch in model_architectures)
|
||||
|
||||
|
||||
def yarn_get_mscale(scale: float = 1, mscale: float = 1) -> float:
|
||||
if scale <= 1:
|
||||
return 1.0
|
||||
|
||||
@@ -37,6 +37,7 @@ class EngineScoreMixin:
|
||||
query_embed_overrides: Optional[List[torch.Tensor]] = None,
|
||||
item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None,
|
||||
return_pooled_hidden_states: bool = False,
|
||||
score_extraction_token_id: Optional[int] = None,
|
||||
temperature: float = 1.0,
|
||||
return_token_logprobs: bool = False,
|
||||
) -> ScoreResult:
|
||||
@@ -53,6 +54,11 @@ class EngineScoreMixin:
|
||||
For SequenceClassification models, returns the pooled class logits directly from
|
||||
the classification head. label_token_ids is optional and ignored.
|
||||
|
||||
Setwise scoring is expressed via ``score_extraction_token_id``: pass a single
|
||||
item containing the whole candidate block with one extraction token per
|
||||
candidate, and the head is pooled AT those positions, so ``scores`` becomes
|
||||
the ``[N x num_labels]`` per-candidate matrix.
|
||||
|
||||
Args:
|
||||
query: The query text or pre-tokenized token IDs.
|
||||
items: The item text(s) or pre-tokenized token IDs.
|
||||
@@ -63,6 +69,11 @@ class EngineScoreMixin:
|
||||
embed_override_token_id: Placeholder token ID used to locate override positions.
|
||||
query_embed_overrides: Embedding vectors replacing placeholder tokens in query.
|
||||
item_embed_overrides: Per-item embedding vectors replacing placeholder tokens in items.
|
||||
score_extraction_token_id: SequenceClassification-only. When set, pool the
|
||||
head at every occurrence of this token per sequence instead of the last
|
||||
token; ``scores`` becomes nested — one ``[Ni x num_labels]`` matrix per
|
||||
item (``len(scores) == len(items)``), where ``Ni`` is the number of
|
||||
extraction tokens (candidates) in item ``i``.
|
||||
return_pooled_hidden_states: Whether to include raw pooled transformer
|
||||
hidden states (before the task head) in the result. Only supported
|
||||
for non-generation models (SequenceClassification, RewardModel).
|
||||
@@ -72,8 +83,10 @@ class EngineScoreMixin:
|
||||
each item's candidates. Only supported for CausalLM, not raw logits.
|
||||
|
||||
Returns:
|
||||
ScoreResult with scores (one list per item), prompt token count, and
|
||||
optional pooled_hidden_states tensors.
|
||||
ScoreResult with scores (flat ``[num_items x num_labels]`` for pointwise
|
||||
scoring, or nested ``[num_items][Ni x num_labels]`` when
|
||||
score_extraction_token_id is set), prompt token count, and optional
|
||||
pooled_hidden_states tensors.
|
||||
"""
|
||||
return self.loop.run_until_complete(
|
||||
self.tokenizer_manager.score_request(
|
||||
@@ -85,6 +98,7 @@ class EngineScoreMixin:
|
||||
embed_override_token_id=embed_override_token_id,
|
||||
query_embed_overrides=query_embed_overrides,
|
||||
item_embed_overrides=item_embed_overrides,
|
||||
score_extraction_token_id=score_extraction_token_id,
|
||||
request=None,
|
||||
return_pooled_hidden_states=return_pooled_hidden_states,
|
||||
temperature=temperature,
|
||||
@@ -103,6 +117,7 @@ class EngineScoreMixin:
|
||||
query_embed_overrides: Optional[List[torch.Tensor]] = None,
|
||||
item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None,
|
||||
return_pooled_hidden_states: bool = False,
|
||||
score_extraction_token_id: Optional[int] = None,
|
||||
temperature: float = 1.0,
|
||||
return_token_logprobs: bool = False,
|
||||
) -> ScoreResult:
|
||||
@@ -116,6 +131,7 @@ class EngineScoreMixin:
|
||||
embed_override_token_id=embed_override_token_id,
|
||||
query_embed_overrides=query_embed_overrides,
|
||||
item_embed_overrides=item_embed_overrides,
|
||||
score_extraction_token_id=score_extraction_token_id,
|
||||
request=None,
|
||||
return_pooled_hidden_states=return_pooled_hidden_states,
|
||||
temperature=temperature,
|
||||
|
||||
@@ -1406,14 +1406,32 @@ class ScoringRequest(BaseModel):
|
||||
return_token_logprobs: bool = False
|
||||
item_first: bool = False
|
||||
return_pooled_hidden_states: bool = False
|
||||
|
||||
# Setwise readout (SequenceClassification-only): when set, the head is pooled
|
||||
# AT every occurrence of this token in each `query + item` sequence instead of
|
||||
# the last token, and `scores` is returned nested (one `[Nᵢ x num_labels]`
|
||||
# matrix per item). --enable-mis fuses items; otherwise each is scored alone.
|
||||
score_extraction_token: Optional[str] = None
|
||||
|
||||
model: str = DEFAULT_MODEL_NAME
|
||||
|
||||
|
||||
class ScoringResponse(BaseModel):
|
||||
scores: List[
|
||||
List[float]
|
||||
] # List of lists of probabilities, each in the order of label_token_ids
|
||||
pooled_hidden_states: Optional[List[Optional[List[float]]]] = None
|
||||
# Pointwise (no `score_extraction_token`): a single [num_rows x num_labels]
|
||||
# matrix. Setwise: nested [num_items][Nᵢ x num_labels]. The nesting depth
|
||||
# disambiguates the two shapes.
|
||||
scores: Union[
|
||||
List[List[float]], # pointwise: [num_rows x num_labels]
|
||||
List[List[List[float]]], # setwise: [num_items][Nᵢ x num_labels]
|
||||
]
|
||||
# Parallel to `scores`: flat [num_rows x hidden] (pointwise) or one such
|
||||
# matrix per item (setwise).
|
||||
pooled_hidden_states: Optional[
|
||||
Union[
|
||||
List[Optional[List[float]]], # pointwise: [num_rows x hidden]
|
||||
List[List[Optional[List[float]]]], # setwise: [num_items][Nᵢ x hidden]
|
||||
]
|
||||
] = None
|
||||
token_logprobs: Optional[List[List[float]]] = None
|
||||
model: str
|
||||
usage: Optional[UsageInfo] = None
|
||||
|
||||
@@ -67,6 +67,16 @@ class OpenAIServingScore(OpenAIServingBase):
|
||||
else None
|
||||
)
|
||||
|
||||
# Resolve the extraction token to an id here (this process owns the
|
||||
# tokenizer); score_request then scans for it and pools the head there.
|
||||
score_extraction_token_id = None
|
||||
if request.score_extraction_token is not None:
|
||||
score_extraction_token_id = (
|
||||
self.tokenizer_manager._resolve_score_extraction_token_id(
|
||||
request.score_extraction_token
|
||||
)
|
||||
)
|
||||
|
||||
result = await self.tokenizer_manager.score_request(
|
||||
query=request.query,
|
||||
items=request.items,
|
||||
@@ -76,18 +86,23 @@ class OpenAIServingScore(OpenAIServingBase):
|
||||
embed_override_token_id=request.embed_override_token_id,
|
||||
query_embed_overrides=query_embed_overrides,
|
||||
item_embed_overrides=item_embed_overrides,
|
||||
score_extraction_token_id=score_extraction_token_id,
|
||||
request=raw_request,
|
||||
return_pooled_hidden_states=request.return_pooled_hidden_states,
|
||||
temperature=request.temperature,
|
||||
return_token_logprobs=request.return_token_logprobs,
|
||||
)
|
||||
|
||||
phs_as_lists = None
|
||||
if result.pooled_hidden_states is not None:
|
||||
phs_as_lists = [
|
||||
t.tolist() if t is not None else None
|
||||
for t in result.pooled_hidden_states
|
||||
]
|
||||
# pooled_hidden_states is flat (pointwise / single-set setwise) or
|
||||
# nested per item (multi-item setwise); convert tensors at any depth.
|
||||
def _tensors_to_lists(value):
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, list):
|
||||
return [_tensors_to_lists(v) for v in value]
|
||||
return value.tolist()
|
||||
|
||||
phs_as_lists = _tensors_to_lists(result.pooled_hidden_states)
|
||||
|
||||
response = ScoringResponse(
|
||||
scores=result.scores,
|
||||
|
||||
@@ -988,8 +988,12 @@ class OutputLogprobProcessor:
|
||||
|
||||
# Handle token_ids logprobs if requested
|
||||
if needs_token_ids_logprobs:
|
||||
# no_copy_to_cpu avoids the .item()/.tolist() CUDA sync that regresses
|
||||
# prefill-only Score throughput at high concurrency.
|
||||
(
|
||||
result.token_ids_logprobs_val,
|
||||
result.token_ids_logprobs_idx,
|
||||
) = get_token_ids_logprobs_batch_optimized(logprobs, token_ids_logprobs)
|
||||
) = get_token_ids_logprobs(
|
||||
logprobs, token_ids_logprobs, no_copy_to_cpu=True
|
||||
)
|
||||
return result
|
||||
|
||||
@@ -35,7 +35,8 @@ class EmbeddingPoolerOutput:
|
||||
pooled_hidden_states: Raw transformer hidden states *before* the
|
||||
task-specific head, present only when
|
||||
``forward_batch.return_pooled_hidden_states`` is True. Tensor
|
||||
(standard path) or list of tensors (MIS path, one per delimiter).
|
||||
(standard path) or list of tensors (one per position — MIS: one per
|
||||
delimiter; setwise / multi-position readout: one per pooled position).
|
||||
"""
|
||||
|
||||
# Pooler can return list[tensor] instead of tensor if the dimension of each tensor in the batch is different
|
||||
@@ -114,6 +115,45 @@ def pool_at_delimiter_positions(
|
||||
return list(data[index_tensor].split(delim_counts))
|
||||
|
||||
|
||||
def pool_at_positions(
|
||||
data: torch.Tensor,
|
||||
per_request_indices: List[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
device: torch.device,
|
||||
) -> List[torch.Tensor]:
|
||||
"""Pool a tensor exactly AT the given token positions for every request.
|
||||
|
||||
Unlike ``pool_at_delimiter_positions`` (which pools at delimiter - 1), this
|
||||
pools at the position itself with no offset shift and no discarded entry.
|
||||
|
||||
Args:
|
||||
data: 2-D tensor [total_tokens, dim] — hidden states or logits.
|
||||
per_request_indices: Per-request anchor positions (CPU tensors), each
|
||||
relative to the start of that request's sequence.
|
||||
forward_batch: Forward batch with extend_seq_lens_cpu populated.
|
||||
device: Device for the index tensor.
|
||||
|
||||
Returns:
|
||||
One tensor per request, shaped [num_anchors, dim].
|
||||
"""
|
||||
all_index_tensors: List[torch.Tensor] = []
|
||||
counts: List[int] = []
|
||||
offset = 0
|
||||
for req_idx, req_seq_len in enumerate(forward_batch.extend_seq_lens_cpu):
|
||||
indices_tensor = per_request_indices[req_idx]
|
||||
n = len(indices_tensor)
|
||||
if n > 0:
|
||||
all_index_tensors.append(indices_tensor + offset)
|
||||
counts.append(n)
|
||||
offset += req_seq_len
|
||||
|
||||
if all_index_tensors:
|
||||
index_tensor = torch.cat(all_index_tensors).to(device, non_blocking=True)
|
||||
else:
|
||||
index_tensor = torch.empty(0, dtype=torch.long, device=device)
|
||||
return list(data[index_tensor].split(counts))
|
||||
|
||||
|
||||
def score_and_pool(
|
||||
score_head: nn.Module,
|
||||
pooler: Pooler,
|
||||
@@ -123,6 +163,10 @@ def score_and_pool(
|
||||
) -> EmbeddingPoolerOutput:
|
||||
"""Apply a classification/score head with MIS and pooled-hidden-states support.
|
||||
|
||||
Multi-position readout path (``token_indices_to_pool`` on forward_batch, e.g.
|
||||
setwise scoring): pool the head AT each caller-specified position (no offset,
|
||||
no discarded row), then split per-request.
|
||||
|
||||
MIS path (pre-computed delimiter indices on forward_batch): extract hidden
|
||||
states at positions just before each delimiter, apply the score head, then
|
||||
split per-request.
|
||||
@@ -132,6 +176,29 @@ def score_and_pool(
|
||||
When ``forward_batch.return_pooled_hidden_states`` is True, the raw pooled
|
||||
hidden states (before the score head) are included in the output.
|
||||
"""
|
||||
if (
|
||||
forward_batch.token_indices_to_pool is not None
|
||||
and forward_batch.is_prefill_only
|
||||
):
|
||||
# Setwise readout: pool the head AT each requested position (no -1 shift,
|
||||
# no discarded boundary row). Concat to call the head once, then split.
|
||||
per_request_phs = pool_at_positions(
|
||||
hidden_states,
|
||||
forward_batch.token_indices_to_pool,
|
||||
forward_batch,
|
||||
input_ids.device,
|
||||
)
|
||||
phs_flat = torch.cat(per_request_phs, dim=0)
|
||||
scores_flat = score_head(phs_flat)
|
||||
pos_counts = [t.shape[0] for t in per_request_phs]
|
||||
per_request_scores = list(scores_flat.split(pos_counts))
|
||||
return EmbeddingPoolerOutput(
|
||||
embeddings=per_request_scores,
|
||||
pooled_hidden_states=(
|
||||
per_request_phs if forward_batch.return_pooled_hidden_states else None
|
||||
),
|
||||
)
|
||||
|
||||
if (
|
||||
forward_batch.multi_item_delimiter_indices is not None
|
||||
and forward_batch.is_prefill_only
|
||||
|
||||
@@ -1233,6 +1233,12 @@ class EmbeddingReqInput:
|
||||
# Batch-level: List[List[int]] (one per request). After __getitem__: List[int].
|
||||
multi_item_delimiter_indices: Optional[Union[List[List[int]], List[int]]] = None
|
||||
|
||||
# Token positions the pooler reads out AT (head / hidden states), instead of
|
||||
# the default LAST/CLS pooling. Generic multi-position primitive (setwise
|
||||
# scoring today). Unlike MIS, pooling is AT the position (no delimiter - 1).
|
||||
# Batch-level: List[List[int]] (one per request). After __getitem__: List[int].
|
||||
token_indices_to_pool: Optional[Union[List[List[int]], List[int]]] = None
|
||||
|
||||
def regenerate_rid(self):
|
||||
"""Generate a new request ID and return it."""
|
||||
if isinstance(self.rid, list):
|
||||
@@ -1359,6 +1365,11 @@ class EmbeddingReqInput:
|
||||
if self.multi_item_delimiter_indices is not None
|
||||
else None
|
||||
),
|
||||
token_indices_to_pool=(
|
||||
self.token_indices_to_pool[i]
|
||||
if self.token_indices_to_pool is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
else:
|
||||
sub = EmbeddingReqInput(
|
||||
@@ -1390,6 +1401,11 @@ class EmbeddingReqInput:
|
||||
if self.multi_item_delimiter_indices is not None
|
||||
else None
|
||||
),
|
||||
token_indices_to_pool=(
|
||||
self.token_indices_to_pool[i]
|
||||
if self.token_indices_to_pool is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
cache[i] = sub
|
||||
return sub
|
||||
@@ -1419,7 +1435,8 @@ class TokenizedEmbeddingReqInput(BaseReq, kw_only=True):
|
||||
return_pooled_hidden_states: bool = False
|
||||
# Pre-computed delimiter indices for multi-item scoring
|
||||
multi_item_delimiter_indices: Optional[List[int]] = None
|
||||
|
||||
# Token positions for setwise pooling readout
|
||||
token_indices_to_pool: Optional[List[int]] = None
|
||||
# For observability
|
||||
# Pickled Optional[Union[APIServerReqTimeStats, DPControllerReqTimeStats]]
|
||||
time_stats: Optional[PickleWrapper] = None
|
||||
|
||||
@@ -1025,6 +1025,7 @@ class Req(ReqDllmMixin):
|
||||
] = None,
|
||||
return_pooled_hidden_states: bool = False,
|
||||
multi_item_delimiter_indices: Optional[List[int]] = None,
|
||||
token_indices_to_pool: Optional[List[int]] = None,
|
||||
session_id: Optional[str] = None,
|
||||
cache_salt: Optional[str] = None,
|
||||
):
|
||||
@@ -1056,6 +1057,7 @@ class Req(ReqDllmMixin):
|
||||
self.input_embeds = input_embeds
|
||||
self.positional_embed_overrides = positional_embed_overrides
|
||||
self.multi_item_delimiter_indices = multi_item_delimiter_indices
|
||||
self.token_indices_to_pool = token_indices_to_pool
|
||||
|
||||
# For req-level memory management
|
||||
self.kv = ReqKvInfo()
|
||||
|
||||
@@ -3431,6 +3431,7 @@ class Scheduler(
|
||||
time_stats=recv_req.time_stats,
|
||||
return_pooled_hidden_states=recv_req.return_pooled_hidden_states,
|
||||
multi_item_delimiter_indices=recv_req.multi_item_delimiter_indices,
|
||||
token_indices_to_pool=recv_req.token_indices_to_pool,
|
||||
)
|
||||
req.tokenizer = self.tokenizer
|
||||
self._maybe_namespace_elastic_radix_cache(req)
|
||||
@@ -3934,6 +3935,15 @@ class Scheduler(
|
||||
if self.enable_lora and not self.can_schedule_lora_req(req, running_loras):
|
||||
continue
|
||||
|
||||
# A forward batch runs one pooling mode, so setwise readout requests
|
||||
# (token_indices_to_pool) cannot share a batch with last-token ones.
|
||||
# Admit only the first request's mode; the other stays queued.
|
||||
if adder.can_run_list and (
|
||||
(req.token_indices_to_pool is not None)
|
||||
!= (adder.can_run_list[0].token_indices_to_pool is not None)
|
||||
):
|
||||
continue
|
||||
|
||||
running_bs = len(running_batch.reqs)
|
||||
candidate_beam_width = (
|
||||
req.beam_group.beam_width if req.beam_group is not None else None
|
||||
|
||||
@@ -1561,6 +1561,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
http_worker_ipc=obj.http_worker_ipc,
|
||||
return_pooled_hidden_states=obj.return_pooled_hidden_states,
|
||||
multi_item_delimiter_indices=obj.multi_item_delimiter_indices,
|
||||
token_indices_to_pool=obj.token_indices_to_pool,
|
||||
)
|
||||
|
||||
tokenized_obj.time_stats = self.rid_to_state[obj.rid].time_stats
|
||||
|
||||
@@ -1,29 +1,41 @@
|
||||
import logging
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
from typing import Any, Dict, List, Optional, Tuple, TypeAlias, Union
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.configs.model_config import is_cross_encoding_pooler_model
|
||||
from sglang.srt.configs.model_config import (
|
||||
is_cross_encoding_pooler_model,
|
||||
is_score_and_pool_model,
|
||||
)
|
||||
from sglang.srt.constants import MIS_DELIMITER_TOKEN_ID
|
||||
from sglang.srt.managers.embed_types import PositionalEmbeds
|
||||
from sglang.srt.managers.io_struct import EmbeddingReqInput, GenerateReqInput
|
||||
from sglang.srt.runtime_context import get_exec
|
||||
from sglang.srt.runtime_context import get_exec, get_memory, get_schedule, get_serving
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
ScoreRow: TypeAlias = List[float]
|
||||
ScoreMatrix: TypeAlias = List[ScoreRow]
|
||||
# Pointwise: [num_items, num_labels]. Setwise (multi-position): one
|
||||
# [N_i, num_labels] matrix per item.
|
||||
ScoreOutput: TypeAlias = Union[ScoreMatrix, List[ScoreMatrix]]
|
||||
|
||||
PooledHiddenStateRows: TypeAlias = List[Optional[torch.Tensor]]
|
||||
PooledHiddenStateOutput: TypeAlias = Union[
|
||||
PooledHiddenStateRows, List[Optional[List[torch.Tensor]]]
|
||||
]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ScoreResult:
|
||||
scores: List[List[float]]
|
||||
scores: ScoreOutput
|
||||
prompt_tokens: int = 0
|
||||
# Per-item pooled hidden states (pre-head transformer output).
|
||||
# CPU tensors when return_pooled_hidden_states=True; kept as tensors so
|
||||
# in-process consumers (gRPC, engine API) avoid a .tolist() round-trip.
|
||||
# The HTTP path converts to lists in serving_score.py before JSON serialization.
|
||||
# Same layout as scores: one tensor per item (not a single packed 2D tensor).
|
||||
pooled_hidden_states: Optional[List[Optional[torch.Tensor]]] = None
|
||||
# Pre-head CPU tensors parallel to scores; kept as tensors so in-process
|
||||
# consumers avoid a .tolist() round-trip (the HTTP path converts before JSON).
|
||||
pooled_hidden_states: Optional[PooledHiddenStateOutput] = None
|
||||
# uncalibrated full-vocabulary logprobs, in each item's candidate order
|
||||
token_logprobs: Optional[List[List[float]]] = None
|
||||
|
||||
@@ -197,7 +209,7 @@ class TokenizerManagerScoreMixin:
|
||||
if return_pooled_hidden_states:
|
||||
raw_phs = single_result.get("pooled_hidden_state")
|
||||
if raw_phs is not None and len(raw_phs) == expected_count:
|
||||
phs_list = raw_phs[1:]
|
||||
phs_list = list(raw_phs[1:])
|
||||
|
||||
return ScoreResult(
|
||||
scores=scores,
|
||||
@@ -206,21 +218,110 @@ class TokenizerManagerScoreMixin:
|
||||
token_logprobs=token_logprobs,
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Multi-position pooling result helpers (setwise scoring): the head is read
|
||||
# at every score-extraction token instead of the last token. Shared by the
|
||||
# batched and fused (--enable-mis) result processors.
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _multi_position_score_rows(
|
||||
self, embedding: Any, apply_softmax: bool
|
||||
) -> List[List[float]]:
|
||||
"""Validate a 2-D multi-position result embedding and return its per-token rows."""
|
||||
embedding_tensor = torch.as_tensor(embedding)
|
||||
if embedding_tensor.ndim != 2:
|
||||
# A 1-D result means the model pooled a single vector (reward/embedding
|
||||
# head) instead of per-position scores; ValueError -> clean 400.
|
||||
raise ValueError(
|
||||
"Multi-position scoring expected a 2-D "
|
||||
"[num_positions, num_labels] result, but got shape "
|
||||
f"{tuple(embedding_tensor.shape)}. This model may not support "
|
||||
"score_extraction_token_id (only SequenceClassification heads that "
|
||||
"pool per position do)."
|
||||
)
|
||||
if apply_softmax:
|
||||
return torch.softmax(embedding_tensor, dim=-1).tolist()
|
||||
return embedding if isinstance(embedding, list) else embedding_tensor.tolist()
|
||||
|
||||
def _multi_position_phs_matrix(self, phs: Any, expected_rows: int) -> torch.Tensor:
|
||||
"""Validate multi-position pooled hidden states are 2-D with one row per position."""
|
||||
phs_tensor = torch.as_tensor(phs)
|
||||
if phs_tensor.ndim != 2 or phs_tensor.shape[0] != expected_rows:
|
||||
raise ValueError(
|
||||
"Multi-position pooled hidden states must be a 2-D tensor with "
|
||||
"one row per score position."
|
||||
)
|
||||
return phs_tensor
|
||||
|
||||
def _process_multi_item_extraction_results(
|
||||
self,
|
||||
results: Any,
|
||||
per_item_anchor_counts: List[int],
|
||||
apply_softmax: bool,
|
||||
return_pooled_hidden_states: bool = False,
|
||||
) -> ScoreResult:
|
||||
"""Process a fused multi-item score-extraction request (``--enable-mis``).
|
||||
|
||||
The fused sequence returns one ``[ΣNᵢ, num_labels]`` matrix of every
|
||||
anchor's logits in item order; split it back into one ``[Nᵢ, num_labels]``
|
||||
matrix per item using per_item_anchor_counts.
|
||||
"""
|
||||
single_result = results[0] if isinstance(results, list) else results
|
||||
meta_info = single_result.get("meta_info", {})
|
||||
request_id = meta_info.get("id", "<unknown>")
|
||||
prompt_tokens = meta_info.get("prompt_tokens", 0)
|
||||
|
||||
embedding = single_result.get("embedding")
|
||||
if embedding is None:
|
||||
raise ValueError("Embedding not found in the result.")
|
||||
|
||||
rows = self._multi_position_score_rows(embedding, apply_softmax)
|
||||
total_anchors = sum(per_item_anchor_counts)
|
||||
if len(rows) != total_anchors:
|
||||
raise RuntimeError(
|
||||
f"Expected {total_anchors} anchor rows across "
|
||||
f"{len(per_item_anchor_counts)} items, but got {len(rows)}. "
|
||||
f"Request ID: {request_id}"
|
||||
)
|
||||
|
||||
scores = []
|
||||
offset = 0
|
||||
for count in per_item_anchor_counts:
|
||||
scores.append(rows[offset : offset + count])
|
||||
offset += count
|
||||
|
||||
phs_grouped = None
|
||||
if return_pooled_hidden_states:
|
||||
raw_phs = single_result.get("pooled_hidden_state")
|
||||
if raw_phs is not None:
|
||||
phs_matrix = self._multi_position_phs_matrix(raw_phs, total_anchors)
|
||||
phs_grouped = [
|
||||
list(group) for group in phs_matrix.split(per_item_anchor_counts)
|
||||
]
|
||||
|
||||
return ScoreResult(
|
||||
scores=scores,
|
||||
prompt_tokens=prompt_tokens,
|
||||
pooled_hidden_states=phs_grouped,
|
||||
)
|
||||
|
||||
def _process_single_item_scoring_results(
|
||||
self,
|
||||
results: Any,
|
||||
label_token_ids: Optional[List[List[int]]],
|
||||
apply_softmax: bool,
|
||||
return_pooled_hidden_states: bool = False,
|
||||
per_item_matrix: bool = False,
|
||||
temperature: float = 1.0,
|
||||
return_token_logprobs: bool = False,
|
||||
) -> ScoreResult:
|
||||
"""
|
||||
Process results from single-item scoring request.
|
||||
|
||||
For generation (CausalLM) models: reads output_token_ids_logprobs.
|
||||
For non-generation (SequenceClassification) models: reads the embedding field
|
||||
which contains pooled class logits from the classification head.
|
||||
CausalLM models read output_token_ids_logprobs; SequenceClassification
|
||||
models read the embedding field (pooled class logits from the head).
|
||||
When per_item_matrix (setwise), each result's 2-D matrix becomes one
|
||||
item's entry so ``scores`` is nested; otherwise each result is one row.
|
||||
"""
|
||||
scores = []
|
||||
phs_list = []
|
||||
@@ -259,24 +360,29 @@ class TokenizerManagerScoreMixin:
|
||||
|
||||
prompt_tokens += result.get("meta_info", {}).get("prompt_tokens", 0)
|
||||
|
||||
if apply_softmax:
|
||||
scores_tensor = torch.as_tensor(embedding, dtype=torch.float64)
|
||||
embedding = torch.softmax(
|
||||
(scores_tensor - scores_tensor.max()) / temperature, dim=-1
|
||||
).tolist()
|
||||
|
||||
# The classification head produces per-token logits, which the pooler reduces
|
||||
# into a single vector per input. That vector is returned in the `.embeddings`
|
||||
# field — not as semantic embeddings, but as pooled classification logits.
|
||||
# The field name is reused for compatibility with the existing
|
||||
# EmbeddingPoolerOutput API.
|
||||
scores.append(embedding)
|
||||
if per_item_matrix:
|
||||
rows = self._multi_position_score_rows(embedding, apply_softmax)
|
||||
scores.append(rows)
|
||||
else:
|
||||
if apply_softmax:
|
||||
scores_tensor = torch.as_tensor(embedding, dtype=torch.float64)
|
||||
embedding = torch.softmax(
|
||||
(scores_tensor - scores_tensor.max()) / temperature, dim=-1
|
||||
).tolist()
|
||||
# Pooled classification logits, reusing the EmbeddingPoolerOutput field.
|
||||
scores.append(embedding)
|
||||
|
||||
if return_pooled_hidden_states:
|
||||
phs = result.get("pooled_hidden_state")
|
||||
phs_list.append(phs)
|
||||
if phs is not None:
|
||||
if per_item_matrix and phs is not None:
|
||||
phs_list.append(
|
||||
list(self._multi_position_phs_matrix(phs, len(rows)))
|
||||
)
|
||||
has_phs = True
|
||||
else:
|
||||
phs_list.append(phs)
|
||||
if phs is not None:
|
||||
has_phs = True
|
||||
|
||||
return ScoreResult(
|
||||
scores=scores,
|
||||
@@ -458,6 +564,190 @@ class TokenizerManagerScoreMixin:
|
||||
None,
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Multi-position pooling readout (setwise scoring)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _resolve_score_extraction_token_id(self, token: str) -> int:
|
||||
"""Resolve the score-extraction token string to a dedicated token id.
|
||||
|
||||
Runs in the tokenizer-manager process; used by the HTTP score handler to
|
||||
turn a client token string into the id ``score_request`` scans for.
|
||||
"""
|
||||
if self.tokenizer is None:
|
||||
raise ValueError(
|
||||
"A tokenizer is required to resolve score_extraction_token."
|
||||
)
|
||||
token_id = self.tokenizer.convert_tokens_to_ids(token)
|
||||
unk_id = self.tokenizer.unk_token_id
|
||||
if token_id is None or (unk_id is not None and token_id == unk_id):
|
||||
raise ValueError(
|
||||
f"score_extraction_token {token!r} did not resolve to a dedicated "
|
||||
f"token id."
|
||||
)
|
||||
return token_id
|
||||
|
||||
def _validate_score_extraction(
|
||||
self,
|
||||
is_generation: bool,
|
||||
item_first: bool,
|
||||
) -> None:
|
||||
"""Validate that multi-position pooling readout is applicable.
|
||||
|
||||
Supported only by SequenceClassification models whose forward routes the
|
||||
head through ``score_and_pool`` (per-position pooling, in
|
||||
``is_score_and_pool_model``). Generation, cross-encoder, reward, and
|
||||
embedding models pool a single vector and are rejected here before
|
||||
inference. Requires radix cache and chunked prefill off and no auto-truncate,
|
||||
since pooling positions are full-prompt coordinates.
|
||||
"""
|
||||
if is_generation:
|
||||
raise ValueError(
|
||||
"score_extraction_token_id is only supported for "
|
||||
"SequenceClassification models, not generation (CausalLM) models."
|
||||
)
|
||||
architectures = self.model_config.hf_config.architectures or []
|
||||
if not is_score_and_pool_model(architectures):
|
||||
raise ValueError(
|
||||
"score_extraction_token_id is only supported for "
|
||||
"SequenceClassification models that pool the head per position "
|
||||
"(score_and_pool); model architecture(s) "
|
||||
f"{architectures} do not (cross-encoder, reward, and embedding "
|
||||
"models pool a single vector and ignore the readout positions)."
|
||||
)
|
||||
if item_first:
|
||||
raise ValueError(
|
||||
"item_first is not supported with score_extraction_token_id."
|
||||
)
|
||||
if not get_memory().disable_radix_cache:
|
||||
raise ValueError(
|
||||
"score_extraction_token_id requires --disable-radix-cache because "
|
||||
"pooling positions are relative to the full prompt."
|
||||
)
|
||||
if get_schedule().chunked_prefill_size != -1:
|
||||
raise ValueError(
|
||||
"score_extraction_token_id requires --chunked-prefill-size -1 "
|
||||
"because all pooling positions must be processed in one prefill."
|
||||
)
|
||||
if get_serving().allow_auto_truncate:
|
||||
raise ValueError(
|
||||
"score_extraction_token_id is not supported with "
|
||||
"--allow-auto-truncate: pooling positions are computed from the "
|
||||
"full prompt, and truncating the sequence would leave them "
|
||||
"pointing past the truncated length."
|
||||
)
|
||||
|
||||
def _reject_query_prefix_anchor(
|
||||
self,
|
||||
query_token_ids: List[int],
|
||||
score_extraction_token_id: int,
|
||||
) -> None:
|
||||
"""Reject a shared query prefix that itself contains the extraction token.
|
||||
|
||||
The non-MIS path scans the whole ``query + item`` sequence, so a query-side
|
||||
hit would emit an extra, misaligned score row instead of one per candidate.
|
||||
"""
|
||||
if score_extraction_token_id in query_token_ids:
|
||||
raise ValueError(
|
||||
"score-extraction token found in the shared query prefix; it must "
|
||||
"appear only within items, one per candidate."
|
||||
)
|
||||
|
||||
def _build_score_extraction_text_inputs(
|
||||
self,
|
||||
query: str,
|
||||
items: Union[str, List[str]],
|
||||
score_extraction_token_id: int,
|
||||
) -> List[List[int]]:
|
||||
"""Tokenize each complete query-item prompt once for extraction scanning."""
|
||||
items_list = [items] if isinstance(items, str) else items
|
||||
if not all(isinstance(item, str) for item in items_list):
|
||||
raise ValueError(
|
||||
"query and items must both be text or both be token IDs when "
|
||||
"score_extraction_token_id is set."
|
||||
)
|
||||
if query:
|
||||
self._reject_query_prefix_anchor(
|
||||
self.tokenizer.encode(query), score_extraction_token_id
|
||||
)
|
||||
return [self.tokenizer.encode(f"{query}{item}") for item in items_list]
|
||||
|
||||
def _resolve_score_extraction_indices(
|
||||
self,
|
||||
input_ids: List[List[int]],
|
||||
score_extraction_token_id: int,
|
||||
) -> List[List[int]]:
|
||||
"""Scan each built sequence for the score-extraction token.
|
||||
|
||||
The head is later pooled AT these positions (via ``token_indices_to_pool``)
|
||||
instead of the last token. Raises ValueError if any sequence has no token,
|
||||
which would silently drop that sequence's scores.
|
||||
"""
|
||||
token_indices_to_pool = [
|
||||
[i for i, t in enumerate(seq) if t == score_extraction_token_id]
|
||||
for seq in input_ids
|
||||
]
|
||||
if any(not seq_indices for seq_indices in token_indices_to_pool):
|
||||
raise ValueError(
|
||||
f"No score-extraction token (id={score_extraction_token_id}) "
|
||||
"found in one or more input(s); the client must render one per "
|
||||
"candidate."
|
||||
)
|
||||
return token_indices_to_pool
|
||||
|
||||
def _anchor_counts_per_item(
|
||||
self,
|
||||
delimiter_indices: List[int],
|
||||
anchor_positions: List[int],
|
||||
) -> List[int]:
|
||||
"""Count extraction-token anchors falling inside each fused item's block.
|
||||
|
||||
``--enable-mis`` fuses items as ``query <delim> item0 <delim> item1 …``, so
|
||||
item ``i`` spans ``(delimiter_indices[i], delimiter_indices[i + 1])``.
|
||||
Raises ValueError (clean 400 before inference) if an item has no anchor or
|
||||
an anchor falls outside every item block.
|
||||
"""
|
||||
counts = [
|
||||
sum(1 for p in anchor_positions if lo < p < hi)
|
||||
for lo, hi in zip(delimiter_indices, delimiter_indices[1:])
|
||||
]
|
||||
# An anchor in the shared query prefix is not attributable to any item and
|
||||
# would leave the pooled forward with more rows than sum(counts); reject it.
|
||||
if sum(counts) != len(anchor_positions):
|
||||
raise ValueError(
|
||||
"score-extraction token found outside the candidate items (e.g. in "
|
||||
"the shared query prefix) under --enable-mis multi-item scoring; it "
|
||||
"must appear only within items, one per candidate."
|
||||
)
|
||||
if any(c == 0 for c in counts):
|
||||
raise ValueError(
|
||||
"Each item (candidate set) must contain at least one "
|
||||
"score-extraction token under --enable-mis multi-item scoring."
|
||||
)
|
||||
return counts
|
||||
|
||||
def _resolve_multi_position_pooling(
|
||||
self,
|
||||
input_ids: List[List[int]],
|
||||
score_extraction_token_id: int,
|
||||
use_multi_item_scoring: bool,
|
||||
delimiter_indices: Optional[List[int]],
|
||||
) -> Tuple[List[List[int]], Optional[List[int]]]:
|
||||
"""Resolve where the head is pooled for a score-extraction request.
|
||||
|
||||
Returns ``(token_indices_to_pool, per_item_anchor_counts)``; the latter is
|
||||
None outside ``--enable-mis`` and otherwise buckets anchors back per item.
|
||||
"""
|
||||
token_indices_to_pool = self._resolve_score_extraction_indices(
|
||||
input_ids, score_extraction_token_id
|
||||
)
|
||||
per_item_anchor_counts = None
|
||||
if use_multi_item_scoring:
|
||||
per_item_anchor_counts = self._anchor_counts_per_item(
|
||||
delimiter_indices, token_indices_to_pool[0]
|
||||
)
|
||||
return token_indices_to_pool, per_item_anchor_counts
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Main entry point
|
||||
# ------------------------------------------------------------------
|
||||
@@ -474,6 +764,7 @@ class TokenizerManagerScoreMixin:
|
||||
item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None,
|
||||
request: Optional[Any] = None,
|
||||
return_pooled_hidden_states: bool = False,
|
||||
score_extraction_token_id: Optional[int] = None,
|
||||
temperature: float = 1.0,
|
||||
return_token_logprobs: bool = False,
|
||||
) -> ScoreResult:
|
||||
@@ -495,6 +786,13 @@ class TokenizerManagerScoreMixin:
|
||||
- Generation (CausalLM): Requires label_token_ids; returns logprob-based scores.
|
||||
- SequenceClassification: label_token_ids is optional; returns pooled class logits.
|
||||
|
||||
Setwise scoring (SequenceClassification-only) is expressed via
|
||||
score_extraction_token_id: when set, the head is pooled AT every occurrence
|
||||
of this token in each ``query + item`` sequence instead of the last token,
|
||||
and ``scores`` is returned nested (one ``[Nᵢ x num_labels]`` matrix per
|
||||
item). With ``--enable-mis`` the items are fused into one multi-item
|
||||
sequence; otherwise each item is scored independently in one batch.
|
||||
|
||||
return_pooled_hidden_states is only supported for non-generation models
|
||||
(SequenceClassification, RewardModel); raises ValueError for CausalLM.
|
||||
"""
|
||||
@@ -548,6 +846,12 @@ class TokenizerManagerScoreMixin:
|
||||
f"Token ID {token_id} is out of vocabulary (vocab size: {vocab_size})"
|
||||
)
|
||||
|
||||
# Normalize an omitted query by item type so path selection sees a concrete
|
||||
# prefix: [] for pre-tokenized items (token-ID path), "" for text items.
|
||||
if query is None:
|
||||
items_are_token_ids = isinstance(items, list) and isinstance(items[0], list)
|
||||
query = [] if items_are_token_ids else ""
|
||||
|
||||
has_embeds = (
|
||||
query_embed_overrides is not None or item_embed_overrides is not None
|
||||
)
|
||||
@@ -566,12 +870,19 @@ class TokenizerManagerScoreMixin:
|
||||
# Check if multi-item scoring is enabled
|
||||
use_multi_item_scoring = get_exec().features.enable_mis
|
||||
|
||||
# Setwise readout: pool the head at every score_extraction_token_id.
|
||||
use_score_extraction = score_extraction_token_id is not None
|
||||
if use_score_extraction:
|
||||
self._validate_score_extraction(is_generation, item_first)
|
||||
|
||||
input_ids = None
|
||||
text_prompts = None
|
||||
positional_embed_overrides = None
|
||||
delimiter_indices = None
|
||||
|
||||
use_text_prompts = isinstance(query, str) and not has_embeds
|
||||
use_text_prompts = (
|
||||
isinstance(query, str) and not has_embeds and not use_score_extraction
|
||||
)
|
||||
|
||||
if use_text_prompts:
|
||||
# Both query and items are text
|
||||
@@ -605,6 +916,10 @@ class TokenizerManagerScoreMixin:
|
||||
):
|
||||
# Both query and items are token IDs — tokenize text inputs if needed for embed overrides
|
||||
query_ids, items_ids = query, items
|
||||
if use_score_extraction and not use_multi_item_scoring:
|
||||
# Non-MIS concatenates query + item, so a query-prefix anchor would
|
||||
# emit a misaligned row; reject it (MIS confines anchors per item).
|
||||
self._reject_query_prefix_anchor(query_ids, score_extraction_token_id)
|
||||
_, input_ids, positional_embed_overrides, delimiter_indices = (
|
||||
self._build_token_id_inputs(
|
||||
query_ids,
|
||||
@@ -616,9 +931,25 @@ class TokenizerManagerScoreMixin:
|
||||
item_embed_overrides,
|
||||
)
|
||||
)
|
||||
elif has_embeds:
|
||||
# Text inputs with embed overrides — need to tokenize first to resolve positions
|
||||
query_ids, items_ids = self._batch_tokenize_query_and_items(query, items)
|
||||
elif (
|
||||
use_score_extraction
|
||||
and not use_multi_item_scoring
|
||||
and not has_embeds
|
||||
and (query is None or isinstance(query, str))
|
||||
):
|
||||
# Single-set setwise: text (or omitted) query + text items, one
|
||||
# sequence per item. --enable-mis instead fuses the items below.
|
||||
input_ids = self._build_score_extraction_text_inputs(
|
||||
query or "", items, score_extraction_token_id
|
||||
)
|
||||
elif has_embeds or use_score_extraction:
|
||||
# Tokenize text inputs to token IDs, then build via the token-id path
|
||||
# (embed overrides need positions; score extraction needs token IDs).
|
||||
query_ids, items_ids = self._batch_tokenize_query_and_items(
|
||||
query or "" if use_multi_item_scoring else query, items
|
||||
)
|
||||
if use_score_extraction and not use_multi_item_scoring:
|
||||
self._reject_query_prefix_anchor(query_ids, score_extraction_token_id)
|
||||
_, input_ids, positional_embed_overrides, delimiter_indices = (
|
||||
self._build_token_id_inputs(
|
||||
query_ids,
|
||||
@@ -635,6 +966,21 @@ class TokenizerManagerScoreMixin:
|
||||
"Invalid combination of query/items types for score_request."
|
||||
)
|
||||
|
||||
# Setwise readout: scan each sequence for the extraction token (positions
|
||||
# the pooler reads the head at); under --enable-mis also return per-item
|
||||
# anchor counts to split the fused score matrix back per item.
|
||||
token_indices_to_pool = None
|
||||
per_item_anchor_counts = None
|
||||
if use_score_extraction:
|
||||
token_indices_to_pool, per_item_anchor_counts = (
|
||||
self._resolve_multi_position_pooling(
|
||||
input_ids,
|
||||
score_extraction_token_id,
|
||||
use_multi_item_scoring,
|
||||
delimiter_indices,
|
||||
)
|
||||
)
|
||||
|
||||
if return_pooled_hidden_states:
|
||||
if is_generation:
|
||||
raise ValueError(
|
||||
@@ -684,11 +1030,22 @@ class TokenizerManagerScoreMixin:
|
||||
positional_embed_overrides=positional_embed_overrides,
|
||||
return_pooled_hidden_states=return_pooled_hidden_states,
|
||||
multi_item_delimiter_indices=mis_delimiter_indices,
|
||||
token_indices_to_pool=token_indices_to_pool,
|
||||
)
|
||||
|
||||
results = await self.generate_request(batch_request, request).__anext__()
|
||||
|
||||
if use_multi_item_scoring:
|
||||
if use_multi_item_scoring and use_score_extraction:
|
||||
# Multi-item setwise: the items were fused into one sequence and the
|
||||
# head pooled at every anchor. Split the flat [ΣNᵢ, num_labels] result
|
||||
# back into one matrix per item (nested scores).
|
||||
return self._process_multi_item_extraction_results(
|
||||
results,
|
||||
per_item_anchor_counts,
|
||||
apply_softmax,
|
||||
return_pooled_hidden_states,
|
||||
)
|
||||
elif use_multi_item_scoring:
|
||||
# Multi-item scoring: extract scores from input_token_ids_logprobs or embedding
|
||||
return self._process_multi_item_scoring_results(
|
||||
results,
|
||||
@@ -701,14 +1058,18 @@ class TokenizerManagerScoreMixin:
|
||||
return_token_logprobs,
|
||||
)
|
||||
else:
|
||||
# Single-item scoring: process each result separately
|
||||
# Single-item scoring: process each result separately. With
|
||||
# score_extraction_token_id, each item is scored as an independent
|
||||
# query+item sequence and its 2-D embedding ([positions x num_labels])
|
||||
# is kept as one score matrix per item (nested scores).
|
||||
return self._process_single_item_scoring_results(
|
||||
results,
|
||||
label_token_ids,
|
||||
apply_softmax,
|
||||
return_pooled_hidden_states,
|
||||
temperature,
|
||||
return_token_logprobs,
|
||||
per_item_matrix=use_score_extraction,
|
||||
temperature=temperature,
|
||||
return_token_logprobs=return_token_logprobs,
|
||||
)
|
||||
|
||||
def _convert_logprobs_to_scores(
|
||||
|
||||
@@ -571,6 +571,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
# Pre-computed delimiter indices for multi-item scoring (CPU tensors, one per request)
|
||||
multi_item_delimiter_indices: Optional[List[torch.Tensor]] = None
|
||||
|
||||
# Setwise pooling readout positions (CPU tensors, one per request)
|
||||
token_indices_to_pool: Optional[List[torch.Tensor]] = None
|
||||
|
||||
# === Borrowed from ScheduleBatch: compound (carry their own device tensors) ===
|
||||
# Sampling info
|
||||
sampling_info: SamplingBatchInfo = None
|
||||
@@ -1158,6 +1161,18 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
for r in batch.reqs
|
||||
]
|
||||
|
||||
# Setwise readout: pool the head AT token_indices_to_pool instead of
|
||||
# the last token. The scheduler keeps readout batches homogeneous
|
||||
# (get_new_batch_prefill), so build only when every request carries
|
||||
# the field and otherwise fall back to standard pooling.
|
||||
if batch.reqs and all(
|
||||
r.token_indices_to_pool is not None for r in batch.reqs
|
||||
):
|
||||
self.token_indices_to_pool = [
|
||||
torch.tensor(r.token_indices_to_pool, dtype=torch.int64)
|
||||
for r in batch.reqs
|
||||
]
|
||||
|
||||
token_type_ids = [
|
||||
r.token_type_ids for r in batch.reqs if r.token_type_ids is not None
|
||||
]
|
||||
|
||||
@@ -1900,6 +1900,7 @@ class ModelRunner:
|
||||
and not isinstance(self.prefill_cuda_graph_runner, EagerRunner)
|
||||
and self.prefill_cuda_graph_runner is not None
|
||||
and self.prefill_cuda_graph_runner.can_run_graph(forward_batch)
|
||||
and forward_batch.token_indices_to_pool is None
|
||||
and _prefill_cuda_graph_allows_context_parallel(
|
||||
self.prefill_cuda_graph_runner, forward_batch
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user