[Score API] Setwise Scoring Support (#38965)
This commit is contained in:
@@ -0,0 +1,271 @@
|
||||
"""Engine benchmark for setwise (multi-position) scoring.
|
||||
|
||||
Benchmarks `Engine.async_score(..., score_extraction_token_id=...)` — the setwise
|
||||
path that pools the classification head at each candidate's anchor token and
|
||||
returns an ``[N, num_labels]`` matrix in a single forward pass.
|
||||
|
||||
Unlike the general ``bench_score.py`` (label-token / MIS scoring), this is a
|
||||
focused, self-contained harness: it sweeps the **set size** (number of
|
||||
candidates packed into one item) and reports latency percentiles and throughput
|
||||
(requests/s and candidates/s) so you can see how a single-forward setwise pass
|
||||
scales with N.
|
||||
|
||||
It talks to the Engine directly (no HTTP) and requires the setwise deployment
|
||||
constraints (``--disable-radix-cache``, ``--chunked-prefill-size -1``), which it
|
||||
sets automatically.
|
||||
|
||||
Usage:
|
||||
# Default sweep over set sizes on the target model
|
||||
python bench_setwise_score.py --model-path /path/to/seqcls_model
|
||||
|
||||
# Custom sweep + concurrency + per-candidate length
|
||||
python bench_setwise_score.py --model-path /path/to/model \
|
||||
--set-sizes 8 16 32 64 --concurrency 16 --num-requests 100 \
|
||||
--item-tokens 180 --query-tokens 64
|
||||
|
||||
# Include pooled hidden states in the response (per-anchor pre-head vectors)
|
||||
python bench_setwise_score.py --model-path /path/to/model --return-pooled-hidden-states
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import csv
|
||||
import random
|
||||
import statistics
|
||||
import time
|
||||
from typing import List, Optional
|
||||
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Benchmark setwise scoring via Engine.async_score",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model-path",
|
||||
type=str,
|
||||
required=True,
|
||||
help="Path to a SequenceClassification model (e.g. Qwen3ForSequenceClassification).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--score-extraction-token",
|
||||
type=str,
|
||||
default="<|object_ref_start|>",
|
||||
help="Anchor token pooled at each candidate (default: <|object_ref_start|>).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--set-sizes",
|
||||
type=int,
|
||||
nargs="+",
|
||||
default=[8, 16, 32, 64],
|
||||
help="Candidates per set (each is one anchor in one item). Swept in order.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--item-tokens",
|
||||
type=int,
|
||||
default=180,
|
||||
help="Content tokens per candidate before its anchor (default: 180).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--query-tokens",
|
||||
type=int,
|
||||
default=0,
|
||||
help="Shared query-prefix tokens (default: 0).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-requests",
|
||||
type=int,
|
||||
default=50,
|
||||
help="Timed requests per set size (default: 50).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--concurrency",
|
||||
type=int,
|
||||
default=8,
|
||||
help="Max in-flight requests (default: 8).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--warmup",
|
||||
type=int,
|
||||
default=3,
|
||||
help="Warmup requests per set size, not timed (default: 3).",
|
||||
)
|
||||
parser.add_argument("--apply-softmax", action="store_true")
|
||||
parser.add_argument("--return-pooled-hidden-states", action="store_true")
|
||||
|
||||
# Engine configuration
|
||||
parser.add_argument("--dtype", type=str, default="float16")
|
||||
parser.add_argument("--mem-fraction-static", type=float, default=0.5)
|
||||
parser.add_argument("--max-prefill-tokens", type=int, default=60000)
|
||||
parser.add_argument("--attention-backend", type=str, default="flashinfer")
|
||||
parser.add_argument("--seed", type=int, default=0)
|
||||
parser.add_argument(
|
||||
"--output", "-o", type=str, default=None, help="Optional CSV output path."
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _random_content(num_tokens: int, anchor_id: int, lo: int = 10, hi: int = 30000):
|
||||
"""Random content token ids that never collide with the anchor id."""
|
||||
ids = [random.randint(lo, hi) for _ in range(num_tokens)]
|
||||
return [t + 1 if t == anchor_id else t for t in ids]
|
||||
|
||||
|
||||
def _build_setwise_request(args, anchor_id: int, n_candidates: int):
|
||||
"""One setwise request: shared query prefix + ONE item with N anchors."""
|
||||
query_ids = (
|
||||
_random_content(args.query_tokens, anchor_id) if args.query_tokens else []
|
||||
)
|
||||
item_ids: List[int] = []
|
||||
for _ in range(n_candidates):
|
||||
item_ids.extend(_random_content(args.item_tokens, anchor_id))
|
||||
item_ids.append(anchor_id)
|
||||
return query_ids, [item_ids]
|
||||
|
||||
|
||||
def _percentile(sorted_vals: List[float], pct: float) -> float:
|
||||
if not sorted_vals:
|
||||
return 0.0
|
||||
k = max(0, min(len(sorted_vals) - 1, round((pct / 100.0) * (len(sorted_vals) - 1))))
|
||||
return sorted_vals[k]
|
||||
|
||||
|
||||
async def _run_set_size(engine, args, anchor_id: int, n: int) -> Optional[dict]:
|
||||
# Pre-build requests so generation cost is outside the timed section.
|
||||
requests = [
|
||||
_build_setwise_request(args, anchor_id, n)
|
||||
for _ in range(args.num_requests + args.warmup)
|
||||
]
|
||||
prompt_tokens = len(requests[0][0]) + len(requests[0][1][0])
|
||||
|
||||
async def _one(query_ids, items):
|
||||
t0 = time.perf_counter()
|
||||
res = await engine.async_score(
|
||||
query=query_ids,
|
||||
items=items,
|
||||
score_extraction_token_id=anchor_id,
|
||||
apply_softmax=args.apply_softmax,
|
||||
return_pooled_hidden_states=args.return_pooled_hidden_states,
|
||||
)
|
||||
# Setwise scores are nested (one [Ni x num_labels] matrix per item) and
|
||||
# this harness sends exactly one item, so the per-candidate row count is
|
||||
# len(res.scores[0]), not len(res.scores) (which is always 1 = num items).
|
||||
return time.perf_counter() - t0, len(res.scores[0]) if res.scores else 0
|
||||
|
||||
# Warmup (also validates the row count == N).
|
||||
for q, it in requests[: args.warmup]:
|
||||
_, rows = await _one(q, it)
|
||||
if rows != n:
|
||||
print(f"[warn] set_size={n}: expected {n} score rows, got {rows}")
|
||||
|
||||
sem = asyncio.Semaphore(args.concurrency)
|
||||
|
||||
async def _guarded(q, it):
|
||||
async with sem:
|
||||
return await _one(q, it)
|
||||
|
||||
timed = requests[args.warmup :]
|
||||
wall_start = time.perf_counter()
|
||||
results = await asyncio.gather(*(_guarded(q, it) for q, it in timed))
|
||||
wall = time.perf_counter() - wall_start
|
||||
|
||||
latencies_ms = sorted(1000.0 * lat for lat, _ in results)
|
||||
n_req = len(results)
|
||||
return {
|
||||
"set_size": n,
|
||||
"prompt_tokens": prompt_tokens,
|
||||
"requests": n_req,
|
||||
"concurrency": args.concurrency,
|
||||
"avg_ms": statistics.mean(latencies_ms),
|
||||
"p50_ms": _percentile(latencies_ms, 50),
|
||||
"p95_ms": _percentile(latencies_ms, 95),
|
||||
"p99_ms": _percentile(latencies_ms, 99),
|
||||
"req_per_s": n_req / wall if wall > 0 else 0.0,
|
||||
"cand_per_s": (n_req * n) / wall if wall > 0 else 0.0,
|
||||
}
|
||||
|
||||
|
||||
def _print_table(rows: List[dict]):
|
||||
headers = [
|
||||
"set_size",
|
||||
"prompt_tokens",
|
||||
"requests",
|
||||
"avg_ms",
|
||||
"p50_ms",
|
||||
"p95_ms",
|
||||
"p99_ms",
|
||||
"req_per_s",
|
||||
"cand_per_s",
|
||||
]
|
||||
print("\n" + " | ".join(f"{h:>13}" for h in headers))
|
||||
print("-" * (16 * len(headers)))
|
||||
for r in rows:
|
||||
cells = []
|
||||
for h in headers:
|
||||
v = r[h]
|
||||
cells.append(f"{v:>13.2f}" if isinstance(v, float) else f"{v:>13}")
|
||||
print(" | ".join(cells))
|
||||
|
||||
|
||||
async def main():
|
||||
args = parse_args()
|
||||
random.seed(args.seed)
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(args.model_path, trust_remote_code=True)
|
||||
anchor_id = tokenizer.convert_tokens_to_ids(args.score_extraction_token)
|
||||
unk_id = tokenizer.unk_token_id
|
||||
if anchor_id is None or (unk_id is not None and anchor_id == unk_id):
|
||||
raise ValueError(
|
||||
f"score_extraction_token {args.score_extraction_token!r} did not resolve "
|
||||
f"to a dedicated token id."
|
||||
)
|
||||
|
||||
from sglang.srt.entrypoints.engine import Engine
|
||||
|
||||
# Setwise deployment constraints: positions are full-prompt coordinates, so
|
||||
# radix reuse / chunked prefill must be off (the engine also validates this).
|
||||
engine = Engine(
|
||||
model_path=args.model_path,
|
||||
dtype=args.dtype,
|
||||
mem_fraction_static=args.mem_fraction_static,
|
||||
max_prefill_tokens=args.max_prefill_tokens,
|
||||
attention_backend=args.attention_backend,
|
||||
disable_radix_cache=True,
|
||||
chunked_prefill_size=-1,
|
||||
)
|
||||
|
||||
print(
|
||||
f"Setwise scoring benchmark | model={args.model_path}\n"
|
||||
f"anchor={args.score_extraction_token!r} (id={anchor_id}) | "
|
||||
f"item_tokens={args.item_tokens} query_tokens={args.query_tokens} | "
|
||||
f"concurrency={args.concurrency} num_requests={args.num_requests}"
|
||||
)
|
||||
|
||||
rows = []
|
||||
try:
|
||||
for n in args.set_sizes:
|
||||
row = await _run_set_size(engine, args, anchor_id, n)
|
||||
if row is not None:
|
||||
rows.append(row)
|
||||
print(
|
||||
f"set_size={n:>4}: avg={row['avg_ms']:.1f}ms "
|
||||
f"p95={row['p95_ms']:.1f}ms "
|
||||
f"{row['req_per_s']:.1f} req/s {row['cand_per_s']:.0f} cand/s"
|
||||
)
|
||||
finally:
|
||||
engine.shutdown()
|
||||
|
||||
_print_table(rows)
|
||||
|
||||
if args.output and rows:
|
||||
with open(args.output, "w", newline="") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=list(rows[0].keys()))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
print(f"\nWrote {len(rows)} rows to {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -2398,6 +2398,20 @@ def is_cross_encoding_pooler_model(model_architectures: List[str]) -> bool:
|
||||
return any(arch in _cross_encoding_pooler_archs for arch in model_architectures)
|
||||
|
||||
|
||||
# SequenceClassification models whose forward routes the head through
|
||||
# score_and_pool (per-position pooling); only these support setwise readout
|
||||
# (token_indices_to_pool). Keep in sync with callers of layers.pooler.score_and_pool.
|
||||
_score_and_pool_archs = [
|
||||
"LlamaForSequenceClassification",
|
||||
"Qwen2ForSequenceClassification",
|
||||
"Qwen3ForSequenceClassification",
|
||||
]
|
||||
|
||||
|
||||
def is_score_and_pool_model(model_architectures: List[str]) -> bool:
|
||||
return any(arch in _score_and_pool_archs for arch in model_architectures)
|
||||
|
||||
|
||||
def yarn_get_mscale(scale: float = 1, mscale: float = 1) -> float:
|
||||
if scale <= 1:
|
||||
return 1.0
|
||||
|
||||
@@ -37,6 +37,7 @@ class EngineScoreMixin:
|
||||
query_embed_overrides: Optional[List[torch.Tensor]] = None,
|
||||
item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None,
|
||||
return_pooled_hidden_states: bool = False,
|
||||
score_extraction_token_id: Optional[int] = None,
|
||||
temperature: float = 1.0,
|
||||
return_token_logprobs: bool = False,
|
||||
) -> ScoreResult:
|
||||
@@ -53,6 +54,11 @@ class EngineScoreMixin:
|
||||
For SequenceClassification models, returns the pooled class logits directly from
|
||||
the classification head. label_token_ids is optional and ignored.
|
||||
|
||||
Setwise scoring is expressed via ``score_extraction_token_id``: pass a single
|
||||
item containing the whole candidate block with one extraction token per
|
||||
candidate, and the head is pooled AT those positions, so ``scores`` becomes
|
||||
the ``[N x num_labels]`` per-candidate matrix.
|
||||
|
||||
Args:
|
||||
query: The query text or pre-tokenized token IDs.
|
||||
items: The item text(s) or pre-tokenized token IDs.
|
||||
@@ -63,6 +69,11 @@ class EngineScoreMixin:
|
||||
embed_override_token_id: Placeholder token ID used to locate override positions.
|
||||
query_embed_overrides: Embedding vectors replacing placeholder tokens in query.
|
||||
item_embed_overrides: Per-item embedding vectors replacing placeholder tokens in items.
|
||||
score_extraction_token_id: SequenceClassification-only. When set, pool the
|
||||
head at every occurrence of this token per sequence instead of the last
|
||||
token; ``scores`` becomes nested — one ``[Ni x num_labels]`` matrix per
|
||||
item (``len(scores) == len(items)``), where ``Ni`` is the number of
|
||||
extraction tokens (candidates) in item ``i``.
|
||||
return_pooled_hidden_states: Whether to include raw pooled transformer
|
||||
hidden states (before the task head) in the result. Only supported
|
||||
for non-generation models (SequenceClassification, RewardModel).
|
||||
@@ -72,8 +83,10 @@ class EngineScoreMixin:
|
||||
each item's candidates. Only supported for CausalLM, not raw logits.
|
||||
|
||||
Returns:
|
||||
ScoreResult with scores (one list per item), prompt token count, and
|
||||
optional pooled_hidden_states tensors.
|
||||
ScoreResult with scores (flat ``[num_items x num_labels]`` for pointwise
|
||||
scoring, or nested ``[num_items][Ni x num_labels]`` when
|
||||
score_extraction_token_id is set), prompt token count, and optional
|
||||
pooled_hidden_states tensors.
|
||||
"""
|
||||
return self.loop.run_until_complete(
|
||||
self.tokenizer_manager.score_request(
|
||||
@@ -85,6 +98,7 @@ class EngineScoreMixin:
|
||||
embed_override_token_id=embed_override_token_id,
|
||||
query_embed_overrides=query_embed_overrides,
|
||||
item_embed_overrides=item_embed_overrides,
|
||||
score_extraction_token_id=score_extraction_token_id,
|
||||
request=None,
|
||||
return_pooled_hidden_states=return_pooled_hidden_states,
|
||||
temperature=temperature,
|
||||
@@ -103,6 +117,7 @@ class EngineScoreMixin:
|
||||
query_embed_overrides: Optional[List[torch.Tensor]] = None,
|
||||
item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None,
|
||||
return_pooled_hidden_states: bool = False,
|
||||
score_extraction_token_id: Optional[int] = None,
|
||||
temperature: float = 1.0,
|
||||
return_token_logprobs: bool = False,
|
||||
) -> ScoreResult:
|
||||
@@ -116,6 +131,7 @@ class EngineScoreMixin:
|
||||
embed_override_token_id=embed_override_token_id,
|
||||
query_embed_overrides=query_embed_overrides,
|
||||
item_embed_overrides=item_embed_overrides,
|
||||
score_extraction_token_id=score_extraction_token_id,
|
||||
request=None,
|
||||
return_pooled_hidden_states=return_pooled_hidden_states,
|
||||
temperature=temperature,
|
||||
|
||||
@@ -1406,14 +1406,32 @@ class ScoringRequest(BaseModel):
|
||||
return_token_logprobs: bool = False
|
||||
item_first: bool = False
|
||||
return_pooled_hidden_states: bool = False
|
||||
|
||||
# Setwise readout (SequenceClassification-only): when set, the head is pooled
|
||||
# AT every occurrence of this token in each `query + item` sequence instead of
|
||||
# the last token, and `scores` is returned nested (one `[Nᵢ x num_labels]`
|
||||
# matrix per item). --enable-mis fuses items; otherwise each is scored alone.
|
||||
score_extraction_token: Optional[str] = None
|
||||
|
||||
model: str = DEFAULT_MODEL_NAME
|
||||
|
||||
|
||||
class ScoringResponse(BaseModel):
|
||||
scores: List[
|
||||
List[float]
|
||||
] # List of lists of probabilities, each in the order of label_token_ids
|
||||
pooled_hidden_states: Optional[List[Optional[List[float]]]] = None
|
||||
# Pointwise (no `score_extraction_token`): a single [num_rows x num_labels]
|
||||
# matrix. Setwise: nested [num_items][Nᵢ x num_labels]. The nesting depth
|
||||
# disambiguates the two shapes.
|
||||
scores: Union[
|
||||
List[List[float]], # pointwise: [num_rows x num_labels]
|
||||
List[List[List[float]]], # setwise: [num_items][Nᵢ x num_labels]
|
||||
]
|
||||
# Parallel to `scores`: flat [num_rows x hidden] (pointwise) or one such
|
||||
# matrix per item (setwise).
|
||||
pooled_hidden_states: Optional[
|
||||
Union[
|
||||
List[Optional[List[float]]], # pointwise: [num_rows x hidden]
|
||||
List[List[Optional[List[float]]]], # setwise: [num_items][Nᵢ x hidden]
|
||||
]
|
||||
] = None
|
||||
token_logprobs: Optional[List[List[float]]] = None
|
||||
model: str
|
||||
usage: Optional[UsageInfo] = None
|
||||
|
||||
@@ -67,6 +67,16 @@ class OpenAIServingScore(OpenAIServingBase):
|
||||
else None
|
||||
)
|
||||
|
||||
# Resolve the extraction token to an id here (this process owns the
|
||||
# tokenizer); score_request then scans for it and pools the head there.
|
||||
score_extraction_token_id = None
|
||||
if request.score_extraction_token is not None:
|
||||
score_extraction_token_id = (
|
||||
self.tokenizer_manager._resolve_score_extraction_token_id(
|
||||
request.score_extraction_token
|
||||
)
|
||||
)
|
||||
|
||||
result = await self.tokenizer_manager.score_request(
|
||||
query=request.query,
|
||||
items=request.items,
|
||||
@@ -76,18 +86,23 @@ class OpenAIServingScore(OpenAIServingBase):
|
||||
embed_override_token_id=request.embed_override_token_id,
|
||||
query_embed_overrides=query_embed_overrides,
|
||||
item_embed_overrides=item_embed_overrides,
|
||||
score_extraction_token_id=score_extraction_token_id,
|
||||
request=raw_request,
|
||||
return_pooled_hidden_states=request.return_pooled_hidden_states,
|
||||
temperature=request.temperature,
|
||||
return_token_logprobs=request.return_token_logprobs,
|
||||
)
|
||||
|
||||
phs_as_lists = None
|
||||
if result.pooled_hidden_states is not None:
|
||||
phs_as_lists = [
|
||||
t.tolist() if t is not None else None
|
||||
for t in result.pooled_hidden_states
|
||||
]
|
||||
# pooled_hidden_states is flat (pointwise / single-set setwise) or
|
||||
# nested per item (multi-item setwise); convert tensors at any depth.
|
||||
def _tensors_to_lists(value):
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, list):
|
||||
return [_tensors_to_lists(v) for v in value]
|
||||
return value.tolist()
|
||||
|
||||
phs_as_lists = _tensors_to_lists(result.pooled_hidden_states)
|
||||
|
||||
response = ScoringResponse(
|
||||
scores=result.scores,
|
||||
|
||||
@@ -988,8 +988,12 @@ class OutputLogprobProcessor:
|
||||
|
||||
# Handle token_ids logprobs if requested
|
||||
if needs_token_ids_logprobs:
|
||||
# no_copy_to_cpu avoids the .item()/.tolist() CUDA sync that regresses
|
||||
# prefill-only Score throughput at high concurrency.
|
||||
(
|
||||
result.token_ids_logprobs_val,
|
||||
result.token_ids_logprobs_idx,
|
||||
) = get_token_ids_logprobs_batch_optimized(logprobs, token_ids_logprobs)
|
||||
) = get_token_ids_logprobs(
|
||||
logprobs, token_ids_logprobs, no_copy_to_cpu=True
|
||||
)
|
||||
return result
|
||||
|
||||
@@ -35,7 +35,8 @@ class EmbeddingPoolerOutput:
|
||||
pooled_hidden_states: Raw transformer hidden states *before* the
|
||||
task-specific head, present only when
|
||||
``forward_batch.return_pooled_hidden_states`` is True. Tensor
|
||||
(standard path) or list of tensors (MIS path, one per delimiter).
|
||||
(standard path) or list of tensors (one per position — MIS: one per
|
||||
delimiter; setwise / multi-position readout: one per pooled position).
|
||||
"""
|
||||
|
||||
# Pooler can return list[tensor] instead of tensor if the dimension of each tensor in the batch is different
|
||||
@@ -114,6 +115,45 @@ def pool_at_delimiter_positions(
|
||||
return list(data[index_tensor].split(delim_counts))
|
||||
|
||||
|
||||
def pool_at_positions(
|
||||
data: torch.Tensor,
|
||||
per_request_indices: List[torch.Tensor],
|
||||
forward_batch: ForwardBatch,
|
||||
device: torch.device,
|
||||
) -> List[torch.Tensor]:
|
||||
"""Pool a tensor exactly AT the given token positions for every request.
|
||||
|
||||
Unlike ``pool_at_delimiter_positions`` (which pools at delimiter - 1), this
|
||||
pools at the position itself with no offset shift and no discarded entry.
|
||||
|
||||
Args:
|
||||
data: 2-D tensor [total_tokens, dim] — hidden states or logits.
|
||||
per_request_indices: Per-request anchor positions (CPU tensors), each
|
||||
relative to the start of that request's sequence.
|
||||
forward_batch: Forward batch with extend_seq_lens_cpu populated.
|
||||
device: Device for the index tensor.
|
||||
|
||||
Returns:
|
||||
One tensor per request, shaped [num_anchors, dim].
|
||||
"""
|
||||
all_index_tensors: List[torch.Tensor] = []
|
||||
counts: List[int] = []
|
||||
offset = 0
|
||||
for req_idx, req_seq_len in enumerate(forward_batch.extend_seq_lens_cpu):
|
||||
indices_tensor = per_request_indices[req_idx]
|
||||
n = len(indices_tensor)
|
||||
if n > 0:
|
||||
all_index_tensors.append(indices_tensor + offset)
|
||||
counts.append(n)
|
||||
offset += req_seq_len
|
||||
|
||||
if all_index_tensors:
|
||||
index_tensor = torch.cat(all_index_tensors).to(device, non_blocking=True)
|
||||
else:
|
||||
index_tensor = torch.empty(0, dtype=torch.long, device=device)
|
||||
return list(data[index_tensor].split(counts))
|
||||
|
||||
|
||||
def score_and_pool(
|
||||
score_head: nn.Module,
|
||||
pooler: Pooler,
|
||||
@@ -123,6 +163,10 @@ def score_and_pool(
|
||||
) -> EmbeddingPoolerOutput:
|
||||
"""Apply a classification/score head with MIS and pooled-hidden-states support.
|
||||
|
||||
Multi-position readout path (``token_indices_to_pool`` on forward_batch, e.g.
|
||||
setwise scoring): pool the head AT each caller-specified position (no offset,
|
||||
no discarded row), then split per-request.
|
||||
|
||||
MIS path (pre-computed delimiter indices on forward_batch): extract hidden
|
||||
states at positions just before each delimiter, apply the score head, then
|
||||
split per-request.
|
||||
@@ -132,6 +176,29 @@ def score_and_pool(
|
||||
When ``forward_batch.return_pooled_hidden_states`` is True, the raw pooled
|
||||
hidden states (before the score head) are included in the output.
|
||||
"""
|
||||
if (
|
||||
forward_batch.token_indices_to_pool is not None
|
||||
and forward_batch.is_prefill_only
|
||||
):
|
||||
# Setwise readout: pool the head AT each requested position (no -1 shift,
|
||||
# no discarded boundary row). Concat to call the head once, then split.
|
||||
per_request_phs = pool_at_positions(
|
||||
hidden_states,
|
||||
forward_batch.token_indices_to_pool,
|
||||
forward_batch,
|
||||
input_ids.device,
|
||||
)
|
||||
phs_flat = torch.cat(per_request_phs, dim=0)
|
||||
scores_flat = score_head(phs_flat)
|
||||
pos_counts = [t.shape[0] for t in per_request_phs]
|
||||
per_request_scores = list(scores_flat.split(pos_counts))
|
||||
return EmbeddingPoolerOutput(
|
||||
embeddings=per_request_scores,
|
||||
pooled_hidden_states=(
|
||||
per_request_phs if forward_batch.return_pooled_hidden_states else None
|
||||
),
|
||||
)
|
||||
|
||||
if (
|
||||
forward_batch.multi_item_delimiter_indices is not None
|
||||
and forward_batch.is_prefill_only
|
||||
|
||||
@@ -1233,6 +1233,12 @@ class EmbeddingReqInput:
|
||||
# Batch-level: List[List[int]] (one per request). After __getitem__: List[int].
|
||||
multi_item_delimiter_indices: Optional[Union[List[List[int]], List[int]]] = None
|
||||
|
||||
# Token positions the pooler reads out AT (head / hidden states), instead of
|
||||
# the default LAST/CLS pooling. Generic multi-position primitive (setwise
|
||||
# scoring today). Unlike MIS, pooling is AT the position (no delimiter - 1).
|
||||
# Batch-level: List[List[int]] (one per request). After __getitem__: List[int].
|
||||
token_indices_to_pool: Optional[Union[List[List[int]], List[int]]] = None
|
||||
|
||||
def regenerate_rid(self):
|
||||
"""Generate a new request ID and return it."""
|
||||
if isinstance(self.rid, list):
|
||||
@@ -1359,6 +1365,11 @@ class EmbeddingReqInput:
|
||||
if self.multi_item_delimiter_indices is not None
|
||||
else None
|
||||
),
|
||||
token_indices_to_pool=(
|
||||
self.token_indices_to_pool[i]
|
||||
if self.token_indices_to_pool is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
else:
|
||||
sub = EmbeddingReqInput(
|
||||
@@ -1390,6 +1401,11 @@ class EmbeddingReqInput:
|
||||
if self.multi_item_delimiter_indices is not None
|
||||
else None
|
||||
),
|
||||
token_indices_to_pool=(
|
||||
self.token_indices_to_pool[i]
|
||||
if self.token_indices_to_pool is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
cache[i] = sub
|
||||
return sub
|
||||
@@ -1419,7 +1435,8 @@ class TokenizedEmbeddingReqInput(BaseReq, kw_only=True):
|
||||
return_pooled_hidden_states: bool = False
|
||||
# Pre-computed delimiter indices for multi-item scoring
|
||||
multi_item_delimiter_indices: Optional[List[int]] = None
|
||||
|
||||
# Token positions for setwise pooling readout
|
||||
token_indices_to_pool: Optional[List[int]] = None
|
||||
# For observability
|
||||
# Pickled Optional[Union[APIServerReqTimeStats, DPControllerReqTimeStats]]
|
||||
time_stats: Optional[PickleWrapper] = None
|
||||
|
||||
@@ -1025,6 +1025,7 @@ class Req(ReqDllmMixin):
|
||||
] = None,
|
||||
return_pooled_hidden_states: bool = False,
|
||||
multi_item_delimiter_indices: Optional[List[int]] = None,
|
||||
token_indices_to_pool: Optional[List[int]] = None,
|
||||
session_id: Optional[str] = None,
|
||||
cache_salt: Optional[str] = None,
|
||||
):
|
||||
@@ -1056,6 +1057,7 @@ class Req(ReqDllmMixin):
|
||||
self.input_embeds = input_embeds
|
||||
self.positional_embed_overrides = positional_embed_overrides
|
||||
self.multi_item_delimiter_indices = multi_item_delimiter_indices
|
||||
self.token_indices_to_pool = token_indices_to_pool
|
||||
|
||||
# For req-level memory management
|
||||
self.kv = ReqKvInfo()
|
||||
|
||||
@@ -3431,6 +3431,7 @@ class Scheduler(
|
||||
time_stats=recv_req.time_stats,
|
||||
return_pooled_hidden_states=recv_req.return_pooled_hidden_states,
|
||||
multi_item_delimiter_indices=recv_req.multi_item_delimiter_indices,
|
||||
token_indices_to_pool=recv_req.token_indices_to_pool,
|
||||
)
|
||||
req.tokenizer = self.tokenizer
|
||||
self._maybe_namespace_elastic_radix_cache(req)
|
||||
@@ -3934,6 +3935,15 @@ class Scheduler(
|
||||
if self.enable_lora and not self.can_schedule_lora_req(req, running_loras):
|
||||
continue
|
||||
|
||||
# A forward batch runs one pooling mode, so setwise readout requests
|
||||
# (token_indices_to_pool) cannot share a batch with last-token ones.
|
||||
# Admit only the first request's mode; the other stays queued.
|
||||
if adder.can_run_list and (
|
||||
(req.token_indices_to_pool is not None)
|
||||
!= (adder.can_run_list[0].token_indices_to_pool is not None)
|
||||
):
|
||||
continue
|
||||
|
||||
running_bs = len(running_batch.reqs)
|
||||
candidate_beam_width = (
|
||||
req.beam_group.beam_width if req.beam_group is not None else None
|
||||
|
||||
@@ -1561,6 +1561,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
http_worker_ipc=obj.http_worker_ipc,
|
||||
return_pooled_hidden_states=obj.return_pooled_hidden_states,
|
||||
multi_item_delimiter_indices=obj.multi_item_delimiter_indices,
|
||||
token_indices_to_pool=obj.token_indices_to_pool,
|
||||
)
|
||||
|
||||
tokenized_obj.time_stats = self.rid_to_state[obj.rid].time_stats
|
||||
|
||||
@@ -1,29 +1,41 @@
|
||||
import logging
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
from typing import Any, Dict, List, Optional, Tuple, TypeAlias, Union
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.configs.model_config import is_cross_encoding_pooler_model
|
||||
from sglang.srt.configs.model_config import (
|
||||
is_cross_encoding_pooler_model,
|
||||
is_score_and_pool_model,
|
||||
)
|
||||
from sglang.srt.constants import MIS_DELIMITER_TOKEN_ID
|
||||
from sglang.srt.managers.embed_types import PositionalEmbeds
|
||||
from sglang.srt.managers.io_struct import EmbeddingReqInput, GenerateReqInput
|
||||
from sglang.srt.runtime_context import get_exec
|
||||
from sglang.srt.runtime_context import get_exec, get_memory, get_schedule, get_serving
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
ScoreRow: TypeAlias = List[float]
|
||||
ScoreMatrix: TypeAlias = List[ScoreRow]
|
||||
# Pointwise: [num_items, num_labels]. Setwise (multi-position): one
|
||||
# [N_i, num_labels] matrix per item.
|
||||
ScoreOutput: TypeAlias = Union[ScoreMatrix, List[ScoreMatrix]]
|
||||
|
||||
PooledHiddenStateRows: TypeAlias = List[Optional[torch.Tensor]]
|
||||
PooledHiddenStateOutput: TypeAlias = Union[
|
||||
PooledHiddenStateRows, List[Optional[List[torch.Tensor]]]
|
||||
]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ScoreResult:
|
||||
scores: List[List[float]]
|
||||
scores: ScoreOutput
|
||||
prompt_tokens: int = 0
|
||||
# Per-item pooled hidden states (pre-head transformer output).
|
||||
# CPU tensors when return_pooled_hidden_states=True; kept as tensors so
|
||||
# in-process consumers (gRPC, engine API) avoid a .tolist() round-trip.
|
||||
# The HTTP path converts to lists in serving_score.py before JSON serialization.
|
||||
# Same layout as scores: one tensor per item (not a single packed 2D tensor).
|
||||
pooled_hidden_states: Optional[List[Optional[torch.Tensor]]] = None
|
||||
# Pre-head CPU tensors parallel to scores; kept as tensors so in-process
|
||||
# consumers avoid a .tolist() round-trip (the HTTP path converts before JSON).
|
||||
pooled_hidden_states: Optional[PooledHiddenStateOutput] = None
|
||||
# uncalibrated full-vocabulary logprobs, in each item's candidate order
|
||||
token_logprobs: Optional[List[List[float]]] = None
|
||||
|
||||
@@ -197,7 +209,7 @@ class TokenizerManagerScoreMixin:
|
||||
if return_pooled_hidden_states:
|
||||
raw_phs = single_result.get("pooled_hidden_state")
|
||||
if raw_phs is not None and len(raw_phs) == expected_count:
|
||||
phs_list = raw_phs[1:]
|
||||
phs_list = list(raw_phs[1:])
|
||||
|
||||
return ScoreResult(
|
||||
scores=scores,
|
||||
@@ -206,21 +218,110 @@ class TokenizerManagerScoreMixin:
|
||||
token_logprobs=token_logprobs,
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Multi-position pooling result helpers (setwise scoring): the head is read
|
||||
# at every score-extraction token instead of the last token. Shared by the
|
||||
# batched and fused (--enable-mis) result processors.
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _multi_position_score_rows(
|
||||
self, embedding: Any, apply_softmax: bool
|
||||
) -> List[List[float]]:
|
||||
"""Validate a 2-D multi-position result embedding and return its per-token rows."""
|
||||
embedding_tensor = torch.as_tensor(embedding)
|
||||
if embedding_tensor.ndim != 2:
|
||||
# A 1-D result means the model pooled a single vector (reward/embedding
|
||||
# head) instead of per-position scores; ValueError -> clean 400.
|
||||
raise ValueError(
|
||||
"Multi-position scoring expected a 2-D "
|
||||
"[num_positions, num_labels] result, but got shape "
|
||||
f"{tuple(embedding_tensor.shape)}. This model may not support "
|
||||
"score_extraction_token_id (only SequenceClassification heads that "
|
||||
"pool per position do)."
|
||||
)
|
||||
if apply_softmax:
|
||||
return torch.softmax(embedding_tensor, dim=-1).tolist()
|
||||
return embedding if isinstance(embedding, list) else embedding_tensor.tolist()
|
||||
|
||||
def _multi_position_phs_matrix(self, phs: Any, expected_rows: int) -> torch.Tensor:
|
||||
"""Validate multi-position pooled hidden states are 2-D with one row per position."""
|
||||
phs_tensor = torch.as_tensor(phs)
|
||||
if phs_tensor.ndim != 2 or phs_tensor.shape[0] != expected_rows:
|
||||
raise ValueError(
|
||||
"Multi-position pooled hidden states must be a 2-D tensor with "
|
||||
"one row per score position."
|
||||
)
|
||||
return phs_tensor
|
||||
|
||||
def _process_multi_item_extraction_results(
|
||||
self,
|
||||
results: Any,
|
||||
per_item_anchor_counts: List[int],
|
||||
apply_softmax: bool,
|
||||
return_pooled_hidden_states: bool = False,
|
||||
) -> ScoreResult:
|
||||
"""Process a fused multi-item score-extraction request (``--enable-mis``).
|
||||
|
||||
The fused sequence returns one ``[ΣNᵢ, num_labels]`` matrix of every
|
||||
anchor's logits in item order; split it back into one ``[Nᵢ, num_labels]``
|
||||
matrix per item using per_item_anchor_counts.
|
||||
"""
|
||||
single_result = results[0] if isinstance(results, list) else results
|
||||
meta_info = single_result.get("meta_info", {})
|
||||
request_id = meta_info.get("id", "<unknown>")
|
||||
prompt_tokens = meta_info.get("prompt_tokens", 0)
|
||||
|
||||
embedding = single_result.get("embedding")
|
||||
if embedding is None:
|
||||
raise ValueError("Embedding not found in the result.")
|
||||
|
||||
rows = self._multi_position_score_rows(embedding, apply_softmax)
|
||||
total_anchors = sum(per_item_anchor_counts)
|
||||
if len(rows) != total_anchors:
|
||||
raise RuntimeError(
|
||||
f"Expected {total_anchors} anchor rows across "
|
||||
f"{len(per_item_anchor_counts)} items, but got {len(rows)}. "
|
||||
f"Request ID: {request_id}"
|
||||
)
|
||||
|
||||
scores = []
|
||||
offset = 0
|
||||
for count in per_item_anchor_counts:
|
||||
scores.append(rows[offset : offset + count])
|
||||
offset += count
|
||||
|
||||
phs_grouped = None
|
||||
if return_pooled_hidden_states:
|
||||
raw_phs = single_result.get("pooled_hidden_state")
|
||||
if raw_phs is not None:
|
||||
phs_matrix = self._multi_position_phs_matrix(raw_phs, total_anchors)
|
||||
phs_grouped = [
|
||||
list(group) for group in phs_matrix.split(per_item_anchor_counts)
|
||||
]
|
||||
|
||||
return ScoreResult(
|
||||
scores=scores,
|
||||
prompt_tokens=prompt_tokens,
|
||||
pooled_hidden_states=phs_grouped,
|
||||
)
|
||||
|
||||
def _process_single_item_scoring_results(
|
||||
self,
|
||||
results: Any,
|
||||
label_token_ids: Optional[List[List[int]]],
|
||||
apply_softmax: bool,
|
||||
return_pooled_hidden_states: bool = False,
|
||||
per_item_matrix: bool = False,
|
||||
temperature: float = 1.0,
|
||||
return_token_logprobs: bool = False,
|
||||
) -> ScoreResult:
|
||||
"""
|
||||
Process results from single-item scoring request.
|
||||
|
||||
For generation (CausalLM) models: reads output_token_ids_logprobs.
|
||||
For non-generation (SequenceClassification) models: reads the embedding field
|
||||
which contains pooled class logits from the classification head.
|
||||
CausalLM models read output_token_ids_logprobs; SequenceClassification
|
||||
models read the embedding field (pooled class logits from the head).
|
||||
When per_item_matrix (setwise), each result's 2-D matrix becomes one
|
||||
item's entry so ``scores`` is nested; otherwise each result is one row.
|
||||
"""
|
||||
scores = []
|
||||
phs_list = []
|
||||
@@ -259,24 +360,29 @@ class TokenizerManagerScoreMixin:
|
||||
|
||||
prompt_tokens += result.get("meta_info", {}).get("prompt_tokens", 0)
|
||||
|
||||
if apply_softmax:
|
||||
scores_tensor = torch.as_tensor(embedding, dtype=torch.float64)
|
||||
embedding = torch.softmax(
|
||||
(scores_tensor - scores_tensor.max()) / temperature, dim=-1
|
||||
).tolist()
|
||||
|
||||
# The classification head produces per-token logits, which the pooler reduces
|
||||
# into a single vector per input. That vector is returned in the `.embeddings`
|
||||
# field — not as semantic embeddings, but as pooled classification logits.
|
||||
# The field name is reused for compatibility with the existing
|
||||
# EmbeddingPoolerOutput API.
|
||||
scores.append(embedding)
|
||||
if per_item_matrix:
|
||||
rows = self._multi_position_score_rows(embedding, apply_softmax)
|
||||
scores.append(rows)
|
||||
else:
|
||||
if apply_softmax:
|
||||
scores_tensor = torch.as_tensor(embedding, dtype=torch.float64)
|
||||
embedding = torch.softmax(
|
||||
(scores_tensor - scores_tensor.max()) / temperature, dim=-1
|
||||
).tolist()
|
||||
# Pooled classification logits, reusing the EmbeddingPoolerOutput field.
|
||||
scores.append(embedding)
|
||||
|
||||
if return_pooled_hidden_states:
|
||||
phs = result.get("pooled_hidden_state")
|
||||
phs_list.append(phs)
|
||||
if phs is not None:
|
||||
if per_item_matrix and phs is not None:
|
||||
phs_list.append(
|
||||
list(self._multi_position_phs_matrix(phs, len(rows)))
|
||||
)
|
||||
has_phs = True
|
||||
else:
|
||||
phs_list.append(phs)
|
||||
if phs is not None:
|
||||
has_phs = True
|
||||
|
||||
return ScoreResult(
|
||||
scores=scores,
|
||||
@@ -458,6 +564,190 @@ class TokenizerManagerScoreMixin:
|
||||
None,
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Multi-position pooling readout (setwise scoring)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _resolve_score_extraction_token_id(self, token: str) -> int:
|
||||
"""Resolve the score-extraction token string to a dedicated token id.
|
||||
|
||||
Runs in the tokenizer-manager process; used by the HTTP score handler to
|
||||
turn a client token string into the id ``score_request`` scans for.
|
||||
"""
|
||||
if self.tokenizer is None:
|
||||
raise ValueError(
|
||||
"A tokenizer is required to resolve score_extraction_token."
|
||||
)
|
||||
token_id = self.tokenizer.convert_tokens_to_ids(token)
|
||||
unk_id = self.tokenizer.unk_token_id
|
||||
if token_id is None or (unk_id is not None and token_id == unk_id):
|
||||
raise ValueError(
|
||||
f"score_extraction_token {token!r} did not resolve to a dedicated "
|
||||
f"token id."
|
||||
)
|
||||
return token_id
|
||||
|
||||
def _validate_score_extraction(
|
||||
self,
|
||||
is_generation: bool,
|
||||
item_first: bool,
|
||||
) -> None:
|
||||
"""Validate that multi-position pooling readout is applicable.
|
||||
|
||||
Supported only by SequenceClassification models whose forward routes the
|
||||
head through ``score_and_pool`` (per-position pooling, in
|
||||
``is_score_and_pool_model``). Generation, cross-encoder, reward, and
|
||||
embedding models pool a single vector and are rejected here before
|
||||
inference. Requires radix cache and chunked prefill off and no auto-truncate,
|
||||
since pooling positions are full-prompt coordinates.
|
||||
"""
|
||||
if is_generation:
|
||||
raise ValueError(
|
||||
"score_extraction_token_id is only supported for "
|
||||
"SequenceClassification models, not generation (CausalLM) models."
|
||||
)
|
||||
architectures = self.model_config.hf_config.architectures or []
|
||||
if not is_score_and_pool_model(architectures):
|
||||
raise ValueError(
|
||||
"score_extraction_token_id is only supported for "
|
||||
"SequenceClassification models that pool the head per position "
|
||||
"(score_and_pool); model architecture(s) "
|
||||
f"{architectures} do not (cross-encoder, reward, and embedding "
|
||||
"models pool a single vector and ignore the readout positions)."
|
||||
)
|
||||
if item_first:
|
||||
raise ValueError(
|
||||
"item_first is not supported with score_extraction_token_id."
|
||||
)
|
||||
if not get_memory().disable_radix_cache:
|
||||
raise ValueError(
|
||||
"score_extraction_token_id requires --disable-radix-cache because "
|
||||
"pooling positions are relative to the full prompt."
|
||||
)
|
||||
if get_schedule().chunked_prefill_size != -1:
|
||||
raise ValueError(
|
||||
"score_extraction_token_id requires --chunked-prefill-size -1 "
|
||||
"because all pooling positions must be processed in one prefill."
|
||||
)
|
||||
if get_serving().allow_auto_truncate:
|
||||
raise ValueError(
|
||||
"score_extraction_token_id is not supported with "
|
||||
"--allow-auto-truncate: pooling positions are computed from the "
|
||||
"full prompt, and truncating the sequence would leave them "
|
||||
"pointing past the truncated length."
|
||||
)
|
||||
|
||||
def _reject_query_prefix_anchor(
|
||||
self,
|
||||
query_token_ids: List[int],
|
||||
score_extraction_token_id: int,
|
||||
) -> None:
|
||||
"""Reject a shared query prefix that itself contains the extraction token.
|
||||
|
||||
The non-MIS path scans the whole ``query + item`` sequence, so a query-side
|
||||
hit would emit an extra, misaligned score row instead of one per candidate.
|
||||
"""
|
||||
if score_extraction_token_id in query_token_ids:
|
||||
raise ValueError(
|
||||
"score-extraction token found in the shared query prefix; it must "
|
||||
"appear only within items, one per candidate."
|
||||
)
|
||||
|
||||
def _build_score_extraction_text_inputs(
|
||||
self,
|
||||
query: str,
|
||||
items: Union[str, List[str]],
|
||||
score_extraction_token_id: int,
|
||||
) -> List[List[int]]:
|
||||
"""Tokenize each complete query-item prompt once for extraction scanning."""
|
||||
items_list = [items] if isinstance(items, str) else items
|
||||
if not all(isinstance(item, str) for item in items_list):
|
||||
raise ValueError(
|
||||
"query and items must both be text or both be token IDs when "
|
||||
"score_extraction_token_id is set."
|
||||
)
|
||||
if query:
|
||||
self._reject_query_prefix_anchor(
|
||||
self.tokenizer.encode(query), score_extraction_token_id
|
||||
)
|
||||
return [self.tokenizer.encode(f"{query}{item}") for item in items_list]
|
||||
|
||||
def _resolve_score_extraction_indices(
|
||||
self,
|
||||
input_ids: List[List[int]],
|
||||
score_extraction_token_id: int,
|
||||
) -> List[List[int]]:
|
||||
"""Scan each built sequence for the score-extraction token.
|
||||
|
||||
The head is later pooled AT these positions (via ``token_indices_to_pool``)
|
||||
instead of the last token. Raises ValueError if any sequence has no token,
|
||||
which would silently drop that sequence's scores.
|
||||
"""
|
||||
token_indices_to_pool = [
|
||||
[i for i, t in enumerate(seq) if t == score_extraction_token_id]
|
||||
for seq in input_ids
|
||||
]
|
||||
if any(not seq_indices for seq_indices in token_indices_to_pool):
|
||||
raise ValueError(
|
||||
f"No score-extraction token (id={score_extraction_token_id}) "
|
||||
"found in one or more input(s); the client must render one per "
|
||||
"candidate."
|
||||
)
|
||||
return token_indices_to_pool
|
||||
|
||||
def _anchor_counts_per_item(
|
||||
self,
|
||||
delimiter_indices: List[int],
|
||||
anchor_positions: List[int],
|
||||
) -> List[int]:
|
||||
"""Count extraction-token anchors falling inside each fused item's block.
|
||||
|
||||
``--enable-mis`` fuses items as ``query <delim> item0 <delim> item1 …``, so
|
||||
item ``i`` spans ``(delimiter_indices[i], delimiter_indices[i + 1])``.
|
||||
Raises ValueError (clean 400 before inference) if an item has no anchor or
|
||||
an anchor falls outside every item block.
|
||||
"""
|
||||
counts = [
|
||||
sum(1 for p in anchor_positions if lo < p < hi)
|
||||
for lo, hi in zip(delimiter_indices, delimiter_indices[1:])
|
||||
]
|
||||
# An anchor in the shared query prefix is not attributable to any item and
|
||||
# would leave the pooled forward with more rows than sum(counts); reject it.
|
||||
if sum(counts) != len(anchor_positions):
|
||||
raise ValueError(
|
||||
"score-extraction token found outside the candidate items (e.g. in "
|
||||
"the shared query prefix) under --enable-mis multi-item scoring; it "
|
||||
"must appear only within items, one per candidate."
|
||||
)
|
||||
if any(c == 0 for c in counts):
|
||||
raise ValueError(
|
||||
"Each item (candidate set) must contain at least one "
|
||||
"score-extraction token under --enable-mis multi-item scoring."
|
||||
)
|
||||
return counts
|
||||
|
||||
def _resolve_multi_position_pooling(
|
||||
self,
|
||||
input_ids: List[List[int]],
|
||||
score_extraction_token_id: int,
|
||||
use_multi_item_scoring: bool,
|
||||
delimiter_indices: Optional[List[int]],
|
||||
) -> Tuple[List[List[int]], Optional[List[int]]]:
|
||||
"""Resolve where the head is pooled for a score-extraction request.
|
||||
|
||||
Returns ``(token_indices_to_pool, per_item_anchor_counts)``; the latter is
|
||||
None outside ``--enable-mis`` and otherwise buckets anchors back per item.
|
||||
"""
|
||||
token_indices_to_pool = self._resolve_score_extraction_indices(
|
||||
input_ids, score_extraction_token_id
|
||||
)
|
||||
per_item_anchor_counts = None
|
||||
if use_multi_item_scoring:
|
||||
per_item_anchor_counts = self._anchor_counts_per_item(
|
||||
delimiter_indices, token_indices_to_pool[0]
|
||||
)
|
||||
return token_indices_to_pool, per_item_anchor_counts
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Main entry point
|
||||
# ------------------------------------------------------------------
|
||||
@@ -474,6 +764,7 @@ class TokenizerManagerScoreMixin:
|
||||
item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None,
|
||||
request: Optional[Any] = None,
|
||||
return_pooled_hidden_states: bool = False,
|
||||
score_extraction_token_id: Optional[int] = None,
|
||||
temperature: float = 1.0,
|
||||
return_token_logprobs: bool = False,
|
||||
) -> ScoreResult:
|
||||
@@ -495,6 +786,13 @@ class TokenizerManagerScoreMixin:
|
||||
- Generation (CausalLM): Requires label_token_ids; returns logprob-based scores.
|
||||
- SequenceClassification: label_token_ids is optional; returns pooled class logits.
|
||||
|
||||
Setwise scoring (SequenceClassification-only) is expressed via
|
||||
score_extraction_token_id: when set, the head is pooled AT every occurrence
|
||||
of this token in each ``query + item`` sequence instead of the last token,
|
||||
and ``scores`` is returned nested (one ``[Nᵢ x num_labels]`` matrix per
|
||||
item). With ``--enable-mis`` the items are fused into one multi-item
|
||||
sequence; otherwise each item is scored independently in one batch.
|
||||
|
||||
return_pooled_hidden_states is only supported for non-generation models
|
||||
(SequenceClassification, RewardModel); raises ValueError for CausalLM.
|
||||
"""
|
||||
@@ -548,6 +846,12 @@ class TokenizerManagerScoreMixin:
|
||||
f"Token ID {token_id} is out of vocabulary (vocab size: {vocab_size})"
|
||||
)
|
||||
|
||||
# Normalize an omitted query by item type so path selection sees a concrete
|
||||
# prefix: [] for pre-tokenized items (token-ID path), "" for text items.
|
||||
if query is None:
|
||||
items_are_token_ids = isinstance(items, list) and isinstance(items[0], list)
|
||||
query = [] if items_are_token_ids else ""
|
||||
|
||||
has_embeds = (
|
||||
query_embed_overrides is not None or item_embed_overrides is not None
|
||||
)
|
||||
@@ -566,12 +870,19 @@ class TokenizerManagerScoreMixin:
|
||||
# Check if multi-item scoring is enabled
|
||||
use_multi_item_scoring = get_exec().features.enable_mis
|
||||
|
||||
# Setwise readout: pool the head at every score_extraction_token_id.
|
||||
use_score_extraction = score_extraction_token_id is not None
|
||||
if use_score_extraction:
|
||||
self._validate_score_extraction(is_generation, item_first)
|
||||
|
||||
input_ids = None
|
||||
text_prompts = None
|
||||
positional_embed_overrides = None
|
||||
delimiter_indices = None
|
||||
|
||||
use_text_prompts = isinstance(query, str) and not has_embeds
|
||||
use_text_prompts = (
|
||||
isinstance(query, str) and not has_embeds and not use_score_extraction
|
||||
)
|
||||
|
||||
if use_text_prompts:
|
||||
# Both query and items are text
|
||||
@@ -605,6 +916,10 @@ class TokenizerManagerScoreMixin:
|
||||
):
|
||||
# Both query and items are token IDs — tokenize text inputs if needed for embed overrides
|
||||
query_ids, items_ids = query, items
|
||||
if use_score_extraction and not use_multi_item_scoring:
|
||||
# Non-MIS concatenates query + item, so a query-prefix anchor would
|
||||
# emit a misaligned row; reject it (MIS confines anchors per item).
|
||||
self._reject_query_prefix_anchor(query_ids, score_extraction_token_id)
|
||||
_, input_ids, positional_embed_overrides, delimiter_indices = (
|
||||
self._build_token_id_inputs(
|
||||
query_ids,
|
||||
@@ -616,9 +931,25 @@ class TokenizerManagerScoreMixin:
|
||||
item_embed_overrides,
|
||||
)
|
||||
)
|
||||
elif has_embeds:
|
||||
# Text inputs with embed overrides — need to tokenize first to resolve positions
|
||||
query_ids, items_ids = self._batch_tokenize_query_and_items(query, items)
|
||||
elif (
|
||||
use_score_extraction
|
||||
and not use_multi_item_scoring
|
||||
and not has_embeds
|
||||
and (query is None or isinstance(query, str))
|
||||
):
|
||||
# Single-set setwise: text (or omitted) query + text items, one
|
||||
# sequence per item. --enable-mis instead fuses the items below.
|
||||
input_ids = self._build_score_extraction_text_inputs(
|
||||
query or "", items, score_extraction_token_id
|
||||
)
|
||||
elif has_embeds or use_score_extraction:
|
||||
# Tokenize text inputs to token IDs, then build via the token-id path
|
||||
# (embed overrides need positions; score extraction needs token IDs).
|
||||
query_ids, items_ids = self._batch_tokenize_query_and_items(
|
||||
query or "" if use_multi_item_scoring else query, items
|
||||
)
|
||||
if use_score_extraction and not use_multi_item_scoring:
|
||||
self._reject_query_prefix_anchor(query_ids, score_extraction_token_id)
|
||||
_, input_ids, positional_embed_overrides, delimiter_indices = (
|
||||
self._build_token_id_inputs(
|
||||
query_ids,
|
||||
@@ -635,6 +966,21 @@ class TokenizerManagerScoreMixin:
|
||||
"Invalid combination of query/items types for score_request."
|
||||
)
|
||||
|
||||
# Setwise readout: scan each sequence for the extraction token (positions
|
||||
# the pooler reads the head at); under --enable-mis also return per-item
|
||||
# anchor counts to split the fused score matrix back per item.
|
||||
token_indices_to_pool = None
|
||||
per_item_anchor_counts = None
|
||||
if use_score_extraction:
|
||||
token_indices_to_pool, per_item_anchor_counts = (
|
||||
self._resolve_multi_position_pooling(
|
||||
input_ids,
|
||||
score_extraction_token_id,
|
||||
use_multi_item_scoring,
|
||||
delimiter_indices,
|
||||
)
|
||||
)
|
||||
|
||||
if return_pooled_hidden_states:
|
||||
if is_generation:
|
||||
raise ValueError(
|
||||
@@ -684,11 +1030,22 @@ class TokenizerManagerScoreMixin:
|
||||
positional_embed_overrides=positional_embed_overrides,
|
||||
return_pooled_hidden_states=return_pooled_hidden_states,
|
||||
multi_item_delimiter_indices=mis_delimiter_indices,
|
||||
token_indices_to_pool=token_indices_to_pool,
|
||||
)
|
||||
|
||||
results = await self.generate_request(batch_request, request).__anext__()
|
||||
|
||||
if use_multi_item_scoring:
|
||||
if use_multi_item_scoring and use_score_extraction:
|
||||
# Multi-item setwise: the items were fused into one sequence and the
|
||||
# head pooled at every anchor. Split the flat [ΣNᵢ, num_labels] result
|
||||
# back into one matrix per item (nested scores).
|
||||
return self._process_multi_item_extraction_results(
|
||||
results,
|
||||
per_item_anchor_counts,
|
||||
apply_softmax,
|
||||
return_pooled_hidden_states,
|
||||
)
|
||||
elif use_multi_item_scoring:
|
||||
# Multi-item scoring: extract scores from input_token_ids_logprobs or embedding
|
||||
return self._process_multi_item_scoring_results(
|
||||
results,
|
||||
@@ -701,14 +1058,18 @@ class TokenizerManagerScoreMixin:
|
||||
return_token_logprobs,
|
||||
)
|
||||
else:
|
||||
# Single-item scoring: process each result separately
|
||||
# Single-item scoring: process each result separately. With
|
||||
# score_extraction_token_id, each item is scored as an independent
|
||||
# query+item sequence and its 2-D embedding ([positions x num_labels])
|
||||
# is kept as one score matrix per item (nested scores).
|
||||
return self._process_single_item_scoring_results(
|
||||
results,
|
||||
label_token_ids,
|
||||
apply_softmax,
|
||||
return_pooled_hidden_states,
|
||||
temperature,
|
||||
return_token_logprobs,
|
||||
per_item_matrix=use_score_extraction,
|
||||
temperature=temperature,
|
||||
return_token_logprobs=return_token_logprobs,
|
||||
)
|
||||
|
||||
def _convert_logprobs_to_scores(
|
||||
|
||||
@@ -571,6 +571,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
# Pre-computed delimiter indices for multi-item scoring (CPU tensors, one per request)
|
||||
multi_item_delimiter_indices: Optional[List[torch.Tensor]] = None
|
||||
|
||||
# Setwise pooling readout positions (CPU tensors, one per request)
|
||||
token_indices_to_pool: Optional[List[torch.Tensor]] = None
|
||||
|
||||
# === Borrowed from ScheduleBatch: compound (carry their own device tensors) ===
|
||||
# Sampling info
|
||||
sampling_info: SamplingBatchInfo = None
|
||||
@@ -1158,6 +1161,18 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
for r in batch.reqs
|
||||
]
|
||||
|
||||
# Setwise readout: pool the head AT token_indices_to_pool instead of
|
||||
# the last token. The scheduler keeps readout batches homogeneous
|
||||
# (get_new_batch_prefill), so build only when every request carries
|
||||
# the field and otherwise fall back to standard pooling.
|
||||
if batch.reqs and all(
|
||||
r.token_indices_to_pool is not None for r in batch.reqs
|
||||
):
|
||||
self.token_indices_to_pool = [
|
||||
torch.tensor(r.token_indices_to_pool, dtype=torch.int64)
|
||||
for r in batch.reqs
|
||||
]
|
||||
|
||||
token_type_ids = [
|
||||
r.token_type_ids for r in batch.reqs if r.token_type_ids is not None
|
||||
]
|
||||
|
||||
@@ -1900,6 +1900,7 @@ class ModelRunner:
|
||||
and not isinstance(self.prefill_cuda_graph_runner, EagerRunner)
|
||||
and self.prefill_cuda_graph_runner is not None
|
||||
and self.prefill_cuda_graph_runner.can_run_graph(forward_batch)
|
||||
and forward_batch.token_indices_to_pool is None
|
||||
and _prefill_cuda_graph_allows_context_parallel(
|
||||
self.prefill_cuda_graph_runner, forward_batch
|
||||
)
|
||||
|
||||
@@ -0,0 +1,401 @@
|
||||
"""HuggingFace parity tests for setwise (multi-position) scoring.
|
||||
|
||||
Setwise scoring pools the SequenceClassification head at every occurrence of a
|
||||
``score_extraction_token`` (one per candidate) instead of the last token. These
|
||||
tests verify the *numerical* correctness of that readout against a HuggingFace
|
||||
reference (run the checkpoint through HF, gather the final hidden state at the
|
||||
anchor positions, apply the model's own classification head), so a wrong pooling
|
||||
position or head application changes the numbers, not just the shape.
|
||||
|
||||
The engine runs ``float16`` on ``flashinfer`` (its scoring backend of record);
|
||||
the HF golden is ``float32`` and compared with an fp16-appropriate tolerance.
|
||||
The anchor token ``<|object_ref_start|>`` is a Qwen3 special token (single
|
||||
dedicated id). All model/dtype/tolerance knobs are overridable via env.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import unittest
|
||||
|
||||
import torch
|
||||
from transformers import AutoModelForSequenceClassification, AutoTokenizer
|
||||
|
||||
from sglang.srt.entrypoints.engine import Engine
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(est_time=200, stage="base-b", runner_config="1-gpu-small")
|
||||
|
||||
_SEQCLS_MODEL = os.environ.get(
|
||||
"TEST_CLASSIFICATION_BASE_MODEL",
|
||||
"tomaarsen/Qwen3-Reranker-0.6B-seq-cls",
|
||||
)
|
||||
_ANCHOR_TOKEN = os.environ.get("TEST_SCORE_EXTRACTION_TOKEN", "<|object_ref_start|>")
|
||||
# float16 on flashinfer (no float32 prefill kernel); HF golden is float32 and the
|
||||
# tolerance below covers the fp16-vs-fp32 gap. All overridable via env.
|
||||
_DTYPE = os.environ.get("TEST_SEQCLS_DTYPE", "float16")
|
||||
_ATOL = float(os.environ.get("TEST_SETWISE_ATOL", "0.2"))
|
||||
_RTOL = float(os.environ.get("TEST_SETWISE_RTOL", "0.05"))
|
||||
|
||||
|
||||
class TestSetwiseScoringHFParity(CustomTestCase):
|
||||
"""Setwise engine scores must match an HF head-at-anchor reference."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.tokenizer = AutoTokenizer.from_pretrained(_SEQCLS_MODEL)
|
||||
cls.anchor_id = cls.tokenizer.convert_tokens_to_ids(_ANCHOR_TOKEN)
|
||||
assert (
|
||||
cls.anchor_id is not None and cls.anchor_id != cls.tokenizer.unk_token_id
|
||||
), f"{_ANCHOR_TOKEN!r} did not resolve to a dedicated token id"
|
||||
|
||||
cls.engine = Engine(
|
||||
model_path=_SEQCLS_MODEL,
|
||||
disable_radix_cache=True,
|
||||
chunked_prefill_size=-1,
|
||||
attention_backend="flashinfer",
|
||||
dtype=_DTYPE,
|
||||
mem_fraction_static=0.15,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
if getattr(cls, "engine", None) is not None:
|
||||
cls.engine.shutdown()
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# HF reference
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _hf_setwise_reference(self, prompt: str):
|
||||
"""Reference: HF hidden state at each anchor -> classification head.
|
||||
|
||||
Returns (ref_scores [N, num_labels] as list-of-lists, anchor_positions).
|
||||
Tokenized exactly like the engine (``tokenizer.encode`` with default
|
||||
special tokens) so the anchor positions line up.
|
||||
"""
|
||||
input_ids = self.tokenizer.encode(prompt)
|
||||
anchor_positions = [i for i, t in enumerate(input_ids) if t == self.anchor_id]
|
||||
self.assertGreater(len(anchor_positions), 0, "prompt has no anchor tokens")
|
||||
|
||||
model = AutoModelForSequenceClassification.from_pretrained(
|
||||
_SEQCLS_MODEL, torch_dtype=torch.float32
|
||||
).eval()
|
||||
try:
|
||||
ids = torch.tensor([input_ids], dtype=torch.long)
|
||||
with torch.no_grad():
|
||||
outputs = model(input_ids=ids, output_hidden_states=True)
|
||||
last_hidden = outputs.hidden_states[-1][0] # [seq, hidden]
|
||||
|
||||
# The classification head is `score` on Qwen/Llama SeqCls models.
|
||||
head = getattr(model, "score", None) or getattr(model, "classifier")
|
||||
anchor_hidden = last_hidden[anchor_positions].to(head.weight.dtype)
|
||||
ref = head(anchor_hidden) # [N, num_labels]
|
||||
return ref.tolist(), anchor_positions
|
||||
finally:
|
||||
model.cpu()
|
||||
del model
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
def _hf_anchor_hidden_states(self, prompt: str):
|
||||
"""Reference: HF final hidden state at each anchor position (pre-head).
|
||||
|
||||
Returns (hidden [N, hidden] as list-of-lists, anchor_positions).
|
||||
"""
|
||||
input_ids = self.tokenizer.encode(prompt)
|
||||
anchor_positions = [i for i, t in enumerate(input_ids) if t == self.anchor_id]
|
||||
self.assertGreater(len(anchor_positions), 0, "prompt has no anchor tokens")
|
||||
|
||||
model = AutoModelForSequenceClassification.from_pretrained(
|
||||
_SEQCLS_MODEL, torch_dtype=torch.float32
|
||||
).eval()
|
||||
try:
|
||||
ids = torch.tensor([input_ids], dtype=torch.long)
|
||||
with torch.no_grad():
|
||||
outputs = model(input_ids=ids, output_hidden_states=True)
|
||||
last_hidden = outputs.hidden_states[-1][0] # [seq, hidden]
|
||||
return last_hidden[anchor_positions].tolist(), anchor_positions
|
||||
finally:
|
||||
model.cpu()
|
||||
del model
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
def _assert_close(self, ref, sgl, atol=_ATOL, rtol=_RTOL):
|
||||
self.assertEqual(len(ref), len(sgl), "row count mismatch")
|
||||
for i, (rrow, srow) in enumerate(zip(ref, sgl)):
|
||||
self.assertEqual(len(rrow), len(srow), f"row {i} width mismatch")
|
||||
for j, (r, s) in enumerate(zip(rrow, srow)):
|
||||
self.assertLessEqual(
|
||||
abs(r - s),
|
||||
atol + rtol * abs(r),
|
||||
f"row {i} label {j}: HF={r:.6f} SGLang={s:.6f}",
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Tests
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _build_prompt(self, n_anchors: int) -> str:
|
||||
candidates = " ".join(f"Candidate {i}." for i in range(n_anchors))
|
||||
return f"Rank the candidates. {candidates} Scores:" + (
|
||||
_ANCHOR_TOKEN * n_anchors
|
||||
)
|
||||
|
||||
def test_setwise_scores_match_hf_reference(self):
|
||||
"""Engine setwise logits == HF head applied at each anchor position."""
|
||||
prompt = self._build_prompt(3)
|
||||
ref, anchor_positions = self._hf_setwise_reference(prompt)
|
||||
|
||||
# `scores` is nested per item (one [N x num_labels] matrix per item); a
|
||||
# single item here yields one matrix with one row per anchor.
|
||||
sgl = self.engine.score(
|
||||
query="",
|
||||
items=[prompt],
|
||||
apply_softmax=False,
|
||||
score_extraction_token_id=self.anchor_id,
|
||||
).scores
|
||||
|
||||
self.assertEqual(len(sgl), 1) # one item
|
||||
item = sgl[0]
|
||||
self.assertEqual(len(item), len(anchor_positions)) # one row per anchor
|
||||
self._assert_close(ref, item)
|
||||
|
||||
def test_setwise_multiple_items_match_hf(self):
|
||||
"""Multiple items -> one HF-matching matrix per item (independent, no MIS)."""
|
||||
prompt0 = self._build_prompt(3)
|
||||
prompt1 = self._build_prompt(2)
|
||||
ref0, anchors0 = self._hf_setwise_reference(prompt0)
|
||||
ref1, anchors1 = self._hf_setwise_reference(prompt1)
|
||||
|
||||
sgl = self.engine.score(
|
||||
query="",
|
||||
items=[prompt0, prompt1],
|
||||
apply_softmax=False,
|
||||
score_extraction_token_id=self.anchor_id,
|
||||
).scores
|
||||
|
||||
self.assertEqual(len(sgl), 2) # one matrix per item
|
||||
self.assertEqual(len(sgl[0]), len(anchors0))
|
||||
self.assertEqual(len(sgl[1]), len(anchors1))
|
||||
self._assert_close(ref0, sgl[0])
|
||||
self._assert_close(ref1, sgl[1])
|
||||
|
||||
def test_setwise_single_anchor_matches_hf(self):
|
||||
"""A single anchor is the degenerate case and must still match HF."""
|
||||
prompt = self._build_prompt(1)
|
||||
ref, _ = self._hf_setwise_reference(prompt)
|
||||
|
||||
sgl = self.engine.score(
|
||||
query="",
|
||||
items=[prompt],
|
||||
apply_softmax=False,
|
||||
score_extraction_token_id=self.anchor_id,
|
||||
).scores
|
||||
|
||||
self.assertEqual(len(sgl), 1) # one item
|
||||
self.assertEqual(len(sgl[0]), 1) # one anchor row
|
||||
self._assert_close(ref, sgl[0])
|
||||
|
||||
def test_setwise_candidate_ranking_matches_hf(self):
|
||||
"""Ranking (argsort over label 0) agrees with HF — robust to fp noise."""
|
||||
prompt = self._build_prompt(5)
|
||||
ref, _ = self._hf_setwise_reference(prompt)
|
||||
|
||||
# Single item -> unwrap its [N x num_labels] matrix.
|
||||
sgl = self.engine.score(
|
||||
query="",
|
||||
items=[prompt],
|
||||
apply_softmax=False,
|
||||
score_extraction_token_id=self.anchor_id,
|
||||
).scores[0]
|
||||
|
||||
ref_order = sorted(range(len(ref)), key=lambda i: ref[i][0])
|
||||
sgl_order = sorted(range(len(sgl)), key=lambda i: sgl[i][0])
|
||||
self.assertEqual(ref_order, sgl_order)
|
||||
|
||||
def test_setwise_pooled_hidden_states_match_hf(self):
|
||||
"""return_pooled_hidden_states -> per-anchor pre-head vectors == HF."""
|
||||
prompt = self._build_prompt(3)
|
||||
ref_hidden, anchor_positions = self._hf_anchor_hidden_states(prompt)
|
||||
|
||||
result = self.engine.score(
|
||||
query="",
|
||||
items=[prompt],
|
||||
apply_softmax=False,
|
||||
score_extraction_token_id=self.anchor_id,
|
||||
return_pooled_hidden_states=True,
|
||||
)
|
||||
|
||||
phs = result.pooled_hidden_states
|
||||
self.assertIsNotNone(phs)
|
||||
# Nested per item, parallel to scores: a single item -> one group of
|
||||
# pre-head vectors, one per anchor, aligned 1:1 with the score rows.
|
||||
self.assertEqual(len(phs), 1) # one item
|
||||
self.assertEqual(len(result.scores), 1)
|
||||
item_phs = phs[0]
|
||||
self.assertEqual(len(item_phs), len(anchor_positions))
|
||||
self.assertEqual(len(item_phs), len(result.scores[0]))
|
||||
sgl_hidden = [vec.tolist() for vec in item_phs]
|
||||
self._assert_close(ref_hidden, sgl_hidden)
|
||||
|
||||
def test_setwise_tokenized_items_omitted_query_match_hf(self):
|
||||
"""Token-ID items with the query omitted take the token-ID path (query -> [])."""
|
||||
prompt = self._build_prompt(3)
|
||||
ref, anchor_positions = self._hf_setwise_reference(prompt)
|
||||
token_ids = self.tokenizer.encode(prompt)
|
||||
|
||||
sgl = self.engine.score(
|
||||
items=[token_ids], # query omitted -> normalized to []
|
||||
apply_softmax=False,
|
||||
score_extraction_token_id=self.anchor_id,
|
||||
).scores
|
||||
|
||||
self.assertEqual(len(sgl), 1)
|
||||
self.assertEqual(len(sgl[0]), len(anchor_positions))
|
||||
self._assert_close(ref, sgl[0])
|
||||
|
||||
def test_mixed_setwise_and_pointwise_concurrent_batch(self):
|
||||
"""Setwise and pointwise requests batched together must both succeed.
|
||||
|
||||
Both share the ``/v1/score`` queue; the scheduler partitions prefill
|
||||
batches by pooling mode, so mixed traffic must not abort the setwise
|
||||
requests (pre-fix a mixed batch returned HTTP 400 for them).
|
||||
"""
|
||||
setwise_prompt = self._build_prompt(3)
|
||||
|
||||
async def _run():
|
||||
coros = []
|
||||
for _ in range(6):
|
||||
coros.append(
|
||||
self.engine.async_score(
|
||||
query="",
|
||||
items=[setwise_prompt],
|
||||
apply_softmax=False,
|
||||
score_extraction_token_id=self.anchor_id,
|
||||
)
|
||||
)
|
||||
coros.append(
|
||||
self.engine.async_score(
|
||||
query="",
|
||||
items=["Candidate A.", "Candidate B."],
|
||||
apply_softmax=False,
|
||||
)
|
||||
)
|
||||
return await asyncio.gather(*coros, return_exceptions=True)
|
||||
|
||||
results = self.engine.loop.run_until_complete(_run())
|
||||
|
||||
errors = [r for r in results if isinstance(r, BaseException)]
|
||||
self.assertEqual(errors, [], f"mixed batch produced errors: {errors}")
|
||||
for i, r in enumerate(results):
|
||||
if i % 2 == 0: # setwise: one item, 3 anchor rows (nested)
|
||||
self.assertEqual(len(r.scores), 1)
|
||||
self.assertEqual(len(r.scores[0]), 3)
|
||||
else: # pointwise: one score row per item (flat)
|
||||
self.assertEqual(len(r.scores), 2)
|
||||
|
||||
|
||||
class TestSetwiseMultiItemMISScoring(CustomTestCase):
|
||||
"""Multi-item setwise scoring under ``--enable-mis`` (fused sequence).
|
||||
|
||||
The fused path carries the standard MIS delimiter tokens, so it is not
|
||||
bit-identical to independent HF scoring (inherent to MIS). These tests assert
|
||||
the nested per-item shape and block-diagonal set isolation rather than HF
|
||||
parity, which ``TestSetwiseScoringHFParity`` covers for the batched path.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.tokenizer = AutoTokenizer.from_pretrained(_SEQCLS_MODEL)
|
||||
cls.anchor_id = cls.tokenizer.convert_tokens_to_ids(_ANCHOR_TOKEN)
|
||||
cls.engine = Engine(
|
||||
model_path=_SEQCLS_MODEL,
|
||||
disable_radix_cache=True,
|
||||
chunked_prefill_size=-1,
|
||||
enable_mis=True,
|
||||
attention_backend="flashinfer",
|
||||
dtype=_DTYPE,
|
||||
mem_fraction_static=0.15,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
if getattr(cls, "engine", None) is not None:
|
||||
cls.engine.shutdown()
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
@staticmethod
|
||||
def _set_prompt(n_anchors: int) -> str:
|
||||
candidates = " ".join(f"Candidate {i}." for i in range(n_anchors))
|
||||
return f"Rank the candidates. {candidates} Scores:" + (
|
||||
_ANCHOR_TOKEN * n_anchors
|
||||
)
|
||||
|
||||
# atol=5e-2 is a floor set by fp16, not slack: one ULP on these logits is
|
||||
# ~0.002-0.004, and fusing set0 with another set changes reduction order and
|
||||
# stacks several such steps (observed drift ~0.012). 5e-2 clears that while
|
||||
# still catching real cross-set leakage, which would diverge far more.
|
||||
def _assert_matrix_close(self, a, b, atol=5e-2):
|
||||
self.assertEqual(len(a), len(b), "row count mismatch")
|
||||
for i, (ra, rb) in enumerate(zip(a, b)):
|
||||
self.assertEqual(len(ra), len(rb), f"row {i} width mismatch")
|
||||
for x, y in zip(ra, rb):
|
||||
self.assertLessEqual(abs(x - y), atol, f"{x} vs {y}")
|
||||
|
||||
def test_mis_multi_item_nested_shape(self):
|
||||
"""Fused multi-item -> one [Nᵢ x num_labels] matrix per item."""
|
||||
scores = self.engine.score(
|
||||
query="Rank:",
|
||||
items=[self._set_prompt(3), self._set_prompt(2)],
|
||||
apply_softmax=False,
|
||||
score_extraction_token_id=self.anchor_id,
|
||||
).scores
|
||||
|
||||
self.assertEqual(len(scores), 2) # one matrix per item
|
||||
self.assertEqual(len(scores[0]), 3) # 3 candidates
|
||||
self.assertEqual(len(scores[1]), 2) # 2 candidates
|
||||
num_labels = len(scores[0][0])
|
||||
self.assertTrue(
|
||||
all(len(row) == num_labels for matrix in scores for row in matrix)
|
||||
)
|
||||
|
||||
def test_mis_set_isolation(self):
|
||||
"""Block-diagonal mask: a set's scores are unchanged by other sets.
|
||||
|
||||
The first set is scored alone and again fused with a second set; the
|
||||
block-diagonal mask means the first set attends only over the shared query
|
||||
prefix and itself, so its matrix must be identical either way.
|
||||
"""
|
||||
set0 = self._set_prompt(3)
|
||||
alone = self.engine.score(
|
||||
query="Rank:",
|
||||
items=[set0],
|
||||
apply_softmax=False,
|
||||
score_extraction_token_id=self.anchor_id,
|
||||
).scores
|
||||
fused = self.engine.score(
|
||||
query="Rank:",
|
||||
items=[set0, self._set_prompt(2)],
|
||||
apply_softmax=False,
|
||||
score_extraction_token_id=self.anchor_id,
|
||||
).scores
|
||||
|
||||
self.assertEqual(len(alone), 1)
|
||||
self.assertEqual(len(fused), 2)
|
||||
# set0 is the first item in both requests -> identical matrix.
|
||||
self._assert_matrix_close(alone[0], fused[0])
|
||||
|
||||
def test_mis_rejects_item_without_anchor(self):
|
||||
"""An item carrying no extraction token is rejected under --enable-mis."""
|
||||
with self.assertRaisesRegex(ValueError, "at least one"):
|
||||
self.engine.score(
|
||||
query="Rank:",
|
||||
items=[self._set_prompt(2), "no anchors here"],
|
||||
apply_softmax=False,
|
||||
score_extraction_token_id=self.anchor_id,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -25,6 +25,7 @@ register_cpu_ci(est_time=10, suite="base-a-test-cpu")
|
||||
def _make_forward_batch(
|
||||
extend_seq_lens,
|
||||
multi_item_delimiter_indices=None,
|
||||
token_indices_to_pool=None,
|
||||
return_pooled_hidden_states=False,
|
||||
is_prefill_only=True,
|
||||
):
|
||||
@@ -33,6 +34,7 @@ def _make_forward_batch(
|
||||
extend_seq_lens=torch.tensor(extend_seq_lens, dtype=torch.long),
|
||||
extend_seq_lens_cpu=extend_seq_lens,
|
||||
multi_item_delimiter_indices=multi_item_delimiter_indices,
|
||||
token_indices_to_pool=token_indices_to_pool,
|
||||
dimensions=None,
|
||||
return_pooled_hidden_states=return_pooled_hidden_states,
|
||||
is_prefill_only=is_prefill_only,
|
||||
@@ -186,6 +188,85 @@ class TestScoreAndPool(CustomTestCase):
|
||||
self.assertEqual(len(out.embeddings), 1)
|
||||
self.assertEqual(out.embeddings[0].shape, (0, self.num_labels))
|
||||
|
||||
def test_setwise_returns_per_request_list(self):
|
||||
"""Anchor positions provided -> one [num_anchors, num_labels] tensor per request."""
|
||||
# Sequence: query + items + suffix; anchors at positions 8, 10, 12.
|
||||
input_ids = torch.arange(14)
|
||||
hidden = torch.randn(len(input_ids), self.hidden_dim)
|
||||
fb = _make_forward_batch(
|
||||
extend_seq_lens=[len(input_ids)],
|
||||
token_indices_to_pool=[torch.tensor([8, 10, 12])],
|
||||
)
|
||||
|
||||
out = score_and_pool(self.score_head, self.pooler, hidden, fb, input_ids)
|
||||
|
||||
self.assertIsInstance(out.embeddings, list)
|
||||
self.assertEqual(len(out.embeddings), 1)
|
||||
# 3 anchors -> 3 score rows (no query-boundary row, unlike MIS).
|
||||
self.assertEqual(out.embeddings[0].shape, (3, self.num_labels))
|
||||
|
||||
def test_setwise_extracts_at_anchor_positions(self):
|
||||
"""Setwise pools AT the anchor index (no delimiter-1 shift)."""
|
||||
input_ids = torch.tensor([0, 1, 2, 3, 4, 5])
|
||||
hidden = (
|
||||
torch.arange(len(input_ids))
|
||||
.unsqueeze(1)
|
||||
.float()
|
||||
.expand(-1, self.hidden_dim)
|
||||
.clone()
|
||||
)
|
||||
fb = _make_forward_batch(
|
||||
extend_seq_lens=[len(input_ids)],
|
||||
token_indices_to_pool=[torch.tensor([2, 5])],
|
||||
)
|
||||
|
||||
identity_head = nn.Linear(self.hidden_dim, self.hidden_dim, bias=False)
|
||||
nn.init.eye_(identity_head.weight)
|
||||
|
||||
out = score_and_pool(identity_head, self.pooler, hidden, fb, input_ids)
|
||||
|
||||
scores = out.embeddings[0]
|
||||
# Pooled exactly at indices 2 and 5 (not 1 and 4).
|
||||
torch.testing.assert_close(scores[0], hidden[2])
|
||||
torch.testing.assert_close(scores[1], hidden[5])
|
||||
|
||||
def test_setwise_batched_splits_per_request(self):
|
||||
"""Two batched setwise requests -> a list of length 2."""
|
||||
req1 = list(range(10, 17)) # 7 tokens, anchors at 5, 6
|
||||
req2 = list(range(20, 24)) # 4 tokens, anchor at 3
|
||||
input_ids = torch.tensor(req1 + req2)
|
||||
hidden = torch.randn(len(input_ids), self.hidden_dim)
|
||||
fb = _make_forward_batch(
|
||||
extend_seq_lens=[len(req1), len(req2)],
|
||||
token_indices_to_pool=[
|
||||
torch.tensor([5, 6]),
|
||||
torch.tensor([3]),
|
||||
],
|
||||
)
|
||||
|
||||
out = score_and_pool(self.score_head, self.pooler, hidden, fb, input_ids)
|
||||
|
||||
self.assertIsInstance(out.embeddings, list)
|
||||
self.assertEqual(len(out.embeddings), 2)
|
||||
self.assertEqual(out.embeddings[0].shape, (2, self.num_labels))
|
||||
self.assertEqual(out.embeddings[1].shape, (1, self.num_labels))
|
||||
|
||||
def test_setwise_returns_pooled_hidden_states(self):
|
||||
"""return_pooled_hidden_states -> per-anchor pre-head hidden states."""
|
||||
input_ids = torch.arange(6)
|
||||
hidden = torch.randn(len(input_ids), self.hidden_dim)
|
||||
fb = _make_forward_batch(
|
||||
extend_seq_lens=[len(input_ids)],
|
||||
token_indices_to_pool=[torch.tensor([3, 5])],
|
||||
return_pooled_hidden_states=True,
|
||||
)
|
||||
|
||||
out = score_and_pool(self.score_head, self.pooler, hidden, fb, input_ids)
|
||||
|
||||
self.assertIsNotNone(out.pooled_hidden_states)
|
||||
self.assertEqual(len(out.pooled_hidden_states), 1)
|
||||
self.assertEqual(out.pooled_hidden_states[0].shape, (2, self.hidden_dim))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -0,0 +1,536 @@
|
||||
"""Unit tests for setwise result grouping on
|
||||
``TokenizerManagerScoreMixin._process_single_item_scoring_results`` and
|
||||
``_process_multi_item_extraction_results``.
|
||||
|
||||
Setwise scoring pools the SequenceClassification head at every occurrence of a
|
||||
``score_extraction_token_id``, so each item carries a 2-D
|
||||
``[num_positions x num_labels]`` embedding kept grouped (``per_item_matrix=True``)
|
||||
as nested ``scores`` ``[num_items][Nᵢ x num_labels]``. These tests cover the
|
||||
grouping, validation, anchor bucketing, and the serialization round-trip on CPU.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.managers.tokenizer_manager_score_mixin import (
|
||||
ScoreResult,
|
||||
TokenizerManagerScoreMixin,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_context
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
# Config that satisfies _validate_score_extraction (dedicated setwise endpoint).
|
||||
_VALID_SETWISE_CONFIG = dict(
|
||||
disable_radix_cache=True,
|
||||
chunked_prefill_size=-1,
|
||||
allow_auto_truncate=False,
|
||||
)
|
||||
|
||||
|
||||
class _ClsHarness(TokenizerManagerScoreMixin):
|
||||
"""Bare SequenceClassification harness for the pure result parser."""
|
||||
|
||||
is_generation = False
|
||||
|
||||
def __init__(self, architecture="Qwen3ForSequenceClassification"):
|
||||
self.model_config = SimpleNamespace(
|
||||
hf_config=SimpleNamespace(architectures=[architecture])
|
||||
)
|
||||
|
||||
|
||||
class TestSingleItemScoringResults(CustomTestCase):
|
||||
@staticmethod
|
||||
def _result(embedding, phs=None, prompt_tokens=7):
|
||||
r = {
|
||||
"meta_info": {"id": "rid-1", "prompt_tokens": prompt_tokens},
|
||||
"embedding": embedding,
|
||||
}
|
||||
if phs is not None:
|
||||
r["pooled_hidden_state"] = phs
|
||||
return r
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.h = _ClsHarness()
|
||||
# _validate_score_extraction reads resolved config bags, so publish a
|
||||
# valid setwise config; validation tests re-publish with one field flipped.
|
||||
self._saved_server_args = get_context()._server_args
|
||||
self.addCleanup(self._restore_server_args)
|
||||
self._publish(**_VALID_SETWISE_CONFIG)
|
||||
|
||||
def _restore_server_args(self):
|
||||
if self._saved_server_args is not None:
|
||||
get_context().set_server_args(self._saved_server_args)
|
||||
|
||||
def _publish(self, **fields):
|
||||
override = get_context().override_server_args(**fields)
|
||||
override.install()
|
||||
self.addCleanup(override.restore)
|
||||
|
||||
# ---- default (per-item) CLS behavior -------------------
|
||||
|
||||
def test_pointwise_one_row_per_item(self):
|
||||
results = [
|
||||
self._result([0.1, 0.2]),
|
||||
self._result([0.3, 0.4]),
|
||||
]
|
||||
res = self.h._process_single_item_scoring_results(
|
||||
results, label_token_ids=None, apply_softmax=False
|
||||
)
|
||||
self.assertIsInstance(res, ScoreResult)
|
||||
self.assertEqual(res.scores, [[0.1, 0.2], [0.3, 0.4]])
|
||||
self.assertEqual(res.prompt_tokens, 14)
|
||||
|
||||
def test_mis_pooled_hidden_states_are_list_of_item_tensors(self):
|
||||
# Standard MIS returns one fused result with a row at each delimiter.
|
||||
# The first row is the query boundary and is discarded; the public
|
||||
# ScoreResult contract is one CPU tensor per remaining item.
|
||||
phs = torch.arange(12).reshape(3, 4)
|
||||
result = self._result([[0.0, 0.0], [1.0, 0.0], [0.0, 1.0]], phs=phs)
|
||||
res = self.h._process_multi_item_scoring_results(
|
||||
[result],
|
||||
items=["item0", "item1"],
|
||||
label_token_ids=None,
|
||||
apply_softmax=False,
|
||||
return_pooled_hidden_states=True,
|
||||
)
|
||||
|
||||
self.assertEqual(res.scores, [[1.0, 0.0], [0.0, 1.0]])
|
||||
self.assertIsInstance(res.pooled_hidden_states, list)
|
||||
self.assertEqual(len(res.pooled_hidden_states), 2)
|
||||
torch.testing.assert_close(res.pooled_hidden_states[0], phs[1])
|
||||
torch.testing.assert_close(res.pooled_hidden_states[1], phs[2])
|
||||
|
||||
# ---- setwise: per_item_matrix=True ----------------------------------
|
||||
|
||||
def test_setwise_one_matrix_per_item(self):
|
||||
# One item (single big candidate block), 3 extraction positions -> one
|
||||
# [3 x num_labels] matrix, nested under the item.
|
||||
emb = [[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]]
|
||||
res = self.h._process_single_item_scoring_results(
|
||||
[self._result(emb)],
|
||||
label_token_ids=None,
|
||||
apply_softmax=False,
|
||||
per_item_matrix=True,
|
||||
)
|
||||
self.assertEqual(res.scores, [emb]) # [num_items][N x num_labels]
|
||||
self.assertEqual(len(res.scores), 1)
|
||||
self.assertEqual(len(res.scores[0]), 3)
|
||||
|
||||
def test_setwise_apply_softmax_over_labels(self):
|
||||
emb = [[0.0, 0.0], [2.0, 2.0]]
|
||||
res = self.h._process_single_item_scoring_results(
|
||||
[self._result(emb)],
|
||||
label_token_ids=None,
|
||||
apply_softmax=True,
|
||||
per_item_matrix=True,
|
||||
)
|
||||
for row in res.scores[0]:
|
||||
self.assertAlmostEqual(sum(row), 1.0, places=5)
|
||||
self.assertAlmostEqual(row[0], 0.5, places=5)
|
||||
|
||||
def test_setwise_multiple_items_grouped_per_item(self):
|
||||
# Two candidate sets scored independently -> one matrix per item (nested).
|
||||
res = self.h._process_single_item_scoring_results(
|
||||
[self._result([[1.0, 0.0], [0.0, 1.0]]), self._result([[0.5, 0.5]])],
|
||||
label_token_ids=None,
|
||||
apply_softmax=False,
|
||||
per_item_matrix=True,
|
||||
)
|
||||
self.assertEqual(res.scores, [[[1.0, 0.0], [0.0, 1.0]], [[0.5, 0.5]]])
|
||||
|
||||
def test_setwise_groups_pooled_hidden_states_per_item(self):
|
||||
emb = [[0.1, 0.2], [0.3, 0.4]]
|
||||
phs = torch.randn(2, 4) # one pre-head vector per position
|
||||
res = self.h._process_single_item_scoring_results(
|
||||
[self._result(emb, phs=phs)],
|
||||
label_token_ids=None,
|
||||
apply_softmax=False,
|
||||
return_pooled_hidden_states=True,
|
||||
per_item_matrix=True,
|
||||
)
|
||||
self.assertIsNotNone(res.pooled_hidden_states)
|
||||
# One item -> one matrix of 2 vectors.
|
||||
self.assertEqual(len(res.pooled_hidden_states), 1)
|
||||
self.assertEqual(len(res.pooled_hidden_states[0]), 2)
|
||||
for v in res.pooled_hidden_states[0]:
|
||||
self.assertEqual(tuple(v.shape), (4,))
|
||||
|
||||
def test_setwise_no_phs_when_not_requested(self):
|
||||
emb = [[0.1, 0.2]]
|
||||
phs = torch.randn(1, 4)
|
||||
res = self.h._process_single_item_scoring_results(
|
||||
[self._result(emb, phs=phs)],
|
||||
label_token_ids=None,
|
||||
apply_softmax=False,
|
||||
return_pooled_hidden_states=False,
|
||||
per_item_matrix=True,
|
||||
)
|
||||
self.assertIsNone(res.pooled_hidden_states)
|
||||
|
||||
def test_setwise_rejects_non_matrix_scores(self):
|
||||
with self.assertRaisesRegex(ValueError, "expected a 2-D"):
|
||||
self.h._process_single_item_scoring_results(
|
||||
[self._result([0.1, 0.2])],
|
||||
label_token_ids=None,
|
||||
apply_softmax=False,
|
||||
per_item_matrix=True,
|
||||
)
|
||||
|
||||
def test_setwise_rejects_misaligned_pooled_hidden_states(self):
|
||||
with self.assertRaisesRegex(ValueError, "one row per score position"):
|
||||
self.h._process_single_item_scoring_results(
|
||||
[self._result([[0.1], [0.2]], phs=torch.randn(1, 4))],
|
||||
label_token_ids=None,
|
||||
apply_softmax=False,
|
||||
return_pooled_hidden_states=True,
|
||||
per_item_matrix=True,
|
||||
)
|
||||
|
||||
def test_validation_rejects_radix_cache(self):
|
||||
self._publish(disable_radix_cache=False, chunked_prefill_size=-1)
|
||||
with self.assertRaisesRegex(ValueError, "--disable-radix-cache"):
|
||||
self.h._validate_score_extraction(False, False)
|
||||
|
||||
def test_validation_rejects_chunked_prefill(self):
|
||||
self._publish(disable_radix_cache=True, chunked_prefill_size=2048)
|
||||
with self.assertRaisesRegex(ValueError, "--chunked-prefill-size -1"):
|
||||
self.h._validate_score_extraction(False, False)
|
||||
|
||||
def test_validation_rejects_cross_encoder(self):
|
||||
harness = _ClsHarness(architecture="BertForSequenceClassification")
|
||||
with self.assertRaisesRegex(ValueError, "cross-encoder"):
|
||||
harness._validate_score_extraction(False, False)
|
||||
|
||||
def test_validation_rejects_reward_model(self):
|
||||
# Reward models (last-token pooling, not score_and_pool) ignore the readout
|
||||
# positions -> rejected up front, not after a wasted forward.
|
||||
harness = _ClsHarness(architecture="InternLM2ForRewardModel")
|
||||
with self.assertRaisesRegex(ValueError, "pool the head per position"):
|
||||
harness._validate_score_extraction(False, False)
|
||||
|
||||
def test_validation_rejects_embedding_model(self):
|
||||
# A base embedding model likewise does not route through score_and_pool.
|
||||
harness = _ClsHarness(architecture="Qwen3Model")
|
||||
with self.assertRaisesRegex(ValueError, "pool the head per position"):
|
||||
harness._validate_score_extraction(False, False)
|
||||
|
||||
def test_validation_rejects_auto_truncate(self):
|
||||
self._publish(**{**_VALID_SETWISE_CONFIG, "allow_auto_truncate": True})
|
||||
with self.assertRaisesRegex(ValueError, "--allow-auto-truncate"):
|
||||
self.h._validate_score_extraction(False, False)
|
||||
|
||||
def test_validation_allows_multi_item(self):
|
||||
# Multiple items (candidate sets) are allowed with or without --enable-mis;
|
||||
# both return one score matrix per item.
|
||||
self.h._validate_score_extraction(False, False)
|
||||
|
||||
def test_anchor_counts_per_item_buckets_by_delimiter(self):
|
||||
# Fused: q <d0> item0(2 anchors) <d1> item1(1 anchor) <d2>.
|
||||
# delimiter_indices point at the delimiter tokens.
|
||||
delimiter_indices = [3, 8, 12]
|
||||
anchor_positions = [5, 7, 10] # 5,7 in item0 (3<p<8); 10 in item1 (8<p<12)
|
||||
counts = self.h._anchor_counts_per_item(delimiter_indices, anchor_positions)
|
||||
self.assertEqual(counts, [2, 1])
|
||||
|
||||
def test_anchor_counts_per_item_rejects_item_without_anchor(self):
|
||||
delimiter_indices = [3, 8, 12]
|
||||
anchor_positions = [5, 7] # nothing in item1
|
||||
with self.assertRaisesRegex(ValueError, "at least one"):
|
||||
self.h._anchor_counts_per_item(delimiter_indices, anchor_positions)
|
||||
|
||||
def test_anchor_counts_per_item_rejects_anchor_outside_items(self):
|
||||
# An anchor before the first delimiter (shared query prefix) is not
|
||||
# attributable to any item and must be rejected up front.
|
||||
delimiter_indices = [3, 8, 12]
|
||||
anchor_positions = [1, 5, 7, 10] # position 1 is in the query region
|
||||
with self.assertRaisesRegex(ValueError, "outside the candidate items"):
|
||||
self.h._anchor_counts_per_item(delimiter_indices, anchor_positions)
|
||||
|
||||
def test_multi_item_extraction_groups_scores_per_item(self):
|
||||
# Fused request returns one flat [ΣNᵢ, num_labels] embedding; split back
|
||||
# into one matrix per item using the per-item anchor counts.
|
||||
res = self.h._process_multi_item_extraction_results(
|
||||
[self._result([[1.0, 0.0], [0.0, 1.0], [0.5, 0.5]])],
|
||||
per_item_anchor_counts=[2, 1],
|
||||
apply_softmax=False,
|
||||
)
|
||||
self.assertEqual(res.scores, [[[1.0, 0.0], [0.0, 1.0]], [[0.5, 0.5]]])
|
||||
self.assertEqual(res.prompt_tokens, 7)
|
||||
|
||||
def test_multi_item_extraction_softmax_over_labels(self):
|
||||
res = self.h._process_multi_item_extraction_results(
|
||||
[self._result([[0.0, 0.0], [2.0, 2.0]])],
|
||||
per_item_anchor_counts=[1, 1],
|
||||
apply_softmax=True,
|
||||
)
|
||||
for item in res.scores:
|
||||
for row in item:
|
||||
self.assertAlmostEqual(sum(row), 1.0, places=5)
|
||||
|
||||
def test_multi_item_extraction_groups_pooled_hidden_states(self):
|
||||
res = self.h._process_multi_item_extraction_results(
|
||||
[self._result([[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]], phs=torch.randn(3, 4))],
|
||||
per_item_anchor_counts=[2, 1],
|
||||
apply_softmax=False,
|
||||
return_pooled_hidden_states=True,
|
||||
)
|
||||
self.assertIsNotNone(res.pooled_hidden_states)
|
||||
self.assertEqual(len(res.pooled_hidden_states), 2)
|
||||
self.assertEqual(len(res.pooled_hidden_states[0]), 2)
|
||||
self.assertEqual(len(res.pooled_hidden_states[1]), 1)
|
||||
self.assertEqual(tuple(res.pooled_hidden_states[0][0].shape), (4,))
|
||||
|
||||
def test_multi_item_extraction_rejects_row_count_mismatch(self):
|
||||
with self.assertRaisesRegex(RuntimeError, "anchor rows"):
|
||||
self.h._process_multi_item_extraction_results(
|
||||
[self._result([[1.0, 0.0], [0.0, 1.0]])],
|
||||
per_item_anchor_counts=[2, 1],
|
||||
apply_softmax=False,
|
||||
)
|
||||
|
||||
def test_text_extraction_tokenizes_combined_prompt_once(self):
|
||||
class _RecordingTokenizer:
|
||||
def __init__(self):
|
||||
self.inputs = []
|
||||
|
||||
def encode(self, text):
|
||||
self.inputs.append(text)
|
||||
return [len(text)]
|
||||
|
||||
self.h.tokenizer = _RecordingTokenizer()
|
||||
# anchor id 999 never appears (encode returns [len(text)]), so the
|
||||
# query-prefix guard passes; the query is scanned once and each item is
|
||||
# combined-encoded once.
|
||||
input_ids = self.h._build_score_extraction_text_inputs("query", ["a", "b"], 999)
|
||||
self.assertEqual(self.h.tokenizer.inputs, ["query", "querya", "queryb"])
|
||||
self.assertEqual(input_ids, [[6], [6]])
|
||||
|
||||
def test_text_extraction_rejects_extraction_token_in_query(self):
|
||||
# A query-side anchor would emit an extra, misaligned row. anchor id 5 ==
|
||||
# len("query"), so encode("query") -> [5] contains it and must reject.
|
||||
class _LenTokenizer:
|
||||
def encode(self, text):
|
||||
return [len(text)]
|
||||
|
||||
self.h.tokenizer = _LenTokenizer()
|
||||
with self.assertRaisesRegex(ValueError, "query prefix"):
|
||||
self.h._build_score_extraction_text_inputs("query", ["a"], 5)
|
||||
|
||||
def test_reject_query_prefix_anchor_rejects_pretokenized_query(self):
|
||||
# A pre-tokenized query whose ids contain the extraction token is rejected:
|
||||
# the non-MIS path scans the whole query + item sequence.
|
||||
with self.assertRaisesRegex(ValueError, "query prefix"):
|
||||
self.h._reject_query_prefix_anchor([1, 7, 2], score_extraction_token_id=7)
|
||||
|
||||
def test_reject_query_prefix_anchor_allows_clean_query(self):
|
||||
# A query prefix without the extraction token passes.
|
||||
self.h._reject_query_prefix_anchor([1, 2, 3], score_extraction_token_id=7)
|
||||
|
||||
def test_resolve_indices_returns_positions_per_sequence(self):
|
||||
# anchor id = 7; positions are per-sequence, in order.
|
||||
indices = self.h._resolve_score_extraction_indices(
|
||||
[[1, 7, 2, 7], [7, 3]], score_extraction_token_id=7
|
||||
)
|
||||
self.assertEqual(indices, [[1, 3], [0]])
|
||||
|
||||
def test_resolve_indices_rejects_any_sequence_without_anchor(self):
|
||||
# Second sequence has no anchor -> must fail (not silently drop its rows),
|
||||
# even though another sequence does contain the anchor.
|
||||
with self.assertRaisesRegex(ValueError, "one or more"):
|
||||
self.h._resolve_score_extraction_indices(
|
||||
[[1, 7, 2], [3, 4]], score_extraction_token_id=7
|
||||
)
|
||||
|
||||
# ---- _resolve_multi_position_pooling (extraction-token pooling) -------
|
||||
|
||||
def test_resolve_multi_position_pooling_batched_returns_no_counts(self):
|
||||
# No MIS: one sequence per item; anchors resolved per sequence and no
|
||||
# per-item anchor counts (the batched path splits by result, not counts).
|
||||
token_indices, counts = self.h._resolve_multi_position_pooling(
|
||||
[[1, 7, 2, 7], [7, 3]],
|
||||
score_extraction_token_id=7,
|
||||
use_multi_item_scoring=False,
|
||||
delimiter_indices=None,
|
||||
)
|
||||
self.assertEqual(token_indices, [[1, 3], [0]])
|
||||
self.assertIsNone(counts)
|
||||
|
||||
def test_resolve_multi_position_pooling_mis_buckets_anchors_by_item(self):
|
||||
# MIS: items fused into one sequence with delimiters; anchors are scanned
|
||||
# over the fused sequence and bucketed back to items via the delimiters.
|
||||
# Fused (D=0, anchor=7): q q <D> 7 9 7 <D> 9 7 <D>
|
||||
# positions: 0 1 2 3 4 5 6 7 8 9
|
||||
# delimiters at 2,6,9; anchors at 3,5,8 -> item0 has 2, item1 has 1.
|
||||
fused = [1, 1, 0, 7, 9, 7, 0, 9, 7, 0]
|
||||
token_indices, counts = self.h._resolve_multi_position_pooling(
|
||||
[fused],
|
||||
score_extraction_token_id=7,
|
||||
use_multi_item_scoring=True,
|
||||
delimiter_indices=[2, 6, 9],
|
||||
)
|
||||
self.assertEqual(token_indices, [[3, 5, 8]])
|
||||
self.assertEqual(counts, [2, 1])
|
||||
|
||||
def test_resolve_multi_position_pooling_mis_rejects_item_without_anchor(self):
|
||||
# An item with no anchor inside its fused block is rejected.
|
||||
# Fused (D=0, anchor=7): q <D> 7 <D> 9 <D> -> item1 (positions 3<p<5) empty.
|
||||
fused = [1, 0, 7, 0, 9, 0]
|
||||
with self.assertRaisesRegex(ValueError, "at least one"):
|
||||
self.h._resolve_multi_position_pooling(
|
||||
[fused],
|
||||
score_extraction_token_id=7,
|
||||
use_multi_item_scoring=True,
|
||||
delimiter_indices=[1, 3, 5],
|
||||
)
|
||||
|
||||
def test_resolve_multi_position_pooling_mis_rejects_query_side_anchor(self):
|
||||
# The extraction token also appears in the query prefix -> rejected before
|
||||
# inference.
|
||||
# Fused (D=0, anchor=7): q 7 <D> 7 <D> 7 <D> (query anchor at position 1)
|
||||
# positions: 0 1 2 3 4 5 6
|
||||
# delimiters at 2,4,6; in-item anchors at 3,5 but the scan also finds 1.
|
||||
fused = [1, 7, 0, 7, 0, 7, 0]
|
||||
with self.assertRaisesRegex(ValueError, "outside the candidate items"):
|
||||
self.h._resolve_multi_position_pooling(
|
||||
[fused],
|
||||
score_extraction_token_id=7,
|
||||
use_multi_item_scoring=True,
|
||||
delimiter_indices=[2, 4, 6],
|
||||
)
|
||||
|
||||
# ---- _multi_position_score_rows / _multi_position_phs_matrix helpers --
|
||||
|
||||
def test_multi_position_score_rows_preserves_list_values_without_softmax(self):
|
||||
# No softmax returns the original list as-is (no float round-trip).
|
||||
emb = [[0.1, 0.2], [0.3, 0.4]]
|
||||
rows = self.h._multi_position_score_rows(emb, apply_softmax=False)
|
||||
self.assertIs(rows, emb)
|
||||
|
||||
def test_multi_position_score_rows_softmax_over_labels(self):
|
||||
rows = self.h._multi_position_score_rows(
|
||||
[[0.0, 0.0], [2.0, 2.0]], apply_softmax=True
|
||||
)
|
||||
for row in rows:
|
||||
self.assertAlmostEqual(sum(row), 1.0, places=5)
|
||||
self.assertAlmostEqual(row[0], 0.5, places=5)
|
||||
|
||||
def test_multi_position_score_rows_rejects_non_matrix(self):
|
||||
with self.assertRaisesRegex(ValueError, "expected a 2-D"):
|
||||
self.h._multi_position_score_rows([0.1, 0.2], apply_softmax=False)
|
||||
|
||||
def test_multi_position_phs_matrix_rejects_row_count_mismatch(self):
|
||||
with self.assertRaisesRegex(ValueError, "one row per score position"):
|
||||
self.h._multi_position_phs_matrix(torch.randn(2, 4), expected_rows=3)
|
||||
|
||||
def test_multi_position_phs_matrix_rejects_non_matrix(self):
|
||||
with self.assertRaisesRegex(ValueError, "one row per score position"):
|
||||
self.h._multi_position_phs_matrix(torch.randn(4), expected_rows=4)
|
||||
|
||||
|
||||
class TestSetwisePooledHiddenStatesRoundTrip(CustomTestCase):
|
||||
"""Validate per-position pooled hidden states survive scheduler serialization.
|
||||
|
||||
The pooler's ``[num_positions, hidden]`` tensor is packed by
|
||||
``stream_output_embedding`` (stacked when shapes match, list when they differ)
|
||||
and reconstructed on the receiver by index. These tests exercise that real
|
||||
transmission path, then confirm the parser groups it as one item's vectors.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _run_stream_output_embedding(phs_per_req):
|
||||
# Lazy import so the heavier scheduler module loads only when this runs.
|
||||
from sglang.srt.managers.scheduler_components.output_streamer import (
|
||||
SchedulerOutputStreamer,
|
||||
)
|
||||
|
||||
captured = []
|
||||
reqs = []
|
||||
for idx, phs in enumerate(phs_per_req):
|
||||
reqs.append(
|
||||
SimpleNamespace(
|
||||
finished=lambda: True,
|
||||
rid=f"rid-{idx}",
|
||||
http_worker_ipc=None,
|
||||
finished_reason=SimpleNamespace(to_json=lambda: {"type": "stop"}),
|
||||
embedding=[[0.0, 0.0] for _ in range(phs.shape[0])],
|
||||
origin_input_ids=[1, 2, 3],
|
||||
cached_tokens=0,
|
||||
time_stats=SimpleNamespace(),
|
||||
retraction_count=0,
|
||||
pooled_hidden_state=phs,
|
||||
)
|
||||
)
|
||||
|
||||
fake_self = SimpleNamespace(
|
||||
get_cached_tokens_details=lambda req: None,
|
||||
send_to_detokenizer=SimpleNamespace(
|
||||
send_output=lambda obj: captured.append(obj)
|
||||
),
|
||||
)
|
||||
SchedulerOutputStreamer._stream_output_embedding(fake_self, reqs)
|
||||
out = captured[0]
|
||||
|
||||
# Mirror the receiver-side reconstruction in tokenizer_manager.py: a
|
||||
# single stacked ``[stacked(N, ...)]`` (len 1, N > 1) is unwrapped back to
|
||||
# per-request indexing regardless of wire format.
|
||||
phs = out.pooled_hidden_states
|
||||
if phs is not None and len(phs) == 1 and len(reqs) > 1:
|
||||
out.pooled_hidden_states = phs[0]
|
||||
return out
|
||||
|
||||
def test_uniform_anchor_counts_stack_and_reconstruct(self):
|
||||
# Two requests, each with 2 anchors -> stackable; receiver [i] must recover
|
||||
# each request's [num_positions, hidden] matrix intact.
|
||||
phs0 = torch.arange(8).float().reshape(2, 4)
|
||||
phs1 = (torch.arange(8).float() + 100).reshape(2, 4)
|
||||
out = self._run_stream_output_embedding([phs0, phs1])
|
||||
torch.testing.assert_close(torch.as_tensor(out.pooled_hidden_states[0]), phs0)
|
||||
torch.testing.assert_close(torch.as_tensor(out.pooled_hidden_states[1]), phs1)
|
||||
|
||||
def test_ragged_anchor_counts_keep_list(self):
|
||||
# Different anchor counts cannot be stacked; the packer must fall back to a
|
||||
# list so per-request indexing stays correct.
|
||||
phs0 = torch.randn(3, 4)
|
||||
phs1 = torch.randn(1, 4)
|
||||
out = self._run_stream_output_embedding([phs0, phs1])
|
||||
self.assertEqual(torch.as_tensor(out.pooled_hidden_states[0]).shape, (3, 4))
|
||||
self.assertEqual(torch.as_tensor(out.pooled_hidden_states[1]).shape, (1, 4))
|
||||
torch.testing.assert_close(torch.as_tensor(out.pooled_hidden_states[0]), phs0)
|
||||
torch.testing.assert_close(torch.as_tensor(out.pooled_hidden_states[1]), phs1)
|
||||
|
||||
def test_full_roundtrip_into_parser_groups_per_item(self):
|
||||
# pooler 2-D PHS -> stream_output_embedding -> receiver [i] -> parser:
|
||||
# the ScoreResult must hold the item's vectors grouped as one item.
|
||||
phs = torch.arange(6).float().reshape(3, 2) # 3 positions, hidden=2
|
||||
out = self._run_stream_output_embedding([phs])
|
||||
received = out.pooled_hidden_states[0] # receiver-side indexing
|
||||
|
||||
parser = _ClsHarness()
|
||||
result = {
|
||||
"meta_info": {"id": "rid-0", "prompt_tokens": 5},
|
||||
"embedding": [[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]],
|
||||
"pooled_hidden_state": received,
|
||||
}
|
||||
res = parser._process_single_item_scoring_results(
|
||||
[result],
|
||||
label_token_ids=None,
|
||||
apply_softmax=False,
|
||||
return_pooled_hidden_states=True,
|
||||
per_item_matrix=True,
|
||||
)
|
||||
self.assertIsNotNone(res.pooled_hidden_states)
|
||||
# One item -> one group of 3 position vectors.
|
||||
self.assertEqual(len(res.pooled_hidden_states), 1)
|
||||
self.assertEqual(len(res.pooled_hidden_states[0]), 3)
|
||||
for i in range(3):
|
||||
torch.testing.assert_close(res.pooled_hidden_states[0][i], phs[i])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1044,6 +1044,7 @@ class TestSchedulerMmTransportBoundary(CustomTestCase):
|
||||
time_stats=None,
|
||||
return_pooled_hidden_states=False,
|
||||
multi_item_delimiter_indices=None,
|
||||
token_indices_to_pool=None,
|
||||
mm_inputs=object(),
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user