diff --git a/benchmark/prefill_only/bench_setwise_score.py b/benchmark/prefill_only/bench_setwise_score.py new file mode 100644 index 0000000000..735c5b4365 --- /dev/null +++ b/benchmark/prefill_only/bench_setwise_score.py @@ -0,0 +1,271 @@ +"""Engine benchmark for setwise (multi-position) scoring. + +Benchmarks `Engine.async_score(..., score_extraction_token_id=...)` — the setwise +path that pools the classification head at each candidate's anchor token and +returns an ``[N, num_labels]`` matrix in a single forward pass. + +Unlike the general ``bench_score.py`` (label-token / MIS scoring), this is a +focused, self-contained harness: it sweeps the **set size** (number of +candidates packed into one item) and reports latency percentiles and throughput +(requests/s and candidates/s) so you can see how a single-forward setwise pass +scales with N. + +It talks to the Engine directly (no HTTP) and requires the setwise deployment +constraints (``--disable-radix-cache``, ``--chunked-prefill-size -1``), which it +sets automatically. + +Usage: + # Default sweep over set sizes on the target model + python bench_setwise_score.py --model-path /path/to/seqcls_model + + # Custom sweep + concurrency + per-candidate length + python bench_setwise_score.py --model-path /path/to/model \ + --set-sizes 8 16 32 64 --concurrency 16 --num-requests 100 \ + --item-tokens 180 --query-tokens 64 + + # Include pooled hidden states in the response (per-anchor pre-head vectors) + python bench_setwise_score.py --model-path /path/to/model --return-pooled-hidden-states +""" + +import argparse +import asyncio +import csv +import random +import statistics +import time +from typing import List, Optional + +from transformers import AutoTokenizer + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Benchmark setwise scoring via Engine.async_score", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--model-path", + type=str, + required=True, + help="Path to a SequenceClassification model (e.g. Qwen3ForSequenceClassification).", + ) + parser.add_argument( + "--score-extraction-token", + type=str, + default="<|object_ref_start|>", + help="Anchor token pooled at each candidate (default: <|object_ref_start|>).", + ) + parser.add_argument( + "--set-sizes", + type=int, + nargs="+", + default=[8, 16, 32, 64], + help="Candidates per set (each is one anchor in one item). Swept in order.", + ) + parser.add_argument( + "--item-tokens", + type=int, + default=180, + help="Content tokens per candidate before its anchor (default: 180).", + ) + parser.add_argument( + "--query-tokens", + type=int, + default=0, + help="Shared query-prefix tokens (default: 0).", + ) + parser.add_argument( + "--num-requests", + type=int, + default=50, + help="Timed requests per set size (default: 50).", + ) + parser.add_argument( + "--concurrency", + type=int, + default=8, + help="Max in-flight requests (default: 8).", + ) + parser.add_argument( + "--warmup", + type=int, + default=3, + help="Warmup requests per set size, not timed (default: 3).", + ) + parser.add_argument("--apply-softmax", action="store_true") + parser.add_argument("--return-pooled-hidden-states", action="store_true") + + # Engine configuration + parser.add_argument("--dtype", type=str, default="float16") + parser.add_argument("--mem-fraction-static", type=float, default=0.5) + parser.add_argument("--max-prefill-tokens", type=int, default=60000) + parser.add_argument("--attention-backend", type=str, default="flashinfer") + parser.add_argument("--seed", type=int, default=0) + parser.add_argument( + "--output", "-o", type=str, default=None, help="Optional CSV output path." + ) + return parser.parse_args() + + +def _random_content(num_tokens: int, anchor_id: int, lo: int = 10, hi: int = 30000): + """Random content token ids that never collide with the anchor id.""" + ids = [random.randint(lo, hi) for _ in range(num_tokens)] + return [t + 1 if t == anchor_id else t for t in ids] + + +def _build_setwise_request(args, anchor_id: int, n_candidates: int): + """One setwise request: shared query prefix + ONE item with N anchors.""" + query_ids = ( + _random_content(args.query_tokens, anchor_id) if args.query_tokens else [] + ) + item_ids: List[int] = [] + for _ in range(n_candidates): + item_ids.extend(_random_content(args.item_tokens, anchor_id)) + item_ids.append(anchor_id) + return query_ids, [item_ids] + + +def _percentile(sorted_vals: List[float], pct: float) -> float: + if not sorted_vals: + return 0.0 + k = max(0, min(len(sorted_vals) - 1, round((pct / 100.0) * (len(sorted_vals) - 1)))) + return sorted_vals[k] + + +async def _run_set_size(engine, args, anchor_id: int, n: int) -> Optional[dict]: + # Pre-build requests so generation cost is outside the timed section. + requests = [ + _build_setwise_request(args, anchor_id, n) + for _ in range(args.num_requests + args.warmup) + ] + prompt_tokens = len(requests[0][0]) + len(requests[0][1][0]) + + async def _one(query_ids, items): + t0 = time.perf_counter() + res = await engine.async_score( + query=query_ids, + items=items, + score_extraction_token_id=anchor_id, + apply_softmax=args.apply_softmax, + return_pooled_hidden_states=args.return_pooled_hidden_states, + ) + # Setwise scores are nested (one [Ni x num_labels] matrix per item) and + # this harness sends exactly one item, so the per-candidate row count is + # len(res.scores[0]), not len(res.scores) (which is always 1 = num items). + return time.perf_counter() - t0, len(res.scores[0]) if res.scores else 0 + + # Warmup (also validates the row count == N). + for q, it in requests[: args.warmup]: + _, rows = await _one(q, it) + if rows != n: + print(f"[warn] set_size={n}: expected {n} score rows, got {rows}") + + sem = asyncio.Semaphore(args.concurrency) + + async def _guarded(q, it): + async with sem: + return await _one(q, it) + + timed = requests[args.warmup :] + wall_start = time.perf_counter() + results = await asyncio.gather(*(_guarded(q, it) for q, it in timed)) + wall = time.perf_counter() - wall_start + + latencies_ms = sorted(1000.0 * lat for lat, _ in results) + n_req = len(results) + return { + "set_size": n, + "prompt_tokens": prompt_tokens, + "requests": n_req, + "concurrency": args.concurrency, + "avg_ms": statistics.mean(latencies_ms), + "p50_ms": _percentile(latencies_ms, 50), + "p95_ms": _percentile(latencies_ms, 95), + "p99_ms": _percentile(latencies_ms, 99), + "req_per_s": n_req / wall if wall > 0 else 0.0, + "cand_per_s": (n_req * n) / wall if wall > 0 else 0.0, + } + + +def _print_table(rows: List[dict]): + headers = [ + "set_size", + "prompt_tokens", + "requests", + "avg_ms", + "p50_ms", + "p95_ms", + "p99_ms", + "req_per_s", + "cand_per_s", + ] + print("\n" + " | ".join(f"{h:>13}" for h in headers)) + print("-" * (16 * len(headers))) + for r in rows: + cells = [] + for h in headers: + v = r[h] + cells.append(f"{v:>13.2f}" if isinstance(v, float) else f"{v:>13}") + print(" | ".join(cells)) + + +async def main(): + args = parse_args() + random.seed(args.seed) + + tokenizer = AutoTokenizer.from_pretrained(args.model_path, trust_remote_code=True) + anchor_id = tokenizer.convert_tokens_to_ids(args.score_extraction_token) + unk_id = tokenizer.unk_token_id + if anchor_id is None or (unk_id is not None and anchor_id == unk_id): + raise ValueError( + f"score_extraction_token {args.score_extraction_token!r} did not resolve " + f"to a dedicated token id." + ) + + from sglang.srt.entrypoints.engine import Engine + + # Setwise deployment constraints: positions are full-prompt coordinates, so + # radix reuse / chunked prefill must be off (the engine also validates this). + engine = Engine( + model_path=args.model_path, + dtype=args.dtype, + mem_fraction_static=args.mem_fraction_static, + max_prefill_tokens=args.max_prefill_tokens, + attention_backend=args.attention_backend, + disable_radix_cache=True, + chunked_prefill_size=-1, + ) + + print( + f"Setwise scoring benchmark | model={args.model_path}\n" + f"anchor={args.score_extraction_token!r} (id={anchor_id}) | " + f"item_tokens={args.item_tokens} query_tokens={args.query_tokens} | " + f"concurrency={args.concurrency} num_requests={args.num_requests}" + ) + + rows = [] + try: + for n in args.set_sizes: + row = await _run_set_size(engine, args, anchor_id, n) + if row is not None: + rows.append(row) + print( + f"set_size={n:>4}: avg={row['avg_ms']:.1f}ms " + f"p95={row['p95_ms']:.1f}ms " + f"{row['req_per_s']:.1f} req/s {row['cand_per_s']:.0f} cand/s" + ) + finally: + engine.shutdown() + + _print_table(rows) + + if args.output and rows: + with open(args.output, "w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=list(rows[0].keys())) + writer.writeheader() + writer.writerows(rows) + print(f"\nWrote {len(rows)} rows to {args.output}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/sglang/srt/configs/model_config.py b/python/sglang/srt/configs/model_config.py index 937e2d702e..1f8dff1b0f 100644 --- a/python/sglang/srt/configs/model_config.py +++ b/python/sglang/srt/configs/model_config.py @@ -2398,6 +2398,20 @@ def is_cross_encoding_pooler_model(model_architectures: List[str]) -> bool: return any(arch in _cross_encoding_pooler_archs for arch in model_architectures) +# SequenceClassification models whose forward routes the head through +# score_and_pool (per-position pooling); only these support setwise readout +# (token_indices_to_pool). Keep in sync with callers of layers.pooler.score_and_pool. +_score_and_pool_archs = [ + "LlamaForSequenceClassification", + "Qwen2ForSequenceClassification", + "Qwen3ForSequenceClassification", +] + + +def is_score_and_pool_model(model_architectures: List[str]) -> bool: + return any(arch in _score_and_pool_archs for arch in model_architectures) + + def yarn_get_mscale(scale: float = 1, mscale: float = 1) -> float: if scale <= 1: return 1.0 diff --git a/python/sglang/srt/entrypoints/engine_score_mixin.py b/python/sglang/srt/entrypoints/engine_score_mixin.py index 3f96419ae7..b2e305b6ff 100644 --- a/python/sglang/srt/entrypoints/engine_score_mixin.py +++ b/python/sglang/srt/entrypoints/engine_score_mixin.py @@ -37,6 +37,7 @@ class EngineScoreMixin: query_embed_overrides: Optional[List[torch.Tensor]] = None, item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None, return_pooled_hidden_states: bool = False, + score_extraction_token_id: Optional[int] = None, temperature: float = 1.0, return_token_logprobs: bool = False, ) -> ScoreResult: @@ -53,6 +54,11 @@ class EngineScoreMixin: For SequenceClassification models, returns the pooled class logits directly from the classification head. label_token_ids is optional and ignored. + Setwise scoring is expressed via ``score_extraction_token_id``: pass a single + item containing the whole candidate block with one extraction token per + candidate, and the head is pooled AT those positions, so ``scores`` becomes + the ``[N x num_labels]`` per-candidate matrix. + Args: query: The query text or pre-tokenized token IDs. items: The item text(s) or pre-tokenized token IDs. @@ -63,6 +69,11 @@ class EngineScoreMixin: embed_override_token_id: Placeholder token ID used to locate override positions. query_embed_overrides: Embedding vectors replacing placeholder tokens in query. item_embed_overrides: Per-item embedding vectors replacing placeholder tokens in items. + score_extraction_token_id: SequenceClassification-only. When set, pool the + head at every occurrence of this token per sequence instead of the last + token; ``scores`` becomes nested — one ``[Ni x num_labels]`` matrix per + item (``len(scores) == len(items)``), where ``Ni`` is the number of + extraction tokens (candidates) in item ``i``. return_pooled_hidden_states: Whether to include raw pooled transformer hidden states (before the task head) in the result. Only supported for non-generation models (SequenceClassification, RewardModel). @@ -72,8 +83,10 @@ class EngineScoreMixin: each item's candidates. Only supported for CausalLM, not raw logits. Returns: - ScoreResult with scores (one list per item), prompt token count, and - optional pooled_hidden_states tensors. + ScoreResult with scores (flat ``[num_items x num_labels]`` for pointwise + scoring, or nested ``[num_items][Ni x num_labels]`` when + score_extraction_token_id is set), prompt token count, and optional + pooled_hidden_states tensors. """ return self.loop.run_until_complete( self.tokenizer_manager.score_request( @@ -85,6 +98,7 @@ class EngineScoreMixin: embed_override_token_id=embed_override_token_id, query_embed_overrides=query_embed_overrides, item_embed_overrides=item_embed_overrides, + score_extraction_token_id=score_extraction_token_id, request=None, return_pooled_hidden_states=return_pooled_hidden_states, temperature=temperature, @@ -103,6 +117,7 @@ class EngineScoreMixin: query_embed_overrides: Optional[List[torch.Tensor]] = None, item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None, return_pooled_hidden_states: bool = False, + score_extraction_token_id: Optional[int] = None, temperature: float = 1.0, return_token_logprobs: bool = False, ) -> ScoreResult: @@ -116,6 +131,7 @@ class EngineScoreMixin: embed_override_token_id=embed_override_token_id, query_embed_overrides=query_embed_overrides, item_embed_overrides=item_embed_overrides, + score_extraction_token_id=score_extraction_token_id, request=None, return_pooled_hidden_states=return_pooled_hidden_states, temperature=temperature, diff --git a/python/sglang/srt/entrypoints/openai/protocol.py b/python/sglang/srt/entrypoints/openai/protocol.py index 243b40e6dd..6215400b07 100644 --- a/python/sglang/srt/entrypoints/openai/protocol.py +++ b/python/sglang/srt/entrypoints/openai/protocol.py @@ -1406,14 +1406,32 @@ class ScoringRequest(BaseModel): return_token_logprobs: bool = False item_first: bool = False return_pooled_hidden_states: bool = False + + # Setwise readout (SequenceClassification-only): when set, the head is pooled + # AT every occurrence of this token in each `query + item` sequence instead of + # the last token, and `scores` is returned nested (one `[Nᵢ x num_labels]` + # matrix per item). --enable-mis fuses items; otherwise each is scored alone. + score_extraction_token: Optional[str] = None + model: str = DEFAULT_MODEL_NAME class ScoringResponse(BaseModel): - scores: List[ - List[float] - ] # List of lists of probabilities, each in the order of label_token_ids - pooled_hidden_states: Optional[List[Optional[List[float]]]] = None + # Pointwise (no `score_extraction_token`): a single [num_rows x num_labels] + # matrix. Setwise: nested [num_items][Nᵢ x num_labels]. The nesting depth + # disambiguates the two shapes. + scores: Union[ + List[List[float]], # pointwise: [num_rows x num_labels] + List[List[List[float]]], # setwise: [num_items][Nᵢ x num_labels] + ] + # Parallel to `scores`: flat [num_rows x hidden] (pointwise) or one such + # matrix per item (setwise). + pooled_hidden_states: Optional[ + Union[ + List[Optional[List[float]]], # pointwise: [num_rows x hidden] + List[List[Optional[List[float]]]], # setwise: [num_items][Nᵢ x hidden] + ] + ] = None token_logprobs: Optional[List[List[float]]] = None model: str usage: Optional[UsageInfo] = None diff --git a/python/sglang/srt/entrypoints/openai/serving_score.py b/python/sglang/srt/entrypoints/openai/serving_score.py index 79cb15b9a4..0ed3bc695f 100644 --- a/python/sglang/srt/entrypoints/openai/serving_score.py +++ b/python/sglang/srt/entrypoints/openai/serving_score.py @@ -67,6 +67,16 @@ class OpenAIServingScore(OpenAIServingBase): else None ) + # Resolve the extraction token to an id here (this process owns the + # tokenizer); score_request then scans for it and pools the head there. + score_extraction_token_id = None + if request.score_extraction_token is not None: + score_extraction_token_id = ( + self.tokenizer_manager._resolve_score_extraction_token_id( + request.score_extraction_token + ) + ) + result = await self.tokenizer_manager.score_request( query=request.query, items=request.items, @@ -76,18 +86,23 @@ class OpenAIServingScore(OpenAIServingBase): embed_override_token_id=request.embed_override_token_id, query_embed_overrides=query_embed_overrides, item_embed_overrides=item_embed_overrides, + score_extraction_token_id=score_extraction_token_id, request=raw_request, return_pooled_hidden_states=request.return_pooled_hidden_states, temperature=request.temperature, return_token_logprobs=request.return_token_logprobs, ) - phs_as_lists = None - if result.pooled_hidden_states is not None: - phs_as_lists = [ - t.tolist() if t is not None else None - for t in result.pooled_hidden_states - ] + # pooled_hidden_states is flat (pointwise / single-set setwise) or + # nested per item (multi-item setwise); convert tensors at any depth. + def _tensors_to_lists(value): + if value is None: + return None + if isinstance(value, list): + return [_tensors_to_lists(v) for v in value] + return value.tolist() + + phs_as_lists = _tensors_to_lists(result.pooled_hidden_states) response = ScoringResponse( scores=result.scores, diff --git a/python/sglang/srt/layers/logprob_processor.py b/python/sglang/srt/layers/logprob_processor.py index 7844ed4e42..d336895f8d 100644 --- a/python/sglang/srt/layers/logprob_processor.py +++ b/python/sglang/srt/layers/logprob_processor.py @@ -988,8 +988,12 @@ class OutputLogprobProcessor: # Handle token_ids logprobs if requested if needs_token_ids_logprobs: + # no_copy_to_cpu avoids the .item()/.tolist() CUDA sync that regresses + # prefill-only Score throughput at high concurrency. ( result.token_ids_logprobs_val, result.token_ids_logprobs_idx, - ) = get_token_ids_logprobs_batch_optimized(logprobs, token_ids_logprobs) + ) = get_token_ids_logprobs( + logprobs, token_ids_logprobs, no_copy_to_cpu=True + ) return result diff --git a/python/sglang/srt/layers/pooler.py b/python/sglang/srt/layers/pooler.py index 4557bc5c94..fb7af7538f 100644 --- a/python/sglang/srt/layers/pooler.py +++ b/python/sglang/srt/layers/pooler.py @@ -35,7 +35,8 @@ class EmbeddingPoolerOutput: pooled_hidden_states: Raw transformer hidden states *before* the task-specific head, present only when ``forward_batch.return_pooled_hidden_states`` is True. Tensor - (standard path) or list of tensors (MIS path, one per delimiter). + (standard path) or list of tensors (one per position — MIS: one per + delimiter; setwise / multi-position readout: one per pooled position). """ # Pooler can return list[tensor] instead of tensor if the dimension of each tensor in the batch is different @@ -114,6 +115,45 @@ def pool_at_delimiter_positions( return list(data[index_tensor].split(delim_counts)) +def pool_at_positions( + data: torch.Tensor, + per_request_indices: List[torch.Tensor], + forward_batch: ForwardBatch, + device: torch.device, +) -> List[torch.Tensor]: + """Pool a tensor exactly AT the given token positions for every request. + + Unlike ``pool_at_delimiter_positions`` (which pools at delimiter - 1), this + pools at the position itself with no offset shift and no discarded entry. + + Args: + data: 2-D tensor [total_tokens, dim] — hidden states or logits. + per_request_indices: Per-request anchor positions (CPU tensors), each + relative to the start of that request's sequence. + forward_batch: Forward batch with extend_seq_lens_cpu populated. + device: Device for the index tensor. + + Returns: + One tensor per request, shaped [num_anchors, dim]. + """ + all_index_tensors: List[torch.Tensor] = [] + counts: List[int] = [] + offset = 0 + for req_idx, req_seq_len in enumerate(forward_batch.extend_seq_lens_cpu): + indices_tensor = per_request_indices[req_idx] + n = len(indices_tensor) + if n > 0: + all_index_tensors.append(indices_tensor + offset) + counts.append(n) + offset += req_seq_len + + if all_index_tensors: + index_tensor = torch.cat(all_index_tensors).to(device, non_blocking=True) + else: + index_tensor = torch.empty(0, dtype=torch.long, device=device) + return list(data[index_tensor].split(counts)) + + def score_and_pool( score_head: nn.Module, pooler: Pooler, @@ -123,6 +163,10 @@ def score_and_pool( ) -> EmbeddingPoolerOutput: """Apply a classification/score head with MIS and pooled-hidden-states support. + Multi-position readout path (``token_indices_to_pool`` on forward_batch, e.g. + setwise scoring): pool the head AT each caller-specified position (no offset, + no discarded row), then split per-request. + MIS path (pre-computed delimiter indices on forward_batch): extract hidden states at positions just before each delimiter, apply the score head, then split per-request. @@ -132,6 +176,29 @@ def score_and_pool( When ``forward_batch.return_pooled_hidden_states`` is True, the raw pooled hidden states (before the score head) are included in the output. """ + if ( + forward_batch.token_indices_to_pool is not None + and forward_batch.is_prefill_only + ): + # Setwise readout: pool the head AT each requested position (no -1 shift, + # no discarded boundary row). Concat to call the head once, then split. + per_request_phs = pool_at_positions( + hidden_states, + forward_batch.token_indices_to_pool, + forward_batch, + input_ids.device, + ) + phs_flat = torch.cat(per_request_phs, dim=0) + scores_flat = score_head(phs_flat) + pos_counts = [t.shape[0] for t in per_request_phs] + per_request_scores = list(scores_flat.split(pos_counts)) + return EmbeddingPoolerOutput( + embeddings=per_request_scores, + pooled_hidden_states=( + per_request_phs if forward_batch.return_pooled_hidden_states else None + ), + ) + if ( forward_batch.multi_item_delimiter_indices is not None and forward_batch.is_prefill_only diff --git a/python/sglang/srt/managers/io_struct.py b/python/sglang/srt/managers/io_struct.py index 3e5042a7bb..e246f2123e 100644 --- a/python/sglang/srt/managers/io_struct.py +++ b/python/sglang/srt/managers/io_struct.py @@ -1233,6 +1233,12 @@ class EmbeddingReqInput: # Batch-level: List[List[int]] (one per request). After __getitem__: List[int]. multi_item_delimiter_indices: Optional[Union[List[List[int]], List[int]]] = None + # Token positions the pooler reads out AT (head / hidden states), instead of + # the default LAST/CLS pooling. Generic multi-position primitive (setwise + # scoring today). Unlike MIS, pooling is AT the position (no delimiter - 1). + # Batch-level: List[List[int]] (one per request). After __getitem__: List[int]. + token_indices_to_pool: Optional[Union[List[List[int]], List[int]]] = None + def regenerate_rid(self): """Generate a new request ID and return it.""" if isinstance(self.rid, list): @@ -1359,6 +1365,11 @@ class EmbeddingReqInput: if self.multi_item_delimiter_indices is not None else None ), + token_indices_to_pool=( + self.token_indices_to_pool[i] + if self.token_indices_to_pool is not None + else None + ), ) else: sub = EmbeddingReqInput( @@ -1390,6 +1401,11 @@ class EmbeddingReqInput: if self.multi_item_delimiter_indices is not None else None ), + token_indices_to_pool=( + self.token_indices_to_pool[i] + if self.token_indices_to_pool is not None + else None + ), ) cache[i] = sub return sub @@ -1419,7 +1435,8 @@ class TokenizedEmbeddingReqInput(BaseReq, kw_only=True): return_pooled_hidden_states: bool = False # Pre-computed delimiter indices for multi-item scoring multi_item_delimiter_indices: Optional[List[int]] = None - + # Token positions for setwise pooling readout + token_indices_to_pool: Optional[List[int]] = None # For observability # Pickled Optional[Union[APIServerReqTimeStats, DPControllerReqTimeStats]] time_stats: Optional[PickleWrapper] = None diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index 1c6b2575c4..5841cf0aa8 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -1025,6 +1025,7 @@ class Req(ReqDllmMixin): ] = None, return_pooled_hidden_states: bool = False, multi_item_delimiter_indices: Optional[List[int]] = None, + token_indices_to_pool: Optional[List[int]] = None, session_id: Optional[str] = None, cache_salt: Optional[str] = None, ): @@ -1056,6 +1057,7 @@ class Req(ReqDllmMixin): self.input_embeds = input_embeds self.positional_embed_overrides = positional_embed_overrides self.multi_item_delimiter_indices = multi_item_delimiter_indices + self.token_indices_to_pool = token_indices_to_pool # For req-level memory management self.kv = ReqKvInfo() diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 2236dc138d..f74ce7307d 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -3431,6 +3431,7 @@ class Scheduler( time_stats=recv_req.time_stats, return_pooled_hidden_states=recv_req.return_pooled_hidden_states, multi_item_delimiter_indices=recv_req.multi_item_delimiter_indices, + token_indices_to_pool=recv_req.token_indices_to_pool, ) req.tokenizer = self.tokenizer self._maybe_namespace_elastic_radix_cache(req) @@ -3934,6 +3935,15 @@ class Scheduler( if self.enable_lora and not self.can_schedule_lora_req(req, running_loras): continue + # A forward batch runs one pooling mode, so setwise readout requests + # (token_indices_to_pool) cannot share a batch with last-token ones. + # Admit only the first request's mode; the other stays queued. + if adder.can_run_list and ( + (req.token_indices_to_pool is not None) + != (adder.can_run_list[0].token_indices_to_pool is not None) + ): + continue + running_bs = len(running_batch.reqs) candidate_beam_width = ( req.beam_group.beam_width if req.beam_group is not None else None diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index f6164c6d01..e3d04a9e06 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -1561,6 +1561,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): http_worker_ipc=obj.http_worker_ipc, return_pooled_hidden_states=obj.return_pooled_hidden_states, multi_item_delimiter_indices=obj.multi_item_delimiter_indices, + token_indices_to_pool=obj.token_indices_to_pool, ) tokenized_obj.time_stats = self.rid_to_state[obj.rid].time_stats diff --git a/python/sglang/srt/managers/tokenizer_manager_score_mixin.py b/python/sglang/srt/managers/tokenizer_manager_score_mixin.py index 589ea1dda7..73e93c9305 100644 --- a/python/sglang/srt/managers/tokenizer_manager_score_mixin.py +++ b/python/sglang/srt/managers/tokenizer_manager_score_mixin.py @@ -1,29 +1,41 @@ import logging import math from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, TypeAlias, Union import torch -from sglang.srt.configs.model_config import is_cross_encoding_pooler_model +from sglang.srt.configs.model_config import ( + is_cross_encoding_pooler_model, + is_score_and_pool_model, +) from sglang.srt.constants import MIS_DELIMITER_TOKEN_ID from sglang.srt.managers.embed_types import PositionalEmbeds from sglang.srt.managers.io_struct import EmbeddingReqInput, GenerateReqInput -from sglang.srt.runtime_context import get_exec +from sglang.srt.runtime_context import get_exec, get_memory, get_schedule, get_serving logger = logging.getLogger(__name__) +ScoreRow: TypeAlias = List[float] +ScoreMatrix: TypeAlias = List[ScoreRow] +# Pointwise: [num_items, num_labels]. Setwise (multi-position): one +# [N_i, num_labels] matrix per item. +ScoreOutput: TypeAlias = Union[ScoreMatrix, List[ScoreMatrix]] + +PooledHiddenStateRows: TypeAlias = List[Optional[torch.Tensor]] +PooledHiddenStateOutput: TypeAlias = Union[ + PooledHiddenStateRows, List[Optional[List[torch.Tensor]]] +] + + @dataclass(frozen=True, slots=True) class ScoreResult: - scores: List[List[float]] + scores: ScoreOutput prompt_tokens: int = 0 - # Per-item pooled hidden states (pre-head transformer output). - # CPU tensors when return_pooled_hidden_states=True; kept as tensors so - # in-process consumers (gRPC, engine API) avoid a .tolist() round-trip. - # The HTTP path converts to lists in serving_score.py before JSON serialization. - # Same layout as scores: one tensor per item (not a single packed 2D tensor). - pooled_hidden_states: Optional[List[Optional[torch.Tensor]]] = None + # Pre-head CPU tensors parallel to scores; kept as tensors so in-process + # consumers avoid a .tolist() round-trip (the HTTP path converts before JSON). + pooled_hidden_states: Optional[PooledHiddenStateOutput] = None # uncalibrated full-vocabulary logprobs, in each item's candidate order token_logprobs: Optional[List[List[float]]] = None @@ -197,7 +209,7 @@ class TokenizerManagerScoreMixin: if return_pooled_hidden_states: raw_phs = single_result.get("pooled_hidden_state") if raw_phs is not None and len(raw_phs) == expected_count: - phs_list = raw_phs[1:] + phs_list = list(raw_phs[1:]) return ScoreResult( scores=scores, @@ -206,21 +218,110 @@ class TokenizerManagerScoreMixin: token_logprobs=token_logprobs, ) + # ------------------------------------------------------------------ + # Multi-position pooling result helpers (setwise scoring): the head is read + # at every score-extraction token instead of the last token. Shared by the + # batched and fused (--enable-mis) result processors. + # ------------------------------------------------------------------ + + def _multi_position_score_rows( + self, embedding: Any, apply_softmax: bool + ) -> List[List[float]]: + """Validate a 2-D multi-position result embedding and return its per-token rows.""" + embedding_tensor = torch.as_tensor(embedding) + if embedding_tensor.ndim != 2: + # A 1-D result means the model pooled a single vector (reward/embedding + # head) instead of per-position scores; ValueError -> clean 400. + raise ValueError( + "Multi-position scoring expected a 2-D " + "[num_positions, num_labels] result, but got shape " + f"{tuple(embedding_tensor.shape)}. This model may not support " + "score_extraction_token_id (only SequenceClassification heads that " + "pool per position do)." + ) + if apply_softmax: + return torch.softmax(embedding_tensor, dim=-1).tolist() + return embedding if isinstance(embedding, list) else embedding_tensor.tolist() + + def _multi_position_phs_matrix(self, phs: Any, expected_rows: int) -> torch.Tensor: + """Validate multi-position pooled hidden states are 2-D with one row per position.""" + phs_tensor = torch.as_tensor(phs) + if phs_tensor.ndim != 2 or phs_tensor.shape[0] != expected_rows: + raise ValueError( + "Multi-position pooled hidden states must be a 2-D tensor with " + "one row per score position." + ) + return phs_tensor + + def _process_multi_item_extraction_results( + self, + results: Any, + per_item_anchor_counts: List[int], + apply_softmax: bool, + return_pooled_hidden_states: bool = False, + ) -> ScoreResult: + """Process a fused multi-item score-extraction request (``--enable-mis``). + + The fused sequence returns one ``[ΣNᵢ, num_labels]`` matrix of every + anchor's logits in item order; split it back into one ``[Nᵢ, num_labels]`` + matrix per item using per_item_anchor_counts. + """ + single_result = results[0] if isinstance(results, list) else results + meta_info = single_result.get("meta_info", {}) + request_id = meta_info.get("id", "") + prompt_tokens = meta_info.get("prompt_tokens", 0) + + embedding = single_result.get("embedding") + if embedding is None: + raise ValueError("Embedding not found in the result.") + + rows = self._multi_position_score_rows(embedding, apply_softmax) + total_anchors = sum(per_item_anchor_counts) + if len(rows) != total_anchors: + raise RuntimeError( + f"Expected {total_anchors} anchor rows across " + f"{len(per_item_anchor_counts)} items, but got {len(rows)}. " + f"Request ID: {request_id}" + ) + + scores = [] + offset = 0 + for count in per_item_anchor_counts: + scores.append(rows[offset : offset + count]) + offset += count + + phs_grouped = None + if return_pooled_hidden_states: + raw_phs = single_result.get("pooled_hidden_state") + if raw_phs is not None: + phs_matrix = self._multi_position_phs_matrix(raw_phs, total_anchors) + phs_grouped = [ + list(group) for group in phs_matrix.split(per_item_anchor_counts) + ] + + return ScoreResult( + scores=scores, + prompt_tokens=prompt_tokens, + pooled_hidden_states=phs_grouped, + ) + def _process_single_item_scoring_results( self, results: Any, label_token_ids: Optional[List[List[int]]], apply_softmax: bool, return_pooled_hidden_states: bool = False, + per_item_matrix: bool = False, temperature: float = 1.0, return_token_logprobs: bool = False, ) -> ScoreResult: """ Process results from single-item scoring request. - For generation (CausalLM) models: reads output_token_ids_logprobs. - For non-generation (SequenceClassification) models: reads the embedding field - which contains pooled class logits from the classification head. + CausalLM models read output_token_ids_logprobs; SequenceClassification + models read the embedding field (pooled class logits from the head). + When per_item_matrix (setwise), each result's 2-D matrix becomes one + item's entry so ``scores`` is nested; otherwise each result is one row. """ scores = [] phs_list = [] @@ -259,24 +360,29 @@ class TokenizerManagerScoreMixin: prompt_tokens += result.get("meta_info", {}).get("prompt_tokens", 0) - if apply_softmax: - scores_tensor = torch.as_tensor(embedding, dtype=torch.float64) - embedding = torch.softmax( - (scores_tensor - scores_tensor.max()) / temperature, dim=-1 - ).tolist() - - # The classification head produces per-token logits, which the pooler reduces - # into a single vector per input. That vector is returned in the `.embeddings` - # field — not as semantic embeddings, but as pooled classification logits. - # The field name is reused for compatibility with the existing - # EmbeddingPoolerOutput API. - scores.append(embedding) + if per_item_matrix: + rows = self._multi_position_score_rows(embedding, apply_softmax) + scores.append(rows) + else: + if apply_softmax: + scores_tensor = torch.as_tensor(embedding, dtype=torch.float64) + embedding = torch.softmax( + (scores_tensor - scores_tensor.max()) / temperature, dim=-1 + ).tolist() + # Pooled classification logits, reusing the EmbeddingPoolerOutput field. + scores.append(embedding) if return_pooled_hidden_states: phs = result.get("pooled_hidden_state") - phs_list.append(phs) - if phs is not None: + if per_item_matrix and phs is not None: + phs_list.append( + list(self._multi_position_phs_matrix(phs, len(rows))) + ) has_phs = True + else: + phs_list.append(phs) + if phs is not None: + has_phs = True return ScoreResult( scores=scores, @@ -458,6 +564,190 @@ class TokenizerManagerScoreMixin: None, ) + # ------------------------------------------------------------------ + # Multi-position pooling readout (setwise scoring) + # ------------------------------------------------------------------ + + def _resolve_score_extraction_token_id(self, token: str) -> int: + """Resolve the score-extraction token string to a dedicated token id. + + Runs in the tokenizer-manager process; used by the HTTP score handler to + turn a client token string into the id ``score_request`` scans for. + """ + if self.tokenizer is None: + raise ValueError( + "A tokenizer is required to resolve score_extraction_token." + ) + token_id = self.tokenizer.convert_tokens_to_ids(token) + unk_id = self.tokenizer.unk_token_id + if token_id is None or (unk_id is not None and token_id == unk_id): + raise ValueError( + f"score_extraction_token {token!r} did not resolve to a dedicated " + f"token id." + ) + return token_id + + def _validate_score_extraction( + self, + is_generation: bool, + item_first: bool, + ) -> None: + """Validate that multi-position pooling readout is applicable. + + Supported only by SequenceClassification models whose forward routes the + head through ``score_and_pool`` (per-position pooling, in + ``is_score_and_pool_model``). Generation, cross-encoder, reward, and + embedding models pool a single vector and are rejected here before + inference. Requires radix cache and chunked prefill off and no auto-truncate, + since pooling positions are full-prompt coordinates. + """ + if is_generation: + raise ValueError( + "score_extraction_token_id is only supported for " + "SequenceClassification models, not generation (CausalLM) models." + ) + architectures = self.model_config.hf_config.architectures or [] + if not is_score_and_pool_model(architectures): + raise ValueError( + "score_extraction_token_id is only supported for " + "SequenceClassification models that pool the head per position " + "(score_and_pool); model architecture(s) " + f"{architectures} do not (cross-encoder, reward, and embedding " + "models pool a single vector and ignore the readout positions)." + ) + if item_first: + raise ValueError( + "item_first is not supported with score_extraction_token_id." + ) + if not get_memory().disable_radix_cache: + raise ValueError( + "score_extraction_token_id requires --disable-radix-cache because " + "pooling positions are relative to the full prompt." + ) + if get_schedule().chunked_prefill_size != -1: + raise ValueError( + "score_extraction_token_id requires --chunked-prefill-size -1 " + "because all pooling positions must be processed in one prefill." + ) + if get_serving().allow_auto_truncate: + raise ValueError( + "score_extraction_token_id is not supported with " + "--allow-auto-truncate: pooling positions are computed from the " + "full prompt, and truncating the sequence would leave them " + "pointing past the truncated length." + ) + + def _reject_query_prefix_anchor( + self, + query_token_ids: List[int], + score_extraction_token_id: int, + ) -> None: + """Reject a shared query prefix that itself contains the extraction token. + + The non-MIS path scans the whole ``query + item`` sequence, so a query-side + hit would emit an extra, misaligned score row instead of one per candidate. + """ + if score_extraction_token_id in query_token_ids: + raise ValueError( + "score-extraction token found in the shared query prefix; it must " + "appear only within items, one per candidate." + ) + + def _build_score_extraction_text_inputs( + self, + query: str, + items: Union[str, List[str]], + score_extraction_token_id: int, + ) -> List[List[int]]: + """Tokenize each complete query-item prompt once for extraction scanning.""" + items_list = [items] if isinstance(items, str) else items + if not all(isinstance(item, str) for item in items_list): + raise ValueError( + "query and items must both be text or both be token IDs when " + "score_extraction_token_id is set." + ) + if query: + self._reject_query_prefix_anchor( + self.tokenizer.encode(query), score_extraction_token_id + ) + return [self.tokenizer.encode(f"{query}{item}") for item in items_list] + + def _resolve_score_extraction_indices( + self, + input_ids: List[List[int]], + score_extraction_token_id: int, + ) -> List[List[int]]: + """Scan each built sequence for the score-extraction token. + + The head is later pooled AT these positions (via ``token_indices_to_pool``) + instead of the last token. Raises ValueError if any sequence has no token, + which would silently drop that sequence's scores. + """ + token_indices_to_pool = [ + [i for i, t in enumerate(seq) if t == score_extraction_token_id] + for seq in input_ids + ] + if any(not seq_indices for seq_indices in token_indices_to_pool): + raise ValueError( + f"No score-extraction token (id={score_extraction_token_id}) " + "found in one or more input(s); the client must render one per " + "candidate." + ) + return token_indices_to_pool + + def _anchor_counts_per_item( + self, + delimiter_indices: List[int], + anchor_positions: List[int], + ) -> List[int]: + """Count extraction-token anchors falling inside each fused item's block. + + ``--enable-mis`` fuses items as ``query item0 item1 …``, so + item ``i`` spans ``(delimiter_indices[i], delimiter_indices[i + 1])``. + Raises ValueError (clean 400 before inference) if an item has no anchor or + an anchor falls outside every item block. + """ + counts = [ + sum(1 for p in anchor_positions if lo < p < hi) + for lo, hi in zip(delimiter_indices, delimiter_indices[1:]) + ] + # An anchor in the shared query prefix is not attributable to any item and + # would leave the pooled forward with more rows than sum(counts); reject it. + if sum(counts) != len(anchor_positions): + raise ValueError( + "score-extraction token found outside the candidate items (e.g. in " + "the shared query prefix) under --enable-mis multi-item scoring; it " + "must appear only within items, one per candidate." + ) + if any(c == 0 for c in counts): + raise ValueError( + "Each item (candidate set) must contain at least one " + "score-extraction token under --enable-mis multi-item scoring." + ) + return counts + + def _resolve_multi_position_pooling( + self, + input_ids: List[List[int]], + score_extraction_token_id: int, + use_multi_item_scoring: bool, + delimiter_indices: Optional[List[int]], + ) -> Tuple[List[List[int]], Optional[List[int]]]: + """Resolve where the head is pooled for a score-extraction request. + + Returns ``(token_indices_to_pool, per_item_anchor_counts)``; the latter is + None outside ``--enable-mis`` and otherwise buckets anchors back per item. + """ + token_indices_to_pool = self._resolve_score_extraction_indices( + input_ids, score_extraction_token_id + ) + per_item_anchor_counts = None + if use_multi_item_scoring: + per_item_anchor_counts = self._anchor_counts_per_item( + delimiter_indices, token_indices_to_pool[0] + ) + return token_indices_to_pool, per_item_anchor_counts + # ------------------------------------------------------------------ # Main entry point # ------------------------------------------------------------------ @@ -474,6 +764,7 @@ class TokenizerManagerScoreMixin: item_embed_overrides: Optional[List[Optional[List[torch.Tensor]]]] = None, request: Optional[Any] = None, return_pooled_hidden_states: bool = False, + score_extraction_token_id: Optional[int] = None, temperature: float = 1.0, return_token_logprobs: bool = False, ) -> ScoreResult: @@ -495,6 +786,13 @@ class TokenizerManagerScoreMixin: - Generation (CausalLM): Requires label_token_ids; returns logprob-based scores. - SequenceClassification: label_token_ids is optional; returns pooled class logits. + Setwise scoring (SequenceClassification-only) is expressed via + score_extraction_token_id: when set, the head is pooled AT every occurrence + of this token in each ``query + item`` sequence instead of the last token, + and ``scores`` is returned nested (one ``[Nᵢ x num_labels]`` matrix per + item). With ``--enable-mis`` the items are fused into one multi-item + sequence; otherwise each item is scored independently in one batch. + return_pooled_hidden_states is only supported for non-generation models (SequenceClassification, RewardModel); raises ValueError for CausalLM. """ @@ -548,6 +846,12 @@ class TokenizerManagerScoreMixin: f"Token ID {token_id} is out of vocabulary (vocab size: {vocab_size})" ) + # Normalize an omitted query by item type so path selection sees a concrete + # prefix: [] for pre-tokenized items (token-ID path), "" for text items. + if query is None: + items_are_token_ids = isinstance(items, list) and isinstance(items[0], list) + query = [] if items_are_token_ids else "" + has_embeds = ( query_embed_overrides is not None or item_embed_overrides is not None ) @@ -566,12 +870,19 @@ class TokenizerManagerScoreMixin: # Check if multi-item scoring is enabled use_multi_item_scoring = get_exec().features.enable_mis + # Setwise readout: pool the head at every score_extraction_token_id. + use_score_extraction = score_extraction_token_id is not None + if use_score_extraction: + self._validate_score_extraction(is_generation, item_first) + input_ids = None text_prompts = None positional_embed_overrides = None delimiter_indices = None - use_text_prompts = isinstance(query, str) and not has_embeds + use_text_prompts = ( + isinstance(query, str) and not has_embeds and not use_score_extraction + ) if use_text_prompts: # Both query and items are text @@ -605,6 +916,10 @@ class TokenizerManagerScoreMixin: ): # Both query and items are token IDs — tokenize text inputs if needed for embed overrides query_ids, items_ids = query, items + if use_score_extraction and not use_multi_item_scoring: + # Non-MIS concatenates query + item, so a query-prefix anchor would + # emit a misaligned row; reject it (MIS confines anchors per item). + self._reject_query_prefix_anchor(query_ids, score_extraction_token_id) _, input_ids, positional_embed_overrides, delimiter_indices = ( self._build_token_id_inputs( query_ids, @@ -616,9 +931,25 @@ class TokenizerManagerScoreMixin: item_embed_overrides, ) ) - elif has_embeds: - # Text inputs with embed overrides — need to tokenize first to resolve positions - query_ids, items_ids = self._batch_tokenize_query_and_items(query, items) + elif ( + use_score_extraction + and not use_multi_item_scoring + and not has_embeds + and (query is None or isinstance(query, str)) + ): + # Single-set setwise: text (or omitted) query + text items, one + # sequence per item. --enable-mis instead fuses the items below. + input_ids = self._build_score_extraction_text_inputs( + query or "", items, score_extraction_token_id + ) + elif has_embeds or use_score_extraction: + # Tokenize text inputs to token IDs, then build via the token-id path + # (embed overrides need positions; score extraction needs token IDs). + query_ids, items_ids = self._batch_tokenize_query_and_items( + query or "" if use_multi_item_scoring else query, items + ) + if use_score_extraction and not use_multi_item_scoring: + self._reject_query_prefix_anchor(query_ids, score_extraction_token_id) _, input_ids, positional_embed_overrides, delimiter_indices = ( self._build_token_id_inputs( query_ids, @@ -635,6 +966,21 @@ class TokenizerManagerScoreMixin: "Invalid combination of query/items types for score_request." ) + # Setwise readout: scan each sequence for the extraction token (positions + # the pooler reads the head at); under --enable-mis also return per-item + # anchor counts to split the fused score matrix back per item. + token_indices_to_pool = None + per_item_anchor_counts = None + if use_score_extraction: + token_indices_to_pool, per_item_anchor_counts = ( + self._resolve_multi_position_pooling( + input_ids, + score_extraction_token_id, + use_multi_item_scoring, + delimiter_indices, + ) + ) + if return_pooled_hidden_states: if is_generation: raise ValueError( @@ -684,11 +1030,22 @@ class TokenizerManagerScoreMixin: positional_embed_overrides=positional_embed_overrides, return_pooled_hidden_states=return_pooled_hidden_states, multi_item_delimiter_indices=mis_delimiter_indices, + token_indices_to_pool=token_indices_to_pool, ) results = await self.generate_request(batch_request, request).__anext__() - if use_multi_item_scoring: + if use_multi_item_scoring and use_score_extraction: + # Multi-item setwise: the items were fused into one sequence and the + # head pooled at every anchor. Split the flat [ΣNᵢ, num_labels] result + # back into one matrix per item (nested scores). + return self._process_multi_item_extraction_results( + results, + per_item_anchor_counts, + apply_softmax, + return_pooled_hidden_states, + ) + elif use_multi_item_scoring: # Multi-item scoring: extract scores from input_token_ids_logprobs or embedding return self._process_multi_item_scoring_results( results, @@ -701,14 +1058,18 @@ class TokenizerManagerScoreMixin: return_token_logprobs, ) else: - # Single-item scoring: process each result separately + # Single-item scoring: process each result separately. With + # score_extraction_token_id, each item is scored as an independent + # query+item sequence and its 2-D embedding ([positions x num_labels]) + # is kept as one score matrix per item (nested scores). return self._process_single_item_scoring_results( results, label_token_ids, apply_softmax, return_pooled_hidden_states, - temperature, - return_token_logprobs, + per_item_matrix=use_score_extraction, + temperature=temperature, + return_token_logprobs=return_token_logprobs, ) def _convert_logprobs_to_scores( diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index 45d6c19f47..1859893321 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -571,6 +571,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): # Pre-computed delimiter indices for multi-item scoring (CPU tensors, one per request) multi_item_delimiter_indices: Optional[List[torch.Tensor]] = None + # Setwise pooling readout positions (CPU tensors, one per request) + token_indices_to_pool: Optional[List[torch.Tensor]] = None + # === Borrowed from ScheduleBatch: compound (carry their own device tensors) === # Sampling info sampling_info: SamplingBatchInfo = None @@ -1158,6 +1161,18 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): for r in batch.reqs ] + # Setwise readout: pool the head AT token_indices_to_pool instead of + # the last token. The scheduler keeps readout batches homogeneous + # (get_new_batch_prefill), so build only when every request carries + # the field and otherwise fall back to standard pooling. + if batch.reqs and all( + r.token_indices_to_pool is not None for r in batch.reqs + ): + self.token_indices_to_pool = [ + torch.tensor(r.token_indices_to_pool, dtype=torch.int64) + for r in batch.reqs + ] + token_type_ids = [ r.token_type_ids for r in batch.reqs if r.token_type_ids is not None ] diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index 64f92832a2..387a9a3a82 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -1900,6 +1900,7 @@ class ModelRunner: and not isinstance(self.prefill_cuda_graph_runner, EagerRunner) and self.prefill_cuda_graph_runner is not None and self.prefill_cuda_graph_runner.can_run_graph(forward_batch) + and forward_batch.token_indices_to_pool is None and _prefill_cuda_graph_allows_context_parallel( self.prefill_cuda_graph_runner, forward_batch ) diff --git a/test/registered/e2e/scoring/test_setwise_scoring.py b/test/registered/e2e/scoring/test_setwise_scoring.py new file mode 100644 index 0000000000..669d2876ba --- /dev/null +++ b/test/registered/e2e/scoring/test_setwise_scoring.py @@ -0,0 +1,401 @@ +"""HuggingFace parity tests for setwise (multi-position) scoring. + +Setwise scoring pools the SequenceClassification head at every occurrence of a +``score_extraction_token`` (one per candidate) instead of the last token. These +tests verify the *numerical* correctness of that readout against a HuggingFace +reference (run the checkpoint through HF, gather the final hidden state at the +anchor positions, apply the model's own classification head), so a wrong pooling +position or head application changes the numbers, not just the shape. + +The engine runs ``float16`` on ``flashinfer`` (its scoring backend of record); +the HF golden is ``float32`` and compared with an fp16-appropriate tolerance. +The anchor token ``<|object_ref_start|>`` is a Qwen3 special token (single +dedicated id). All model/dtype/tolerance knobs are overridable via env. +""" + +import asyncio +import os +import unittest + +import torch +from transformers import AutoModelForSequenceClassification, AutoTokenizer + +from sglang.srt.entrypoints.engine import Engine +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=200, stage="base-b", runner_config="1-gpu-small") + +_SEQCLS_MODEL = os.environ.get( + "TEST_CLASSIFICATION_BASE_MODEL", + "tomaarsen/Qwen3-Reranker-0.6B-seq-cls", +) +_ANCHOR_TOKEN = os.environ.get("TEST_SCORE_EXTRACTION_TOKEN", "<|object_ref_start|>") +# float16 on flashinfer (no float32 prefill kernel); HF golden is float32 and the +# tolerance below covers the fp16-vs-fp32 gap. All overridable via env. +_DTYPE = os.environ.get("TEST_SEQCLS_DTYPE", "float16") +_ATOL = float(os.environ.get("TEST_SETWISE_ATOL", "0.2")) +_RTOL = float(os.environ.get("TEST_SETWISE_RTOL", "0.05")) + + +class TestSetwiseScoringHFParity(CustomTestCase): + """Setwise engine scores must match an HF head-at-anchor reference.""" + + @classmethod + def setUpClass(cls): + cls.tokenizer = AutoTokenizer.from_pretrained(_SEQCLS_MODEL) + cls.anchor_id = cls.tokenizer.convert_tokens_to_ids(_ANCHOR_TOKEN) + assert ( + cls.anchor_id is not None and cls.anchor_id != cls.tokenizer.unk_token_id + ), f"{_ANCHOR_TOKEN!r} did not resolve to a dedicated token id" + + cls.engine = Engine( + model_path=_SEQCLS_MODEL, + disable_radix_cache=True, + chunked_prefill_size=-1, + attention_backend="flashinfer", + dtype=_DTYPE, + mem_fraction_static=0.15, + ) + + @classmethod + def tearDownClass(cls): + if getattr(cls, "engine", None) is not None: + cls.engine.shutdown() + torch.cuda.empty_cache() + + # ------------------------------------------------------------------ + # HF reference + # ------------------------------------------------------------------ + + def _hf_setwise_reference(self, prompt: str): + """Reference: HF hidden state at each anchor -> classification head. + + Returns (ref_scores [N, num_labels] as list-of-lists, anchor_positions). + Tokenized exactly like the engine (``tokenizer.encode`` with default + special tokens) so the anchor positions line up. + """ + input_ids = self.tokenizer.encode(prompt) + anchor_positions = [i for i, t in enumerate(input_ids) if t == self.anchor_id] + self.assertGreater(len(anchor_positions), 0, "prompt has no anchor tokens") + + model = AutoModelForSequenceClassification.from_pretrained( + _SEQCLS_MODEL, torch_dtype=torch.float32 + ).eval() + try: + ids = torch.tensor([input_ids], dtype=torch.long) + with torch.no_grad(): + outputs = model(input_ids=ids, output_hidden_states=True) + last_hidden = outputs.hidden_states[-1][0] # [seq, hidden] + + # The classification head is `score` on Qwen/Llama SeqCls models. + head = getattr(model, "score", None) or getattr(model, "classifier") + anchor_hidden = last_hidden[anchor_positions].to(head.weight.dtype) + ref = head(anchor_hidden) # [N, num_labels] + return ref.tolist(), anchor_positions + finally: + model.cpu() + del model + torch.cuda.empty_cache() + + def _hf_anchor_hidden_states(self, prompt: str): + """Reference: HF final hidden state at each anchor position (pre-head). + + Returns (hidden [N, hidden] as list-of-lists, anchor_positions). + """ + input_ids = self.tokenizer.encode(prompt) + anchor_positions = [i for i, t in enumerate(input_ids) if t == self.anchor_id] + self.assertGreater(len(anchor_positions), 0, "prompt has no anchor tokens") + + model = AutoModelForSequenceClassification.from_pretrained( + _SEQCLS_MODEL, torch_dtype=torch.float32 + ).eval() + try: + ids = torch.tensor([input_ids], dtype=torch.long) + with torch.no_grad(): + outputs = model(input_ids=ids, output_hidden_states=True) + last_hidden = outputs.hidden_states[-1][0] # [seq, hidden] + return last_hidden[anchor_positions].tolist(), anchor_positions + finally: + model.cpu() + del model + torch.cuda.empty_cache() + + def _assert_close(self, ref, sgl, atol=_ATOL, rtol=_RTOL): + self.assertEqual(len(ref), len(sgl), "row count mismatch") + for i, (rrow, srow) in enumerate(zip(ref, sgl)): + self.assertEqual(len(rrow), len(srow), f"row {i} width mismatch") + for j, (r, s) in enumerate(zip(rrow, srow)): + self.assertLessEqual( + abs(r - s), + atol + rtol * abs(r), + f"row {i} label {j}: HF={r:.6f} SGLang={s:.6f}", + ) + + # ------------------------------------------------------------------ + # Tests + # ------------------------------------------------------------------ + + def _build_prompt(self, n_anchors: int) -> str: + candidates = " ".join(f"Candidate {i}." for i in range(n_anchors)) + return f"Rank the candidates. {candidates} Scores:" + ( + _ANCHOR_TOKEN * n_anchors + ) + + def test_setwise_scores_match_hf_reference(self): + """Engine setwise logits == HF head applied at each anchor position.""" + prompt = self._build_prompt(3) + ref, anchor_positions = self._hf_setwise_reference(prompt) + + # `scores` is nested per item (one [N x num_labels] matrix per item); a + # single item here yields one matrix with one row per anchor. + sgl = self.engine.score( + query="", + items=[prompt], + apply_softmax=False, + score_extraction_token_id=self.anchor_id, + ).scores + + self.assertEqual(len(sgl), 1) # one item + item = sgl[0] + self.assertEqual(len(item), len(anchor_positions)) # one row per anchor + self._assert_close(ref, item) + + def test_setwise_multiple_items_match_hf(self): + """Multiple items -> one HF-matching matrix per item (independent, no MIS).""" + prompt0 = self._build_prompt(3) + prompt1 = self._build_prompt(2) + ref0, anchors0 = self._hf_setwise_reference(prompt0) + ref1, anchors1 = self._hf_setwise_reference(prompt1) + + sgl = self.engine.score( + query="", + items=[prompt0, prompt1], + apply_softmax=False, + score_extraction_token_id=self.anchor_id, + ).scores + + self.assertEqual(len(sgl), 2) # one matrix per item + self.assertEqual(len(sgl[0]), len(anchors0)) + self.assertEqual(len(sgl[1]), len(anchors1)) + self._assert_close(ref0, sgl[0]) + self._assert_close(ref1, sgl[1]) + + def test_setwise_single_anchor_matches_hf(self): + """A single anchor is the degenerate case and must still match HF.""" + prompt = self._build_prompt(1) + ref, _ = self._hf_setwise_reference(prompt) + + sgl = self.engine.score( + query="", + items=[prompt], + apply_softmax=False, + score_extraction_token_id=self.anchor_id, + ).scores + + self.assertEqual(len(sgl), 1) # one item + self.assertEqual(len(sgl[0]), 1) # one anchor row + self._assert_close(ref, sgl[0]) + + def test_setwise_candidate_ranking_matches_hf(self): + """Ranking (argsort over label 0) agrees with HF — robust to fp noise.""" + prompt = self._build_prompt(5) + ref, _ = self._hf_setwise_reference(prompt) + + # Single item -> unwrap its [N x num_labels] matrix. + sgl = self.engine.score( + query="", + items=[prompt], + apply_softmax=False, + score_extraction_token_id=self.anchor_id, + ).scores[0] + + ref_order = sorted(range(len(ref)), key=lambda i: ref[i][0]) + sgl_order = sorted(range(len(sgl)), key=lambda i: sgl[i][0]) + self.assertEqual(ref_order, sgl_order) + + def test_setwise_pooled_hidden_states_match_hf(self): + """return_pooled_hidden_states -> per-anchor pre-head vectors == HF.""" + prompt = self._build_prompt(3) + ref_hidden, anchor_positions = self._hf_anchor_hidden_states(prompt) + + result = self.engine.score( + query="", + items=[prompt], + apply_softmax=False, + score_extraction_token_id=self.anchor_id, + return_pooled_hidden_states=True, + ) + + phs = result.pooled_hidden_states + self.assertIsNotNone(phs) + # Nested per item, parallel to scores: a single item -> one group of + # pre-head vectors, one per anchor, aligned 1:1 with the score rows. + self.assertEqual(len(phs), 1) # one item + self.assertEqual(len(result.scores), 1) + item_phs = phs[0] + self.assertEqual(len(item_phs), len(anchor_positions)) + self.assertEqual(len(item_phs), len(result.scores[0])) + sgl_hidden = [vec.tolist() for vec in item_phs] + self._assert_close(ref_hidden, sgl_hidden) + + def test_setwise_tokenized_items_omitted_query_match_hf(self): + """Token-ID items with the query omitted take the token-ID path (query -> []).""" + prompt = self._build_prompt(3) + ref, anchor_positions = self._hf_setwise_reference(prompt) + token_ids = self.tokenizer.encode(prompt) + + sgl = self.engine.score( + items=[token_ids], # query omitted -> normalized to [] + apply_softmax=False, + score_extraction_token_id=self.anchor_id, + ).scores + + self.assertEqual(len(sgl), 1) + self.assertEqual(len(sgl[0]), len(anchor_positions)) + self._assert_close(ref, sgl[0]) + + def test_mixed_setwise_and_pointwise_concurrent_batch(self): + """Setwise and pointwise requests batched together must both succeed. + + Both share the ``/v1/score`` queue; the scheduler partitions prefill + batches by pooling mode, so mixed traffic must not abort the setwise + requests (pre-fix a mixed batch returned HTTP 400 for them). + """ + setwise_prompt = self._build_prompt(3) + + async def _run(): + coros = [] + for _ in range(6): + coros.append( + self.engine.async_score( + query="", + items=[setwise_prompt], + apply_softmax=False, + score_extraction_token_id=self.anchor_id, + ) + ) + coros.append( + self.engine.async_score( + query="", + items=["Candidate A.", "Candidate B."], + apply_softmax=False, + ) + ) + return await asyncio.gather(*coros, return_exceptions=True) + + results = self.engine.loop.run_until_complete(_run()) + + errors = [r for r in results if isinstance(r, BaseException)] + self.assertEqual(errors, [], f"mixed batch produced errors: {errors}") + for i, r in enumerate(results): + if i % 2 == 0: # setwise: one item, 3 anchor rows (nested) + self.assertEqual(len(r.scores), 1) + self.assertEqual(len(r.scores[0]), 3) + else: # pointwise: one score row per item (flat) + self.assertEqual(len(r.scores), 2) + + +class TestSetwiseMultiItemMISScoring(CustomTestCase): + """Multi-item setwise scoring under ``--enable-mis`` (fused sequence). + + The fused path carries the standard MIS delimiter tokens, so it is not + bit-identical to independent HF scoring (inherent to MIS). These tests assert + the nested per-item shape and block-diagonal set isolation rather than HF + parity, which ``TestSetwiseScoringHFParity`` covers for the batched path. + """ + + @classmethod + def setUpClass(cls): + cls.tokenizer = AutoTokenizer.from_pretrained(_SEQCLS_MODEL) + cls.anchor_id = cls.tokenizer.convert_tokens_to_ids(_ANCHOR_TOKEN) + cls.engine = Engine( + model_path=_SEQCLS_MODEL, + disable_radix_cache=True, + chunked_prefill_size=-1, + enable_mis=True, + attention_backend="flashinfer", + dtype=_DTYPE, + mem_fraction_static=0.15, + ) + + @classmethod + def tearDownClass(cls): + if getattr(cls, "engine", None) is not None: + cls.engine.shutdown() + torch.cuda.empty_cache() + + @staticmethod + def _set_prompt(n_anchors: int) -> str: + candidates = " ".join(f"Candidate {i}." for i in range(n_anchors)) + return f"Rank the candidates. {candidates} Scores:" + ( + _ANCHOR_TOKEN * n_anchors + ) + + # atol=5e-2 is a floor set by fp16, not slack: one ULP on these logits is + # ~0.002-0.004, and fusing set0 with another set changes reduction order and + # stacks several such steps (observed drift ~0.012). 5e-2 clears that while + # still catching real cross-set leakage, which would diverge far more. + def _assert_matrix_close(self, a, b, atol=5e-2): + self.assertEqual(len(a), len(b), "row count mismatch") + for i, (ra, rb) in enumerate(zip(a, b)): + self.assertEqual(len(ra), len(rb), f"row {i} width mismatch") + for x, y in zip(ra, rb): + self.assertLessEqual(abs(x - y), atol, f"{x} vs {y}") + + def test_mis_multi_item_nested_shape(self): + """Fused multi-item -> one [Nᵢ x num_labels] matrix per item.""" + scores = self.engine.score( + query="Rank:", + items=[self._set_prompt(3), self._set_prompt(2)], + apply_softmax=False, + score_extraction_token_id=self.anchor_id, + ).scores + + self.assertEqual(len(scores), 2) # one matrix per item + self.assertEqual(len(scores[0]), 3) # 3 candidates + self.assertEqual(len(scores[1]), 2) # 2 candidates + num_labels = len(scores[0][0]) + self.assertTrue( + all(len(row) == num_labels for matrix in scores for row in matrix) + ) + + def test_mis_set_isolation(self): + """Block-diagonal mask: a set's scores are unchanged by other sets. + + The first set is scored alone and again fused with a second set; the + block-diagonal mask means the first set attends only over the shared query + prefix and itself, so its matrix must be identical either way. + """ + set0 = self._set_prompt(3) + alone = self.engine.score( + query="Rank:", + items=[set0], + apply_softmax=False, + score_extraction_token_id=self.anchor_id, + ).scores + fused = self.engine.score( + query="Rank:", + items=[set0, self._set_prompt(2)], + apply_softmax=False, + score_extraction_token_id=self.anchor_id, + ).scores + + self.assertEqual(len(alone), 1) + self.assertEqual(len(fused), 2) + # set0 is the first item in both requests -> identical matrix. + self._assert_matrix_close(alone[0], fused[0]) + + def test_mis_rejects_item_without_anchor(self): + """An item carrying no extraction token is rejected under --enable-mis.""" + with self.assertRaisesRegex(ValueError, "at least one"): + self.engine.score( + query="Rank:", + items=[self._set_prompt(2), "no anchors here"], + apply_softmax=False, + score_extraction_token_id=self.anchor_id, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layers/test_pooler_score_and_pool.py b/test/registered/unit/layers/test_pooler_score_and_pool.py index 4b2ce14a4b..80fded6d1d 100644 --- a/test/registered/unit/layers/test_pooler_score_and_pool.py +++ b/test/registered/unit/layers/test_pooler_score_and_pool.py @@ -25,6 +25,7 @@ register_cpu_ci(est_time=10, suite="base-a-test-cpu") def _make_forward_batch( extend_seq_lens, multi_item_delimiter_indices=None, + token_indices_to_pool=None, return_pooled_hidden_states=False, is_prefill_only=True, ): @@ -33,6 +34,7 @@ def _make_forward_batch( extend_seq_lens=torch.tensor(extend_seq_lens, dtype=torch.long), extend_seq_lens_cpu=extend_seq_lens, multi_item_delimiter_indices=multi_item_delimiter_indices, + token_indices_to_pool=token_indices_to_pool, dimensions=None, return_pooled_hidden_states=return_pooled_hidden_states, is_prefill_only=is_prefill_only, @@ -186,6 +188,85 @@ class TestScoreAndPool(CustomTestCase): self.assertEqual(len(out.embeddings), 1) self.assertEqual(out.embeddings[0].shape, (0, self.num_labels)) + def test_setwise_returns_per_request_list(self): + """Anchor positions provided -> one [num_anchors, num_labels] tensor per request.""" + # Sequence: query + items + suffix; anchors at positions 8, 10, 12. + input_ids = torch.arange(14) + hidden = torch.randn(len(input_ids), self.hidden_dim) + fb = _make_forward_batch( + extend_seq_lens=[len(input_ids)], + token_indices_to_pool=[torch.tensor([8, 10, 12])], + ) + + out = score_and_pool(self.score_head, self.pooler, hidden, fb, input_ids) + + self.assertIsInstance(out.embeddings, list) + self.assertEqual(len(out.embeddings), 1) + # 3 anchors -> 3 score rows (no query-boundary row, unlike MIS). + self.assertEqual(out.embeddings[0].shape, (3, self.num_labels)) + + def test_setwise_extracts_at_anchor_positions(self): + """Setwise pools AT the anchor index (no delimiter-1 shift).""" + input_ids = torch.tensor([0, 1, 2, 3, 4, 5]) + hidden = ( + torch.arange(len(input_ids)) + .unsqueeze(1) + .float() + .expand(-1, self.hidden_dim) + .clone() + ) + fb = _make_forward_batch( + extend_seq_lens=[len(input_ids)], + token_indices_to_pool=[torch.tensor([2, 5])], + ) + + identity_head = nn.Linear(self.hidden_dim, self.hidden_dim, bias=False) + nn.init.eye_(identity_head.weight) + + out = score_and_pool(identity_head, self.pooler, hidden, fb, input_ids) + + scores = out.embeddings[0] + # Pooled exactly at indices 2 and 5 (not 1 and 4). + torch.testing.assert_close(scores[0], hidden[2]) + torch.testing.assert_close(scores[1], hidden[5]) + + def test_setwise_batched_splits_per_request(self): + """Two batched setwise requests -> a list of length 2.""" + req1 = list(range(10, 17)) # 7 tokens, anchors at 5, 6 + req2 = list(range(20, 24)) # 4 tokens, anchor at 3 + input_ids = torch.tensor(req1 + req2) + hidden = torch.randn(len(input_ids), self.hidden_dim) + fb = _make_forward_batch( + extend_seq_lens=[len(req1), len(req2)], + token_indices_to_pool=[ + torch.tensor([5, 6]), + torch.tensor([3]), + ], + ) + + out = score_and_pool(self.score_head, self.pooler, hidden, fb, input_ids) + + self.assertIsInstance(out.embeddings, list) + self.assertEqual(len(out.embeddings), 2) + self.assertEqual(out.embeddings[0].shape, (2, self.num_labels)) + self.assertEqual(out.embeddings[1].shape, (1, self.num_labels)) + + def test_setwise_returns_pooled_hidden_states(self): + """return_pooled_hidden_states -> per-anchor pre-head hidden states.""" + input_ids = torch.arange(6) + hidden = torch.randn(len(input_ids), self.hidden_dim) + fb = _make_forward_batch( + extend_seq_lens=[len(input_ids)], + token_indices_to_pool=[torch.tensor([3, 5])], + return_pooled_hidden_states=True, + ) + + out = score_and_pool(self.score_head, self.pooler, hidden, fb, input_ids) + + self.assertIsNotNone(out.pooled_hidden_states) + self.assertEqual(len(out.pooled_hidden_states), 1) + self.assertEqual(out.pooled_hidden_states[0].shape, (2, self.hidden_dim)) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/managers/test_setwise_score_mixin.py b/test/registered/unit/managers/test_setwise_score_mixin.py new file mode 100644 index 0000000000..52f409cd69 --- /dev/null +++ b/test/registered/unit/managers/test_setwise_score_mixin.py @@ -0,0 +1,536 @@ +"""Unit tests for setwise result grouping on +``TokenizerManagerScoreMixin._process_single_item_scoring_results`` and +``_process_multi_item_extraction_results``. + +Setwise scoring pools the SequenceClassification head at every occurrence of a +``score_extraction_token_id``, so each item carries a 2-D +``[num_positions x num_labels]`` embedding kept grouped (``per_item_matrix=True``) +as nested ``scores`` ``[num_items][Nᵢ x num_labels]``. These tests cover the +grouping, validation, anchor bucketing, and the serialization round-trip on CPU. +""" + +import unittest +from types import SimpleNamespace + +import torch + +from sglang.srt.managers.tokenizer_manager_score_mixin import ( + ScoreResult, + TokenizerManagerScoreMixin, +) +from sglang.srt.runtime_context import get_context +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +# Config that satisfies _validate_score_extraction (dedicated setwise endpoint). +_VALID_SETWISE_CONFIG = dict( + disable_radix_cache=True, + chunked_prefill_size=-1, + allow_auto_truncate=False, +) + + +class _ClsHarness(TokenizerManagerScoreMixin): + """Bare SequenceClassification harness for the pure result parser.""" + + is_generation = False + + def __init__(self, architecture="Qwen3ForSequenceClassification"): + self.model_config = SimpleNamespace( + hf_config=SimpleNamespace(architectures=[architecture]) + ) + + +class TestSingleItemScoringResults(CustomTestCase): + @staticmethod + def _result(embedding, phs=None, prompt_tokens=7): + r = { + "meta_info": {"id": "rid-1", "prompt_tokens": prompt_tokens}, + "embedding": embedding, + } + if phs is not None: + r["pooled_hidden_state"] = phs + return r + + def setUp(self): + super().setUp() + self.h = _ClsHarness() + # _validate_score_extraction reads resolved config bags, so publish a + # valid setwise config; validation tests re-publish with one field flipped. + self._saved_server_args = get_context()._server_args + self.addCleanup(self._restore_server_args) + self._publish(**_VALID_SETWISE_CONFIG) + + def _restore_server_args(self): + if self._saved_server_args is not None: + get_context().set_server_args(self._saved_server_args) + + def _publish(self, **fields): + override = get_context().override_server_args(**fields) + override.install() + self.addCleanup(override.restore) + + # ---- default (per-item) CLS behavior ------------------- + + def test_pointwise_one_row_per_item(self): + results = [ + self._result([0.1, 0.2]), + self._result([0.3, 0.4]), + ] + res = self.h._process_single_item_scoring_results( + results, label_token_ids=None, apply_softmax=False + ) + self.assertIsInstance(res, ScoreResult) + self.assertEqual(res.scores, [[0.1, 0.2], [0.3, 0.4]]) + self.assertEqual(res.prompt_tokens, 14) + + def test_mis_pooled_hidden_states_are_list_of_item_tensors(self): + # Standard MIS returns one fused result with a row at each delimiter. + # The first row is the query boundary and is discarded; the public + # ScoreResult contract is one CPU tensor per remaining item. + phs = torch.arange(12).reshape(3, 4) + result = self._result([[0.0, 0.0], [1.0, 0.0], [0.0, 1.0]], phs=phs) + res = self.h._process_multi_item_scoring_results( + [result], + items=["item0", "item1"], + label_token_ids=None, + apply_softmax=False, + return_pooled_hidden_states=True, + ) + + self.assertEqual(res.scores, [[1.0, 0.0], [0.0, 1.0]]) + self.assertIsInstance(res.pooled_hidden_states, list) + self.assertEqual(len(res.pooled_hidden_states), 2) + torch.testing.assert_close(res.pooled_hidden_states[0], phs[1]) + torch.testing.assert_close(res.pooled_hidden_states[1], phs[2]) + + # ---- setwise: per_item_matrix=True ---------------------------------- + + def test_setwise_one_matrix_per_item(self): + # One item (single big candidate block), 3 extraction positions -> one + # [3 x num_labels] matrix, nested under the item. + emb = [[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]] + res = self.h._process_single_item_scoring_results( + [self._result(emb)], + label_token_ids=None, + apply_softmax=False, + per_item_matrix=True, + ) + self.assertEqual(res.scores, [emb]) # [num_items][N x num_labels] + self.assertEqual(len(res.scores), 1) + self.assertEqual(len(res.scores[0]), 3) + + def test_setwise_apply_softmax_over_labels(self): + emb = [[0.0, 0.0], [2.0, 2.0]] + res = self.h._process_single_item_scoring_results( + [self._result(emb)], + label_token_ids=None, + apply_softmax=True, + per_item_matrix=True, + ) + for row in res.scores[0]: + self.assertAlmostEqual(sum(row), 1.0, places=5) + self.assertAlmostEqual(row[0], 0.5, places=5) + + def test_setwise_multiple_items_grouped_per_item(self): + # Two candidate sets scored independently -> one matrix per item (nested). + res = self.h._process_single_item_scoring_results( + [self._result([[1.0, 0.0], [0.0, 1.0]]), self._result([[0.5, 0.5]])], + label_token_ids=None, + apply_softmax=False, + per_item_matrix=True, + ) + self.assertEqual(res.scores, [[[1.0, 0.0], [0.0, 1.0]], [[0.5, 0.5]]]) + + def test_setwise_groups_pooled_hidden_states_per_item(self): + emb = [[0.1, 0.2], [0.3, 0.4]] + phs = torch.randn(2, 4) # one pre-head vector per position + res = self.h._process_single_item_scoring_results( + [self._result(emb, phs=phs)], + label_token_ids=None, + apply_softmax=False, + return_pooled_hidden_states=True, + per_item_matrix=True, + ) + self.assertIsNotNone(res.pooled_hidden_states) + # One item -> one matrix of 2 vectors. + self.assertEqual(len(res.pooled_hidden_states), 1) + self.assertEqual(len(res.pooled_hidden_states[0]), 2) + for v in res.pooled_hidden_states[0]: + self.assertEqual(tuple(v.shape), (4,)) + + def test_setwise_no_phs_when_not_requested(self): + emb = [[0.1, 0.2]] + phs = torch.randn(1, 4) + res = self.h._process_single_item_scoring_results( + [self._result(emb, phs=phs)], + label_token_ids=None, + apply_softmax=False, + return_pooled_hidden_states=False, + per_item_matrix=True, + ) + self.assertIsNone(res.pooled_hidden_states) + + def test_setwise_rejects_non_matrix_scores(self): + with self.assertRaisesRegex(ValueError, "expected a 2-D"): + self.h._process_single_item_scoring_results( + [self._result([0.1, 0.2])], + label_token_ids=None, + apply_softmax=False, + per_item_matrix=True, + ) + + def test_setwise_rejects_misaligned_pooled_hidden_states(self): + with self.assertRaisesRegex(ValueError, "one row per score position"): + self.h._process_single_item_scoring_results( + [self._result([[0.1], [0.2]], phs=torch.randn(1, 4))], + label_token_ids=None, + apply_softmax=False, + return_pooled_hidden_states=True, + per_item_matrix=True, + ) + + def test_validation_rejects_radix_cache(self): + self._publish(disable_radix_cache=False, chunked_prefill_size=-1) + with self.assertRaisesRegex(ValueError, "--disable-radix-cache"): + self.h._validate_score_extraction(False, False) + + def test_validation_rejects_chunked_prefill(self): + self._publish(disable_radix_cache=True, chunked_prefill_size=2048) + with self.assertRaisesRegex(ValueError, "--chunked-prefill-size -1"): + self.h._validate_score_extraction(False, False) + + def test_validation_rejects_cross_encoder(self): + harness = _ClsHarness(architecture="BertForSequenceClassification") + with self.assertRaisesRegex(ValueError, "cross-encoder"): + harness._validate_score_extraction(False, False) + + def test_validation_rejects_reward_model(self): + # Reward models (last-token pooling, not score_and_pool) ignore the readout + # positions -> rejected up front, not after a wasted forward. + harness = _ClsHarness(architecture="InternLM2ForRewardModel") + with self.assertRaisesRegex(ValueError, "pool the head per position"): + harness._validate_score_extraction(False, False) + + def test_validation_rejects_embedding_model(self): + # A base embedding model likewise does not route through score_and_pool. + harness = _ClsHarness(architecture="Qwen3Model") + with self.assertRaisesRegex(ValueError, "pool the head per position"): + harness._validate_score_extraction(False, False) + + def test_validation_rejects_auto_truncate(self): + self._publish(**{**_VALID_SETWISE_CONFIG, "allow_auto_truncate": True}) + with self.assertRaisesRegex(ValueError, "--allow-auto-truncate"): + self.h._validate_score_extraction(False, False) + + def test_validation_allows_multi_item(self): + # Multiple items (candidate sets) are allowed with or without --enable-mis; + # both return one score matrix per item. + self.h._validate_score_extraction(False, False) + + def test_anchor_counts_per_item_buckets_by_delimiter(self): + # Fused: q item0(2 anchors) item1(1 anchor) . + # delimiter_indices point at the delimiter tokens. + delimiter_indices = [3, 8, 12] + anchor_positions = [5, 7, 10] # 5,7 in item0 (3 [5] contains it and must reject. + class _LenTokenizer: + def encode(self, text): + return [len(text)] + + self.h.tokenizer = _LenTokenizer() + with self.assertRaisesRegex(ValueError, "query prefix"): + self.h._build_score_extraction_text_inputs("query", ["a"], 5) + + def test_reject_query_prefix_anchor_rejects_pretokenized_query(self): + # A pre-tokenized query whose ids contain the extraction token is rejected: + # the non-MIS path scans the whole query + item sequence. + with self.assertRaisesRegex(ValueError, "query prefix"): + self.h._reject_query_prefix_anchor([1, 7, 2], score_extraction_token_id=7) + + def test_reject_query_prefix_anchor_allows_clean_query(self): + # A query prefix without the extraction token passes. + self.h._reject_query_prefix_anchor([1, 2, 3], score_extraction_token_id=7) + + def test_resolve_indices_returns_positions_per_sequence(self): + # anchor id = 7; positions are per-sequence, in order. + indices = self.h._resolve_score_extraction_indices( + [[1, 7, 2, 7], [7, 3]], score_extraction_token_id=7 + ) + self.assertEqual(indices, [[1, 3], [0]]) + + def test_resolve_indices_rejects_any_sequence_without_anchor(self): + # Second sequence has no anchor -> must fail (not silently drop its rows), + # even though another sequence does contain the anchor. + with self.assertRaisesRegex(ValueError, "one or more"): + self.h._resolve_score_extraction_indices( + [[1, 7, 2], [3, 4]], score_extraction_token_id=7 + ) + + # ---- _resolve_multi_position_pooling (extraction-token pooling) ------- + + def test_resolve_multi_position_pooling_batched_returns_no_counts(self): + # No MIS: one sequence per item; anchors resolved per sequence and no + # per-item anchor counts (the batched path splits by result, not counts). + token_indices, counts = self.h._resolve_multi_position_pooling( + [[1, 7, 2, 7], [7, 3]], + score_extraction_token_id=7, + use_multi_item_scoring=False, + delimiter_indices=None, + ) + self.assertEqual(token_indices, [[1, 3], [0]]) + self.assertIsNone(counts) + + def test_resolve_multi_position_pooling_mis_buckets_anchors_by_item(self): + # MIS: items fused into one sequence with delimiters; anchors are scanned + # over the fused sequence and bucketed back to items via the delimiters. + # Fused (D=0, anchor=7): q q 7 9 7 9 7 + # positions: 0 1 2 3 4 5 6 7 8 9 + # delimiters at 2,6,9; anchors at 3,5,8 -> item0 has 2, item1 has 1. + fused = [1, 1, 0, 7, 9, 7, 0, 9, 7, 0] + token_indices, counts = self.h._resolve_multi_position_pooling( + [fused], + score_extraction_token_id=7, + use_multi_item_scoring=True, + delimiter_indices=[2, 6, 9], + ) + self.assertEqual(token_indices, [[3, 5, 8]]) + self.assertEqual(counts, [2, 1]) + + def test_resolve_multi_position_pooling_mis_rejects_item_without_anchor(self): + # An item with no anchor inside its fused block is rejected. + # Fused (D=0, anchor=7): q 7 9 -> item1 (positions 3 rejected before + # inference. + # Fused (D=0, anchor=7): q 7 7 7 (query anchor at position 1) + # positions: 0 1 2 3 4 5 6 + # delimiters at 2,4,6; in-item anchors at 3,5 but the scan also finds 1. + fused = [1, 7, 0, 7, 0, 7, 0] + with self.assertRaisesRegex(ValueError, "outside the candidate items"): + self.h._resolve_multi_position_pooling( + [fused], + score_extraction_token_id=7, + use_multi_item_scoring=True, + delimiter_indices=[2, 4, 6], + ) + + # ---- _multi_position_score_rows / _multi_position_phs_matrix helpers -- + + def test_multi_position_score_rows_preserves_list_values_without_softmax(self): + # No softmax returns the original list as-is (no float round-trip). + emb = [[0.1, 0.2], [0.3, 0.4]] + rows = self.h._multi_position_score_rows(emb, apply_softmax=False) + self.assertIs(rows, emb) + + def test_multi_position_score_rows_softmax_over_labels(self): + rows = self.h._multi_position_score_rows( + [[0.0, 0.0], [2.0, 2.0]], apply_softmax=True + ) + for row in rows: + self.assertAlmostEqual(sum(row), 1.0, places=5) + self.assertAlmostEqual(row[0], 0.5, places=5) + + def test_multi_position_score_rows_rejects_non_matrix(self): + with self.assertRaisesRegex(ValueError, "expected a 2-D"): + self.h._multi_position_score_rows([0.1, 0.2], apply_softmax=False) + + def test_multi_position_phs_matrix_rejects_row_count_mismatch(self): + with self.assertRaisesRegex(ValueError, "one row per score position"): + self.h._multi_position_phs_matrix(torch.randn(2, 4), expected_rows=3) + + def test_multi_position_phs_matrix_rejects_non_matrix(self): + with self.assertRaisesRegex(ValueError, "one row per score position"): + self.h._multi_position_phs_matrix(torch.randn(4), expected_rows=4) + + +class TestSetwisePooledHiddenStatesRoundTrip(CustomTestCase): + """Validate per-position pooled hidden states survive scheduler serialization. + + The pooler's ``[num_positions, hidden]`` tensor is packed by + ``stream_output_embedding`` (stacked when shapes match, list when they differ) + and reconstructed on the receiver by index. These tests exercise that real + transmission path, then confirm the parser groups it as one item's vectors. + """ + + @staticmethod + def _run_stream_output_embedding(phs_per_req): + # Lazy import so the heavier scheduler module loads only when this runs. + from sglang.srt.managers.scheduler_components.output_streamer import ( + SchedulerOutputStreamer, + ) + + captured = [] + reqs = [] + for idx, phs in enumerate(phs_per_req): + reqs.append( + SimpleNamespace( + finished=lambda: True, + rid=f"rid-{idx}", + http_worker_ipc=None, + finished_reason=SimpleNamespace(to_json=lambda: {"type": "stop"}), + embedding=[[0.0, 0.0] for _ in range(phs.shape[0])], + origin_input_ids=[1, 2, 3], + cached_tokens=0, + time_stats=SimpleNamespace(), + retraction_count=0, + pooled_hidden_state=phs, + ) + ) + + fake_self = SimpleNamespace( + get_cached_tokens_details=lambda req: None, + send_to_detokenizer=SimpleNamespace( + send_output=lambda obj: captured.append(obj) + ), + ) + SchedulerOutputStreamer._stream_output_embedding(fake_self, reqs) + out = captured[0] + + # Mirror the receiver-side reconstruction in tokenizer_manager.py: a + # single stacked ``[stacked(N, ...)]`` (len 1, N > 1) is unwrapped back to + # per-request indexing regardless of wire format. + phs = out.pooled_hidden_states + if phs is not None and len(phs) == 1 and len(reqs) > 1: + out.pooled_hidden_states = phs[0] + return out + + def test_uniform_anchor_counts_stack_and_reconstruct(self): + # Two requests, each with 2 anchors -> stackable; receiver [i] must recover + # each request's [num_positions, hidden] matrix intact. + phs0 = torch.arange(8).float().reshape(2, 4) + phs1 = (torch.arange(8).float() + 100).reshape(2, 4) + out = self._run_stream_output_embedding([phs0, phs1]) + torch.testing.assert_close(torch.as_tensor(out.pooled_hidden_states[0]), phs0) + torch.testing.assert_close(torch.as_tensor(out.pooled_hidden_states[1]), phs1) + + def test_ragged_anchor_counts_keep_list(self): + # Different anchor counts cannot be stacked; the packer must fall back to a + # list so per-request indexing stays correct. + phs0 = torch.randn(3, 4) + phs1 = torch.randn(1, 4) + out = self._run_stream_output_embedding([phs0, phs1]) + self.assertEqual(torch.as_tensor(out.pooled_hidden_states[0]).shape, (3, 4)) + self.assertEqual(torch.as_tensor(out.pooled_hidden_states[1]).shape, (1, 4)) + torch.testing.assert_close(torch.as_tensor(out.pooled_hidden_states[0]), phs0) + torch.testing.assert_close(torch.as_tensor(out.pooled_hidden_states[1]), phs1) + + def test_full_roundtrip_into_parser_groups_per_item(self): + # pooler 2-D PHS -> stream_output_embedding -> receiver [i] -> parser: + # the ScoreResult must hold the item's vectors grouped as one item. + phs = torch.arange(6).float().reshape(3, 2) # 3 positions, hidden=2 + out = self._run_stream_output_embedding([phs]) + received = out.pooled_hidden_states[0] # receiver-side indexing + + parser = _ClsHarness() + result = { + "meta_info": {"id": "rid-0", "prompt_tokens": 5}, + "embedding": [[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]], + "pooled_hidden_state": received, + } + res = parser._process_single_item_scoring_results( + [result], + label_token_ids=None, + apply_softmax=False, + return_pooled_hidden_states=True, + per_item_matrix=True, + ) + self.assertIsNotNone(res.pooled_hidden_states) + # One item -> one group of 3 position vectors. + self.assertEqual(len(res.pooled_hidden_states), 1) + self.assertEqual(len(res.pooled_hidden_states[0]), 3) + for i in range(3): + torch.testing.assert_close(res.pooled_hidden_states[0][i], phs[i]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/multimodal/test_gpu_feature_transport.py b/test/registered/unit/multimodal/test_gpu_feature_transport.py index 670e68a04e..b9efb7e263 100644 --- a/test/registered/unit/multimodal/test_gpu_feature_transport.py +++ b/test/registered/unit/multimodal/test_gpu_feature_transport.py @@ -1044,6 +1044,7 @@ class TestSchedulerMmTransportBoundary(CustomTestCase): time_stats=None, return_pooled_hidden_states=False, multi_item_delimiter_indices=None, + token_indices_to_pool=None, mm_inputs=object(), )