minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[misc] Trim restating comments and docstrings in srt/managers (#35622)

Browse Source
This commit is contained in:
Liangsheng Yin
2026-08-20 14:18:40 -07:00
committed by GitHub
parent ad367d72b0
commit 5a100d9086
4 changed files with 98 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/managers/schedule_batch.py
+1 -14
View File
@@ -372,9 +372,6 @@ class MultimodalDataItem:
return len([item for item in flatten_nested_list(l) if item is not None]) == 0
def set_pad_value(self):
"""
Set the pad value after first hashing the data
"""
if self.pad_value is not None:
return
@@ -722,9 +719,6 @@ class MultimodalInputs:
return image_tokens, audio_tokens, video_tokens
def merge(self, other: MultimodalInputs):
"""
merge image inputs when requests are being merged
"""
# args needed to be merged
optional_args = [
@@ -1233,11 +1227,7 @@ class Req(ReqDllmMixin):
return self.kv_committed_len
def update_spec_correct_drafts_histogram(self, num_correct_drafts: int):
"""Update the speculative decoding acceptance histogram.
Args:
num_correct_drafts: Number of correct draft tokens (no bonus) in this step.
"""
"""Record one step accepted draft count (excludes bonus token) into the histogram."""
if len(self.spec_correct_drafts_histogram) <= num_correct_drafts:
self.spec_correct_drafts_histogram.extend(
[0] * (num_correct_drafts - len(self.spec_correct_drafts_histogram) + 1)
@@ -1459,9 +1449,6 @@ class Req(ReqDllmMixin):
return self.tokenizer.decode(self.output_ids[-tail_len:])
def check_match_stop_str_prefix(self) -> bool:
"""
Check if the suffix of tail_str overlaps with any stop_str prefix
"""
if not self.sampling_params.stop_strs:
return False
python/sglang/srt/managers/tokenizer_manager.py
+2 -60
View File
@@ -833,13 +833,6 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
def _detect_input_format(
self, texts: Union[str, List[str]], is_cross_encoder: bool
) -> InputFormat:
"""Detect the format of input texts for proper tokenization handling.
Returns:
- InputFormat.SINGLE_STRING: Regular single text like "Hello world"
- InputFormat.BATCH_STRINGS: Regular batch like ["Hello", "World"]
- InputFormat.CROSS_ENCODER_PAIRS: Cross-encoder pairs like [["query", "document"]]
"""
if isinstance(texts, str):
return InputFormat.SINGLE_STRING
@@ -894,38 +887,6 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
Tuple[List[int], Optional[List[int]]],
Tuple[List[List[int]], Optional[List[List[int]]]],
]:
"""
Tokenize text(s) using the appropriate tokenizer strategy.
This method handles multiple input formats and chooses between async dynamic
batch tokenizer (for single texts only) and regular tokenizer.
Args:
texts: Text input in various formats:
Regular cases:
- Single string: "How are you?"
- Batch of strings: ["Hello", "World", "How are you?"]
Cross-encoder cases (sentence pairs for similarity/ranking):
- Single pair: [["query text", "document text"]]
- Multiple pairs: [["q1", "d1"], ["q2", "d2"], ["q3", "d3"]]
is_cross_encoder: Whether to return token_type_ids for cross-encoder models.
Enables proper handling of sentence pairs with segment IDs.
Returns:
Single input cases:
Tuple[List[int], Optional[List[int]]]: (input_ids, token_type_ids)
Example: ([101, 2129, 102], [0, 0, 0]) for single text
Example: ([101, 2129, 102, 4068, 102], [0, 0, 0, 1, 1]) for cross-encoder pair
Batch input cases:
Tuple[List[List[int]], Optional[List[List[int]]]]: (batch_input_ids, batch_token_type_ids)
Example: ([[101, 2129, 102], [101, 4068, 102]], None) for regular batch
Note: token_type_ids is None unless is_cross_encoder=True.
"""
if not texts or self.tokenizer is None:
raise ValueError("texts cannot be empty and tokenizer must be initialized")
@@ -1490,11 +1451,6 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
token_id: int,
embeds: List[torch.Tensor],
) -> PositionalEmbeds:
"""Resolve placeholder positions in input_ids and create PositionalEmbeds.
Scans input_ids for occurrences of token_id and pairs them with the
provided embedding tensors.
"""
positions = [idx for idx, tok in enumerate(input_ids) if tok == token_id]
if len(positions) != len(embeds):
raise ValueError(
@@ -1692,11 +1648,8 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
state: ReqState,
is_stream: bool,
) -> Optional[dict]:
"""Handle abort/error finish reasons from the scheduler.
Returns the output dict if it should be yielded (stream abort), or None
for normal flow. Raises ValueError or HTTPException for non-stream aborts.
"""
"""Returns the output dict to yield (stream abort), None for normal flow;
raises ValueError/HTTPException for non-stream aborts."""
finish_reason = out["meta_info"]["finish_reason"]
if (
@@ -3433,17 +3386,6 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
def _should_dispatch_to_encoder(
self, obj: Union[GenerateReqInput, EmbeddingReqInput]
) -> bool:
"""Check if the request should be dispatched to encoder for processing.
Returns True if the request should be dispatched to encoder (multiple multimodal items),
False if it should be processed locally (single multimodal item or no multimodal items).
Args:
obj: The request input object
Returns:
bool: True if should dispatch to encoder, False otherwise
"""
if obj.batch_size > 1:
logger.warning(
"Batch request (batch_size=%d) is not supported in EPD disaggregation mode; skipping encoder dispatch.",
python/sglang/srt/managers/tokenizer_manager_score_mixin.py
+7 -116
View File
@@ -38,18 +38,6 @@ class TokenizerManagerScoreMixin:
This is a thin wrapper over `score_request` that treats `prompts` as
already-composed inputs (i.e., no query/item concatenation needed).
Args:
prompts: A single prompt string, a list of prompt strings, or a list of
pre-tokenized prompt token ID sequences.
label_token_ids: Token IDs to compute probabilities for.
apply_softmax: Whether to normalize probabilities using softmax.
request: Optional FastAPI request object.
Returns:
ScoreResult with:
scores: List of score lists, one for each prompt, each in the order of label_token_ids.
prompt_tokens: The number of prompt tokens processed.
"""
# Text prompts
if isinstance(prompts, str) or (
@@ -83,14 +71,6 @@ class TokenizerManagerScoreMixin:
"""
Build a single token sequence for multi-item scoring.
Format: query<delimiter>item1<delimiter>item2<delimiter>item3<delimiter>
Args:
query: Query token IDs
items: List of item token ID sequences
delimiter_token_id: Token ID to use as delimiter
Returns:
Tuple of (combined token sequence, delimiter indices)
"""
combined_sequence = query[:] # Start with query
delimiter_indices = []
@@ -111,16 +91,6 @@ class TokenizerManagerScoreMixin:
query: Optional[Union[str, List[int]]],
items: Optional[Union[str, List[str], List[List[int]]]],
) -> Tuple[List[int], List[List[int]]]:
"""
Tokenize query and items into token IDs.
Args:
query: The query text (str) or pre-tokenized token IDs (List[int]).
items: Item texts or pre-tokenized token IDs.
Returns:
(query_ids, items_ids): query token IDs and list of per-item token IDs.
"""
if isinstance(query, str):
query_ids = self.tokenizer.encode(query)
else:
@@ -153,20 +123,6 @@ class TokenizerManagerScoreMixin:
populated (input_token_ids_logprobs for generation models,
embedding for classification models), then uniformly validates,
skips the query-boundary delimiter, and normalizes.
Args:
results: Results from generate_request
items: List of items being scored
label_token_ids: Token IDs to extract scores for
apply_softmax: Whether to apply softmax normalization
batch_request: The original batch request containing input sequence
return_pooled_hidden_states: Whether to extract pooled hidden states
from the result and include them in the ScoreResult.
Returns:
ScoreResult with per-item scores, prompt token count, and optional
pooled_hidden_states (when return_pooled_hidden_states=True and the
model populated the field).
"""
single_result = results[0] if isinstance(results, list) else results
meta_info = single_result.get("meta_info", {})
@@ -246,15 +202,6 @@ class TokenizerManagerScoreMixin:
For generation (CausalLM) models: reads output_token_ids_logprobs.
For non-generation (SequenceClassification) models: reads the embedding field
which contains pooled class logits from the classification head.
Args:
results: Results from generate_request
label_token_ids: Token IDs to extract scores for (generation models only)
apply_softmax: Whether to apply softmax normalization
return_pooled_hidden_states: Whether to extract pooled hidden states
Returns:
ScoreResult with per-item scores, prompt token count, and optional pooled_hidden_states.
"""
scores = []
phs_list = []
@@ -329,17 +276,7 @@ class TokenizerManagerScoreMixin:
label: str = "input",
) -> Tuple[List[torch.Tensor], List[int]]:
"""Scan token_ids for placeholder occurrences and pair with embeddings.
Args:
token_ids: The token sequence to scan.
embeds: Embedding tensors to place at placeholder positions (None = skip).
embed_override_token_id: The placeholder token ID.
position_offset: Added to each found position (for absolute coordinates).
label: Label for error messages (e.g. "query", "items[2]").
Returns:
(embeds, positions) lists. Empty lists if embeds is None.
"""
Returns empty lists when embeds is None."""
if embeds is None:
return [], []
positions = [
@@ -365,10 +302,7 @@ class TokenizerManagerScoreMixin:
item_position_offset: int,
item_label: str,
) -> Optional[PositionalEmbeds]:
"""Resolve embed overrides for a single query+item pair.
Returns PositionalEmbeds if any overrides exist, None otherwise.
"""
"""Resolve embed overrides for a query+item pair; None when no overrides exist."""
q_embeds, q_positions = self._resolve_overrides_for_sequence(
query,
query_embed_overrides,
@@ -405,11 +339,8 @@ class TokenizerManagerScoreMixin:
) -> Tuple[None, List[List[int]], Optional[list], Optional[List[int]]]:
"""Build input_ids and resolve embed overrides for token-ID inputs.
Works identically for multi-item-scoring and single-item modes — the only difference is
how input_ids are assembled and what position offset each item gets.
Returns:
(text_prompts, input_ids, positional_embed_overrides, delimiter_indices)
Multi-item-scoring and single-item modes differ only in how input_ids
are assembled and what position offset each item gets.
"""
# Both query and items are token IDs
has_embeds = (
@@ -540,28 +471,8 @@ class TokenizerManagerScoreMixin:
- Generation (CausalLM): Requires label_token_ids; returns logprob-based scores.
- SequenceClassification: label_token_ids is optional; returns pooled class logits.
Args:
query: The query text or pre-tokenized query token IDs
items: The item text(s) or pre-tokenized item token IDs
label_token_ids: List of token IDs to compute probabilities for
apply_softmax: Whether to normalize probabilities using softmax
item_first: If True, prepend items to query. Ignored for multi-item scoring.
embed_override_token_id: Placeholder token ID for embedding override positions.
query_embed_overrides: Embedding vectors replacing placeholder tokens in query.
item_embed_overrides: Per-item embedding vectors replacing placeholder tokens in items.
request: Optional FastAPI request object
return_pooled_hidden_states: Whether to include the raw pooled transformer
hidden states (before the task-specific head) in the result. Only
supported for non-generation models (SequenceClassification,
RewardModel). Raises ValueError for CausalLM models.
Returns:
ScoreResult with:
scores: List of score lists, one per item.
prompt_tokens: The number of prompt tokens processed.
pooled_hidden_states: Per-item CPU tensors when
return_pooled_hidden_states=True and the model supports it;
None otherwise.
return_pooled_hidden_states is only supported for non-generation models
(SequenceClassification, RewardModel); raises ValueError for CausalLM.
"""
is_generation = self.is_generation
@@ -734,17 +645,6 @@ class TokenizerManagerScoreMixin:
label_token_ids: List[int],
apply_softmax: bool,
) -> List[float]:
"""
Convert logprobs dictionary to ordered score list.
Args:
logprobs: Dictionary mapping token_id to logprob
label_token_ids: Token IDs in desired order
apply_softmax: Whether to apply softmax normalization
Returns:
List of scores in the same order as label_token_ids
"""
score_list = [
logprobs.get(token_id, float("-inf")) for token_id in label_token_ids
]
@@ -762,16 +662,7 @@ class TokenizerManagerScoreMixin:
def _extract_logprobs_for_tokens(
self, logprobs_data: List, label_token_ids: List[int]
) -> Dict[int, float]:
"""
Extract logprobs for specified token IDs from logprobs data.
Args:
logprobs_data: List of (logprob, token_id, text) tuples
label_token_ids: Token IDs to extract logprobs for
Returns:
Dictionary mapping token_id to logprob
"""
"""Extract logprobs for label_token_ids from (logprob, token_id, text) tuples."""
logprobs = {}
if logprobs_data:
for logprob, token_id, _ in logprobs_data: